Backup Agent Error node check: get node status: not found

Hello,
I’m currently trying to enable backups for a MongoDB. But the backup agent throws following errors. I get the feeling that this is a permission issue.

2022-06-08T07:48:08.000+0000 I got command backup [name: 2022-06-08T07:48:07Z, compression: gzip (level: 6)] <ts: 1654674487>
2022-06-08T07:48:08.000+0000 I got epoch {1654674474 1}
2022-06-08T07:48:08.000+0000 E [backup/2022-06-08T07:48:07Z] node check: get node status: not found

Also I get a lot of

[agentCheckup] get replSetGetStatus: not found

gist to log

The rs.status() command returns

{
  set: 'rs0',
  date: 2022-06-08T07:46:51.094Z,
  myState: 1,
  term: Long("16"),
  syncSourceHost: '',
  syncSourceId: -1,
  heartbeatIntervalMillis: Long("2000"),
  majorityVoteCount: 1,
  writeMajorityCount: 1,
  votingMembersCount: 1,
  writableVotingMembersCount: 1,
  optimes: {
    lastCommittedOpTime: { ts: Timestamp({ t: 1654674409, i: 1 }), t: Long("16") },
    lastCommittedWallTime: 2022-06-08T07:46:49.009Z,
    readConcernMajorityOpTime: { ts: Timestamp({ t: 1654674409, i: 1 }), t: Long("16") },
    appliedOpTime: { ts: Timestamp({ t: 1654674409, i: 1 }), t: Long("16") },
    durableOpTime: { ts: Timestamp({ t: 1654674409, i: 1 }), t: Long("16") },
    lastAppliedWallTime: 2022-06-08T07:46:49.009Z,
    lastDurableWallTime: 2022-06-08T07:46:49.009Z
  },
  lastStableRecoveryTimestamp: Timestamp({ t: 1654674349, i: 1 }),
  electionCandidateMetrics: {
    lastElectionReason: 'electionTimeout',
    lastElectionDate: 2022-06-08T07:44:03.693Z,
    electionTerm: Long("16"),
    lastCommittedOpTimeAtElection: { ts: Timestamp({ t: 0, i: 0 }), t: Long("-1") },
    lastSeenOpTimeAtElection: { ts: Timestamp({ t: 1654674221, i: 2 }), t: Long("15") },
    numVotesNeeded: 1,
    priorityAtElection: 2,
    electionTimeoutMillis: Long("10000"),
    newTermStartDate: 2022-06-08T07:44:03.725Z,
    wMajorityWriteAvailabilityDate: 2022-06-08T07:44:03.738Z
  },
  members: [
    {
      _id: 0,
      name: '20.0.0.12:27017',
      health: 1,
      state: 1,
      stateStr: 'PRIMARY',
      uptime: 180,
      optime: [Object],
      optimeDate: 2022-06-08T07:46:49.000Z,
      lastAppliedWallTime: 2022-06-08T07:46:49.009Z,
      lastDurableWallTime: 2022-06-08T07:46:49.009Z,
      syncSourceHost: '',
      syncSourceId: -1,
      infoMessage: '',
      electionTime: Timestamp({ t: 1654674243, i: 1 }),
      electionDate: 2022-06-08T07:44:03.000Z,
      configVersion: 32866449,
      configTerm: -1,
      self: true,
      lastHeartbeatMessage: ''
    }
  ],
  ok: 1,
  '$clusterTime': {
    clusterTime: Timestamp({ t: 1654674409, i: 1 }),
    signature: {
      hash: Binary(Buffer.from("4a9a219e1008f161772e56810ac9c01cfa9a0dd9", "hex"), 0),
      keyId: Long("7097192070888030213")
    }
  },
  operationTime: Timestamp({ t: 1654674409, i: 1 })
}
1 Like

I did some more digging:

Running the following in the ‘Backup-Agent’ Container

mongo mongodb://${PBM_AGENT_MONGODB_USERNAME}:${PBM_AGENT_MONGODB_PASSWORD}@127.0.0.1:${PBM_MONGODB_PORT}

returns

{
    "set" : "rs0",
    "date" : ISODate("2022-06-10T08:55:04.099Z"),
    "myState" : 1,
    "term" : NumberLong(17),
    "syncSourceHost" : "",
    "syncSourceId" : -1,
    "heartbeatIntervalMillis" : NumberLong(2000),
    "majorityVoteCount" : 1,
    "writeMajorityCount" : 1,
    "votingMembersCount" : 1,
    "writableVotingMembersCount" : 1,
    "optimes" : {
        "lastCommittedOpTime" : {
            "ts" : Timestamp(1654851301, 2),
            "t" : NumberLong(17)
        },
        "lastCommittedWallTime" : ISODate("2022-06-10T08:55:01.110Z"),
        "readConcernMajorityOpTime" : {
            "ts" : Timestamp(1654851301, 2),
            "t" : NumberLong(17)
        },
        "appliedOpTime" : {
            "ts" : Timestamp(1654851301, 2),
            "t" : NumberLong(17)
        },
        "durableOpTime" : {
            "ts" : Timestamp(1654851301, 2),
            "t" : NumberLong(17)
        },
        "lastAppliedWallTime" : ISODate("2022-06-10T08:55:01.110Z"),
        "lastDurableWallTime" : ISODate("2022-06-10T08:55:01.110Z")
    },
    "lastStableRecoveryTimestamp" : Timestamp(1654851256, 15),
    "electionCandidateMetrics" : {
        "lastElectionReason" : "electionTimeout",
        "lastElectionDate" : ISODate("2022-06-10T08:15:20.888Z"),
        "electionTerm" : NumberLong(17),
        "lastCommittedOpTimeAtElection" : {
            "ts" : Timestamp(0, 0),
            "t" : NumberLong(-1)
        },
        "lastSeenOpTimeAtElection" : {
            "ts" : Timestamp(1654848844, 1),
            "t" : NumberLong(16)
        },
        "numVotesNeeded" : 1,
        "priorityAtElection" : 2,
        "electionTimeoutMillis" : NumberLong(10000),
        "newTermStartDate" : ISODate("2022-06-10T08:15:20.949Z"),
        "wMajorityWriteAvailabilityDate" : ISODate("2022-06-10T08:15:20.956Z")
    },
    "members" : [
        {
            "_id" : 0,
            "name" : "20.0.0.12:27017",
            "health" : 1,
            "state" : 1,
            "stateStr" : "PRIMARY",
            "uptime" : 2449,
            "optime" : {
                "ts" : Timestamp(1654851301, 2),
                "t" : NumberLong(17)
            },
            "optimeDate" : ISODate("2022-06-10T08:55:01Z"),
            "lastAppliedWallTime" : ISODate("2022-06-10T08:55:01.110Z"),
            "lastDurableWallTime" : ISODate("2022-06-10T08:55:01.110Z"),
            "syncSourceHost" : "",
            "syncSourceId" : -1,
            "infoMessage" : "",
            "electionTime" : Timestamp(1654848920, 1),
            "electionDate" : ISODate("2022-06-10T08:15:20Z"),
            "configVersion" : 33021609,
            "configTerm" : -1,
            "self" : true,
            "lastHeartbeatMessage" : ""
        }
    ],
    "ok" : 1,
    "$clusterTime" : {
        "clusterTime" : Timestamp(1654851301, 2),
        "signature" : {
            "hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
            "keyId" : NumberLong(0)
        }
    },
    "operationTime" : Timestamp(1654851301, 2)
}

pbm status --mongodb-url mongodb://${PBM_AGENT_MONGODB_USERNAME}:${PBM_AGENT_MONGODB_PASSWORD}@127.0.0.1:${PBM_MONGODB_PORT}

returns

bash-4.2$ pbm status --mongodb-uri=mongodb://${PBM_AGENT_MONGODB_USERNAME}:${PBM_AGENT_MONGODB_PASSWORD}@127.0.0.1:${PBM_MONGODB_PORT}
Cluster:
========
rs0:
  - rs0/20.0.0.12:27017: pbm-agent NOT FOUND FAILED status:
PITR incremental backup:
========================
Status [OFF]Currently running:
==================
(none)Backups:
========
Azure  https://xxxxxxxx.blob.core.windows.net/xxxxxxx
  (none)
bash-4.2$
1 Like