Backup Agent Error node check: get node status: not found

Hello,
I’m currently trying to enable backups for a MongoDB. But the backup agent throws following errors. I get the feeling that this is a permission issue.

2022-06-08T07:48:08.000+0000 I got command backup [name: 2022-06-08T07:48:07Z, compression: gzip (level: 6)] <ts: 1654674487>
2022-06-08T07:48:08.000+0000 I got epoch {1654674474 1}
2022-06-08T07:48:08.000+0000 E [backup/2022-06-08T07:48:07Z] node check: get node status: not found

Also I get a lot of

[agentCheckup] get replSetGetStatus: not found

gist to log

The rs.status() command returns

{
  set: 'rs0',
  date: 2022-06-08T07:46:51.094Z,
  myState: 1,
  term: Long("16"),
  syncSourceHost: '',
  syncSourceId: -1,
  heartbeatIntervalMillis: Long("2000"),
  majorityVoteCount: 1,
  writeMajorityCount: 1,
  votingMembersCount: 1,
  writableVotingMembersCount: 1,
  optimes: {
    lastCommittedOpTime: { ts: Timestamp({ t: 1654674409, i: 1 }), t: Long("16") },
    lastCommittedWallTime: 2022-06-08T07:46:49.009Z,
    readConcernMajorityOpTime: { ts: Timestamp({ t: 1654674409, i: 1 }), t: Long("16") },
    appliedOpTime: { ts: Timestamp({ t: 1654674409, i: 1 }), t: Long("16") },
    durableOpTime: { ts: Timestamp({ t: 1654674409, i: 1 }), t: Long("16") },
    lastAppliedWallTime: 2022-06-08T07:46:49.009Z,
    lastDurableWallTime: 2022-06-08T07:46:49.009Z
  },
  lastStableRecoveryTimestamp: Timestamp({ t: 1654674349, i: 1 }),
  electionCandidateMetrics: {
    lastElectionReason: 'electionTimeout',
    lastElectionDate: 2022-06-08T07:44:03.693Z,
    electionTerm: Long("16"),
    lastCommittedOpTimeAtElection: { ts: Timestamp({ t: 0, i: 0 }), t: Long("-1") },
    lastSeenOpTimeAtElection: { ts: Timestamp({ t: 1654674221, i: 2 }), t: Long("15") },
    numVotesNeeded: 1,
    priorityAtElection: 2,
    electionTimeoutMillis: Long("10000"),
    newTermStartDate: 2022-06-08T07:44:03.725Z,
    wMajorityWriteAvailabilityDate: 2022-06-08T07:44:03.738Z
  },
  members: [
    {
      _id: 0,
      name: '20.0.0.12:27017',
      health: 1,
      state: 1,
      stateStr: 'PRIMARY',
      uptime: 180,
      optime: [Object],
      optimeDate: 2022-06-08T07:46:49.000Z,
      lastAppliedWallTime: 2022-06-08T07:46:49.009Z,
      lastDurableWallTime: 2022-06-08T07:46:49.009Z,
      syncSourceHost: '',
      syncSourceId: -1,
      infoMessage: '',
      electionTime: Timestamp({ t: 1654674243, i: 1 }),
      electionDate: 2022-06-08T07:44:03.000Z,
      configVersion: 32866449,
      configTerm: -1,
      self: true,
      lastHeartbeatMessage: ''
    }
  ],
  ok: 1,
  '$clusterTime': {
    clusterTime: Timestamp({ t: 1654674409, i: 1 }),
    signature: {
      hash: Binary(Buffer.from("4a9a219e1008f161772e56810ac9c01cfa9a0dd9", "hex"), 0),
      keyId: Long("7097192070888030213")
    }
  },
  operationTime: Timestamp({ t: 1654674409, i: 1 })
}
2 Likes

I did some more digging:

Running the following in the ‘Backup-Agent’ Container

mongo mongodb://${PBM_AGENT_MONGODB_USERNAME}:${PBM_AGENT_MONGODB_PASSWORD}@127.0.0.1:${PBM_MONGODB_PORT}

returns

{
    "set" : "rs0",
    "date" : ISODate("2022-06-10T08:55:04.099Z"),
    "myState" : 1,
    "term" : NumberLong(17),
    "syncSourceHost" : "",
    "syncSourceId" : -1,
    "heartbeatIntervalMillis" : NumberLong(2000),
    "majorityVoteCount" : 1,
    "writeMajorityCount" : 1,
    "votingMembersCount" : 1,
    "writableVotingMembersCount" : 1,
    "optimes" : {
        "lastCommittedOpTime" : {
            "ts" : Timestamp(1654851301, 2),
            "t" : NumberLong(17)
        },
        "lastCommittedWallTime" : ISODate("2022-06-10T08:55:01.110Z"),
        "readConcernMajorityOpTime" : {
            "ts" : Timestamp(1654851301, 2),
            "t" : NumberLong(17)
        },
        "appliedOpTime" : {
            "ts" : Timestamp(1654851301, 2),
            "t" : NumberLong(17)
        },
        "durableOpTime" : {
            "ts" : Timestamp(1654851301, 2),
            "t" : NumberLong(17)
        },
        "lastAppliedWallTime" : ISODate("2022-06-10T08:55:01.110Z"),
        "lastDurableWallTime" : ISODate("2022-06-10T08:55:01.110Z")
    },
    "lastStableRecoveryTimestamp" : Timestamp(1654851256, 15),
    "electionCandidateMetrics" : {
        "lastElectionReason" : "electionTimeout",
        "lastElectionDate" : ISODate("2022-06-10T08:15:20.888Z"),
        "electionTerm" : NumberLong(17),
        "lastCommittedOpTimeAtElection" : {
            "ts" : Timestamp(0, 0),
            "t" : NumberLong(-1)
        },
        "lastSeenOpTimeAtElection" : {
            "ts" : Timestamp(1654848844, 1),
            "t" : NumberLong(16)
        },
        "numVotesNeeded" : 1,
        "priorityAtElection" : 2,
        "electionTimeoutMillis" : NumberLong(10000),
        "newTermStartDate" : ISODate("2022-06-10T08:15:20.949Z"),
        "wMajorityWriteAvailabilityDate" : ISODate("2022-06-10T08:15:20.956Z")
    },
    "members" : [
        {
            "_id" : 0,
            "name" : "20.0.0.12:27017",
            "health" : 1,
            "state" : 1,
            "stateStr" : "PRIMARY",
            "uptime" : 2449,
            "optime" : {
                "ts" : Timestamp(1654851301, 2),
                "t" : NumberLong(17)
            },
            "optimeDate" : ISODate("2022-06-10T08:55:01Z"),
            "lastAppliedWallTime" : ISODate("2022-06-10T08:55:01.110Z"),
            "lastDurableWallTime" : ISODate("2022-06-10T08:55:01.110Z"),
            "syncSourceHost" : "",
            "syncSourceId" : -1,
            "infoMessage" : "",
            "electionTime" : Timestamp(1654848920, 1),
            "electionDate" : ISODate("2022-06-10T08:15:20Z"),
            "configVersion" : 33021609,
            "configTerm" : -1,
            "self" : true,
            "lastHeartbeatMessage" : ""
        }
    ],
    "ok" : 1,
    "$clusterTime" : {
        "clusterTime" : Timestamp(1654851301, 2),
        "signature" : {
            "hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
            "keyId" : NumberLong(0)
        }
    },
    "operationTime" : Timestamp(1654851301, 2)
}

pbm status --mongodb-url mongodb://${PBM_AGENT_MONGODB_USERNAME}:${PBM_AGENT_MONGODB_PASSWORD}@127.0.0.1:${PBM_MONGODB_PORT}

returns

bash-4.2$ pbm status --mongodb-uri=mongodb://${PBM_AGENT_MONGODB_USERNAME}:${PBM_AGENT_MONGODB_PASSWORD}@127.0.0.1:${PBM_MONGODB_PORT}
Cluster:
========
rs0:
  - rs0/20.0.0.12:27017: pbm-agent NOT FOUND FAILED status:
PITR incremental backup:
========================
Status [OFF]Currently running:
==================
(none)Backups:
========
Azure  https://xxxxxxxx.blob.core.windows.net/xxxxxxx
  (none)
bash-4.2$
1 Like

For all that come across this post: I had a similar issue with one node of my replicaSet not being recognized by pbm. For me, changing the MongoDB URI from localhost to the nodes IP and restarting pbm did the trick.
Maybe, in general restarting pbm via sudo systemctl restart pbm-agent might work.