Using OVF 2.28.0 appliance.
Added a few mongodb nodes using Grafana PMM Inventory “Add instance”.
This is a replicaset with 3 secondaries and 2 of those are setup using “slaveDelay” config:
1 Primary
1 Secondary, slaveDelay 0 sec
1 Secondary, hidden, slaveDelay 3600 sec
1 Secondary, hidden, slaveDelay 7200 sec
In dashboard ReplSet Summary the replication lag however is 2h (7200 sec) for all 3 secondary nodes.
It should be 0 sec for replica01, 3600 sec for replica02 and 7200 sec replica03.
Checking victoriametrics for mongodb_mongod_replset_member_replication_lag it looks like all nodes have entries for the 0, 3600, 7200 seconds latency:
[
{
"metric": {
"__name__": "mongodb_mongod_replset_member_replication_lag",
"agent_id": "/agent_id/021d9edd-5a99-42a3-8db0-51a4f39577b7",
"agent_type": "mongodb_exporter",
"environment": "prd",
"instance": "/agent_id/021d9edd-5a99-42a3-8db0-51a4f39577b7",
"job": "mongodb_exporter_agent_id_021d9edd-5a99-42a3-8db0-51a4f39577b7_hr-5s",
"name": "vlpr-mongodb-replica01.prd.saas.dpp.companyname.network:27017",
"node_id": "/node_id/75421994-ca41-4d35-af5e-23764bf8f0ec",
"node_name": "vlpr-mongodb-replica01.prd.saas.dpp.companyname.network",
"node_type": "remote",
"service_id": "/service_id/4965e5be-2f20-4691-a81e-63b8dcd097ac",
"service_name": "vlpr-mongodb-replica01.prd.saas.dpp.companyname.network",
"service_type": "mongodb",
"set": "rs0",
"state": "SECONDARY"
},
"value": [
1653587227,
"0"
],
"group": 1
},
{
"metric": {
"__name__": "mongodb_mongod_replset_member_replication_lag",
"agent_id": "/agent_id/021d9edd-5a99-42a3-8db0-51a4f39577b7",
"agent_type": "mongodb_exporter",
"environment": "prd",
"instance": "/agent_id/021d9edd-5a99-42a3-8db0-51a4f39577b7",
"job": "mongodb_exporter_agent_id_021d9edd-5a99-42a3-8db0-51a4f39577b7_hr-5s",
"name": "vlpr-mongodb-replica02.prd.saas.dpp.companyname.network:27017",
"node_id": "/node_id/75421994-ca41-4d35-af5e-23764bf8f0ec",
"node_name": "vlpr-mongodb-replica01.prd.saas.dpp.companyname.network",
"node_type": "remote",
"service_id": "/service_id/4965e5be-2f20-4691-a81e-63b8dcd097ac",
"service_name": "vlpr-mongodb-replica01.prd.saas.dpp.companyname.network",
"service_type": "mongodb",
"set": "rs0",
"state": "SECONDARY"
},
"value": [
1653587227,
"3599"
],
"group": 1
},
{
"metric": {
"__name__": "mongodb_mongod_replset_member_replication_lag",
"agent_id": "/agent_id/021d9edd-5a99-42a3-8db0-51a4f39577b7",
"agent_type": "mongodb_exporter",
"environment": "prd",
"instance": "/agent_id/021d9edd-5a99-42a3-8db0-51a4f39577b7",
"job": "mongodb_exporter_agent_id_021d9edd-5a99-42a3-8db0-51a4f39577b7_hr-5s",
"name": "vlpr2-mongodb-replica03.prd.saas.dpp.companyname.network:27017",
"node_id": "/node_id/75421994-ca41-4d35-af5e-23764bf8f0ec",
"node_name": "vlpr-mongodb-replica01.prd.saas.dpp.companyname.network",
"node_type": "remote",
"service_id": "/service_id/4965e5be-2f20-4691-a81e-63b8dcd097ac",
"service_name": "vlpr-mongodb-replica01.prd.saas.dpp.companyname.network",
"service_type": "mongodb",
"set": "rs0",
"state": "SECONDARY"
},
"value": [
1653587227,
"7200"
],
"group": 1
},
{
"metric": {
"__name__": "mongodb_mongod_replset_member_replication_lag",
"agent_id": "/agent_id/3c726eee-32f3-4e49-ac50-43b5211bc314",
"agent_type": "mongodb_exporter",
"environment": "prd",
"instance": "/agent_id/3c726eee-32f3-4e49-ac50-43b5211bc314",
"job": "mongodb_exporter_agent_id_3c726eee-32f3-4e49-ac50-43b5211bc314_hr-5s",
"name": "vlpr-mongodb-replica01.prd.saas.dpp.companyname.network:27017",
"node_id": "/node_id/132ef510-405d-425f-8fe7-ea64db961533",
"node_name": "vlpr-mongodb-replica02.prd.saas.dpp.companyname.network",
"node_type": "remote",
"service_id": "/service_id/3bb6fb9b-85ba-410c-bf23-9f33f605631f",
"service_name": "vlpr-mongodb-replica02.prd.saas.dpp.companyname.network",
"service_type": "mongodb",
"set": "rs0",
"state": "SECONDARY"
},
"value": [
1653587227,
"0"
],
"group": 1
},
{
"metric": {
"__name__": "mongodb_mongod_replset_member_replication_lag",
"agent_id": "/agent_id/3c726eee-32f3-4e49-ac50-43b5211bc314",
"agent_type": "mongodb_exporter",
"environment": "prd",
"instance": "/agent_id/3c726eee-32f3-4e49-ac50-43b5211bc314",
"job": "mongodb_exporter_agent_id_3c726eee-32f3-4e49-ac50-43b5211bc314_hr-5s",
"name": "vlpr-mongodb-replica02.prd.saas.dpp.companyname.network:27017",
"node_id": "/node_id/132ef510-405d-425f-8fe7-ea64db961533",
"node_name": "vlpr-mongodb-replica02.prd.saas.dpp.companyname.network",
"node_type": "remote",
"service_id": "/service_id/3bb6fb9b-85ba-410c-bf23-9f33f605631f",
"service_name": "vlpr-mongodb-replica02.prd.saas.dpp.companyname.network",
"service_type": "mongodb",
"set": "rs0",
"state": "SECONDARY"
},
"value": [
1653587227,
"3593"
],
"group": 1
},
{
"metric": {
"__name__": "mongodb_mongod_replset_member_replication_lag",
"agent_id": "/agent_id/3c726eee-32f3-4e49-ac50-43b5211bc314",
"agent_type": "mongodb_exporter",
"environment": "prd",
"instance": "/agent_id/3c726eee-32f3-4e49-ac50-43b5211bc314",
"job": "mongodb_exporter_agent_id_3c726eee-32f3-4e49-ac50-43b5211bc314_hr-5s",
"name": "vlpr2-mongodb-replica03.prd.saas.dpp.companyname.network:27017",
"node_id": "/node_id/132ef510-405d-425f-8fe7-ea64db961533",
"node_name": "vlpr-mongodb-replica02.prd.saas.dpp.companyname.network",
"node_type": "remote",
"service_id": "/service_id/3bb6fb9b-85ba-410c-bf23-9f33f605631f",
"service_name": "vlpr-mongodb-replica02.prd.saas.dpp.companyname.network",
"service_type": "mongodb",
"set": "rs0",
"state": "SECONDARY"
},
"value": [
1653587227,
"7200"
],
"group": 1
},
{
"metric": {
"__name__": "mongodb_mongod_replset_member_replication_lag",
"agent_id": "/agent_id/d2cff2a8-d420-4e91-a74f-bcc6b65e6d06",
"agent_type": "mongodb_exporter",
"environment": "prd",
"instance": "/agent_id/d2cff2a8-d420-4e91-a74f-bcc6b65e6d06",
"job": "mongodb_exporter_agent_id_d2cff2a8-d420-4e91-a74f-bcc6b65e6d06_hr-5s",
"name": "vlpr-mongodb-replica01.prd.saas.dpp.companyname.network:27017",
"node_id": "/node_id/e429a74f-713d-41fa-b53f-6dc1e2a8a8f5",
"node_name": "vlpr2-mongodb-replica03.prd.saas.dpp.companyname.network",
"node_type": "remote",
"service_id": "/service_id/1f87ff9b-0888-4035-a3ba-9853a31de3a9",
"service_name": "vlpr2-mongodb-replica03.prd.saas.dpp.companyname.network",
"service_type": "mongodb",
"set": "rs0",
"state": "SECONDARY"
},
"value": [
1653587227,
"0"
],
"group": 1
},
{
"metric": {
"__name__": "mongodb_mongod_replset_member_replication_lag",
"agent_id": "/agent_id/d2cff2a8-d420-4e91-a74f-bcc6b65e6d06",
"agent_type": "mongodb_exporter",
"environment": "prd",
"instance": "/agent_id/d2cff2a8-d420-4e91-a74f-bcc6b65e6d06",
"job": "mongodb_exporter_agent_id_d2cff2a8-d420-4e91-a74f-bcc6b65e6d06_hr-5s",
"name": "vlpr-mongodb-replica02.prd.saas.dpp.companyname.network:27017",
"node_id": "/node_id/e429a74f-713d-41fa-b53f-6dc1e2a8a8f5",
"node_name": "vlpr2-mongodb-replica03.prd.saas.dpp.companyname.network",
"node_type": "remote",
"service_id": "/service_id/1f87ff9b-0888-4035-a3ba-9853a31de3a9",
"service_name": "vlpr2-mongodb-replica03.prd.saas.dpp.companyname.network",
"service_type": "mongodb",
"set": "rs0",
"state": "SECONDARY"
},
"value": [
1653587227,
"3599"
],
"group": 1
},
{
"metric": {
"__name__": "mongodb_mongod_replset_member_replication_lag",
"agent_id": "/agent_id/d2cff2a8-d420-4e91-a74f-bcc6b65e6d06",
"agent_type": "mongodb_exporter",
"environment": "prd",
"instance": "/agent_id/d2cff2a8-d420-4e91-a74f-bcc6b65e6d06",
"job": "mongodb_exporter_agent_id_d2cff2a8-d420-4e91-a74f-bcc6b65e6d06_hr-5s",
"name": "vlpr2-mongodb-replica03.prd.saas.dpp.companyname.network:27017",
"node_id": "/node_id/e429a74f-713d-41fa-b53f-6dc1e2a8a8f5",
"node_name": "vlpr2-mongodb-replica03.prd.saas.dpp.companyname.network",
"node_type": "remote",
"service_id": "/service_id/1f87ff9b-0888-4035-a3ba-9853a31de3a9",
"service_name": "vlpr2-mongodb-replica03.prd.saas.dpp.companyname.network",
"service_type": "mongodb",
"set": "rs0",
"state": "SECONDARY"
},
"value": [
1653587227,
"7200"
],
"group": 1
},
{
"metric": {
"__name__": "mongodb_mongod_replset_member_replication_lag",
"agent_id": "/agent_id/ece12141-60f7-43ed-a4f8-25a95e77e81d",
"agent_type": "mongodb_exporter",
"environment": "prd",
"instance": "/agent_id/ece12141-60f7-43ed-a4f8-25a95e77e81d",
"job": "mongodb_exporter_agent_id_ece12141-60f7-43ed-a4f8-25a95e77e81d_hr-5s",
"name": "vlpr-mongodb-replica01.prd.saas.dpp.companyname.network:27017",
"node_id": "/node_id/7a9a4719-47f6-48c7-a991-550eab48d431",
"node_name": "vlpr-mongodb01.prd.saas.dpp.companyname.network",
"node_type": "remote",
"service_id": "/service_id/fc03c7d0-9e2f-4f87-bddd-85cdb7a3cc15",
"service_name": "vlpr-mongodb01.prd.saas.dpp.companyname.network",
"service_type": "mongodb",
"set": "rs0",
"state": "SECONDARY"
},
"value": [
1653587227,
"0"
],
"group": 1
},
{
"metric": {
"__name__": "mongodb_mongod_replset_member_replication_lag",
"agent_id": "/agent_id/ece12141-60f7-43ed-a4f8-25a95e77e81d",
"agent_type": "mongodb_exporter",
"environment": "prd",
"instance": "/agent_id/ece12141-60f7-43ed-a4f8-25a95e77e81d",
"job": "mongodb_exporter_agent_id_ece12141-60f7-43ed-a4f8-25a95e77e81d_hr-5s",
"name": "vlpr-mongodb-replica02.prd.saas.dpp.companyname.network:27017",
"node_id": "/node_id/7a9a4719-47f6-48c7-a991-550eab48d431",
"node_name": "vlpr-mongodb01.prd.saas.dpp.companyname.network",
"node_type": "remote",
"service_id": "/service_id/fc03c7d0-9e2f-4f87-bddd-85cdb7a3cc15",
"service_name": "vlpr-mongodb01.prd.saas.dpp.companyname.network",
"service_type": "mongodb",
"set": "rs0",
"state": "SECONDARY"
},
"value": [
1653587227,
"3599"
],
"group": 1
},
{
"metric": {
"__name__": "mongodb_mongod_replset_member_replication_lag",
"agent_id": "/agent_id/ece12141-60f7-43ed-a4f8-25a95e77e81d",
"agent_type": "mongodb_exporter",
"environment": "prd",
"instance": "/agent_id/ece12141-60f7-43ed-a4f8-25a95e77e81d",
"job": "mongodb_exporter_agent_id_ece12141-60f7-43ed-a4f8-25a95e77e81d_hr-5s",
"name": "vlpr2-mongodb-replica03.prd.saas.dpp.companyname.network:27017",
"node_id": "/node_id/7a9a4719-47f6-48c7-a991-550eab48d431",
"node_name": "vlpr-mongodb01.prd.saas.dpp.companyname.network",
"node_type": "remote",
"service_id": "/service_id/fc03c7d0-9e2f-4f87-bddd-85cdb7a3cc15",
"service_name": "vlpr-mongodb01.prd.saas.dpp.companyname.network",
"service_type": "mongodb",
"set": "rs0",
"state": "SECONDARY"
},
"value": [
1653587227,
"7200"
],
"group": 1
}
]
MongoDB rs.config() returns config with 1 primary & 3 secondaries with 2 secondaries using slaveDelay:
{
"_id" : "rs0",
"version" : 18,
"protocolVersion" : 1,
"writeConcernMajorityJournalDefault" : false,
"members" : [
{
"_id" : 0,
"host" : "vlpr-mongodb01.prd.saas.dpp.companyname.network:27017",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 1,
"tags" : {
},
"slaveDelay" : 0,
"votes" : 1
},
{
"_id" : 1,
"host" : "vlpr-mongodb-replica01.prd.saas.dpp.companyname.network:27017",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 0,
"tags" : {
},
"slaveDelay" : 0,
"votes" : 0
},
{
"_id" : 3,
"host" : "vlpr-mongodb-replica02.prd.saas.dpp.companyname.network:27017",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : true,
"priority" : 0,
"tags" : {
},
"slaveDelay" : 3600,
"votes" : 0
},
{
"_id" : 4,
"host" : "vlpr2-mongodb-replica03.prd.saas.dpp.companyname.network:27017",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : true,
"priority" : 0,
"tags" : {
},
"slaveDelay" : 7200,
"votes" : 0
}
],
"settings" : {
"chainingAllowed" : true,
"heartbeatIntervalMillis" : 2000,
"heartbeatTimeoutSecs" : 10,
"electionTimeoutMillis" : 10000,
"catchUpTimeoutMillis" : -1,
"catchUpTakeoverDelayMillis" : 30000,
"getLastErrorModes" : {
},
"getLastErrorDefaults" : {
"w" : 1,
"wtimeout" : 0
},
"replicaSetId" : ObjectId("5f7f6f725f331ec974e9c55a")
}
}