Does anyone have a proper working Configuration Percona Server for MongoDB cross-site replication for OpenShift? Perhaps something in GitHub or Gist? The current documentation is very vague. Any help us appreciated.
1 Like
Hi @Wayne_Dovey,
Apart from the docs we have two blog posts:
They’re not specific to Openshift but they may help. If you have any specific problems, please let us know.
2 Likes
Magic! Thanks mate. I have tried the config but I now get the following error on all our my replica site nodes. Any ideas?
"_id" : 5,
"name" : "172.132.89.162:27017",
"health" : 0,
"state" : 8,
"stateStr" : "(not reachable/healthy)",
"uptime" : 0,
"optime" : {
"ts" : Timestamp(0, 0),
"t" : NumberLong(-1)
},
"optimeDurable" : {
"ts" : Timestamp(0, 0),
"t" : NumberLong(-1)
},
"optimeDate" : ISODate("1970-01-01T00:00:00Z"),
"optimeDurableDate" : ISODate("1970-01-01T00:00:00Z"),
"lastHeartbeat" : ISODate("2021-11-04T11:52:29.249Z"),
"lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"),
"pingMs" : NumberLong(0),
"lastHeartbeatMessage" : "replica set IDs do not match, ours: 6183c202955df555e0b64ce0; remote node's: 6183c398459fbd2de4645c09",
1 Like
Could you please share the cr.yaml for each environment? If I have to guess I’d say you haven’t deployed the cluster with unmanaged: true in the replica side.
1 Like
Thanks, I think that might be the issue but please checkout my config
1 Like
The configs look OK to me but the error you posted before means two cluster has their own ReplicaSet. When you deploy an unmanaged cluster, the operator won’t create a replica set config and just wait it to be configured externally.
1 Like
So no luck. Do you know if the AWS Loadbalancer works?
my replicas seem to be unhealthy
{
"_id" : 3,
"name" : "a4152077f49044a6381e3870fae3e9be-1830102077.ap-southeast-2.elb.amazonaws.com:27017",
"health" : 0,
"state" : 6,
"stateStr" : "(not reachable/healthy)",
"uptime" : 0,
"optime" : {
"ts" : Timestamp(0, 0),
"t" : NumberLong(-1)
},
"optimeDurable" : {
"ts" : Timestamp(0, 0),
"t" : NumberLong(-1)
},
"optimeDate" : ISODate("1970-01-01T00:00:00Z"),
"optimeDurableDate" : ISODate("1970-01-01T00:00:00Z"),
"lastHeartbeat" : ISODate("2021-11-05T12:44:41.289Z"),
"lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"),
"pingMs" : NumberLong(0),
"lastHeartbeatMessage" : "",
"authenticated" : false,
"syncSourceHost" : "",
"syncSourceId" : -1,
"infoMessage" : "",
"configVersion" : -1,
"configTerm" : -1
},
{
"_id" : 4,
"name" : "a74b438bcf55847da88acf544445b946-790054600.ap-southeast-2.elb.amazonaws.com:27017",
"health" : 0,
"state" : 6,
"stateStr" : "(not reachable/healthy)",
"uptime" : 0,
"optime" : {
"ts" : Timestamp(0, 0),
"t" : NumberLong(-1)
},
"optimeDurable" : {
"ts" : Timestamp(0, 0),
"t" : NumberLong(-1)
},
"optimeDate" : ISODate("1970-01-01T00:00:00Z"),
"optimeDurableDate" : ISODate("1970-01-01T00:00:00Z"),
"lastHeartbeat" : ISODate("2021-11-05T12:44:43.130Z"),
"lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"),
"pingMs" : NumberLong(0),
"lastHeartbeatMessage" : "",
"authenticated" : false,
"syncSourceHost" : "",
"syncSourceId" : -1,
"infoMessage" : "",
"configVersion" : -1,
"configTerm" : -1
},
{
"_id" : 5,
"name" : "a0a49c3809d5345a986d4d41fc88f03d-638471580.ap-southeast-2.elb.amazonaws.com:27017",
"health" : 0,
"state" : 6,
"stateStr" : "(not reachable/healthy)",
"uptime" : 0,
"optime" : {
"ts" : Timestamp(0, 0),
"t" : NumberLong(-1)
},
"optimeDurable" : {
"ts" : Timestamp(0, 0),
"t" : NumberLong(-1)
},
"optimeDate" : ISODate("1970-01-01T00:00:00Z"),
"optimeDurableDate" : ISODate("1970-01-01T00:00:00Z"),
"lastHeartbeat" : ISODate("2021-11-05T12:44:41.598Z"),
"lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"),
"pingMs" : NumberLong(0),
"lastHeartbeatMessage" : "",
"authenticated" : false,
"syncSourceHost" : "",
"syncSourceId" : -1,
"infoMessage" : "",
"configVersion" : -1,
"configTerm" : -1
}
],
1 Like
Logs of one of the Mongos…
.elb.amazonaws.com:27017"}}
{"t":{"$date":"2021-11-05T12:46:29.253+00:00"},"s":"I", "c":"-", "id":4333222, "ctx":"ReplicaSetMonitor-TaskExecutor","msg":"RSM received failed isMaster","attr":{"host":"a9606f3cd081c4f8eb002265184ada67-1106207869.ap-southeast-2.elb.amazonaws.com:27017","error":"NetworkInterfaceExceededTimeLimit: Couldn't get a connection within the time limit of 731ms","replicaSet":"cfg","isMasterReply":"{}"}}
{"t":{"$date":"2021-11-05T12:46:29.253+00:00"},"s":"I", "c":"NETWORK", "id":4712102, "ctx":"ReplicaSetMonitor-TaskExecutor","msg":"Host failed in replica set","attr":{"replicaSet":"cfg","host":"a9606f3cd081c4f8eb002265184ada67-1106207869.ap-southeast-2.elb.amazonaws.com:27017","error":{"code":202,"codeName":"NetworkInterfaceExceededTimeLimit","errmsg":"Couldn't get a connection within the time limit of 731ms"},"action":{"dropConnections":false,"requestImmediateCheck":false,"outcome":{"host":"a9606f3cd081c4f8eb002265184ada67-1106207869.ap-southeast-2.elb.amazonaws.com:27017","success":false,"errorMessage":"NetworkInterfaceExceededTimeLimit: Couldn't get a connection within the time limit of 731ms"}}}}
{"t":{"$date":"2021-11-05T12:46:29.253+00:00"},"s":"I", "c":"NETWORK", "id":4333213, "ctx":"ReplicaSetMonitor-TaskExecutor","msg":"RSM Topology Change","attr":{"replicaSet":"cfg","newTopologyDescription":"{ id: \"014ce674-2bc6-4196-8781-89c4de4a8896\", topologyType: \"Unknown\", servers: { a1185a8f27fa848bba2c35a4bf4ad973-1414717655.ap-southeast-2.elb.amazonaws.com:27017: { address: \"a1185a8f27fa848bba2c35a4bf4ad973-1414717655.ap-southeast-2.elb.amazonaws.com:27017\", topologyVersion: { processId: ObjectId('61852761853e43b580f8660d'), counter: 0 }, roundTripTime: 228187, type: \"RSGhost\", minWireVersion: 0, maxWireVersion: 9, lastUpdateTime: new Date(1636116387048), logicalSessionTimeoutMinutes: 30, hosts: {}, arbiters: {}, passives: {} }, a347a759e31444ae2b62a46a734bbc8f-1343586652.ap-southeast-2.elb.amazonaws.com:27017: { address: \"a347a759e31444ae2b62a46a734bbc8f-1343586652.ap-southeast-2.elb.amazonaws.com:27017\", topologyVersion: { processId: ObjectId('6185277edcfc77066db01ae5'), counter: 0 }, roundTripTime: 90302, type: \"RSGhost\", minWireVersion: 0, maxWireVersion: 9, lastUpdateTime: new Date(1636116387046), logicalSessionTimeoutMinutes: 30, hosts: {}, arbiters: {}, passives: {} }, a9606f3cd081c4f8eb002265184ada67-1106207869.ap-southeast-2.elb.amazonaws.com:27017: { address: \"a9606f3cd081c4f8eb002265184ada67-1106207869.ap-southeast-2.elb.amazonaws.com:27017\", type: \"Unknown\", minWireVersion: 0, maxWireVersion: 0, lastUpdateTime: new Date(-9223372036854775808), hosts: {}, arbiters: {}, passives: {} } }, compatible: true }","previousTopologyDescription":"{ id: \"014ce674-2bc6-4196-8781-89c4de4a8896\", topologyType: \"Unknown\", servers: { a1185a8f27fa848bba2c35a4bf4ad973-1414717655.ap-southeast-2.elb.amazonaws.com:27017: { address: \"a1185a8f27fa848bba2c35a4bf4ad973-1414717655.ap-southeast-2.elb.amazonaws.com:27017\", topologyVersion: { processId: ObjectId('61852761853e43b580f8660d'), counter: 0 }, roundTripTime: 228187, type: \"RSGhost\", minWireVersion: 0, maxWireVersion: 9, lastUpdateTime: new Date(1636116387048), logicalSessionTimeoutMinutes: 30, hosts: {}, arbiters: {}, passives: {} }, a347a759e31444ae2b62a46a734bbc8f-1343586652.ap-southeast-2.elb.amazonaws.com:27017: { address: \"a347a759e31444ae2b62a46a734bbc8f-1343586652.ap-southeast-2.elb.amazonaws.com:27017\", topologyVersion: { processId: ObjectId('6185277edcfc77066db01ae5'), counter: 0 }, roundTripTime: 90302, type: \"RSGhost\", minWireVersion: 0, maxWireVersion: 9, lastUpdateTime: new Date(1636116387046), logicalSessionTimeoutMinutes: 30, hosts: {}, arbiters: {}, passives: {} }, a9606f3cd081c4f8eb002265184ada67-1106207869.ap-southeast-2.elb.amazonaws.com:27017: { address: \"a9606f3cd081c4f8eb002265184ada67-1106207869.ap-southeast-2.elb.amazonaws.com:27017\", topologyVersion: { processId: ObjectId('618526eabd389c9baac893f3'), counter: 0 }, roundTripTime: 277092, type: \"RSGhost\", minWireVersion: 0, maxWireVersion: 9, lastUpdateTime: new Date(1636116377524), logicalSessionTimeoutMinutes: 30, hosts: {}, arbiters: {}, passives: {} } }, compatible: true }"}}
{"t":{"$date":"2021-11-05T12:46:32.663+00:00"},"s":"W", "c":"NETWORK", "id":23238, "ctx":"ReplicaSetMonitor-TaskExecutor","msg":"The server certificate does not match the remote host name","attr":{"remoteHost":"a9606f3cd081c4f8eb002265184ada67-1106207869.ap-southeast-2.elb.amazonaws.com","certificateNames":"SAN(s): localhost, sydney-cluster-rs0, sydney-cluster-rs0.pacman-app, sydney-cluster-rs0.pacman-app.svc.cluster.local, *.sydney-cluster-rs0, *.sydney-cluster-rs0.pacman-app, *.sydney-cluster-rs0.pacman-app.svc.cluster.local, sydney-cluster-mongos, sydney-cluster-mongos.pacman-app, sydney-cluster-mongos.pacman-app.svc.cluster.local, *.sydney-cluster-mongos, *.sydney-cluster-mongos.pacman-app, *.sydney-cluster-mongos.pacman-app.svc.cluster.local, sydney-cluster-cfg, sydney-cluster-cfg.pacman-app, sydney-cluster-cfg.pacman-app.svc.cluster.local, *.sydney-cluster-cfg, *.sydney-cluster-cfg.pacman-app, *.sydney-cluster-cfg.pacman-app.svc.cluster.local, "}}
{"t":{"$date":"2021-11-05T12:46:32.665+00:00"},"s":"I", "c":"NETWORK", "id":4333213, "ctx":"ReplicaSetMonitor-TaskExecutor","msg":"RSM Topology Change","attr":{"replicaSet":"cfg","newTopologyDescription":"{ id: \"014ce674-2bc6-4196-8781-89c4de4a8896\", topologyType: \"Unknown\", servers: { a1185a8f27fa848bba2c35a4bf4ad973-1414717655.ap-southeast-2.elb.amazonaws.com:27017: { address: \"a1185a8f27fa848bba2c35a4bf4ad973-1414717655.ap-southeast-2.elb.amazonaws.com:27017\", topologyVersion: { processId: ObjectId('61852761853e43b580f8660d'), counter: 0 }, roundTripTime: 228187, type: \"RSGhost\", minWireVersion: 0, maxWireVersion: 9, lastUpdateTime: new Date(1636116387048), logicalSessionTimeoutMinutes: 30, hosts: {}, arbiters: {}, passives: {} }, a347a759e31444ae2b62a46a734bbc8f-1343586652.ap-southeast-2.elb.amazonaws.com:27017: { address: \"a347a759e31444ae2b62a46a734bbc8f-1343586652.ap-southeast-2.elb.amazonaws.com:27017\", topologyVersion: { processId: ObjectId('6185277edcfc77066db01ae5'), counter: 0 }, roundTripTime: 90302, type: \"RSGhost\", minWireVersion: 0, maxWireVersion: 9, lastUpdateTime: new Date(1636116387046), logicalSessionTimeoutMinutes: 30, hosts: {}, arbiters: {}, passives: {} }, a9606f3cd081c4f8eb002265184ada67-1106207869.ap-southeast-2.elb.amazonaws.com:27017: { address: \"a9606f3cd081c4f8eb002265184ada67-1106207869.ap-southeast-2.elb.amazonaws.com:27017\", topologyVersion: { processId: ObjectId('6185279ee8fef90821c0f09c'), counter: 0 }, roundTripTime: 1229, type: \"RSGhost\", minWireVersion: 0, maxWireVersion: 9, lastUpdateTime: new Date(1636116392665), logicalSessionTimeoutMinutes: 30, hosts: {}, arbiters: {}, passives: {} } }, compatible: true }","previousTopologyDescription":"{ id: \"014ce674-2bc6-4196-8781-89c4de4a8896\", topologyType: \"Unknown\", servers: { a1185a8f27fa848bba2c35a4bf4ad973-1414717655.ap-southeast-2.elb.amazonaws.com:27017: { address: \"a1185a8f27fa848bba2c35a4bf4ad973-1414717655.ap-southeast-2.elb.amazonaws.com:27017\", topologyVersion: { processId: ObjectId('61852761853e43b580f8660d'), counter: 0 }, roundTripTime: 228187, type: \"RSGhost\", minWireVersion: 0, maxWireVersion: 9, lastUpdateTime: new Date(1636116387048), logicalSessionTimeoutMinutes: 30, hosts: {}, arbiters: {}, passives: {} }, a347a759e31444ae2b62a46a734bbc8f-1343586652.ap-southeast-2.elb.amazonaws.com:27017: { address: \"a347a759e31444ae2b62a46a734bbc8f-1343586652.ap-southeast-2.elb.amazonaws.com:27017\", topologyVersion: { processId: ObjectId('6185277edcfc77066db01ae5'), counter: 0 }, roundTripTime: 90302, type: \"RSGhost\", minWireVersion: 0, maxWireVersion: 9, lastUpdateTime: new Date(1636116387046), logicalSessionTimeoutMinutes: 30, hosts: {}, arbiters: {}, passives: {} }, a9606f3cd081c4f8eb002265184ada67-1106207869.ap-southeast-2.elb.amazonaws.com:27017: { address: \"a9606f3cd081c4f8eb002265184ada67-1106207869.ap-southeast-2.elb.amazonaws.com:27017\", type: \"Unknown\", minWireVersion: 0, maxWireVersion: 0, lastUpdateTime: new Date(-9223372036854775808), hosts: {}, arbiters: {}, passives: {} } }, compatible: true }"}}
{"t":{"$date":"2021-11-05T12:46:33.593+00:00"},"s":"W", "c":"NETWORK", "id":23238, "ctx":"ReplicaSetMonitor-TaskExecutor","msg":"The server certificate does not match the remote host name","attr":{"remoteHost":"a9606f3cd081c4f8eb002265184ada67-1106207869.ap-southeast-2.elb.amazonaws.com","certificateNames":"SAN(s): localhost, sydney-cluster-rs0, sydney-cluster-rs0.pacman-app, sydney-cluster-rs0.pacman-app.svc.cluster.local, *.sydney-cluster-rs0, *.sydney-cluster-rs0.pacman-app, *.sydney-cluster-rs0.pacman-app.svc.cluster.local, sydney-cluster-mongos, sydney-cluster-mongos.pacman-app, sydney-cluster-mongos.pacman-app.svc.cluster.local, *.sydney-cluster-mongos, *.sydney-cluster-mongos.pacman-app, *.sydney-cluster-mongos.pacman-app.svc.cluster.local, sydney-cluster-cfg, sydney-cluster-cfg.pacman-app, sydney-cluster-cfg.pacman-app.svc.cluster.local, *.sydney-cluster-cfg, *.sydney-cluster-cfg.pacman-app, *.sydney-cluster-cfg.pacman-app.svc.cluster.local, "}}
{"t":{"$date":"2021-11-05T12:46:36.332+00:00"},"s":"I", "c":"NETWORK", "id":4333208, "ctx":"ReplicaSetMonitor-TaskExecutor","msg":"RSM host selection timeout","attr":{"replicaSet":"cfg","error":"FailedToSatisfyReadPreference: Could not find host matching read preference { mode: \"nearest\" } for set cfg"}}
{"t":{"$date":"2021-11-05T12:46:36.332+00:00"},"s":"I", "c":"NETWORK", "id":4333208, "ctx":"ReplicaSetMonitor-TaskExecutor","msg":"RSM host selection timeout","attr":{"replicaSet":"cfg","error":"FailedToSatisfyReadPreference: Could not find host matching read preference { mode: \"nearest\" } for set cfg"}}
{"t":{"$date":"2021-11-05T12:46:36.332+00:00"},"s":"I", "c":"NETWORK", "id":4333208, "ctx":"ReplicaSetMonitor-TaskExecutor","msg":"RSM host selection timeout","attr":{"replicaSet":"cfg","error":"FailedToSatisfyReadPreference: Could not find host matching read preference { mode: \"nearest\" } for set cfg"}}
{"t":{"$date":"2021-11-05T12:46:36.332+00:00"},"s":"I", "c":"NETWORK", "id":4333208, "ctx":"ReplicaSetMonitor-TaskExecutor","msg":"RSM host selection timeout","attr":{"replicaSet":"cfg","error":"FailedToSatisfyReadPreference: Could not find host matching read preference { mode: \"primary\" } for set cfg"}}
{"t":{"$date":"2021-11-05T12:46:36.332+00:00"},"s":"I", "c":"SHARDING", "id":22727, "ctx":"shard-registry-reload","msg":"Error running periodic reload of shard registry","attr":{"error":"FailedToSatisfyReadPreference: could not get updated shard list from config server :: caused by :: Could not find host matching read preference { mode: \"nearest\" } for set cfg","shardRegistryReloadIntervalSeconds":30}}
{"t":{"$date":"2021-11-05T12:46:36.332+00:00"},"s":"W", "c":"SHARDING", "id":23834, "ctx":"mongosMain","msg":"Error initializing sharding state, sleeping for 2 seconds and retrying","attr":{"error":{"code":133,"codeName":"FailedToSatisfyReadPreference","errmsg":"Error loading clusterID :: caused by :: Could not find host matching read preference { mode: \"nearest\" } for set cfg"}}}
{"t":{"$date":"2021-11-05T12:46:36.332+00:00"},"s":"W", "c":"SHARDING", "id":22668, "ctx":"replSetDistLockPinger","msg":"Pinging failed for distributed lock pinger","attr":{"error":{"code":133,"codeName":"FailedToSatisfyReadPreference","errmsg":"Could not find host matching read preference { mode: \"primary\" } for set cfg"}}}
{"t":{"$date":"2021-11-05T12:46:56.532+00:00"},"s":"I", "c":"NETWORK", "id":4333208, "ctx":"ReplicaSetMonitor-TaskExecutor","msg":"RSM host selection timeout","attr":{"replicaSet":"cfg","error":"FailedToSatisfyReadPreference: Could not find host matching read preference { mode: \"nearest\" } for set cfg"}}
{"t":{"$date":"2021-11-05T12:46:58.332+00:00"},"s":"I", "c":"NETWORK", "id":4333208, "ctx":"ReplicaSetMonitor-TaskExecutor","msg":"RSM host selection timeout","attr":{"replicaSet":"cfg","error":"FailedToSatisfyReadPreference: Could not find host matching read preference { mode: \"nearest\" } for set cfg"}}
{"t":{"$date":"2021-11-05T12:46:58.332+00:00"},"s":"W", "c":"SHARDING", "id":23834, "ctx":"mongosMain","msg":"Error initializing sharding state, sleeping for 2 seconds and retrying","attr":{"error":{"code":133,"codeName":"FailedToSatisfyReadPreference","errmsg":"Error loading clusterID :: caused by :: Could not find host matching read preference { mode: \"nearest\" } for set cfg"}}}
1 Like
I’m facing the same issue did you fix it please