NAME READY STATUS RESTARTS AGE minimal-cluster-cfg-0 1/1 Running 122 (7m21s ago) 10h minimal-cluster-mongos-0 0/1 Running 2 (10h ago) 10h minimal-cluster-rs0-0 1/1 Running 122 (7m21s ago) 10h mongo-dev-cfg-0 0/1 CrashLoopBackOff 111 (108s ago) 9h mongo-dev-mongos-0 0/1 Running 0 9h mongo-dev-rs0-0 0/1 CrashLoopBackOff 111 (108s ago) 9h minimal-cluster-cfg-0 {"t":{"$date":"2023-04-06T23:22:00.521+00:00"},"s":"I", "c":"QUERY", "id":6278516, "ctx":"SignalHandler","msg":"Finished shutting down Change Stream Expired Pre-images Remover thread"} {"t":{"$date":"2023-04-06T23:22:00.521+00:00"},"s":"I", "c":"CONTROL", "id":4784929, "ctx":"SignalHandler","msg":"Acquiring the global lock for shutdown"} {"t":{"$date":"2023-04-06T23:22:00.521+00:00"},"s":"I", "c":"CONTROL", "id":4784930, "ctx":"SignalHandler","msg":"Shutting down the storage engine"} {"t":{"$date":"2023-04-06T23:22:00.521+00:00"},"s":"I", "c":"STORAGE", "id":22320, "ctx":"SignalHandler","msg":"Shutting down journal flusher thread"} {"t":{"$date":"2023-04-06T23:22:00.521+00:00"},"s":"I", "c":"STORAGE", "id":22321, "ctx":"SignalHandler","msg":"Finished shutting down journal flusher thread"} {"t":{"$date":"2023-04-06T23:22:00.521+00:00"},"s":"I", "c":"STORAGE", "id":22322, "ctx":"SignalHandler","msg":"Shutting down checkpoint thread"} {"t":{"$date":"2023-04-06T23:22:00.521+00:00"},"s":"I", "c":"STORAGE", "id":22323, "ctx":"SignalHandler","msg":"Finished shutting down checkpoint thread"} {"t":{"$date":"2023-04-06T23:22:00.521+00:00"},"s":"I", "c":"STORAGE", "id":20282, "ctx":"SignalHandler","msg":"Deregistering all the collections"} {"t":{"$date":"2023-04-06T23:22:00.521+00:00"},"s":"I", "c":"STORAGE", "id":22372, "ctx":"OplogVisibilityThread","msg":"Oplog visibility thread shutting down."} {"t":{"$date":"2023-04-06T23:22:00.522+00:00"},"s":"I", "c":"STORAGE", "id":22261, "ctx":"SignalHandler","msg":"Timestamp monitor shutting down"} {"t":{"$date":"2023-04-06T23:22:00.522+00:00"},"s":"I", "c":"STORAGE", "id":22317, "ctx":"SignalHandler","msg":"WiredTigerKVEngine shutting down"} {"t":{"$date":"2023-04-06T23:22:00.522+00:00"},"s":"I", "c":"STORAGE", "id":22318, "ctx":"SignalHandler","msg":"Shutting down session sweeper thread"} {"t":{"$date":"2023-04-06T23:22:00.522+00:00"},"s":"I", "c":"STORAGE", "id":22319, "ctx":"SignalHandler","msg":"Finished shutting down session sweeper thread"} {"t":{"$date":"2023-04-06T23:22:00.523+00:00"},"s":"I", "c":"STORAGE", "id":4795902, "ctx":"SignalHandler","msg":"Closing WiredTiger","attr":{"closeConfig":"leak_memory=true,"}} {"t":{"$date":"2023-04-06T23:22:00.561+00:00"},"s":"I", "c":"STORAGE", "id":4795901, "ctx":"SignalHandler","msg":"WiredTiger closed","attr":{"durationMillis":38}} {"t":{"$date":"2023-04-06T23:22:00.582+00:00"},"s":"I", "c":"STORAGE", "id":22279, "ctx":"SignalHandler","msg":"shutdown: removing fs lock..."} {"t":{"$date":"2023-04-06T23:22:00.582+00:00"},"s":"I", "c":"-", "id":4784931, "ctx":"SignalHandler","msg":"Dropping the scope cache for shutdown"} {"t":{"$date":"2023-04-06T23:22:00.582+00:00"},"s":"I", "c":"FTDC", "id":20626, "ctx":"SignalHandler","msg":"Shutting down full-time diagnostic data capture"} {"t":{"$date":"2023-04-06T23:22:00.585+00:00"},"s":"I", "c":"CONTROL", "id":20565, "ctx":"SignalHandler","msg":"Now exiting"} {"t":{"$date":"2023-04-06T23:22:00.585+00:00"},"s":"I", "c":"CONTROL", "id":23138, "ctx":"SignalHandler","msg":"Shutting down","attr":{"exitCode":0}} minimal-cluster-mongos-0 {"t":{"$date":"2023-04-06T23:25:36.110+00:00"},"s":"I", "c":"NETWORK", "id":51800, "ctx":"conn187853","msg":"client metadata","attr":{"remote":"127.0.0.1:51638","client":"conn187853","doc":{"driver":{"name":"mongo-go-driver","version":"v1.11.2"},"os":{"type":"linux","architecture":"amd64"},"platform":"go1.19.6","mongos":{"host":"minimal-cluster-mongos-0:27017","client":"127.0.0.1:51638","version":"6.0.4-3"}}}} {"t":{"$date":"2023-04-06T23:25:36.110+00:00"},"s":"I", "c":"NETWORK", "id":22943, "ctx":"listener","msg":"Connection accepted","attr":{"remote":"127.0.0.1:51640","uuid":"ccdd4440-8fbd-46fe-8738-a9dc1f72b9d2","connectionId":187854,"connectionCount":3}} {"t":{"$date":"2023-04-06T23:25:36.112+00:00"},"s":"I", "c":"NETWORK", "id":51800, "ctx":"conn187854","msg":"client metadata","attr":{"remote":"127.0.0.1:51640","client":"conn187854","doc":{"driver":{"name":"mongo-go-driver","version":"v1.11.2"},"os":{"type":"linux","architecture":"amd64"},"platform":"go1.19.6","mongos":{"host":"minimal-cluster-mongos-0:27017","client":"127.0.0.1:51640","version":"6.0.4-3"}}}} {"t":{"$date":"2023-04-06T23:25:36.598+00:00"},"s":"I", "c":"CONNPOOL", "id":22576, "ctx":"ReplicaSetMonitor-TaskExecutor","msg":"Connecting","attr":{"hostAndPort":"minimal-cluster-cfg-0.minimal-cluster-cfg.test-psmdb-ns1.svc.cluster.local:27017"}} {"t":{"$date":"2023-04-06T23:25:36.599+00:00"},"s":"I", "c":"-", "id":4333222, "ctx":"ReplicaSetMonitor-TaskExecutor","msg":"RSM received error response","attr":{"host":"minimal-cluster-cfg-0.minimal-cluster-cfg.test-psmdb-ns1.svc.cluster.local:27017","error":"HostUnreachable: Error connecting to minimal-cluster-cfg-0.minimal-cluster-cfg.test-psmdb-ns1.svc.cluster.local:27017 :: caused by :: Could not find address for minimal-cluster-cfg-0.minimal-cluster-cfg.test-psmdb-ns1.svc.cluster.local:27017: SocketException: Host not found (authoritative)","replicaSet":"cfg","response":{}}} {"t":{"$date":"2023-04-06T23:25:36.599+00:00"},"s":"I", "c":"NETWORK", "id":4712102, "ctx":"ReplicaSetMonitor-TaskExecutor","msg":"Host failed in replica set","attr":{"replicaSet":"cfg","host":"minimal-cluster-cfg-0.minimal-cluster-cfg.test-psmdb-ns1.svc.cluster.local:27017","error":{"code":6,"codeName":"HostUnreachable","errmsg":"Error connecting to minimal-cluster-cfg-0.minimal-cluster-cfg.test-psmdb-ns1.svc.cluster.local:27017 :: caused by :: Could not find address for minimal-cluster-cfg-0.minimal-cluster-cfg.test-psmdb-ns1.svc.cluster.local:27017: SocketException: Host not found (authoritative)"},"action":{"dropConnections":true,"requestImmediateCheck":true}}} {"t":{"$date":"2023-04-06T23:25:37.032+00:00"},"s":"I", "c":"-", "id":20883, "ctx":"conn187854","msg":"Interrupted operation as its client disconnected","attr":{"opId":416150}} {"t":{"$date":"2023-04-06T23:25:37.032+00:00"},"s":"I", "c":"-", "id":20883, "ctx":"conn187853","msg":"Interrupted operation as its client disconnected","attr":{"opId":416149}} {"t":{"$date":"2023-04-06T23:25:37.033+00:00"},"s":"I", "c":"NETWORK", "id":22944, "ctx":"conn187852","msg":"Connection ended","attr":{"remote":"127.0.0.1:51632","uuid":"27fe9222-0cf0-4086-89bd-a06e221376be","connectionId":187852,"connectionCount":2}} {"t":{"$date":"2023-04-06T23:25:37.033+00:00"},"s":"I", "c":"COMMAND", "id":51803, "ctx":"conn187854","msg":"Slow query","attr":{"type":"command","ns":"admin.$cmd","command":{"isMaster":1,"helloOk":true,"saslSupportedMechs":"admin.clusterMonitor","speculativeAuthenticate":{"saslStart":1,"mechanism":"SCRAM-SHA-256","payload":{"$binary":{"base64":"biwsbj1jbHVzdGVyTW9uaXRvcixyPTBYNHZKWmZhOHM1UnhFSUtZSTBvaW94UjF3eXlCZ29u","subType":"0"}},"db":"admin","options":{"skipEmptyExchange":true}},"compression":[],"client":{"driver":{"name":"mongo-go-driver","version":"v1.11.2"},"os":{"type":"linux","architecture":"amd64"},"platform":"go1.19.6"},"$readPreference":{"mode":"secondaryPreferred"},"$db":"admin"},"numYields":0,"ok":0,"errMsg":"operation was interrupted","errName":"ClientDisconnect","errCode":279,"reslen":257,"authorization":{"startedUserCacheAcquisitionAttempts":1,"completedUserCacheAcquisitionAttempts":1,"userCacheWaitTimeMicros":920427},"remote":"127.0.0.1:51640","protocol":"op_query","durationMillis":921}} {"t":{"$date":"2023-04-06T23:25:37.033+00:00"},"s":"I", "c":"NETWORK", "id":22944, "ctx":"conn187854","msg":"Connection ended","attr":{"remote":"127.0.0.1:51640","uuid":"ccdd4440-8fbd-46fe-8738-a9dc1f72b9d2","connectionId":187854,"connectionCount":1}} {"t":{"$date":"2023-04-06T23:25:37.033+00:00"},"s":"I", "c":"NETWORK", "id":22944, "ctx":"conn187853","msg":"Connection ended","attr":{"remote":"127.0.0.1:51638","uuid":"6a6a7aa2-ed0e-4abc-894e-b9543f3d9596","connectionId":187853,"connectionCount":0}} {"t":{"$date":"2023-04-06T23:25:37.100+00:00"},"s":"I", "c":"-", "id":4333222, "ctx":"ReplicaSetMonitor-TaskExecutor","msg":"RSM received error response","attr":{"host":"minimal-cluster-cfg-0.minimal-cluster-cfg.test-psmdb-ns1.svc.cluster.local:27017","error":"HostUnreachable: Error connecting to minimal-cluster-cfg-0.minimal-cluster-cfg.test-psmdb-ns1.svc.cluster.local:27017 :: caused by :: Could not find address for minimal-cluster-cfg-0.minimal-cluster-cfg.test-psmdb-ns1.svc.cluster.local:27017: SocketException: Host not found (authoritative)","replicaSet":"cfg","response":{}}} {"t":{"$date":"2023-04-06T23:25:37.100+00:00"},"s":"I", "c":"NETWORK", "id":4712102, "ctx":"ReplicaSetMonitor-TaskExecutor","msg":"Host failed in replica set","attr":{"replicaSet":"cfg","host":"minimal-cluster-cfg-0.minimal-cluster-cfg.test-psmdb-ns1.svc.cluster.local:27017","error":{"code":6,"codeName":"HostUnreachable","errmsg":"Error connecting to minimal-cluster-cfg-0.minimal-cluster-cfg.test-psmdb-ns1.svc.cluster.local:27017 :: caused by :: Could not find address for minimal-cluster-cfg-0.minimal-cluster-cfg.test-psmdb-ns1.svc.cluster.local:27017: SocketException: Host not found (authoritative)"},"action":{"dropConnections":true,"requestImmediateCheck":false,"outcome":{"host":"minimal-cluster-cfg-0.minimal-cluster-cfg.test-psmdb-ns1.svc.cluster.local:27017","success":false,"errorMessage":"HostUnreachable: Error connecting to minimal-cluster-cfg-0.minimal-cluster-cfg.test-psmdb-ns1.svc.cluster.local:27017 :: caused by :: Could not find address for minimal-cluster-cfg-0.minimal-cluster-cfg.test-psmdb-ns1.svc.cluster.local:27017: SocketException: Host not found (authoritative)"}}}} {"t":{"$date":"2023-04-06T23:25:37.110+00:00"},"s":"I", "c":"NETWORK", "id":22943, "ctx":"listener","msg":"Connection accepted","attr":{"remote":"127.0.0.1:51766","uuid":"2f6a75a1-3567-462b-8388-fc78f2ed27af","connectionId":187855,"connectionCount":1}} {"t":{"$date":"2023-04-06T23:25:37.111+00:00"},"s":"I", "c":"NETWORK", "id":22943, "ctx":"listener","msg":"Connection accepted","attr":{"remote":"127.0.0.1:51776","uuid":"0c4edde6-8c4b-4187-87c9-5a50abfff418","connectionId":187856,"connectionCount":2}} {"t":{"$date":"2023-04-06T23:25:37.111+00:00"},"s":"I", "c":"NETWORK", "id":51800, "ctx":"conn187855","msg":"client metadata","attr":{"remote":"127.0.0.1:51766","client":"conn187855","doc":{"driver":{"name":"mongo-go-driver","version":"v1.11.2"},"os":{"type":"linux","architecture":"amd64"},"platform":"go1.19.6","mongos":{"host":"minimal-cluster-mongos-0:27017","client":"127.0.0.1:51766","version":"6.0.4-3"}}}} {"t":{"$date":"2023-04-06T23:25:37.111+00:00"},"s":"I", "c":"NETWORK", "id":51800, "ctx":"conn187856","msg":"client metadata","attr":{"remote":"127.0.0.1:51776","client":"conn187856","doc":{"driver":{"name":"mongo-go-driver","version":"v1.11.2"},"os":{"type":"linux","architecture":"amd64"},"platform":"go1.19.6","mongos":{"host":"minimal-cluster-mongos-0:27017","client":"127.0.0.1:51776","version":"6.0.4-3"}}}} {"t":{"$date":"2023-04-06T23:25:37.112+00:00"},"s":"I", "c":"NETWORK", "id":22943, "ctx":"listener","msg":"Connection accepted","attr":{"remote":"127.0.0.1:51788","uuid":"ce775024-7513-4a19-ae3c-db3f94e8d909","connectionId":187857,"connectionCount":3}} {"t":{"$date":"2023-04-06T23:25:37.112+00:00"},"s":"I", "c":"NETWORK", "id":51800, "ctx":"conn187857","msg":"client metadata","attr":{"remote":"127.0.0.1:51788","client":"conn187857","doc":{"driver":{"name":"mongo-go-driver","version":"v1.11.2"},"os":{"type":"linux","architecture":"amd64"},"platform":"go1.19.6","mongos":{"host":"minimal-cluster-mongos-0:27017","client":"127.0.0.1:51788","version":"6.0.4-3"}}}} minimal-cluster-rs0-0 {"t":{"$date":"2023-04-06T23:22:00.854+00:00"},"s":"I", "c":"STORAGE", "id":22318, "ctx":"SignalHandler","msg":"Shutting down session sweeper thread"} {"t":{"$date":"2023-04-06T23:22:00.854+00:00"},"s":"I", "c":"STORAGE", "id":22319, "ctx":"SignalHandler","msg":"Finished shutting down session sweeper thread"} {"t":{"$date":"2023-04-06T23:22:00.855+00:00"},"s":"I", "c":"STORAGE", "id":22324, "ctx":"SignalHandler","msg":"Closing WiredTiger in preparation for reconfiguring","attr":{"closeConfig":"leak_memory=true,"}} {"t":{"$date":"2023-04-06T23:22:00.880+00:00"},"s":"I", "c":"STORAGE", "id":4795905, "ctx":"SignalHandler","msg":"WiredTiger closed","attr":{"durationMillis":25}} {"t":{"$date":"2023-04-06T23:22:01.000+00:00"},"s":"W", "c":"REPL", "id":6100702, "ctx":"ftdc","msg":"Failed to get last stable recovery timestamp due to {error}","attr":{"error":"lock acquire timeout"}} {"t":{"$date":"2023-04-06T23:22:01.261+00:00"},"s":"I", "c":"STORAGE", "id":4795904, "ctx":"SignalHandler","msg":"WiredTiger re-opened","attr":{"durationMillis":380}} {"t":{"$date":"2023-04-06T23:22:01.261+00:00"},"s":"I", "c":"STORAGE", "id":22325, "ctx":"SignalHandler","msg":"Reconfiguring","attr":{"newConfig":"compatibility=(release=10.0)"}} {"t":{"$date":"2023-04-06T23:22:01.261+00:00"},"s":"I", "c":"STORAGE", "id":4795903, "ctx":"SignalHandler","msg":"Reconfigure complete","attr":{"durationMillis":0}} {"t":{"$date":"2023-04-06T23:22:01.261+00:00"},"s":"I", "c":"STORAGE", "id":4795902, "ctx":"SignalHandler","msg":"Closing WiredTiger","attr":{"closeConfig":"leak_memory=true,"}} {"t":{"$date":"2023-04-06T23:22:01.286+00:00"},"s":"I", "c":"STORAGE", "id":4795901, "ctx":"SignalHandler","msg":"WiredTiger closed","attr":{"durationMillis":25}} {"t":{"$date":"2023-04-06T23:22:01.286+00:00"},"s":"I", "c":"STORAGE", "id":29075, "ctx":"SignalHandler","msg":"Closing KeyDB in preparation for reconfiguring"} {"t":{"$date":"2023-04-06T23:22:01.311+00:00"},"s":"I", "c":"STORAGE", "id":29076, "ctx":"SignalHandler","msg":"KeyDB closed","attr":{"durationMillis":25}} {"t":{"$date":"2023-04-06T23:22:01.357+00:00"},"s":"I", "c":"STORAGE", "id":29077, "ctx":"SignalHandler","msg":"KeyDB re-opened","attr":{"durationMillis":46}} {"t":{"$date":"2023-04-06T23:22:01.357+00:00"},"s":"I", "c":"STORAGE", "id":29078, "ctx":"SignalHandler","msg":"Reconfiguring KeyDB","attr":{"newConfig":"compatibility=(release=10.0)"}} {"t":{"$date":"2023-04-06T23:22:01.358+00:00"},"s":"I", "c":"STORAGE", "id":29079, "ctx":"SignalHandler","msg":"KeyDB reconfigure complete","attr":{"durationMillis":1}} {"t":{"$date":"2023-04-06T23:22:01.374+00:00"},"s":"I", "c":"STORAGE", "id":22279, "ctx":"SignalHandler","msg":"shutdown: removing fs lock..."} {"t":{"$date":"2023-04-06T23:22:01.374+00:00"},"s":"I", "c":"-", "id":4784931, "ctx":"SignalHandler","msg":"Dropping the scope cache for shutdown"} {"t":{"$date":"2023-04-06T23:22:01.374+00:00"},"s":"I", "c":"FTDC", "id":20626, "ctx":"SignalHandler","msg":"Shutting down full-time diagnostic data capture"} {"t":{"$date":"2023-04-06T23:22:01.376+00:00"},"s":"I", "c":"CONTROL", "id":20565, "ctx":"SignalHandler","msg":"Now exiting"} {"t":{"$date":"2023-04-06T23:22:01.376+00:00"},"s":"I", "c":"CONTROL", "id":23138, "ctx":"SignalHandler","msg":"Shutting down","attr":{"exitCode":0}}