Mongod process getting crashed with Got signal: 11 (Segmentation fault)

MongoDB Percona Server for MongoDB
1

Mongod process crashed many time with Got signal: 11 (Segmentation fault) error

h-4.2# mongod --version
db version v4.4.4-6
Build Info: {
“version”: “4.4.4-6”,
“gitVersion”: “f3dd4bc7c7500705a537de40bb4d6127ba498bd3”,
“openSSLVersion”: “OpenSSL 1.0.2k-fips 26 Jan 2017”,
“modules”: ,
“allocator”: “tcmalloc”,
“environment”: {
“distarch”: “x86_64”,
“target_arch”: “x86_64”
}
}

Error logs before crash

{“t”:{"$date":“2021-03-31T13:13:57.110+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22944, “ctx”:“conn5366”,“msg”:“Connection ended”,“attr”:{“remote”:“172.24.5.56:38068”,“connectionId”:5366,“connectionCount”:417}}
{“t”:{"$date":“2021-03-31T13:13:57.110+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22944, “ctx”:“conn5367”,“msg”:“Connection ended”,“attr”:{“remote”:“172.24.5.56:38072”,“connectionId”:5367,“connectionCount”:416}}
{“t”:{"$date":“2021-03-31T13:13:57.112+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22943, “ctx”:“listener”,“msg”:“Connection accepted”,“attr”:{“r emote”:“172.24.5.56:38080”,“connectionId”:5368,“connectionCount”:417}}
{“t”:{"$date":“2021-03-31T13:13:57.112+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22944, “ctx”:“conn5368”,“msg”:“Connection ended”,“attr”:{“remote”:“172.24.5.56:38080”,“connectionId”:5368,“connectionCount”:416}}
{“t”:{"$date":“2021-03-31T13:13:57.113+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22943, “ctx”:“listener”,“msg”:“Connection accepted”,“attr”:{“r emote”:“172.24.5.56:38086”,“connectionId”:5369,“connectionCount”:417}}
{“t”:{"$date":“2021-03-31T13:13:57.113+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:51800, “ctx”:“conn5369”,“msg”:“client metadata”,“attr”:{“remote”:“172.24.5.56:38086”,“client”:“conn5369”,“doc”:{“driver”:{“name”:“mongo-java-driver”,“version”:“3.8.2”},“os”:{“type”:“Linux”,“name”:“Linux”,“architecture”:“amd64”,“version”:“3.10.0-514.26.2.el7.x86_64”},“platform”:“Java/Oracle Corporation/1.8.0_45-b14”}}}
{“t”:{"$date":“2021-03-31T13:13:57.139+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22944, “ctx”:“conn5369”,“msg”:“Connection ended”,“attr”:{“remote”:“172.24.5.56:38086”,“connectionId”:5369,“connectionCount”:416}}
{“t”:{"$date":“2021-03-31T13:13:57.140+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22943, “ctx”:“listener”,“msg”:“Connection accepted”,“attr”:{“r emote”:“172.24.5.56:38094”,“connectionId”:5370,“connectionCount”:417}}
{“t”:{"$date":“2021-03-31T13:13:57.141+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:51800, “ctx”:“conn5370”,“msg”:“client metadata”,“attr”:{“remote”:“172.24.5.56:38094”,“client”:“conn5370”,“doc”:{“driver”:{“name”:“mongo-java-driver”,“version”:“3.8.2”},“os”:{“type”:“Linux”,“name”:“Linux”,“architecture”:“amd64”,“version”:“3.10.0-514.26.2.el7.x86_64”},“platform”:“Java/Oracle Corporation/1.8.0_45-b14”}}}
{“t”:{"$date":“2021-03-31T13:13:57.168+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22944, “ctx”:“conn5370”,“msg”:“Connection ended”,“attr”:{“remote”:“172.24.5.56:38094”,“connectionId”:5370,“connectionCount”:416}}
{“t”:{"$date":“2021-03-31T13:13:57.177+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22944, “ctx”:“conn5364”,“msg”:“Connection ended”,“attr”:{“remote”:“172.24.5.56:38052”,“connectionId”:5364,“connectionCount”:415}}
{“t”:{"$date":“2021-03-31T13:13:57.189+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22944, “ctx”:“conn5363”,“msg”:“Connection ended”,“attr”:{“remote”:“172.24.5.56:38044”,“connectionId”:5363,“connectionCount”:414}}
{“t”:{"$date":“2021-03-31T13:14:01.978+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22943, “ctx”:“listener”,“msg”:“Connection accepted”,“attr”:{“r emote”:“127.0.0.1:58726”,“connectionId”:5371,“connectionCount”:415}}
{“t”:{"$date":“2021-03-31T13:14:01.978+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:51800, “ctx”:“conn5371”,“msg”:“client metadata”,“attr”:{“remote”:“127.0.0.1:58726”,“client”:“conn5371”,“doc”:{“application”:{“name”:“MongoDB Shell”},“driver”:{“name”:“MongoDB Internal Client”,“version”:“4.4.4-6”},“os”:{“type”:“Linux”,“name”:“CentOS Linux release 7.7.1908 (Core)”,“architecture”:“x86_64”,“version”:“Kernel 3.10.0-1062.12.1.el7.x86
_64”}}}}
{“t”:{"$date":“2021-03-31T13:14:01.979+00:00”},“s”:“I”, “c”:“ACCESS”, “id”:20249, “ctx”:“conn5371”,“msg”:“Authentication failed”,“attr”:{“mechanism”:“SCRAM-SHA-256”,“principalName”:“monitor”,“authenticationDatabase”:“admin”,“client”:“127.0.0.1:58726”,“result”:“AuthenticationFailed: Unable to use SCRAM-SHA-256 based authentication for user without any SCRAM-SHA-256 credentials registered”}}
{“t”:{"$date":“2021-03-31T13:14:01.989+00:00”},“s”:“I”, “c”:“ACCESS”, “id”:20250, “ctx”:“conn5371”,“msg”:“Successful authentication”,“attr”:{“mechanism”:“SCRAM-SHA-1”,“principalName”:“monitor”,“authenticationDatabase”:“admin”,“client”:“127.0.0.1:58726”}}
{“t”:{"$date":“2021-03-31T13:14:02.010+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22944, “ctx”:“conn5371”,“msg”:“Connection ended”,“attr”:{“remote”:“127.0.0.1:58726”,“connectionId”:5371,“connectionCount”:414}}
{“t”:{"$date":“2021-03-31T13:14:08.359+00:00”},“s”:“I”, “c”:“QUERY”, “id”:20529, “ctx”:“clientcursormon”,“msg”:“Cursor timed out”,“attr”:{“cursorId”:7904269181106903313,“idleSince”:{"$date":“2021-03-31T13:04:04.681Z”}}}
{“t”:{"$date":“2021-03-31T13:14:48.361+00:00”},“s”:“I”, “c”:“QUERY”, “id”:20529, “ctx”:“clientcursormon”,“msg”:“Cursor timed out”,“attr”:{“cursorId”:4853223127451353051,“idleSince”:{"$date":“2021-03-31T13:04:44.475Z”}}}
{“t”:{"$date":“2021-03-31T13:14:52.361+00:00”},“s”:“I”, “c”:“QUERY”, “id”:20529, “ctx”:“clientcursormon”,“msg”:“Cursor timed out”,“attr”:{“cursorId”:5563441711628392626,“idleSince”:{"$date":“2021-03-31T13:04:50.155Z”}}}
{“t”:{"$date":“2021-03-31T13:14:53.110+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22943, “ctx”:“listener”,“msg”:“Connection accepted”,“attr”:{“remote”:“172.18.1.20:47614”,“connectionId”:5372,“connectionCount”:415}}
{“t”:{"$date":“2021-03-31T13:14:53.111+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22944, “ctx”:“conn5372”,“msg”:“Connection ended”,“attr”:{“remote”:“172.18.1.20:47614”,“connectionId”:5372,“connectionCount”:414}}
{“t”:{"$date":“2021-03-31T13:15:01.371+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22943, “ctx”:“listener”,“msg”:“Connection accepted”,“attr”:{“r emote”:“127.0.0.1:58852”,“connectionId”:5373,“connectionCount”:415}}
{“t”:{"$date":“2021-03-31T13:15:01.371+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:51800, “ctx”:“conn5373”,“msg”:“client metadata”,“attr”:{“remote”:“127.0.0.1:58852”,“client”:“conn5373”,“doc”:{“application”:{“name”:“MongoDB Shell”},“driver”:{“name”:“MongoDB Internal Client”,“version”:“4.4.4-6”},“os”:{“type”:“Linux”,“name”:“CentOS Linux release 7.7.1908 (Core)”,“architecture”:“x86_64”,“version”:“Kernel 3.10.0-1062.12.1.el7.x86_64”}}}}
{“t”:{"$date":“2021-03-31T13:15:01.371+00:00”},“s”:“I”, “c”:“ACCESS”, “id”:20249, “ctx”:“conn5373”,“msg”:“Authentication failed”,“attr”:{“mechanism”:“SCRAM-SHA-256”,“principalName”:“monitor”,“authenticationDatabase”:“admin”,“client”:“127.0.0.1:58852”,“result”:“AuthenticationFailed: Unable to use SCRAM-SHA-256 based authentication for user without any SCRAM-SHA-256 credentials registered”}}
{“t”:{"$date":“2021-03-31T13:15:01.383+00:00”},“s”:“I”, “c”:“ACCESS”, “id”:20250, “ctx”:“conn5373”,“msg”:“Successful authentication”,“attr”:{“mechanism”:“SCRAM-SHA-1”,“principalName”:“monitor”,“authenticationDatabase”:“admin”,“client”:“127.0.0.1:58852”}}
{“t”:{"$date":“2021-03-31T13:15:01.405+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22944, “ctx”:“conn5373”,“msg”:“Connection ended”,“attr”:{“remote”:“127.0.0.1:58852”,“connectionId”:5373,“connectionCount”:414}}
{“t”:{"$date":“2021-03-31T13:15:07.373+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22943, “ctx”:“listener”,“msg”:“Connection accepted”,“attr”:{“remote”:“172.24.6.16:44894”,“connectionId”:5374,“connectionCount”:415}}
{“t”:{"$date":“2021-03-31T13:15:07.373+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22943, “ctx”:“listener”,“msg”:“Connection accepted”,“attr”:{“remote”:“172.24.6.16:44896”,“connectionId”:5375,“connectionCount”:416}}
{“t”:{"$date":“2021-03-31T13:15:07.373+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:51800, “ctx”:“conn5374”,“msg”:“client metadata”,“attr”:{“remote”:“172.24.6.16:44894”,“client”:“conn5374”,“doc”:{“driver”:{“name”:“mongo-java-driver|legacy”,“version”:“4.1.1”},“os”:{“type”:“Linux”,“name”:"Linux",“architecture”:“amd64”,“version”:“3.10.0-1062.9.1.el7.x86_64”},“platform”:“Java/Oracle Corporation/1.8.0_252-b09”}}}
{“t”:{"$date":“2021-03-31T13:15:07.373+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:51800, “ctx”:“conn5375”,“msg”:“client metadata”,“attr”:{“remote”:“172.24.6.16:44896”,“client”:“conn5375”,“doc”:{“driver”:{“name”:“mongo-java-driver|legacy”,“version”:“4.1.1”},“os”:{“type”:“Linux”,“name”:"Linux",“architecture”:“amd64”,“version”:“3.10.0-1062.9.1.el7.x86_64”},“platform”:“Java/Oracle Corporation/1.8.0_252-b09”}}}
{“t”:{"$date":“2021-03-31T13:15:07.401+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22944, “ctx”:“conn5375”,“msg”:“Connection ended”,“attr”:{“remote”:“172.24.6.16:44896”,“connectionId”:5375,“connectionCount”:415}}
{“t”:{"$date":“2021-03-31T13:15:07.401+00:00”},“s”:“I”, “c”:"-", “id”:20883, “ctx”:“conn5374”,“msg”:“Interrupted operation as its client disconnected”,“attr”:{“opId”:296689953}}
{“t”:{"$date":“2021-03-31T13:15:07.401+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22944, “ctx”:“conn5374”,“msg”:“Connection ended”,“attr”:{“remote”:“172.24.6.16:44894”,“connectionId”:5374,“connectionCount”:414}}
{“t”:{"$date":“2021-03-31T13:15:16.490+00:00”},“s”:“F”, “c”:“CONTROL”, “id”:4757800, “ctx”:“thread5363”,“msg”:“Writing fatal message”,“attr”:{“message”:“Invalid access at address: 0”}}
{“t”:{"$date":“2021-03-31T13:15:16.490+00:00”},“s”:“F”, “c”:“CONTROL”, “id”:4757800, “ctx”:“thread5363”,“msg”:“Writing fatal message”,“attr”:{“message”:“Got signal: 11 (Segmentation fault).\n”}}
2

Hi Murali.

I see the final messages are:

“Invalid access at address: 0”
“Got signal: 11 (Segmentation fault).\n

Which looks like a classic null-pointer dereference error. Classic for programming in general I mean, not for MongoDB particularly. I haven’t seen this much with MongoDB.

I don’t think the failed authentication from ~9 secs prior to the crash, the one with connectionId = 5375 is likely to be worth investigating.

Maybe the operation with “opId”:296689953 on connectionId 5374 is the culprit - but there’s no proof of that so far.

Is there a backtrace section from the log file?

1 Like
3

Hi Akira_Kurogane,

No backtrace for the log. will investigate the shared connection id and get back to you.

1 Like
4

Hi Team,
Again today the mongodb got crashed with following error .

{“t”:{“$date”:“2021-07-08T11:58:57.127+00:00”},“s”:“F”, “c”:“-”, “id”:23080, “ctx”:“xxxxxxing-keys-for-HMAC”,“msg”:“\n\n***aborting
after invariant() failure\n\n”}
{“t”:{“$date”:“2021-07-08T11:58:57.127+00:00”},“s”:“F”, “c”:“CONTROL”, “id”:4757800, “ctx”:“xxxxxxing-keys-for-HMAC”,“msg”:“Writing fatal m
essage”,“attr”:{“message”:“Got signal: 6 (Aborted).\n”}}
{“t”:{“$date”:“2021-07-08T11:58:57.313+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:31431, “ctx”:“xxxxxxing-keys-for-HMAC”,“msg”:"BACKTRACE: {bt}
.

Here is the complete log before crashed
emote":“xx.xx.xx.xx:36696”,“connectionId”:67997,“connectionCount”:444}}
{“t”:{“$date”:“2021-07-08T11:53:15.861+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22944, “ctx”:“conn67997”,“msg”:“Connection ended”,“attr”:{“rem
ote”:“xx.xx.xx.xx:36696”,“connectionId”:67997,“connectionCount”:443}}
{“t”:{“$date”:“2021-07-08T11:53:18.515+00:00”},“s”:“I”, “c”:“QUERY”, “id”:20529, “ctx”:“clientcursormon”,“msg”:“Cursor timed out”,“attr”
:{“cursorId”:377223788387066660,“idleSince”:{“$date”:“2021-07-08T11:43:15.140Z”}}}
{“t”:{“$date”:“2021-07-08T11:53:26.516+00:00”},“s”:“I”, “c”:“QUERY”, “id”:20529, “ctx”:“clientcursormon”,“msg”:“Cursor timed out”,“attr”
:{“cursorId”:8859598790519006230,“idleSince”:{“$date”:“2021-07-08T11:43:23.292Z”}}}
{“t”:{“$date”:“2021-07-08T11:53:34.516+00:00”},“s”:“I”, “c”:“QUERY”, “id”:20529, “ctx”:“clientcursormon”,“msg”:“Cursor timed out”,“attr”
:{“cursorId”:4256969131112290776,“idleSince”:{“$date”:“2021-07-08T11:43:34.125Z”}}}
{“t”:{“$date”:“2021-07-08T11:53:34.516+00:00”},“s”:“I”, “c”:“QUERY”, “id”:20529, “ctx”:“clientcursormon”,“msg”:“Cursor timed out”,“attr”
:{“cursorId”:5802009825555165581,“idleSince”:{“$date”:“2021-07-08T11:43:32.539Z”}}}
{“t”:{“$date”:“2021-07-08T11:53:42.517+00:00”},“s”:“I”, “c”:“QUERY”, “id”:20529, “ctx”:“clientcursormon”,“msg”:“Cursor timed out”,“attr”
:{“cursorId”:2313105868267765464,“idleSince”:{“$date”:“2021-07-08T11:43:42.482Z”}}}
{“t”:{“$date”:“2021-07-08T11:53:54.518+00:00”},“s”:“I”, “c”:“QUERY”, “id”:20529, “ctx”:“clientcursormon”,“msg”:“Cursor timed out”,“attr”
:{“cursorId”:3301981260009307571,“idleSince”:{“$date”:“2021-07-08T11:43:53.403Z”}}}
{“t”:{“$date”:“2021-07-08T11:54:06.519+00:00”},“s”:“I”, “c”:“QUERY”, “id”:20529, “ctx”:“clientcursormon”,“msg”:“Cursor timed out”,“attr”
:{“cursorId”:7637104258329324527,“idleSince”:{“$date”:“2021-07-08T11:44:03.965Z”}}}
{“t”:{“$date”:“2021-07-08T11:54:10.519+00:00”},“s”:“I”, “c”:“QUERY”, “id”:20529, “ctx”:“clientcursormon”,“msg”:“Cursor timed out”,“attr”
:{“cursorId”:1204502043730160407,“idleSince”:{“$date”:“2021-07-08T11:44:07.442Z”}}}
{“t”:{“$date”:“2021-07-08T11:54:10.520+00:00”},“s”:“I”, “c”:“QUERY”, “id”:20529, “ctx”:“clientcursormon”,“msg”:“Cursor timed out”,“attr”
:{“cursorId”:7112901260208356327,“idleSince”:{“$date”:“2021-07-08T11:44:09.617Z”}}}
{“t”:{“$date”:“2021-07-08T11:54:14.520+00:00”},“s”:“I”, “c”:“QUERY”, “id”:20529, “ctx”:“clientcursormon”,“msg”:“Cursor timed out”,“attr”
:{“cursorId”:7129352708288117004,“idleSince”:{“$date”:“2021-07-08T11:44:13.696Z”}}}
{“t”:{“$date”:“2021-07-08T11:54:15.860+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22943, “ctx”:“listener”,“msg”:“Connection accepted”,“attr”:{“r
emote”:“xx.xx.xx.xx:37292”,“connectionId”:67998,“connectionCount”:444}}
{“t”:{“$date”:“2021-07-08T11:54:15.861+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22944, “ctx”:“conn67998”,“msg”:“Connection ended”,“attr”:{“rem
ote”:“xx.xx.xx.xx:37292”,“connectionId”:67998,“connectionCount”:443}}
{“t”:{“$date”:“2021-07-08T11:54:22.521+00:00”},“s”:“I”, “c”:“QUERY”, “id”:20529, “ctx”:“clientcursormon”,“msg”:“Cursor timed out”,“attr”
:{“cursorId”:8185101709135123398,“idleSince”:{“$date”:“2021-07-08T11:44:19.955Z”}}}
{“t”:{“$date”:“2021-07-08T11:54:34.521+00:00”},“s”:“I”, “c”:“QUERY”, “id”:20529, “ctx”:“clientcursormon”,“msg”:“Cursor timed out”,“attr”
:{“cursorId”:7761858418928150960,“idleSince”:{“$date”:“2021-07-08T11:44:34.454Z”}}}
{“t”:{“$date”:“2021-07-08T11:54:58.523+00:00”},“s”:“I”, “c”:“QUERY”, “id”:20529, “ctx”:“clientcursormon”,“msg”:“Cursor timed out”,“attr”
:{“cursorId”:6056470634275818920,“idleSince”:{“$date”:“2021-07-08T11:44:55.452Z”}}}
{“t”:{“$date”:“2021-07-08T11:55:01.643+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22943, “ctx”:“listener”,“msg”:“Connection accepted”,“attr”:{“r
emote”:“127.0.0.1:33480”,“connectionId”:67999,“connectionCount”:444}}
{“t”:{“$date”:“2021-07-08T11:55:01.644+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:51800, “ctx”:“conn67999”,“msg”:“client metadata”,“attr”:{“remo
te”:“127.0.0.1:33480”,“client”:“conn67999”,“doc”:{“application”:{“name”:“MongoDB Shell”},“driver”:{“name”:“MongoDB Internal Client”,“version”:
“4.4.4-6”},“os”:{“type”:“Linux”,“name”:“CentOS Linux release 7.7.1908 (Core)”,“architecture”:“x86_64”,“version”:“Kernel 3.10.0-1062.9.1.el7.x8
6_64”}}}}
{“t”:{“$date”:“2021-07-08T11:55:01.644+00:00”},“s”:“I”, “c”:“ACCESS”, “id”:20249, “ctx”:“conn67999”,“msg”:“Authentication failed”,“attr”:
{“mechanism”:“SCRAM-SHA-256”,“principalName”:“xxxxxx”,“authenticationDatabase”:“admin”,“client”:“127.0.0.1:33480”,“result”:“AuthenticationFai
led: Unable to use SCRAM-SHA-256 based authentication for user without any SCRAM-SHA-256 credentials registered”}}
{“t”:{“$date”:“2021-07-08T11:55:01.654+00:00”},“s”:“I”, “c”:“ACCESS”, “id”:20250, “ctx”:“conn67999”,“msg”:“Successful authentication”,“at
tr”:{“mechanism”:“SCRAM-SHA-1”,“principalName”:“xxxxxx”,“authenticationDatabase”:“admin”,“client”:“127.0.0.1:33480”}}
{“t”:{“$date”:“2021-07-08T11:55:01.660+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22944, “ctx”:“conn67999”,“msg”:“Connection ended”,“attr”:{“rem
ote”:“127.0.0.1:33480”,“connectionId”:67999,“connectionCount”:443}}
{“t”:{“$date”:“2021-07-08T11:55:15.860+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22943, “ctx”:“listener”,“msg”:“Connection accepted”,“attr”:{“r
emote”:“xx.xx.xx.xx:37870”,“connectionId”:68000,“connectionCount”:444}}
{“t”:{“$date”:“2021-07-08T11:55:15.861+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22944, “ctx”:“conn68000”,“msg”:“Connection ended”,“attr”:{“rem
ote”:“xx.xx.xx.xx:37870”,“connectionId”:68000,“connectionCount”:443}}
{“t”:{“$date”:“2021-07-08T11:55:42.527+00:00”},“s”:“I”, “c”:“QUERY”, “id”:20529, “ctx”:“clientcursormon”,“msg”:“Cursor timed out”,“attr”
:{“cursorId”:7372937045780781111,“idleSince”:{“$date”:“2021-07-08T11:45:42.287Z”}}}
{“t”:{“$date”:“2021-07-08T11:55:46.527+00:00”},“s”:“I”, “c”:“QUERY”, “id”:20529, “ctx”:“clientcursormon”,“msg”:“Cursor timed out”,“attr”
:{“cursorId”:8124829944653316967,“idleSince”:{“$date”:“2021-07-08T11:45:44.598Z”}}}
{“t”:{“$date”:“2021-07-08T11:55:46.527+00:00”},“s”:“I”, “c”:“QUERY”, “id”:20529, “ctx”:“clientcursormon”,“msg”:“Cursor timed out”,“attr”
:{“cursorId”:8989965420745887868,“idleSince”:{“$date”:“2021-07-08T11:45:44.397Z”}}}
{“t”:{“$date”:“2021-07-08T11:55:50.528+00:00”},“s”:“I”, “c”:“QUERY”, “id”:20529, “ctx”:“clientcursormon”,“msg”:“Cursor timed out”,“attr”
:{“cursorId”:3928038101692991331,“idleSince”:{“$date”:“2021-07-08T11:45:49.224Z”}}}
{“t”:{“$date”:“2021-07-08T11:56:06.529+00:00”},“s”:“I”, “c”:“QUERY”, “id”:20529, “ctx”:“clientcursormon”,“msg”:“Cursor timed out”,“attr”
:{“cursorId”:8475410858587847881,“idleSince”:{“$date”:“2021-07-08T11:46:04.009Z”}}}
{“t”:{“$date”:“2021-07-08T11:56:15.861+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22943, “ctx”:“listener”,“msg”:“Connection accepted”,“attr”:{“r
emote”:“xx.xx.xx.xx:38450”,“connectionId”:68001,“connectionCount”:444}}
{“t”:{“$date”:“2021-07-08T11:56:15.861+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22944, “ctx”:“conn68001”,“msg”:“Connection ended”,“attr”:{“rem
ote”:“xx.xx.xx.xx:38450”,“connectionId”:68001,“connectionCount”:443}}
{“t”:{“$date”:“2021-07-08T11:57:15.860+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22943, “ctx”:“listener”,“msg”:“Connection accepted”,“attr”:{“r
emote”:“xx.xx.xx.xx:39030”,“connectionId”:68002,“connectionCount”:444}}
{“t”:{“$date”:“2021-07-08T11:57:15.861+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22944, “ctx”:“conn68002”,“msg”:“Connection ended”,“attr”:{“rem
ote”:“xx.xx.xx.xx:39030”,“connectionId”:68002,“connectionCount”:443}}
{“t”:{“$date”:“2021-07-08T11:57:18.535+00:00”},“s”:“I”, “c”:“QUERY”, “id”:20529, “ctx”:“clientcursormon”,“msg”:“Cursor timed out”,“attr”
:{“cursorId”:713370189066619809,“idleSince”:{“$date”:“2021-07-08T11:47:17.052Z”}}}
{“t”:{“$date”:“2021-07-08T11:58:15.860+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22943, “ctx”:“listener”,“msg”:“Connection accepted”,“attr”:{“r
emote”:“xx.xx.xx.xx:39610”,“connectionId”:68003,“connectionCount”:444}}
{“t”:{“$date”:“2021-07-08T11:58:15.861+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22944, “ctx”:“conn68003”,“msg”:“Connection ended”,“attr”:{“rem
ote”:“xx.xx.xx.xx:39610”,“connectionId”:68003,“connectionCount”:443}}
{“t”:{“$date”:“2021-07-08T11:58:57.127+00:00”},“s”:“F”, “c”:“-”, “id”:23079, “ctx”:“xxxxxxing-keys-for-HMAC”,“msg”:“Invariant failu
re”,“attr”:{“expr”:“opCtx != nullptr && _opCtx == nullptr”,“file”:“src/mongo/db/client.cpp”,“line”:126}}
{“t”:{“$date”:“2021-07-08T11:58:57.127+00:00”},“s”:“F”, “c”:“-”, “id”:23080, “ctx”:“xxxxxxing-keys-for-HMAC”,“msg”:“\n\n***aborting
after invariant() failure\n\n”}
{“t”:{“$date”:“2021-07-08T11:58:57.127+00:00”},“s”:“F”, “c”:“CONTROL”, “id”:4757800, “ctx”:“xxxxxxing-keys-for-HMAC”,“msg”:“Writing fatal m
essage”,“attr”:{“message”:“Got signal: 6 (Aborted).\n”}}
{“t”:{“$date”:“2021-07-08T11:58:57.313+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:31431, “ctx”:“xxxxxxing-keys-for-HMAC”,“msg”:“BACKTRACE: {bt}
“,“attr”:{“bt”:{“backtrace”:[{“a”:“55FDAE5C8D56”,“b”:“55FDAB722000”,“o”:“2EA6D56”,“s”:”_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStack
TraceImplERKNS1_7OptionsEPNS_14StackTraceSinkE.constprop.608”,“s+”:“1A6”},{“a”:“55FDAE5CA889”,“b”:“55FDAB722000”,“o”:“2EA8889”,“s”:“_ZN5mongo1
5printStackTraceEv”,“s+”:“29”},{“a”:“55FDAE5C7AE6”,“b”:“55FDAB722000”,“o”:“2EA5AE6”,“s”:“ZN5mongo12_GLOBAL__N_116abruptQuitActionEiP9siginfo
tPv”,“s+”:“66”},{“a”:“7F6D53BA85F0”,“b”:“7F6D53B99000”,“o”:“F5F0”,“s”:“_L_unlock_13”,“s+”:“34”},{“a”:“7F6D53801337”,“b”:“7F6D537CB000”,“o”:“36
337”,“s”:“gsignal”,“s+”:“37”},{“a”:“7F6D53802A28”,“b”:“7F6D537CB000”,“o”:“37A28”,“s”:“abort”,“s+”:“148”},{“a”:“55FDAC6EB328”,“b”:“55FDAB722000
“,“o”:“FC9328”,“s”:”_ZN5mongo15invariantFailedEPKcS1_j”,“s+”:“12C”},{“a”:“55FDAC6BD7A2”,“b”:“55FDAB722000”,“o”:“F9B7A2”,“s”:“_ZN5mongo6Client1
9setOperationContextEPNS_16OperationContextE.cold.133”,“s+”:“18”},{“a”:“55FDAE484B65”,“b”:“55FDAB722000”,“o”:“2D62B65”,“s”:“_ZN5mongo14Service
Context20makeOperationContextEPNS_6ClientE”,“s+”:“125”},{“a”:“55FDAE479AF7”,“b”:“55FDAB722000”,“o”:“2D57AF7”,“s”:“_ZN5mongo6Client20makeOperat
ionContextEv”,“s+”:“27”},{“a”:“55FDAE273AB2”,“b”:“55FDAB722000”,“o”:“2B51AB2”,“s”:“ZN5mongo21KeysCollectionManager14PeriodicRunner18_doPeriod
icRefreshEPNS_14ServiceContextESsNS_8DurationISt5ratioILl1ELl1000EEEE",“s+”:“162”},{“a”:“55FDAE275B87”,“b”:“55FDAB722000”,“o”:“2B53B87”,“s”:"
ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN5mongo4stdx6threadC4IZNS3_21KeysCollectionManager14PeriodicRunner5startEPNS3_14ServiceContex
tERKSsNS3_8DurationISt5ratioILl1ELl1000EEEEEUlvE_JELi0EEET_DpOT0_EUlvE_EEEEE6_M_runEv”,“s+”:“A7”},{“a”:“55FDAE8E330F”,“b”:“55FDAB722000”,“o”:"
31C130F",“s”:“execute_native_thread_routine”,“s+”:“F”},{“a”:“7F6D53BA0E65”,“b”:“7F6D53B99000”,“o”:“7E65”,“s”:“start_thread”,“s+”:“C5”},{“a”:“7
F6D538C988D”,“b”:“7F6D537CB000”,“o”:“FE88D”,“s”:“clone”,“s+”:“6D”}],“processInfo”:{“mongodbVersion”:“4.4.4-6”,“gitVersion”:“f3dd4bc7c7500705a5
37de40bb4d6127ba498bd3”,“compiledModules”:,“uname”:{“sysname”:“Linux”,“release”:“3.10.0-1062.9.1.el7.x86_64”,“version”:“#1 SMP Fri Dec 6 15:
49:49 UTC 2019”,“machine”:“x86_64”},“somap”:[{“b”:“55FDAB722000”,“elfType”:3,“buildId”:“9C273C1D1F388E3DF5FF7BAE1D9FD4899A87149D”},{“b”:“7F6D5
3B99000”,“path”:“/lib64/libpthread.so.0”,“elfType”:3,“buildId”:“8B33F7F8C86F8D544C63C5541A8E42B3DDFEF8B1”},{“b”:“7F6D537CB000”,“path”:“/lib64/
libc.so.6”,“elfType”:3,“buildId”:“398944D32CF16A67AF51067A326E6C0CC14F90ED”}]}}}}
{“t”:{“$date”:“2021-07-08T11:58:57.313+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:31427, “ctx”:“xxxxxxing-keys-for-HMAC”,“msg”:" Frame: {frame
}“,“attr”:{“frame”:{“a”:“55FDAE5C8D56”,“b”:“55FDAB722000”,“o”:“2EA6D56”,“s”:”_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplE
RKNS1_7OptionsEPNS_14StackTraceSinkE.constprop.608",“s+”:“1A6”}}}
{“t”:{“$date”:“2021-07-08T11:58:57.313+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:31427, “ctx”:“xxxxxxing-keys-for-HMAC”,“msg”:" Frame: {frame
}“,“attr”:{“frame”:{“a”:“55FDAE5CA889”,“b”:“55FDAB722000”,“o”:“2EA8889”,“s”:”_ZN5mongo15printStackTraceEv",“s+”:“29”}}}
{“t”:{“$date”:“2021-07-08T11:58:57.313+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:31427, “ctx”:“xxxxxxing-keys-for-HMAC”,“msg”:" Frame: {frame
}“,“attr”:{“frame”:{“a”:“55FDAE5C7AE6”,“b”:“55FDAB722000”,“o”:“2EA5AE6”,“s”:”_ZN5mongo12_GLOBAL__N_116abruptQuitActionEiP9siginfo_tPv",“s+”:“6
6”}}}
{“t”:{“$date”:“2021-07-08T11:58:57.313+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:31427, “ctx”:“xxxxxxing-keys-for-HMAC”,“msg”:" Frame: {frame
}“,“attr”:{“frame”:{“a”:“7F6D53BA85F0”,“b”:“7F6D53B99000”,“o”:“F5F0”,“s”:”_L_unlock_13",“s+”:“34”}}}
{“t”:{“$date”:“2021-07-08T11:58:57.313+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:31427, “ctx”:“xxxxxxing-keys-for-HMAC”,“msg”:" Frame: {frame
}“,“attr”:{“frame”:{“a”:“7F6D53801337”,“b”:“7F6D537CB000”,“o”:“36337”,“s”:“gsignal”,“s+”:“37”}}}
{“t”:{”$date":“2021-07-08T11:58:57.313+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:31427, “ctx”:“xxxxxxing-keys-for-HMAC”,“msg”:" Frame: {frame
}“,“attr”:{“frame”:{“a”:“7F6D53802A28”,“b”:“7F6D537CB000”,“o”:“37A28”,“s”:“abort”,“s+”:“148”}}}
{“t”:{”$date":“2021-07-08T11:58:57.313+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:31427, “ctx”:“xxxxxxing-keys-for-HMAC”,“msg”:" Frame: {frame
}“,“attr”:{“frame”:{“a”:“55FDAC6EB328”,“b”:“55FDAB722000”,“o”:“FC9328”,“s”:”_ZN5mongo15invariantFailedEPKcS1_j",“s+”:“12C”}}}
{“t”:{“$date”:“2021-07-08T11:58:57.313+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:31427, “ctx”:“xxxxxxing-keys-for-HMAC”,“msg”:" Frame: {frame
}“,“attr”:{“frame”:{“a”:“55FDAC6BD7A2”,“b”:“55FDAB722000”,“o”:“F9B7A2”,“s”:”_ZN5mongo6Client19setOperationContextEPNS_16OperationContextE.cold
.133",“s+”:“18”}}}
{“t”:{“$date”:“2021-07-08T11:58:57.313+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:31427, “ctx”:“xxxxxxing-keys-for-HMAC”,“msg”:" Frame: {frame
}“,“attr”:{“frame”:{“a”:“55FDAE484B65”,“b”:“55FDAB722000”,“o”:“2D62B65”,“s”:”_ZN5mongo14ServiceContext20makeOperationContextEPNS_6ClientE",“s+
“:“125”}}}
{“t”:{”$date”:“2021-07-08T11:58:57.313+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:31427, “ctx”:“xxxxxxing-keys-for-HMAC”,“msg”:" Frame: {frame
}“,“attr”:{“frame”:{“a”:“55FDAE479AF7”,“b”:“55FDAB722000”,“o”:“2D57AF7”,“s”:”_ZN5mongo6Client20makeOperationContextEv",“s+”:“27”}}}
{“t”:{“$date”:“2021-07-08T11:58:57.313+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:31427, “ctx”:“xxxxxxing-keys-for-HMAC”,“msg”:" Frame: {frame
}“,“attr”:{“frame”:{“a”:“55FDAE273AB2”,“b”:“55FDAB722000”,“o”:“2B51AB2”,“s”:”_ZN5mongo21KeysCollectionManager14PeriodicRunner18_doPeriodicRefr
eshEPNS_14ServiceContextESsNS_8DurationISt5ratioILl1ELl1000EEEE",“s+”:“162”}}}
{“t”:{“$date”:“2021-07-08T11:58:57.313+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:31427, “ctx”:“xxxxxxing-keys-for-HMAC”,“msg”:" Frame: {frame
}“,“attr”:{“frame”:{“a”:“55FDAE275B87”,“b”:“55FDAB722000”,“o”:“2B53B87”,“s”:”_ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN5mongo4stdx6th
readC4IZNS3_21KeysCollectionManager14PeriodicRunner5startEPNS3_14ServiceContextERKSsNS3_8DurationISt5ratioILl1ELl1000EEEEEUlvE_JELi0EEET_DpOT0
_EUlvE_EEEEE6_M_runEv",“s+”:“A7”}}}
{“t”:{“$date”:“2021-07-08T11:58:57.313+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:31427, “ctx”:“xxxxxxing-keys-for-HMAC”,“msg”:" Frame: {frame
}“,“attr”:{“frame”:{“a”:“55FDAE8E330F”,“b”:“55FDAB722000”,“o”:“31C130F”,“s”:“execute_native_thread_routine”,“s+”:“F”}}}
{“t”:{”$date":“2021-07-08T11:58:57.313+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:31427, “ctx”:“xxxxxxing-keys-for-HMAC”,“msg”:" Frame: {frame
}“,“attr”:{“frame”:{“a”:“7F6D53BA0E65”,“b”:“7F6D53B99000”,“o”:“7E65”,“s”:“start_thread”,“s+”:“C5”}}}
{“t”:{”$date":“2021-07-08T11:58:57.313+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:31427, “ctx”:“xxxxxxing-keys-for-HMAC”,“msg”:" Frame: {frame
}",“attr”:{“frame”:{“a”:“7F6D538C988D”,“b”:“7F6D537CB000”,“o”:“FE88D”,“s”:“clone”,“s+”:“6D”}}}

1 Like
5

The backtrace certainly changes things.

I can see that the last normal code function being used before an assertion was thrown was one for mongo::client::setOperationContext().

void Client::setOperationContext(OperationContext* opCtx) {
    // We can only set the OperationContext once before resetting it.
    invariant(opCtx != nullptr && _opCtx == nullptr);
    _opCtx = opCtx;
}

OperationContext objects are widely used though, so that’s not so specific.

Where that OperationContext was being created or referenced from was KeysCollectionManager::PeriodicRunner::_doPeriodicRefresh(). There are two lines inside_doPeriodicRefresh() which use the makeOperationContext() function the backtrace includes, I’m not sure which it would be yet.

It appears that when ServiceContext::makeOperationContext() was called:

  • the following lines managed to pass an opCtx pointer that was null, OR
  • the _opCtx member in the the service context class had somehow already been made null.
    {
        stdx::lock_guard<Client> lk(*client);
        client->setOperationContext(opCtx.get());
    }

And that’s the code in the latest supported version v4.4.6. That’s still the version you’re using, right?

1 Like
6

I am likewise experiencing similar crashes, with both signal 11 and signal 6. However, the backtrace seems to reference LDAP calls. We are primarily using ad security groups to manage user access.

The only consistency that I’ve seen when it happens is that the originating connection is from a Windows client (using java based drivers) that has been open a while and inactive, then maybe a new command results in the crash.

mongo --version
Percona Server for MongoDB shell version v4.0.27-22
git version: 00217efc472160ebce31dd98b772c4393448e4a1
OpenSSL version: OpenSSL 1.1.1g FIPS  21 Apr 2020
allocator: tcmalloc
modules: none
build environment:
    distarch: x86_64
    target_arch: x86_64

2022-12-12T14:36:57.347-0500 F -        [conn1198] Got signal: 11 (Segmentation fault).
 0x555557bfedb1 0x555557bfdfc9 0x555557bfe636 0x7ffff5d22fc0 0x7ffff74a0754 0x7ffff74a9708 0x555557b36bd3 0x555557b5557b3932f 0x555557b38141 0x5555575c2c6c 0x5555575c7ee5 0x5555575c8de0 0x5555575cb7d3 0x5555575c9ff3 0x5555575d911b6d84a 0x55555617b66a 0x555556176ca7 0x555556179b41 0x5555573497c2 0x555556174e90 0x5555561777a5 0x5555561763e7 0x55 0x555557349d25 0x555557b4c2f1 0x555557b4c364 0x7ffff5d18594 0x7ffff5a4c00f
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"555555554000","o":"26AADB1","s":"_ZN5mongo15printStackTraceERSo"},{"b":"555555554000","o":"26A9,"o":"26AA636"},{"b":"7FFFF5D11000","o":"11FC0"},{"b":"7FFFF748F000","o":"11754","s":"ldap_result"},{"b":"7FFFF748Fp_sasl_bind_s"},{"b":"555555554000","o":"25E2BD3","s":"_ZN5mongo8LDAPbindEP4ldapPKcS3_"},{"b":"555555554000","o":"2DAPManagerImpl24borrow_search_connectionEv"},{"b":"555555554000","o":"25E35CF","s":"_ZN5mongo15LDAPManagerImpl9exec_stringIcSt11char_traitsIcESaIcEEERSt6vectorIS6_SaIS6_EE"},{"b":"555555554000","o":"25E532F","s":"_ZN5mongo15LDAPMaNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERS6_"},{"b":"555555554000","o":"25E4141","s":"_ZN5mongo15LDAPMaERKNS_8UserNameERSt13unordered_setINS_8RoleNameESt4hashIS5_ESt8equal_toIS5_ESaIS5_EE"},{"b":"555555554000","o":"206hzManagerExternalStateLocal18getUserDescriptionEPNS_16OperationContextERKNS_8UserNameEPNS_7BSONObjE"},{"b":"5555555"_ZN5mongo24AuthorizationManagerImpl18getUserDescriptionEPNS_16OperationContextERKNS_8UserNameEPNS_7BSONObjE"},{"b"DE0","s":"_ZN5mongo24AuthorizationManagerImpl12_fetchUserV2EPNS_16OperationContextERKNS_8UserNameEPSt10unique_ptrINeIS7_EE"},{"b":"555555554000","o":"20777D3","s":"_ZN5mongo24AuthorizationManagerImpl11acquireUserEPNS_16OperationCo_4UserE"},{"b":"555555554000","o":"2075FF3","s":"_ZN5mongo24AuthorizationManagerImpl28acquireUserForSessionRefreshENS_8UserNameERKSt6vectorIhSaIhEEPPNS_4UserE"},{"b":"555555554000","o":"208511B","s":"_ZN5mongo24AuthorizationSessioNeededEPNS_16OperationContextE"},{"b":"555555554000","o":"C2EB3B","s":"_ZN5mongo23ServiceEntryPointCommon13handleReextERKNS_7MessageERKNS0_5HooksE"},{"b":"555555554000","o":"C1984A","s":"_ZN5mongo23ServiceEntryPointMongod13handleRtextERKNS_7MessageE"},{"b":"555555554000","o":"C2766A","s":"_ZN5mongo19ServiceStateMachine15_processMessageENS0_11T5554000","o":"C22CA7","s":"_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE"},{"b":"555555554000"55554000","o":"1DF57C2","s":"_ZN5mongo9transport26ServiceExecutorSynchronous8scheduleESt8functionIFvvEENS0_15ServicNS0_23ServiceExecutorTaskNameE"},{"b":"555555554000","o":"C20E90","s":"_ZN5mongo19ServiceStateMachine22_scheduleNexardENS_9transport15ServiceExecutor13ScheduleFlagsENS2_23ServiceExecutorTaskNameENS0_9OwnershipE"},{"b":"55555555400mongo19ServiceStateMachine15_sourceCallbackENS_6StatusE"},{"b":"555555554000","o":"C223E7","s":"_ZN5mongo19ServiceSgeENS0_11ThreadGuardE"},{"b":"555555554000","o":"C22D2D","s":"_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_1555554000","o":"C25B41"},{"b":"555555554000","o":"1DF5D25"},{"b":"555555554000","o":"25F82F1"},{"b":"555555554000",F5D11000","o":"7594"},{"b":"7FFFF5952000","o":"FA00F","s":"clone"}],"processInfo":{ "mongodbVersion" : "4.0.27-22",472160ebce31dd98b772c4393448e4a1", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "4.16.13-30"version" : "#1 SMP Wed Oct 3 08:46:00 CDT 2018", "machine" : "x86_64" }, "somap" : [ { "b" : "555555554000", "elfT4CD741FC5D1D9D483CFA7E5176DB61ACCE3A02" }, { "b" : "7FFFF7FFA000", "path" : "linux-vdso.so.1", "elfType" : 3, "buil0329FF2999FFAB9D338A0A0" }, { "b" : "7FFFF7B4F000", "path" : "/lib64/libcurl.so.4", "elfType" : 3, "buildId" : "67F26ED5105E273" }, { "b" : "7FFFF7900000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "392B51B6FBF5401" }, { "b" : "7FFFF76E3000", "path" : "/lib64/libsasl2.so.3", "elfType" : 3, "buildId" : "3DD431575F610904D}, { "b" : "7FFFF748F000", "path" : "/lib64/libldap-2.4.so.2", "elfType" : 3, "buildId" : "93D46EF1D712217F23B20ABC" : "7FFFF7280000", "path" : "/lib64/liblber-2.4.so.2", "elfType" : 3, "buildId" : "1ED4A7C36F4A16CBF49BB3A266D1A47FFF7069000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "21A5C9DED84DBF70DD12C7B3E1C57677A6A8274B0", "path" : "/lib64/libcrypto.so.1.1", "elfType" : 3, "buildId" : "00D972ABB1A4A3A04DE76B83F32FC4A1C2BA081E" }, { th" : "/lib64/libssl.so.1.1", "elfType" : 3, "buildId" : "BAC21E0DB690F555608453A91CAF6692406DD4A6" }, { "b" : "7FFb64/libdl.so.2", "elfType" : 3, "buildId" : "8E165CC915B59318CCFFE3F9A85319FA45DBF011" }, { "b" : "7FFFF64DC000", "1", "elfType" : 3, "buildId" : "5C4CEA4FE1BFC08501036C4DC88C250FED518A22" }, { "b" : "7FFFF6148000", "path" : "/lib: 3, "buildId" : "B3CA9767D9C5EC04A11D118C4BE453CCFB396404" }, { "b" : "7FFFF5F30000", "path" : "/lib64/libgcc_s.sodId" : "D238BBD1DE09FB51325C496CA88F4C6FB7A4EBDF" }, { "b" : "7FFFF5D11000", "path" : "/lib64/libpthread.so.0", "el0CB8F50BD0605EF18AC93CCCE2A50B6996C23C2D" }, { "b" : "7FFFF5952000", "path" : "/lib64/libc.so.6", "elfType" : 3, "b84B064796B20B2171F4E8C276B" }, { "b" : "7FFFF7DD7000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buil20768752AD47D642694D4BA" }, { "b" : "7FFFF572C000", "path" : "/lib64/libnghttp2.so.14", "elfType" : 3, "buildId" : 6DB4FCF47D4C8FFC" }, { "b" : "7FFFF550F000", "path" : "/lib64/libidn2.so.0", "elfType" : 3, "buildId" : "57B415AB374475D" }, { "b" : "7FFFF529E000", "path" : "/lib64/libssh.so.4", "elfType" : 3, "buildId" : "4C13D7FE28EBAA1D62AEAF"b" : "7FFFF508D000", "path" : "/lib64/libpsl.so.5", "elfType" : 3, "buildId" : "8F0F737E788719AD588B90895AD14C3CA64DA0000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "87B3E1642DBC3F29A00E4DD5A5CC0AE79CE0F59C" }, path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "16A14F2B57B8B647F8843E1FA61920C3AB6D641B" }, { "b" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "CDDA7CF0E16A18C8AE748AE99C260FED7D96C6B1" }, { "b" : "7FFFF4/libz.so.1", "elfType" : 3, "buildId" : "74DB165681064BA8C79C43FF4EBA1196003E3FFE" }, { "b" : "7FFFF455A000", "patht.so.0", "elfType" : 3, "buildId" : "C3AA0C92E7E12BD8551DAFE6F2129A90AB6AC177" }, { "b" : "7FFFF4356000", "path" : , "elfType" : 3, "buildId" : "661F078D0571A9BD7F1FD7041EAF82D55066D6C0" }, { "b" : "7FFFF412D000", "path" : "/lib64" : 3, "buildId" : "4FE657CC911CFA5C680FB902556AD97124944473" }, { "b" : "7FFFF3ED4000", "path" : "/lib64/libssl3.sId" : "61CF52DE71D70D8FE74B2B710F420C83090C9297" }, { "b" : "7FFFF3CAC000", "path" : "/lib64/libsmime3.so", "elfTyp06F3C66C895F1E2B5964ACA8B8AC441DCF35" }, { "b" : "7FFFF397F000", "path" : "/lib64/libnss3.so", "elfType" : 3, "buil91A72726A8C347132397807" }, { "b" : "7FFFF374F000", "path" : "/lib64/libnssutil3.so", "elfType" : 3, "buildId" : "2660878C8120358" }, { "b" : "7FFFF354B000", "path" : "/lib64/libplds4.so", "elfType" : 3, "buildId" : "A47F2F5D64F8AE1" }, { "b" : "7FFFF3346000", "path" : "/lib64/libplc4.so", "elfType" : 3, "buildId" : "17A2D8F7A06F1BFF18790B6421: "7FFFF3106000", "path" : "/lib64/libnspr4.so", "elfType" : 3, "buildId" : "BBB5DF2B54FFFC169547FF0B6293F5B742869B000", "path" : "/lib64/libunistring.so.2", "elfType" : 3, "buildId" : "38753C1A812DE48F6FD50150628817AEFCFFD7C0" },"path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "621907DCE16DEE67F915A58854C59F4130836216" }, { "b" : "/lib64/libpcre2-8.so.0", "elfType" : 3, "buildId" : "A61CBDE7C88F99235C750A1216EAA40C08F8B4AD" } ] }}
 mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x555557bfedb1]
 mongod(+0x26A9FC9) [0x555557bfdfc9]
 mongod(+0x26AA636) [0x555557bfe636]
 libpthread.so.0(+0x11FC0) [0x7ffff5d22fc0]
 libldap-2.4.so.2(ldap_result+0x444) [0x7ffff74a0754]
 libldap-2.4.so.2(ldap_sasl_bind_s+0xE8) [0x7ffff74a9708]
 mongod(_ZN5mongo8LDAPbindEP4ldapPKcS3_+0x83) [0x555557b36bd3]
 mongod(_ZN5mongo15LDAPManagerImpl24borrow_search_connectionEv+0x22E) [0x555557b3700e]
 mongod(_ZN5mongo15LDAPManagerImpl9execQueryERNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERSt6vectorIS6_SaIcf]
 mongod(_ZN5mongo15LDAPManagerImpl11mapUserToDNERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERS6_+0xA0F) [
 mongod(_ZN5mongo15LDAPManagerImpl14queryUserRolesERKNS_8UserNameERSt13unordered_setINS_8RoleNameESt4hashIS5_ESt8eq [0x555557b38141]
 mongod(_ZN5mongo30AuthzManagerExternalStateLocal18getUserDescriptionEPNS_16OperationContextERKNS_8UserNameEPNS_7BS2c6c]
 mongod(_ZN5mongo24AuthorizationManagerImpl18getUserDescriptionEPNS_16OperationContextERKNS_8UserNameEPNS_7BSONObjE
 mongod(_ZN5mongo24AuthorizationManagerImpl12_fetchUserV2EPNS_16OperationContextERKNS_8UserNameEPSt10unique_ptrINS_S7_EE+0x50) [0x5555575c8de0]
 mongod(_ZN5mongo24AuthorizationManagerImpl11acquireUserEPNS_16OperationContextERKNS_8UserNameEPPNS_4UserE+0x503) [
 mongod(_ZN5mongo24AuthorizationManagerImpl28acquireUserForSessionRefreshEPNS_16OperationContextERKNS_8UserNameERKSrE+0x43) [0x5555575c9ff3]
 mongod(_ZN5mongo24AuthorizationSessionImpl24_refreshUserInfoAsNeededEPNS_16OperationContextE+0x11B) [0x5555575d911
 mongod(_ZN5mongo23ServiceEntryPointCommon13handleRequestEPNS_16OperationContextERKNS_7MessageERKNS0_5HooksE+0x12B)
 mongod(_ZN5mongo23ServiceEntryPointMongod13handleRequestEPNS_16OperationContextERKNS_7MessageE+0x3A) [0x55555616d8
 mongod(_ZN5mongo19ServiceStateMachine15_processMessageENS0_11ThreadGuardE+0xBA) [0x55555617b66a]
 mongod(_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE+0x97) [0x555556176ca7]
 mongod(+0xC25B41) [0x555556179b41]
 mongod(_ZN5mongo9transport26ServiceExecutorSynchronous8scheduleESt8functionIFvvEENS0_15ServiceExecutor13ScheduleFlTaskNameE+0x1A2) [0x5555573497c2]
 mongod(_ZN5mongo19ServiceStateMachine22_scheduleNextWithGuardENS0_11ThreadGuardENS_9transport15ServiceExecutor13ScExecutorTaskNameENS0_9OwnershipE+0x150) [0x555556174e90]
 mongod(_ZN5mongo19ServiceStateMachine15_sourceCallbackENS_6StatusE+0x815) [0x5555561777a5]
 mongod(_ZN5mongo19ServiceStateMachine14_sourceMessageENS0_11ThreadGuardE+0x357) [0x5555561763e7]
 mongod(_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE+0x11D) [0x555556176d2d]
 mongod(+0xC25B41) [0x555556179b41]
 mongod(+0x1DF5D25) [0x555557349d25]
 mongod(+0x25F82F1) [0x555557b4c2f1]
 mongod(+0x25F8364) [0x555557b4c364]
 libpthread.so.0(+0x7594) [0x7ffff5d18594]
 libc.so.6(clone+0x3F) [0x7ffff5a4c00f]
-----  END BACKTRACE  -----
1 Like