MongoDB crashed by null pointer referencing

MongoDB Percona Distribution for MongoDB
1

Hello, I’m using 4.4.10-11 version of the Percona MongoDB with InMemory storage Engine. Sometimes, I can observed that the MongoDB had been crashed by null pointer referencing.

Here are details of that.

  1. One member for single replicaset not sharding.

  2. Options that used:
    {“t”:{“$date”:“2022-04-01T09:13:36.199+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:21951, “ctx”:“initandlisten”,“msg”:“Options set by command line”,“attr”:{“options”:{“net”:{“bindIp”:“*”,“compression”:{“compressors”:“snappy,zlib”},“port”:27017,“tls”:{“CAFile”:“/etc/mongodb-tls-internal/ca.crt”,“allowInvalidCertificates”:true,“certificateKeyFile”:“/etc/mongodb-tls-internal/client.crt”,“mode”:“preferTLS”}},“replication”:{“replSet”:“db-0-0”},“security”:{“keyFile”:“/etc/mongodb-internal/mongodb.key”,“relaxPermChecks”:true,“transitionToAuth”:true},“setParameter”:{“enableFlowControl”:“false”},“storage”:{“dbPath”:“/data/db”,“engine”:“inMemory”,“inMemory”:{“engineConfig”:{“inMemorySizeGB”:49.531648}}}}}}

  3. Last crash log.
    {“t”:{“$date”:“2022-04-01T09:56:27.358+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22943, “ctx”:“listener”,“msg”:“Connection accepted”,“attr”:{“remote”:“10.128.16.2:45638”,“connectionId”:892,“connectionCount”:18}}
    {“t”:{“$date”:“2022-04-01T09:56:27.358+00:00”},“s”:“I”, “c”:“NETWORK”, “id”:22944, “ctx”:“conn892”,“msg”:“Connection ended”,“attr”:{“remote”:“10.128.16.2:45638”,“connectionId”:892,“connectionCount”:17}}
    {“t”:{“$date”:“2022-04-01T09:56:28.669+00:00”},“s”:“I”, “c”:“STORAGE”, “id”:22402, “ctx”:“OplogCapMaintainerThread-local.oplog.rs”,“msg”:“WiredTiger record store oplog truncation finished”,“attr”:{“durationMillis”:29}}
    {“t”:{“$date”:“2022-04-01T09:56:28.678+00:00”},“s”:“F”, “c”:“CONTROL”, “id”:4757800, “ctx”:“thread893”,“msg”:“Writing fatal message”,“attr”:{“message”:“Invalid access at address: 0”}}
    {“t”:{“$date”:“2022-04-01T09:56:28.678+00:00”},“s”:“F”, “c”:“CONTROL”, “id”:4757800, “ctx”:“thread893”,“msg”:“Writing fatal message”,“attr”:{“message”:“Got signal: 11 (Segmentation fault).\n”}}
    {“t”:{“$date”:“2022-04-01T09:56:28.709+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:31431, “ctx”:“thread893”,“msg”:“BACKTRACE: {bt}”,“attr”:{“bt”:{“backtrace”:[{“a”:“558D1DAAD4CA”,“b”:“558D1AB9F000”,“o”:“2F0E4CA”,“s”:“_ZN5mongo34StackTraceAddressMetadataGenerator4loadEPv”,“s+”:“62A”},{“a”:“558D1DAAEF59”,“b”:“558D1AB9F000”,“o”:“2F0FF59”,“s”:“_ZN5mongo15printStackTraceEv”,“s+”:“29”},{“a”:“558D1DAAC42C”,“b”:“558D1AB9F000”,“o”:“2F0D42C”,“s”:“_ZN5mongo16stackTraceSignalEv”,“s+”:“4BC”},{“a”:“7F2890200B20”,“b”:“7F28901EE000”,“o”:“12B20”,“s”:“funlockfile”,“s+”:“50”},{“a”:“558D1DC2CE5B”,“b”:“558D1AB9F000”,“o”:“308DE5B”,“s”:“_ZN8tcmalloc11ThreadCache21ReleaseToCentralCacheEPNS0_8FreeListEji”,“s+”:“EB”},{“a”:“558D1DC2D085”,“b”:“558D1AB9F000”,“o”:“308E085”,“s”:“_ZN8tcmalloc11ThreadCache11ListTooLongEPNS0_8FreeListEj”,“s+”:“35”},{“a”:“558D1BE121F0”,“b”:“558D1AB9F000”,“o”:“12731F0”,“s”:“__wt_page_out”,“s+”:“6F0”},{“a”:“558D1BD6D0C3”,“b”:“558D1AB9F000”,“o”:“11CE0C3”,“s”:“__wt_evict”,“s+”:“1493”},{“a”:“558D1BD640BB”,“b”:“558D1AB9F000”,“o”:“11C50BB”,“s”:“__wt_evict_thread_chk”,“s+”:“D1B”},{“a”:“558D1BD64548”,“b”:“558D1AB9F000”,“o”:“11C5548”,“s”:“__wt_evict_thread_chk”,“s+”:“11A8”},{“a”:“558D1BD693B4”,“b”:“558D1AB9F000”,“o”:“11CA3B4”,“s”:“__wt_evict_thread_run”,“s+”:“74”},{“a”:“558D1BDCE4A9”,“b”:“558D1AB9F000”,“o”:“122F4A9”,“s”:“__wt_stat_session_clear_single”,“s+”:“79”},{“a”:“7F28901F614A”,“b”:“7F28901EE000”,“o”:“814A”,“s”:“start_thread”,“s+”:“EA”},{“a”:“7F288FF25DC3”,“b”:“7F288FE29000”,“o”:“FCDC3”,“s”:“clone”,“s+”:“43”}],“processInfo”:{“mongodbVersion”:“4.4.10-11”,“gitVersion”:“17077507fedd996277adb88715c384329950898b”,“compiledModules”:,“uname”:{“sysname”:“Linux”,“release”:“4.18.0-305.34.2.el8_4.x86_64”,“version”:“#1 SMP Mon Jan 17 09:42:23 EST 2022”,“machine”:“x86_64”},“somap”:[{“b”:“558D1AB9F000”,“elfType”:3,“buildId”:“9D05E13AE7BD1F7956720B391A84EBE9FAB5AE0B”},{“b”:“7F28901EE000”,“path”:“/lib64/libpthread.so.0”,“elfType”:3,“buildId”:“CF6D86CF965B129EC9DFDCD756741E43233684AB”},{“b”:“7F288FE29000”,“path”:“/lib64/libc.so.6”,“elfType”:3,“buildId”:“A91D7B51B23A64E86B82A91511C446C137D3EC8E”}]}}}}
    {“t”:{“$date”:“2022-04-01T09:56:28.709+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:31427, “ctx”:“thread893”,“msg”:" Frame: {frame}“,“attr”:{“frame”:{“a”:“558D1DAAD4CA”,“b”:“558D1AB9F000”,“o”:“2F0E4CA”,“s”:”_ZN5mongo34StackTraceAddressMetadataGenerator4loadEPv",“s+”:“62A”}}}
    {“t”:{“$date”:“2022-04-01T09:56:28.709+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:31427, “ctx”:“thread893”,“msg”:" Frame: {frame}“,“attr”:{“frame”:{“a”:“558D1DAAEF59”,“b”:“558D1AB9F000”,“o”:“2F0FF59”,“s”:”_ZN5mongo15printStackTraceEv",“s+”:“29”}}}
    {“t”:{“$date”:“2022-04-01T09:56:28.709+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:31427, “ctx”:“thread893”,“msg”:" Frame: {frame}“,“attr”:{“frame”:{“a”:“558D1DAAC42C”,“b”:“558D1AB9F000”,“o”:“2F0D42C”,“s”:”_ZN5mongo16stackTraceSignalEv",“s+”:“4BC”}}}
    {“t”:{“$date”:“2022-04-01T09:56:28.709+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:31427, “ctx”:“thread893”,“msg”:" Frame: {frame}“,“attr”:{“frame”:{“a”:“7F2890200B20”,“b”:“7F28901EE000”,“o”:“12B20”,“s”:“funlockfile”,“s+”:“50”}}}
    {“t”:{”$date":“2022-04-01T09:56:28.709+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:31427, “ctx”:“thread893”,“msg”:" Frame: {frame}“,“attr”:{“frame”:{“a”:“558D1DC2CE5B”,“b”:“558D1AB9F000”,“o”:“308DE5B”,“s”:”_ZN8tcmalloc11ThreadCache21ReleaseToCentralCacheEPNS0_8FreeListEji",“s+”:“EB”}}}
    {“t”:{“$date”:“2022-04-01T09:56:28.709+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:31427, “ctx”:“thread893”,“msg”:" Frame: {frame}“,“attr”:{“frame”:{“a”:“558D1DC2D085”,“b”:“558D1AB9F000”,“o”:“308E085”,“s”:”_ZN8tcmalloc11ThreadCache11ListTooLongEPNS0_8FreeListEj",“s+”:“35”}}}
    {“t”:{“$date”:“2022-04-01T09:56:28.709+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:31427, “ctx”:“thread893”,“msg”:" Frame: {frame}“,“attr”:{“frame”:{“a”:“558D1BE121F0”,“b”:“558D1AB9F000”,“o”:“12731F0”,“s”:”__wt_page_out",“s+”:“6F0”}}}
    {“t”:{“$date”:“2022-04-01T09:56:28.709+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:31427, “ctx”:“thread893”,“msg”:" Frame: {frame}“,“attr”:{“frame”:{“a”:“558D1BD6D0C3”,“b”:“558D1AB9F000”,“o”:“11CE0C3”,“s”:”__wt_evict",“s+”:“1493”}}}
    {“t”:{“$date”:“2022-04-01T09:56:28.709+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:31427, “ctx”:“thread893”,“msg”:" Frame: {frame}“,“attr”:{“frame”:{“a”:“558D1BD640BB”,“b”:“558D1AB9F000”,“o”:“11C50BB”,“s”:”__wt_evict_thread_chk",“s+”:“D1B”}}}
    {“t”:{“$date”:“2022-04-01T09:56:28.709+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:31427, “ctx”:“thread893”,“msg”:" Frame: {frame}“,“attr”:{“frame”:{“a”:“558D1BD64548”,“b”:“558D1AB9F000”,“o”:“11C5548”,“s”:”__wt_evict_thread_chk",“s+”:“11A8”}}}
    {“t”:{“$date”:“2022-04-01T09:56:28.709+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:31427, “ctx”:“thread893”,“msg”:" Frame: {frame}“,“attr”:{“frame”:{“a”:“558D1BD693B4”,“b”:“558D1AB9F000”,“o”:“11CA3B4”,“s”:”__wt_evict_thread_run",“s+”:“74”}}}
    {“t”:{“$date”:“2022-04-01T09:56:28.709+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:31427, “ctx”:“thread893”,“msg”:" Frame: {frame}“,“attr”:{“frame”:{“a”:“558D1BDCE4A9”,“b”:“558D1AB9F000”,“o”:“122F4A9”,“s”:”__wt_stat_session_clear_single",“s+”:“79”}}}
    {“t”:{“$date”:“2022-04-01T09:56:28.709+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:31427, “ctx”:“thread893”,“msg”:" Frame: {frame}“,“attr”:{“frame”:{“a”:“7F28901F614A”,“b”:“7F28901EE000”,“o”:“814A”,“s”:“start_thread”,“s+”:“EA”}}}
    {“t”:{”$date":“2022-04-01T09:56:28.709+00:00”},“s”:“I”, “c”:“CONTROL”, “id”:31427, “ctx”:“thread893”,“msg”:" Frame: {frame}",“attr”:{“frame”:{“a”:“7F288FF25DC3”,“b”:“7F288FE29000”,“o”:“FCDC3”,“s”:“clone”,“s+”:“43”}}}

  4. Single document size is about 80kbytes.

Could you check it?

1 Like
2

Tracking this issue here: [PSMDB-1026] MongoDB crashed by null pointer referencing - Percona JIRA

1 Like
3

Hi @chadr - we’re going to verify this issue on the latest MongoDB release. Version 4.4.10-11 you had back then is already EOL.

4

The backtrace here looks quite similar to https://jira.mongodb.org/browse/WT-7570
A potential fix should be https://jira.mongodb.org/browse/WT-11280