Error reattaching a zstd volume to snappy replica set

MongoDB Percona Server for MongoDB
,
1

Hi team,

I am trying to move the replica set from snappy to zstd and when I tried to switch the volume of one of the zstd compressed volume node (which was done hidden sync), it failed and I tried revert back the instance it is not working either

Could you please help me with it

{"t":{"$date":"2023-01-09T07:43:22.859+00:00"},"s":"I",  "c":"CONTROL",  "id":21951,   "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"auditLog":{"destination":"file","filter":"{ atype: \"authCheck\", \"param.command\": { $in: [ \"delete\" ] } }","format":"JSON","path":"/var/log/mongodb/auditLog.json"},"config":"/etc/mongod.conf","net":{"bindIp":"*","compression":{"compressors":"zstd"},"maxIncomingConnections":25000,"port":27017},"operationProfiling":{"mode":"slowOp","slowOpThresholdMs":1000},"processManagement":{"fork":true,"pidFilePath":"/var/run/mongod.pid"},"replication":{"replSetName":"rs_afds_IR"},"security":{"authorization":"enabled","keyFile":"/var/lib/mongo/mongodb-keyfile","ldap":{"authz":{"queryTemplate":"{USER}?memberOf?base"},"bind":{"method":"simple","queryPassword":"<password>","queryUser":"cn=mongodb.prod@phenompeople.com,ou=users,dc=phenom,dc=onelogin,dc=com"},"servers":"ldap.us.onelogin.com:636","userToDNMapping":"[ { match: \"(.+)\", substitution: \"CN={0},OU=users,DC=phenom,DC=onelogin,DC=com\" } ]"}},"setParameter":{"auditAuthorizationSuccess":"true","authenticationMechanisms":"PLAIN,SCRAM-SHA-1,SCRAM-SHA-256"},"storage":{"dbPath":"/data/mongodb","directoryPerDB":true,"journal":{"enabled":true},"wiredTiger":{"collectionConfig":{"blockCompressor":"zstd"},"engineConfig":{"journalCompressor":"zstd"},"indexConfig":{"prefixCompression":true}}},"systemLog":{"destination":"file","logRotate":"rename","path":"/var/log/mongodb/mongod.log"}}}}
{"t":{"$date":"2023-01-09T07:43:22.860+00:00"},"s":"W",  "c":"STORAGE",  "id":22271,   "ctx":"initandlisten","msg":"Detected unclean shutdown - Lock file is not empty","attr":{"lockFile":"/data/mongodb/mongod.lock"}}
{"t":{"$date":"2023-01-09T07:43:22.860+00:00"},"s":"I",  "c":"STORAGE",  "id":22270,   "ctx":"initandlisten","msg":"Storage engine to use detected by data files","attr":{"dbpath":"/data/mongodb","storageEngine":"wiredTiger"}}
{"t":{"$date":"2023-01-09T07:43:22.860+00:00"},"s":"W",  "c":"STORAGE",  "id":22302,   "ctx":"initandlisten","msg":"Recovering data from the last clean checkpoint."}
{"t":{"$date":"2023-01-09T07:43:22.860+00:00"},"s":"I",  "c":"STORAGE",  "id":22315,   "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=3356M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=zstd),file_manager=(close_idle_time=100000,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],"}}
{"t":{"$date":"2023-01-09T07:43:23.403+00:00"},"s":"I",  "c":"STORAGE",  "id":22430,   "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1673250203:403049][9219:0x7fada53ffbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 93 through 589"}}
{"t":{"$date":"2023-01-09T07:43:23.403+00:00"},"s":"E",  "c":"STORAGE",  "id":22435,   "ctx":"initandlisten","msg":"WiredTiger error","attr":{"error":0,"message":"[1673250203:403183][9219:0x7fada53ffbc0], txn-recover: zstd error: ZSTD_decompress: Unknown frame descriptor"}}
{"t":{"$date":"2023-01-09T07:43:23.403+00:00"},"s":"E",  "c":"STORAGE",  "id":22435,   "ctx":"initandlisten","msg":"WiredTiger error","attr":{"error":-31802,"message":"[1673250203:403239][9219:0x7fada53ffbc0], txn-recover: __wt_txn_recover, 1109: Recovery failed: WT_ERROR: non-specific WiredTiger error"}}
{"t":{"$date":"2023-01-09T07:43:23.403+00:00"},"s":"E",  "c":"STORAGE",  "id":22435,   "ctx":"initandlisten","msg":"WiredTiger error","attr":{"error":0,"message":"[1673250203:403613][9219:0x7fada53ffbc0], connection: __wt_cache_destroy, 364: cache server: exiting with 3 pages in memory and 0 pages evicted"}}
{"t":{"$date":"2023-01-09T07:43:23.403+00:00"},"s":"E",  "c":"STORAGE",  "id":22435,   "ctx":"initandlisten","msg":"WiredTiger error","attr":{"error":0,"message":"[1673250203:403650][9219:0x7fada53ffbc0], connection: __wt_cache_destroy, 367: cache server: exiting with 22473 image bytes in memory"}}
{"t":{"$date":"2023-01-09T07:43:23.403+00:00"},"s":"E",  "c":"STORAGE",  "id":22435,   "ctx":"initandlisten","msg":"WiredTiger error","attr":{"error":0,"message":"[1673250203:403667][9219:0x7fada53ffbc0], connection: __wt_cache_destroy, 370: cache server: exiting with 27617 bytes in memory"}}
{"t":{"$date":"2023-01-09T07:43:23.419+00:00"},"s":"I",  "c":"STORAGE",  "id":22430,   "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1673250203:419258][9219:0x7fada53ffbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 93 through 590"}}
{"t":{"$date":"2023-01-09T07:43:23.419+00:00"},"s":"E",  "c":"STORAGE",  "id":22435,   "ctx":"initandlisten","msg":"WiredTiger error","attr":{"error":0,"message":"[1673250203:419353][9219:0x7fada53ffbc0], txn-recover: zstd error: ZSTD_decompress: Unknown frame descriptor"}}
{"t":{"$date":"2023-01-09T07:43:23.419+00:00"},"s":"E",  "c":"STORAGE",  "id":22435,   "ctx":"initandlisten","msg":"WiredTiger error","attr":{"error":-31802,"message":"[1673250203:419392][9219:0x7fada53ffbc0], txn-recover: __wt_txn_recover, 1109: Recovery failed: WT_ERROR: non-specific WiredTiger error"}}
{"t":{"$date":"2023-01-09T07:43:23.419+00:00"},"s":"E",  "c":"STORAGE",  "id":22435,   "ctx":"initandlisten","msg":"WiredTiger error","attr":{"error":0,"message":"[1673250203:419516][9219:0x7fada53ffbc0], connection: __wt_cache_destroy, 364: cache server: exiting with 3 pages in memory and 0 pages evicted"}}
{"t":{"$date":"2023-01-09T07:43:23.419+00:00"},"s":"E",  "c":"STORAGE",  "id":22435,   "ctx":"initandlisten","msg":"WiredTiger error","attr":{"error":0,"message":"[1673250203:419555][9219:0x7fada53ffbc0], connection: __wt_cache_destroy, 367: cache server: exiting with 22473 image bytes in memory"}}
{"t":{"$date":"2023-01-09T07:43:23.419+00:00"},"s":"E",  "c":"STORAGE",  "id":22435,   "ctx":"initandlisten","msg":"WiredTiger error","attr":{"error":0,"message":"[1673250203:419586][9219:0x7fada53ffbc0], connection: __wt_cache_destroy, 370: cache server: exiting with 27617 bytes in memory"}}
{"t":{"$date":"2023-01-09T07:43:23.431+00:00"},"s":"I",  "c":"STORAGE",  "id":22430,   "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1673250203:431653][9219:0x7fada53ffbc0], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 93 through 591"}}
{"t":{"$date":"2023-01-09T07:43:23.431+00:00"},"s":"E",  "c":"STORAGE",  "id":22435,   "ctx":"initandlisten","msg":"WiredTiger error","attr":{"error":0,"message":"[1673250203:431749][9219:0x7fada53ffbc0], txn-recover: zstd error: ZSTD_decompress: Unknown frame descriptor"}}
{"t":{"$date":"2023-01-09T07:43:23.431+00:00"},"s":"E",  "c":"STORAGE",  "id":22435,   "ctx":"initandlisten","msg":"WiredTiger error","attr":{"error":-31802,"message":"[1673250203:431781][9219:0x7fada53ffbc0], txn-recover: __wt_txn_recover, 1109: Recovery failed: WT_ERROR: non-specific WiredTiger error"}}
{"t":{"$date":"2023-01-09T07:43:23.431+00:00"},"s":"E",  "c":"STORAGE",  "id":22435,   "ctx":"initandlisten","msg":"WiredTiger error","attr":{"error":0,"message":"[1673250203:431920][9219:0x7fada53ffbc0], connection: __wt_cache_destroy, 364: cache server: exiting with 3 pages in memory and 0 pages evicted"}}
{"t":{"$date":"2023-01-09T07:43:23.431+00:00"},"s":"E",  "c":"STORAGE",  "id":22435,   "ctx":"initandlisten","msg":"WiredTiger error","attr":{"error":0,"message":"[1673250203:431963][9219:0x7fada53ffbc0], connection: __wt_cache_destroy, 367: cache server: exiting with 22473 image bytes in memory"}}
{"t":{"$date":"2023-01-09T07:43:23.432+00:00"},"s":"E",  "c":"STORAGE",  "id":22435,   "ctx":"initandlisten","msg":"WiredTiger error","attr":{"error":0,"message":"[1673250203:431998][9219:0x7fada53ffbc0], connection: __wt_cache_destroy, 370: cache server: exiting with 27617 bytes in memory"}}
{"t":{"$date":"2023-01-09T07:43:23.433+00:00"},"s":"W",  "c":"STORAGE",  "id":22347,   "ctx":"initandlisten","msg":"Failed to start up WiredTiger under any compatibility version. This may be due to an unsupported upgrade or downgrade."}
{"t":{"$date":"2023-01-09T07:43:23.433+00:00"},"s":"F",  "c":"-",        "id":23089,   "ctx":"initandlisten","msg":"Fatal assertion","attr":{"msgid":28559,"file":"src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp","line":97}}
{"t":{"$date":"2023-01-09T07:43:23.433+00:00"},"s":"F",  "c":"-",        "id":23090,   "ctx":"initandlisten","msg":"\n\n***aborting after fassert() failure\n\n"}
{"t":{"$date":"2023-01-09T07:43:23.433+00:00"},"s":"F",  "c":"CONTROL",  "id":4757800, "ctx":"initandlisten","msg":"Writing fatal message","attr":{"message":"Got signal: 6 (Aborted).\n"}}
1 Like
2

developed a solution.

By altering the priority or step-down mechanism, we can make the zstd machine the main, change its configuration to resemble the zstd machine, attach the compressed volumes (generated from the primary’s snapshot), and restart the machines.

2 Likes
3

Another solution is to remove the node from replicaset and with zstd volume bring up the server as standalone and then add the node into the cluster.