Hi all,
I updated ubuntu 20.04 via apt dist-upgrade.
The config didn’t change (I updated ubuntu packages on all nodes).
Now one of my nodes cannot join the cluster anymore, and I guess, it has something to do with that Errror:
[ERROR] [MY-000000] [Galera] Unrecognized parameter ‘”socket.ssl_key’
here the full error.log:
INFO: Skipping wsrep-recover for 5936d93f-dd84-11ea-88fe-bec65adf6365:82206641 pair
INFO: Assigning 5936d93f-dd84-11ea-88fe-bec65adf6365:82206641 to wsrep_start_position
2023-02-15T11:17:34.821901Z 0 [Warning] [MY-011068] [Server] The syntax 'wsrep_slave_threads' is deprecated and will be removed in a future release. Please use wsrep_applier_threads instead.
2023-02-15T11:17:34.824437Z 0 [Warning] [MY-010101] [Server] Insecure configuration for --secure-file-priv: Location is accessible to all OS users. Consider choosing a different directory.
2023-02-15T11:17:34.824451Z 0 [Warning] [MY-010097] [Server] Insecure configuration for --secure-log-path: Current value does not restrict location of generated files. Consider setting it to a valid, non-empty path.
2023-02-15T11:17:34.825341Z 0 [System] [MY-010116] [Server] /usr/sbin/mysqld (mysqld 8.0.30-22.1) starting as process 2087
2023-02-15T11:17:34.830073Z 0 [Warning] [MY-010068] [Server] CA certificate /etc/mysql/certs/ca.pem is self signed.
2023-02-15T11:17:34.830119Z 0 [System] [MY-013602] [Server] Channel mysql_main configured to support TLS. Encrypted connections are now supported for this channel.
2023-02-15T11:17:34.830130Z 0 [Note] [MY-000000] [WSREP] New joining cluster node configured to use specified SSL artifacts
2023-02-15T11:17:34.830192Z 0 [Note] [MY-000000] [Galera] Loading provider /usr/lib/galera4/libgalera_smm.so initial position: 5936d93f-dd84-11ea-88fe-bec65adf6365:82206641
2023-02-15T11:17:34.830222Z 0 [Note] [MY-000000] [Galera] wsrep_load(): loading provider library '/usr/lib/galera4/libgalera_smm.so'
2023-02-15T11:17:34.831136Z 0 [Note] [MY-000000] [Galera] wsrep_load(): Galera 4.12(e167906) by Codership Oy <info@codership.com> (modified by Percona <https://percona.com/>) loaded successfully.
2023-02-15T11:17:34.831194Z 0 [Note] [MY-000000] [Galera] CRC-32C: using 64-bit x86 acceleration.
2023-02-15T11:17:34.831584Z 0 [ERROR] [MY-000000] [Galera] Unrecognized parameter '”socket.ssl_key'
2023-02-15T11:17:34.832103Z 0 [Note] [MY-000000] [Galera] Found saved state: 5936d93f-dd84-11ea-88fe-bec65adf6365:82206641, safe_to_bootstrap: 0
2023-02-15T11:17:34.832214Z 0 [Note] [MY-000000] [Galera] GCache DEBUG: opened preamble:
Version: 2
UUID: 5936d93f-dd84-11ea-88fe-bec65adf6365
Seqno: 82169099 - 82206641
Offset: 72307944
Synced: 1
2023-02-15T11:17:34.832228Z 0 [Note] [MY-000000] [Galera] Recovering GCache ring buffer: version: 2, UUID: 5936d93f-dd84-11ea-88fe-bec65adf6365, offset: 72307944
2023-02-15T11:17:34.832353Z 0 [Note] [MY-000000] [Galera] GCache::RingBuffer initial scan... 0.0% ( 0/134217752 bytes) complete.
2023-02-15T11:17:34.866379Z 0 [Note] [MY-000000] [Galera] GCache::RingBuffer initial scan...100.0% (134217752/134217752 bytes) complete.
2023-02-15T11:17:34.867126Z 0 [Note] [MY-000000] [Galera] Recovering GCache ring buffer: found gapless sequence 82169099-82206641
2023-02-15T11:17:34.867200Z 0 [Note] [MY-000000] [Galera] GCache::RingBuffer unused buffers scan... 0.0% ( 0/134216616 bytes) complete.
2023-02-15T11:17:34.870150Z 0 [Note] [MY-000000] [Galera] GCache::RingBuffer unused buffers scan...100.0% (134216616/134216616 bytes) complete.
2023-02-15T11:17:34.870173Z 0 [Note] [MY-000000] [Galera] Recovering GCache ring buffer: found 8/37551 locked buffers
2023-02-15T11:17:34.870181Z 0 [Note] [MY-000000] [Galera] Recovering GCache ring buffer: free space: 2968/134217728
2023-02-15T11:17:34.874345Z 0 [Note] [MY-000000] [Galera] Passing config to GCS: base_dir = /var/lib/mysql/; base_host = 192.168.70.7; base_port = 4567; cert.log_conflicts = no; cert.optimistic_pa = no; debug = no; evs.auto_evict = 0; evs.delay_margin = PT1S; evs.delayed_keep_period = PT30S; evs.inactive_check_period = PT0.5S; evs.inactive_timeout = PT15S; evs.join_retrans_period = PT1S; evs.max_install_timeouts = 3; evs.send_window = 10; evs.stats_report_period = PT1M; evs.suspect_timeout = PT5S; evs.user_send_window = 4; evs.view_forget_timeout = PT24H; gcache.dir = /var/lib/mysql/; gcache.freeze_purge_at_seqno = -1; gcache.keep_pages_count = 0; gcache.keep_pages_size = 0; gcache.mem_size = 0; gcache.name = galera.cache; gcache.page_size = 128M; gcache.recover = yes; gcache.size = 128M; gcomm.thread_prio = ; gcs.fc_debug = 0; gcs.fc_factor = 1.0; gcs.fc_limit = 100; gcs.fc_master_slave = no; gcs.fc_single_primary = no; gcs.max_packet_size = 64500; gcs.max_throttle = 0.25; gcs.recv_q_hard_limit = 9223372036854775807; gcs.recv_q_soft_limit = 0.25; gcs.sync_donor = no; gmcast.segment = 0; gmcast.version = 0; pc.announce_timeout = PT3S; pc.checksum = false; pc.ignore_quorum = false; pc.ignore_sb = false; pc.npvo = false; pc.recovery = true; pc.version = 0; pc.wait_prim = true; pc.wait_prim_timeout = PT30S; pc.weight = 1; protonet.backend = asio; protonet.version = 0; repl.causal_read_timeout = PT30S; repl.commit_order = 3; repl.key_format = FLAT8; repl.max_ws_size = 2147483647; repl.proto_max = 10; socket.checksum = 2; socket.recv_buf_size = auto; socket.send_buf_size = auto; socket.ssl = YES; socket.ssl_ca = /etc/mysql/certs/ca.pem; socket.ssl_cert = /etc/mysql/certs/server-cert.pem; socket.ssl_cipher = ; socket.ssl_compression = YES; socket.ssl_key = /etc/mysql/certs/server-key.pem; socket.ssl_reload = 1;
2023-02-15T11:17:34.896865Z 0 [Note] [MY-000000] [Galera] Service thread queue flushed.
2023-02-15T11:17:34.897099Z 0 [Note] [MY-000000] [Galera] ####### Assign initial position for certification: 5936d93f-dd84-11ea-88fe-bec65adf6365:82206641, protocol version: -1
2023-02-15T11:17:34.897234Z 0 [Note] [MY-000000] [WSREP] Starting replication
2023-02-15T11:17:34.897263Z 0 [Note] [MY-000000] [Galera] Connecting with bootstrap option: 0
2023-02-15T11:17:34.897281Z 0 [Note] [MY-000000] [Galera] Setting GCS initial position to 5936d93f-dd84-11ea-88fe-bec65adf6365:82206641
2023-02-15T11:17:34.897367Z 0 [Note] [MY-000000] [Galera] protonet asio version 0
2023-02-15T11:17:34.897991Z 0 [Note] [MY-000000] [Galera] Using CRC-32C for message checksums.
2023-02-15T11:17:34.898026Z 0 [Note] [MY-000000] [Galera] backend: asio
2023-02-15T11:17:34.898133Z 0 [Note] [MY-000000] [Galera] gcomm thread scheduling priority set to other:0
2023-02-15T11:17:34.898321Z 0 [Note] [MY-000000] [Galera] Fail to access the file (/var/lib/mysql//gvwstate.dat) error (No such file or directory). It is possible if node is booting for first time or re-booting after a graceful shutdown
2023-02-15T11:17:34.898343Z 0 [Note] [MY-000000] [Galera] Restoring primary-component from disk failed. Either node is booting for first time or re-booting after a graceful shutdown
2023-02-15T11:17:34.898522Z 0 [Note] [MY-000000] [Galera] GMCast version 0
2023-02-15T11:17:34.898723Z 0 [Note] [MY-000000] [Galera] (58d1b00f-81d6, 'ssl://0.0.0.0:4567') listening at ssl://0.0.0.0:4567
2023-02-15T11:17:34.898741Z 0 [Note] [MY-000000] [Galera] (58d1b00f-81d6, 'ssl://0.0.0.0:4567') multicast: , ttl: 1
2023-02-15T11:17:34.899024Z 0 [Note] [MY-000000] [Galera] EVS version 1
2023-02-15T11:17:34.899185Z 0 [Note] [MY-000000] [Galera] gcomm: connecting to group 'pxc-cluster', peer '192.168.70.6:,192.168.70.7:,192.168.70.9:'
2023-02-15T11:17:34.907699Z 0 [Note] [MY-000000] [Galera] (58d1b00f-81d6, 'ssl://0.0.0.0:4567') Found matching local endpoint for a connection, blacklisting address ssl://192.168.70.7:4567
2023-02-15T11:17:34.908412Z 0 [Warning] [MY-000000] [Galera] handshake with d6448eae-9431 ssl://192.168.70.9:4567 failed: 'invalid group'
2023-02-15T11:17:34.909246Z 0 [Warning] [MY-000000] [Galera] handshake with 495b8e2e-9b9c ssl://192.168.70.6:4567 failed: 'invalid group'
2023-02-15T11:17:36.407163Z 0 [Warning] [MY-000000] [Galera] handshake with 495b8e2e-9b9c ssl://192.168.70.6:4567 failed: 'invalid group'
2023-02-15T11:17:36.906131Z 0 [Warning] [MY-000000] [Galera] handshake with d6448eae-9431 ssl://192.168.70.9:4567 failed: 'invalid group'
2023-02-15T11:17:37.901098Z 0 [Note] [MY-000000] [Galera] announce period timed out (pc.announce_timeout)
2023-02-15T11:17:37.901348Z 0 [Note] [MY-000000] [Galera] EVS version upgrade 0 -> 1
2023-02-15T11:17:37.901389Z 0 [Note] [MY-000000] [Galera] PC protocol upgrade 0 -> 1
2023-02-15T11:17:37.901483Z 0 [Warning] [MY-000000] [Galera] no nodes coming from prim view, prim not possible
2023-02-15T11:17:37.901552Z 0 [Note] [MY-000000] [Galera] Current view of cluster as seen by this node
view (view_id(NON_PRIM,58d1b00f-81d6,1)
memb {
58d1b00f-81d6,0
}
joined {
}
left {
}
partitioned {
}
)
2023-02-15T11:17:37.907291Z 0 [Warning] [MY-000000] [Galera] handshake with 495b8e2e-9b9c ssl://192.168.70.6:4567 failed: 'invalid group'
2023-02-15T11:17:38.401886Z 0 [Warning] [MY-000000] [Galera] last inactive check more than PT1.5S (3*evs.inactive_check_period) ago (PT3.50289S), skipping check
2023-02-15T11:17:38.405995Z 0 [Warning] [MY-000000] [Galera] handshake with d6448eae-9431 ssl://192.168.70.9:4567 failed: 'invalid group'
2023-02-15T11:17:39.406369Z 0 [Warning] [MY-000000] [Galera] handshake with 495b8e2e-9b9c ssl://192.168.70.6:4567 failed: 'invalid group'
2023-02-15T11:17:39.907149Z 0 [Warning] [MY-000000] [Galera] handshake with d6448eae-9431 ssl://192.168.70.9:4567 failed: 'invalid group'
....
2023-02-15T11:24:06.440582Z 0 [Warning] [MY-000000] [Galera] handshake with 495b8e2e-9b9c ssl://192.168.70.6:4567 failed: 'invalid group'
2023-02-15T11:24:06.939703Z 0 [Warning] [MY-000000] [Galera] handshake with d6448eae-9431 ssl://192.168.70.9:4567 failed: 'invalid group'
2023-02-15T11:24:07.940616Z 0 [Warning] [MY-000000] [Galera] handshake with 495b8e2e-9b9c ssl://192.168.70.6:4567 failed: 'invalid group'
2023-02-15T11:24:07.942243Z 0 [Note] [MY-000000] [Galera] PC protocol downgrade 1 -> 0
2023-02-15T11:24:07.942327Z 0 [Note] [MY-000000] [Galera] Current view of cluster as seen by this node
view ((empty))
2023-02-15T11:24:07.942446Z 0 [ERROR] [MY-000000] [Galera] failed to open gcomm backend connection: 110: failed to reach primary view (pc.wait_prim_timeout): 110 (Connection timed out)
at gcomm/src/pc.cpp:connect():161
2023-02-15T11:24:07.942487Z 0 [ERROR] [MY-000000] [Galera] gcs/src/gcs_core.cpp:gcs_core_open():219: Failed to open backend connection: -110 (Connection timed out)
2023-02-15T11:24:08.942748Z 0 [Note] [MY-000000] [Galera] gcomm: terminating thread
2023-02-15T11:24:08.942891Z 0 [Note] [MY-000000] [Galera] gcomm: joining thread
2023-02-15T11:24:08.943109Z 0 [ERROR] [MY-000000] [Galera] gcs/src/gcs.cpp:gcs_open():1811: Failed to open channel 'pxc-cluster' at 'gcomm://192.168.70.6,192.168.70.7,192.168.70.9': -110 (Connection timed out)
2023-02-15T11:24:08.943163Z 0 [ERROR] [MY-000000] [Galera] gcs connect failed: Connection timed out
2023-02-15T11:24:08.943210Z 0 [ERROR] [MY-000000] [WSREP] Provider/Node (gcomm://192.168.70.6,192.168.70.7,192.168.70.9) failed to establish connection with cluster (reason: 7)
2023-02-15T11:24:08.943258Z 0 [ERROR] [MY-010119] [Server] Aborting
2023-02-15T11:24:08.943795Z 0 [System] [MY-010910] [Server] /usr/sbin/mysqld: Shutdown complete (mysqld 8.0.30-22.1) Percona XtraDB Cluster (GPL), Release rel22, Revision aff6a8b, WSREP version 26.4.3.
2023-02-15T11:24:08.945234Z 0 [Note] [MY-000000] [Galera] dtor state: CLOSED
2023-02-15T11:24:08.945337Z 0 [Note] [MY-000000] [Galera] MemPool(TrxHandleSlave): hit ratio: 0, misses: 0, in use: 0, in pool: 0
2023-02-15T11:24:08.947914Z 0 [Note] [MY-000000] [Galera] apply mon: entered 0
2023-02-15T11:24:08.950234Z 0 [Note] [MY-000000] [Galera] apply mon: entered 0
2023-02-15T11:24:08.952494Z 0 [Note] [MY-000000] [Galera] apply mon: entered 0
2023-02-15T11:24:08.952596Z 0 [Note] [MY-000000] [Galera] cert index usage at exit 0
2023-02-15T11:24:08.952625Z 0 [Note] [MY-000000] [Galera] cert trx map usage at exit 0
2023-02-15T11:24:08.952654Z 0 [Note] [MY-000000] [Galera] deps set usage at exit 0
2023-02-15T11:24:08.952693Z 0 [Note] [MY-000000] [Galera] avg deps dist 0
2023-02-15T11:24:08.952733Z 0 [Note] [MY-000000] [Galera] avg cert interval 0
2023-02-15T11:24:08.952761Z 0 [Note] [MY-000000] [Galera] cert index size 0
2023-02-15T11:24:08.952886Z 0 [Note] [MY-000000] [Galera] Service thread queue flushed.
2023-02-15T11:24:08.952986Z 0 [Note] [MY-000000] [Galera] wsdb trx map usage 0 conn query map usage 0
2023-02-15T11:24:08.953024Z 0 [Note] [MY-000000] [Galera] MemPool(LocalTrxHandle): hit ratio: 0, misses: 0, in use: 0, in pool: 0
2023-02-15T11:24:08.953220Z 0 [Note] [MY-000000] [Galera] Shifting CLOSED -> DESTROYED (TO: 0)
2023-02-15T11:24:08.956120Z 0 [Note] [MY-000000] [Galera] Flushing memory map to disk...
here is the config
/etc/mysql/mysql.conf.d# cat /etc/mysql/mysql.conf.d/mysqld.cnf
# Template my.cnf for PXC
# Edit to your requirements.
[client]
socket=/var/run/mysqld/mysqld.sock
[mysqld]
server-id=3
datadir=/var/lib/mysql
socket=/var/run/mysqld/mysqld.sock
log-error=/var/log/mysql/error.log
pid-file=/var/run/mysqld/mysqld.pid
# Binary log expiration period is 604800 seconds, which equals 7 days
binlog_expire_logs_seconds=604800
######## wsrep ###############
# Path to Galera library
wsrep_provider=/usr/lib/galera4/libgalera_smm.so
wsrep_provider_options=”socket.ssl_key=server-key.pem;socket.ssl_cert=server-cert.pem;socket.ssl_ca=ca.pem”
# Cluster connection URL contains IPs of nodes
#If no IP is found, this implies that a new cluster needs to be created,
#in order to do that you need to bootstrap this node
wsrep_cluster_address=gcomm://192.168.70.6,192.168.70.7,192.168.70.9
# In order for Galera to work correctly binlog format should be ROW
binlog_format=ROW
# Slave thread to use
wsrep_slave_threads=8
wsrep_log_conflicts
# This changes how InnoDB autoincrement locks are managed and is a requirement for Galera
innodb_autoinc_lock_mode=2
# Node IP address
wsrep_node_address=192.168.70.7
# Cluster name
wsrep_cluster_name=pxc-cluster
#If wsrep_node_name is not specified, then system hostname will be used
wsrep_node_name=ngh-db3-prod
#pxc_strict_mode allowed values: DISABLED,PERMISSIVE,ENFORCING,MASTER
pxc_strict_mode=ENFORCING
# SST method
wsrep_sst_method=xtrabackup-v2
[sst]
encrypt=4
ssl-key=server-key.pem
ssl-ca=ca.pem
ssl-cert=server-cert.pem