How Can I Running PXC_MySQL in a docker swarm mode by docker stack deploy

MySQL & MariaDB Percona XtraDB Cluster 5.x
1

Hi, I got deploy error in deploy a pxc_mysql in docker swarm mode. I have been saw the blog post about the how to use the container, but it seen not really running in swarm mode. just docker compose mode. I got success running the containers running is compose mode by using the compose file like below:

version: '3.9'

services:
    pxc_node1:
      container_name: pxc_node1
      #image: perconalab/percona-xtradb-cluster:5.7.41
      image: percona/percona-xtradb-cluster:5.7.41
      networks:
        - mlo_db_bkend
      environment:
        - CLUSTER_JOIN=
      ports:
        - "3306"
        - "4444"
        - "4567"
        - "4568"

      env_file: .env
      #depends_on:
      #node2: 
      # condition: service_healthy
      healthcheck:
         test: mysql -u root --password=${MYSQL_ROOT_PASSWORD} -e 'show databases;' || exit 1
         interval: 10s
         timeout: 3s
         retries: 5
         start_period: 10s

      volumes:
        - vol_pxc_node1:/var/lib/mysql
        # bind mount my local my.cnf
        # - $PWD/my.cnf:/etc/my.cnf

    pxc_node2:
      container_name: pxc_node2
      image: percona/percona-xtradb-cluster:5.7.41
      #image: perconalab/percona-xtradb-cluster:5.7.41
      networks:
        - mlo_db_bkend
      ports:
        - "3306"
        - "4444"
        - "4567"
        - "4568"

      environment:
        - CLUSTER_JOIN=pxc_node1,pxc_node3
      env_file: .env
      depends_on:
        - pxc_node1 
          #  condition: "service_healthy"

      healthcheck:
         test: mysql -u root --password=${MYSQL_ROOT_PASSWORD} -e 'show databases;' || exit 1
         interval: 10s
         timeout: 3s
         retries: 5
         start_period: 10s
      volumes:
        - vol_pxc_node2:/var/lib/mysql

    pxc_node3:
      container_name: pxc_node3
      #image: perconalab/percona-xtradb-cluster:5.7.41
      image: percona/percona-xtradb-cluster:5.7.41
      networks:
      - mlo_db_bkend
      ports:
        - "3306"
        - "4444"
        - "4567"
        - "4568"
      environment:
        - CLUSTER_JOIN=pxc_node1,pxc_node2
      env_file: 
        - .env
      depends_on:
        - pxc_node1
                 #condition: service_healthy
      volumes:
        - vol_pxc_node3:/var/lib/mysql

    proxysql:
      container_name: proxysql
      image: percona/proxysql2:2.5
      networks:
        - mlo_db_front
        - mlo_db_bkend
      ports:
        - "3306:3306"
        - "6032:6032"
        - "6080:6080"
      environment:
        - CLUSTER_JOIN=pxc_node1,pxc_node2,pxc_node3
      env_file: 
        - .env
      #- .env_proxy
      volumes:
        - vol_proxy_sql:/var/lib/proxysql
        - ./proxysql/proxysql.cnf:/etc/proxysql/proxysql.cnf
          #command: chown -R proxysql:root /etc/proxysql
networks:
  mlo_db_bkend:
    driver: overlay
    attachable: true
  mlo_db_front:
    driver: overlay
    attachable: true
volumes:
  vol_pxc_node1:
  vol_pxc_node2:
  vol_pxc_node3:
  vol_proxy_sql:

And I also success deploy in compose mode by using compose file like:

version: '3.9'

services:
    etcdnode:
      container_name: etcdnode
      image: quay.io/coreos/etcd:v3.4.26
      command: /usr/local/bin/etcd
      volumes:
      # note: to test on windows change the path to this
      #- ./usr/share/ca-certificates/:/etc/ssl/certs
      #  - /usr/share/ca-certificates/:/etc/ssl/certs
        - /etcd-data
      env_file: .env_etcd
      networks:
        - mlo_db_bkend
                #ipv4_address: "10.180.17.200"
      ports:
        - "2379"
        - "2380"
        - "4001"
      deploy:
        mode: replicated
        replicas: 1

    pxc_node:
      container_name: pxc_node
      image: perconalab/percona-xtradb-cluster:5.7.41
      #image: percona/percona-xtradb-cluster:5.7.41
      networks:
        - mlo_db_bkend
      ports:
        - "3306"
        - "4444"
        - "4567"
        - "4568"

      env_file: .env
      depends_on:
        - etcdnode 
      # condition: service_healthy
      healthcheck:
         test: mysql -u root --password="123456" -e 'show databases;' || exit 1
         interval: 10s
         timeout: 3s
         retries: 5
         start_period: 10s

      volumes:
        - /var/lib/mysql
        # bind mount my local my.cnf
        # - $PWD/my.cnf:/etc/my.cnf
      deploy:
        update_config:
          parallelism: 1
          delay: 10s
          order: stop-first

    proxysql:
      container_name: proxysql
      image: percona/proxysql2:2.5
      networks:
        - mlo_db_front
        - mlo_db_bkend
      ports:
        - "3306:3306"
        - "6032:6032"
        - "6080:6080"

      env_file: 
        - .env
      #- .env_proxy
      volumes:
        - /var/lib/proxysql
        - ./proxysql/proxysql.cnf:/etc/proxysql/proxysql.cnf
          #command: chown -R proxysql:root /etc/proxysql
      depends_on:
        - etcdnode 
networks:
  mlo_db_bkend:
    driver: overlay
    attachable: true
  mlo_db_front:
    driver: overlay
    attachable: true
    #volumes:
        #vol_pxc_node:
        # vol_proxy_sql:
        #vol_etcd:

but when i deploy in the swarm mode by using docker stack deploy command the above configuration fails! it could not join each other in a cluster.
the first node is boot ok , but when i scale the node to 2 it fails.

my .env file are:

YSQL_ROOT_PASSWORD=123456
#DISCOVERY_SERVICE=etcdnode:2379
DISCOVERY_SERVICE=etcdnode:2379
#CLUSTER_JOIN=pxc-node1,pxc-node2,pxc-node3
CLUSTER_NAME=mysql_cluster01
#MYSQL_PROXY_USER=proxysql_user
#MYSQL_PROXY_PASSWORD=s123456
ETCD_HOST=etcdnode

my .env_etcd

ETCD_DATA_DIR=/etcd-data
ETCD_NAME=etcdnode
#ETCD_NAME=etcd-node-01
ETCD_LISTEN_CLIENT_URLS=http://0.0.0.0:2379
#ETCD_LISTEN_CLIENT_URLS=http://0.0.0.0:2379,http://0.0.0.0:4001
ETCD_ADVERTISE_CLIENT_URLS=http://etcdnode:2379,http://etcdnode:4001
ETCD_LISTEN_PEER_URLS=http://0.0.0.0:2380,http://0.0.0.0:4001
ETCD_INITIAL_ADVERTISE_PEER_URLS=http://etcdnode:2380
ETCD-INITIAL-CLUSTER=etcd0=http://etcdnode:2380
ETCD_INITIAL_CLUSTER_STATE=new
ETCD_INITIAL_CLUSTER_TOKEN=etcd-cluster-1
ALLOW_NONE_AUTHENTICATION=yes
ETCD_ENABLE_V2=true
ETCDCTL_API=2
2

bellow are the logs

[root@rocklinux pxcd]# echo "----------------------------------------------------"
----------------------------------------------------
[root@rocklinux pxcd]# docker service logs -f --tail 500 reg_pxc_node
    | + '[' m = - ']'
    | + CFG=/etc/mysql/node.cnf
    | + wantHelp=
    | + for arg in "$@"
    | + case "$arg" in
    | + vault_secret=/etc/mysql/vault-keyring-secret/keyring_vault.conf
    | + '[' -f /etc/mysql/vault-keyring-secret/keyring_vault.conf ']'
    | + file_env XTRABACKUP_PASSWORD xtrabackup
    | + set +o xtrace
    | + file_env CLUSTERCHECK_PASSWORD clustercheck
    | + set +o xtrace
    | ++ hostname -f
    | Registering in the discovery service
    | + NODE_NAME=fe9efafd2f8e
    | + NODE_PORT=3306
    | + '[' -n '' ']'
    | + '[' -n etcdnode:2379 ']'
    | + echo 'Registering in the discovery service'
    | ++ hostname -I
    | ++ awk ' { print $1 } '
    | + NODE_IP=10.180.0.181
    | + '[' etcd '!=' http ']'
    | + DISCOVERY_SERVICE=http://etcdnode:2379
    | + curl http://etcdnode:2379/v2/keys/pxc-cluster/queue/mysql_cluster01 -XPOST -d value=10.180.0.181 -d ttl=60
    |   % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
    |                                  Dload  Upload   Total   Spent    Left  Speed
100   231  100   206  100    25  51500   6250 --:--:-- --:--:-- --:--:-- 57750
    | {"action":"create","node":{"key":"/pxc-cluster/queue/mysql_cluster01/00000000000000000004","value":"10.180.0.181","expiration":"2023-06-03T09:53:15.366739013Z","ttl":60,"modifiedIndex":4,"createdIndex":4}}
    | ++ curl http://etcdnode:2379/v2/keys/pxc-cluster/queue/mysql_cluster01
    | ++ jq -r '.node.nodes[].value'
    |   % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
    |                                  Dload  Upload   Total   Spent    Left  Speed
100   304  100   304 

u   0     0    98k      0 --:--:-- --:--:-- --:--:--   98k
    | + i=10.180.0.181
    | + i1=
    | + curl http://etcdnode:2379/v2/keys/pxc-cluster/mysql_cluster01/10.180.0.181/ipaddr -XPUT -d value=10.180.0.181 -d ttl=30
    |   % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
    |                                  Dload  Upload   Total   Spent    Left  Speed
100   221  100   196  100    25  49000   6250 --:--:-- --:--:-- --:--:-- 55250
    | {"action":"set","node":{"key":"/pxc-cluster/mysql_cluster01/10.180.0.181/ipaddr","value":"10.180.0.181","expiration":"2023-06-03T09:52:45.399381457Z","ttl":30,"modifiedIndex":5,"createdIndex":5}}
    | + curl http://etcdnode:2379/v2/keys/pxc-cluster/mysql_cluster01/10.180.0.181/hostname -XPUT -d value=fe9efafd2f8e -d ttl=30
    |   % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
    |                                  Dload  Upload   Total   Spent    Left  Speed
100   223  100   198  100    25  49500   6250 --:--:-- --:--:-- --:--:-- 74333
    | {"action":"set","node":{"key":"/pxc-cluster/mysql_cluster01/10.180.0.181/hostname","value":"fe9efafd2f8e","expiration":"2023-06-03T09:52:45.417550116Z","ttl":30,"modifiedIndex":6,"createdIndex":6}}
    | + curl http://etcdnode:2379/v2/keys/pxc-cluster/mysql_cluster01/10.180.0.181 -XPUT -d ttl=30 -d dir=true -d prevExist=true
    |   % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
    |                                  Dload  Upload   Total   Spent    Left  Speed
100   319  100   289  100    30  72250   7500 --:--:-- --:--:-- --:--:-- 79750
    | {"action":"update","node":{"key":"/pxc-cluster/mysql_cluster01/10.180.0.181","dir":true,"expiration":"2023-06-03T09:52:45.433913582Z","ttl":30,"modifiedIndex":7,"createdIndex":5},"prevNode":{"key":"/pxc-cluster/mysql_cluster01/10.180.0.181","dir":true,"modifiedIndex":5,"createdIndex":5}}
    | ++ curl 'http://etcdnode:2379/v2/keys/pxc-cluster/mysql_cluster01/?quorum=true'
    | ++ jq -r '.node.nodes[]?.key'
    | ++ awk -F/ '{print $(NF)}'
    |   % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
    |                                  Dload  Upload   Total   Spent    Left  Speed
100   272  100   272    0     0  68000      0 --:--:-- --:--:-- --:--:-- 90666
    | + i=10.180.0.181
    | + i2=
    | ++ join ,
    | ++ local IFS=,
    | ++ shift
    | +++ tr , '\n'
    | +++ sort -u
    | +++ tr '\n' ,
    | ++ joined=,
    | ++ echo ''
    | + CLUSTER_JOIN=
    | + sed -r 's|^[#]?wsrep_node_address=.*$|wsrep_node_address=10.180.0.181|' /etc/mysql/node.cnf
    | + sed -r 's|^[#]?wsrep_cluster_name=.*$|wsrep_cluster_name=mysql_cluster01|' /etc/mysql/node.cnf
    | + sed -r 's|^[#]?wsrep_cluster_address=.*$|wsrep_cluster_address=gcomm://|' /etc/mysql/node.cnf
    | + sed -r 's|^[#]?wsrep_node_incoming_address=.*$|wsrep_node_incoming_address=fe9efafd2f8e:3306|' /etc/mysql/node.cnf
    | + sed -r 's|^[#]?wsrep_sst_auth=.*$|wsrep_sst_auth='\''xtrabackup:xtrabackup'\''|' /etc/mysql/node.cnf
    | + /usr/bin/clustercheckcron clustercheck clustercheck 1 /var/lib/mysql/clustercheck.log 1
    | + '[' -z '' ']'
    | + '[' mysqld = mysqld -a -z '' ']'
    | + _check_config mysqld
    | + toRun=("$@" --verbose --help --wsrep-provider='none')
    | ++ mysqld --verbose --help --wsrep-provider=none
    | + errors=
    | ++ _get_config datadir mysqld
    | ++ local conf=datadir
    | ++ shift
    | ++ awk '$1 == "datadir" && /^[^ \t]/ { sub(/^[^ \t]+[ \t]+/, ""); print; exit }'
    | +++ mktemp -u
    | ++ mysqld --verbose --help --wsrep-provider=none --log-bin-index=/tmp/tmp.cEOX2wxczF
    | + DATADIR=/var/lib/mysql/
    | + '[' '!' -d /var/lib/mysql//mysql ']'
    | + file_env MYSQL_ROOT_PASSWORD
    | + set +o xtrace
    | + '[' -z '123456' -a -z '' -a -z '' ']'
    | + rm -rf /var/lib/mysql//clustercheck.log
    | + mkdir -p /var/lib/mysql/
    | Initializing database
    | + echo 'Initializing database'
    | + mysqld --initialize-insecure --skip-ssl
    | 2023-06-03T09:52:15.570718Z 0 [Warning] TIMESTAMP with implicit DEFAULT value is deprecated. Please use --explicit_defaults_for_timestamp server option (see documentation for more details).
    | 2023-06-03T09:52:15.570861Z 0 [Warning] WSREP: Node is running in bootstrap/initialize mode. Disabling pxc_strict_mode checks
    | 2023-06-03T09:52:15.884731Z 0 [Warning] InnoDB: New log files created, LSN=45790
    | 2023-06-03T09:52:15.932984Z 0 [Warning] InnoDB: Creating foreign key constraint system tables.
    | 2023-06-03T09:52:15.937867Z 0 [Warning] No existing UUID has been found, so we assume that this is the first time that this server has been started. Generating a new UUID: 524b4c7d-01f4-11ee-bb6a-02420ab400b5.
    | 2023-06-03T09:52:15.938929Z 0 [Warning] Gtid table is not ready to be used. Table 'mysql.gtid_executed' cannot be opened.
    | 2023-06-03T09:52:16.011785Z 1 [Warning] root@localhost is created with an empty password ! Please consider switching off the --initialize-insecure option.
    | + echo 'Database initialized'
    | Database initialized
    | ++ _get_config socket mysqld
    | ++ local conf=socket
    | ++ shift
    | ++ awk '$1 == "socket" && /^[^ \t]/ { sub(/^[^ \t]+[ \t]+/, ""); print; exit }'
    | +++ mktemp -u
    | ++ mysqld --verbose --help --wsrep-provider=none --log-bin-index=/tmp/tmp.bntGBCIMNQ
    | + SOCKET=/tmp/mysql.sock
    | + pid=104
    | + mysql=(mysql --protocol=socket -uroot -hlocalhost --socket="${SOCKET}" --password="")
    | + mysqld --skip-networking --socket=/tmp/mysql.sock
    | + wsrep_local_state_select='SELECT variable_value FROM performance_schema.global_status WHERE variable_name='\''wsrep_local_state_comment'\'''
    | + for i in {120..0}
    | ++ echo 'SELECT variable_value FROM performance_schema.global_status WHERE variable_name='\''wsrep_local_state_comment'\'''
    | ++ mysql --protocol=socket -uroot -hlocalhost --socket=/tmp/mysql.sock --password= -s
    | MySQL init process in progress...
    | + wsrep_local_state=
    | + true
    | + '[' '' = Synced ']'
    | + echo 'MySQL init process in progress...'
    | + sleep 1
    | 2023-06-03T09:52:23.482297Z 0 [Warning] TIMESTAMP with implicit DEFAULT value is deprecated. Please use --explicit_defaults_for_timestamp server option (see documentation for more details).
    | 2023-06-03T09:52:23.485273Z 0 [Note] mysqld (mysqld 5.7.41-44-57) starting as process 104 ...
    | 2023-06-03T09:52:23.489710Z 0 [Note] WSREP: Setting wsrep_ready to false
    | 2023-06-03T09:52:23.489734Z 0 [Note] WSREP: No pre-stored wsrep-start position found. Skipping position initialization.
    | 2023-06-03T09:52:23.489741Z 0 [Note] WSREP: wsrep_load(): loading provider library '/usr/lib64/galera3/libgalera_smm.so'
    | 2023-06-03T09:52:23.490652Z 0 [Note] WSREP: wsrep_load(): Galera 3.65(rf47405c) by Codership Oy <info@codership.com> loaded successfully.
    | 2023-06-03T09:52:23.490729Z 0 [Note] WSREP: CRC-32C: using 64-bit x86 acceleration.
    | 2023-06-03T09:52:23.491038Z 0 [Warning] WSREP: Could not open state file for reading: '/var/lib/mysql//grastate.dat'
    | 2023-06-03T09:52:23.491082Z 0 [Warning] WSREP: No persistent state found. Bootstraping with default state
    | 2023-06-03T09:52:23.491170Z 0 [Note] WSREP: Found saved state: 00000000-0000-0000-0000-000000000000:-1, safe_to_bootstrap: 1
    | 2023-06-03T09:52:23.495345Z 0 [Note] WSREP: Passing config to GCS: base_dir = /var/lib/mysql/; base_host = 10.180.0.181; base_port = 4567; cert.log_conflicts = no; cert.optimistic_pa = yes; debug = no; evs.auto_evict = 0; evs.delay_margin = PT1S; evs.delayed_keep_period = PT30S; evs.inactive_check_period = PT0.5S; evs.inactive_timeout = PT15S; evs.join_retrans_period = PT1S; evs.max_install_timeouts = 3; evs.send_window = 10; evs.stats_report_period = PT1M; evs.suspect_timeout = PT5S; evs.user_send_window = 4; evs.view_forget_timeout = PT24H; gcache.dir = /var/lib/mysql/; gcache.freeze_purge_at_seqno = -1; gcache.keep_pages_count = 0; gcache.keep_pages_size = 0; gcache.mem_size = 0; gcache.name = /var/lib/mysql//galera.cache; gcache.page_size = 128M; gcache.recover = no; gcache.size = 128M; gcomm.thread_prio = ; gcs.fc_debug = 0; gcs.fc_factor = 1; gcs.fc_limit = 100; gcs.fc_master_slave = no; gcs.max_packet_size = 64500; gcs.max_throttle = 0.25; gcs.recv_q_hard_limit = 9223372036854775807; gcs.recv_q_soft_limit = 0.25; gcs.sync_donor = no; gmcast.segment = 0; gmcast.version = 0; pc.announce_timeout = PT3S; pc.checksum = false; pc.ignore_quorum = false; pc.ignore_sb = false; pc.npvo = false; pc.recovery = true; pc.version = 0; pc.wait_prim = true; pc.wait_prim_timeout = PT30S; pc.weight = 1; protonet.backend = asio; protonet.version = 0; repl.causal_read_timeout = PT30S; repl.commit_order = 3; repl.key_format = FLAT8; repl.max_ws_size = 2147483647; repl.proto_max = 9; socket.checksum = 2; socket.recv_buf_size = auto; socket.send_buf_size = auto;
    | 2023-06-03T09:52:23.507030Z 0 [Note] WSREP: GCache history reset: 00000000-0000-0000-0000-000000000000:0 -> 00000000-0000-0000-0000-000000000000:-1
    | 2023-06-03T09:52:23.507430Z 0 [Note] WSREP: Assign initial position for certification: -1, protocol version: -1
    | 2023-06-03T09:52:23.507470Z 0 [Note] WSREP: Preparing to initiate SST/IST
    | 2023-06-03T09:52:23.507479Z 0 [Note] WSREP: Starting replication
    | 2023-06-03T09:52:23.507491Z 0 [Note] WSREP: Setting initial position to 00000000-0000-0000-0000-000000000000:-1
    | 2023-06-03T09:52:23.507995Z 0 [Note] WSREP: Using CRC-32C for message checksums.
    | 2023-06-03T09:52:23.508167Z 0 [Note] WSREP: gcomm thread scheduling priority set to other:0
    | 2023-06-03T09:52:23.508324Z 0 [Note] WSREP: Fail to access the file (/var/lib/mysql//gvwstate.dat) error (No such file or directory). It is possible if node is booting for first time or re-booting after a graceful shutdown
    | 2023-06-03T09:52:23.508358Z 0 [Note] WSREP: Restoring primary-component from disk failed. Either node is booting for first time or re-booting after a graceful shutdown
    | 2023-06-03T09:52:23.508570Z 0 [Note] WSREP: GMCast version 0
    | 2023-06-03T09:52:23.508695Z 0 [Note] WSREP: (56ce78c1, 'tcp://0.0.0.0:4567') listening at tcp://0.0.0.0:4567
    | 2023-06-03T09:52:23.508709Z 0 [Note] WSREP: (56ce78c1, 'tcp://0.0.0.0:4567') multicast: , ttl: 1
    | 2023-06-03T09:52:23.509032Z 0 [Note] WSREP: EVS version 0
    | 2023-06-03T09:52:23.509151Z 0 [Note] WSREP: gcomm: connecting to group 'mysql_cluster01', peer ''
    | 2023-06-03T09:52:23.509184Z 0 [Note] WSREP: start_prim is enabled, turn off pc_recovery
    | 2023-06-03T09:52:23.509480Z 0 [Note] WSREP: Node 56ce78c1 state primary
    | 2023-06-03T09:52:23.509529Z 0 [Note] WSREP: Current view of cluster as seen by this node
    | view (view_id(PRIM,56ce78c1,1)
    | memb {
    |  56ce78c1,0
    |  }
    | joined {
    |  }
    | left {
    |  }
    | partitioned {
    |  }
    | )
    | 2023-06-03T09:52:23.509541Z 0 [Note] WSREP: Save the discovered primary-component to disk
    | 2023-06-03T09:52:23.510442Z 0 [Note] WSREP: gcomm: connected
    | 2023-06-03T09:52:23.510610Z 0 [Note] WSREP: Shifting CLOSED -> OPEN (TO: 0)
    | 2023-06-03T09:52:23.510825Z 0 [Note] WSREP: New COMPONENT: primary = yes, bootstrap = no, my_idx = 0, memb_num = 1
    | 2023-06-03T09:52:23.510942Z 0 [Note] WSREP: Waiting for SST/IST to complete.
    | 2023-06-03T09:52:23.511006Z 0 [Note] WSREP: Starting new group from scratch: 56cedc56-01f4-11ee-b577-47a2f3cad270
    | 2023-06-03T09:52:23.511081Z 0 [Note] WSREP: STATE_EXCHANGE: sent state UUID: 56cedf0d-01f4-11ee-b0f3-035724215dfd
    | 2023-06-03T09:52:23.511096Z 0 [Note] WSREP: STATE EXCHANGE: sent state msg: 56cedf0d-01f4-11ee-b0f3-035724215dfd
    | 2023-06-03T09:52:23.511105Z 0 [Note] WSREP: STATE EXCHANGE: got state msg: 56cedf0d-01f4-11ee-b0f3-035724215dfd from 0 (fe9efafd2f8e)
    | 2023-06-03T09:52:23.511113Z 0 [Note] WSREP: Quorum results:
    |  version    = 6,
    |  component  = PRIMARY,
    |  conf_id    = 0,
    |  members    = 1/1 (primary/total),
    |  act_id     = 0,
    |  last_appl. = -1,
    |  protocols  = 0/9/3 (gcs/repl/appl),
    |  group UUID = 56cedc56-01f4-11ee-b577-47a2f3cad270
    | 2023-06-03T09:52:23.511120Z 0 [Note] WSREP: Flow-control interval: [100, 100]
    | 2023-06-03T09:52:23.511124Z 0 [Note] WSREP: Restored state OPEN -> JOINED (0)
    | 2023-06-03T09:52:23.511141Z 0 [Note] WSREP: Member 0.0 (fe9efafd2f8e) synced with group.
    | 2023-06-03T09:52:23.511148Z 0 [Note] WSREP: Shifting JOINED -> SYNCED (TO: 0)
    | 2023-06-03T09:52:23.511800Z 2 [Note] WSREP: REPL Protocols: 9 (4, 2)
    | 2023-06-03T09:52:23.511851Z 2 [Note] WSREP: REPL Protocols: 9 (4, 2)
    | 2023-06-03T09:52:23.511873Z 2 [Note] WSREP: New cluster view: global state: 56cedc56-01f4-11ee-b577-47a2f3cad270:0, view# 1: Primary, number of nodes: 1, my index: 0, protocol version 3
    | 2023-06-03T09:52:23.511879Z 2 [Note] WSREP: Setting wsrep_ready to true
    | 2023-06-03T09:52:23.511956Z 0 [Note] WSREP: SST complete, seqno: 0
    | 2023-06-03T09:52:23.514036Z 0 [Note] InnoDB: PUNCH HOLE support available
    | 2023-06-03T09:52:23.514085Z 0 [Note] InnoDB: Mutexes and rw_locks use GCC atomic builtins
    | 2023-06-03T09:52:23.514094Z 0 [Note] InnoDB: Uses event mutexes
    | 2023-06-03T09:52:23.514100Z 0 [Note] InnoDB: GCC builtin __atomic_thread_fence() is used for memory barrier
    | 2023-06-03T09:52:23.514105Z 0 [Note] InnoDB: Compressed tables use zlib 1.2.12
    | 2023-06-03T09:52:23.514111Z 0 [Note] InnoDB: Using Linux native AIO
    | 2023-06-03T09:52:23.514462Z 0 [Note] InnoDB: Number of pools: 1
    | 2023-06-03T09:52:23.514686Z 0 [Note] InnoDB: Using CPU crc32 instructions
    | 2023-06-03T09:52:23.519111Z 0 [Note] InnoDB: Initializing buffer pool, total size = 128M, instances = 1, chunk size = 128M
    | 2023-06-03T09:52:23.524395Z 0 [Note] InnoDB: Completed initialization of buffer pool
    | 2023-06-03T09:52:23.528966Z 0 [Note] InnoDB: If the mysqld execution user is authorized, page cleaner thread priority can be changed. See the man page of setpriority().
    | 2023-06-03T09:52:23.543494Z 0 [Note] InnoDB: Crash recovery did not find the parallel doublewrite buffer at /var/lib/mysql/xb_doublewrite
    | 2023-06-03T09:52:23.545041Z 0 [Note] InnoDB: Highest supported file format is Barracuda.
    | 2023-06-03T09:52:23.556570Z 0 [Note] InnoDB: Created parallel doublewrite buffer at /var/lib/mysql/xb_doublewrite, size 3932160 bytes
    | 2023-06-03T09:52:23.591787Z 0 [Note] InnoDB: Creating shared tablespace for temporary tables
    | 2023-06-03T09:52:23.591983Z 0 [Note] InnoDB: Setting file './ibtmp1' size to 12 MB. Physically writing the file full; Please wait ...
    | 2023-06-03T09:52:23.606165Z 0 [Note] InnoDB: File './ibtmp1' size is now 12 MB.
    | 2023-06-03T09:52:23.606819Z 0 [Note] InnoDB: 96 redo rollback segment(s) found. 96 redo rollback segment(s) are active.
    | 2023-06-03T09:52:23.606856Z 0 [Note] InnoDB: 32 non-redo rollback segment(s) are active.
    | 2023-06-03T09:52:23.608776Z 0 [Note] InnoDB: Percona XtraDB (http://www.percona.com) 5.7.41-44 started; log sequence number 2768030
    | 2023-06-03T09:52:23.609483Z 0 [Note] InnoDB: Loading buffer pool(s) from /var/lib/mysql/ib_buffer_pool
    | 2023-06-03T09:52:23.609769Z 0 [Note] Plugin 'FEDERATED' is disabled.
    | 2023-06-03T09:52:23.630532Z 0 [Note] InnoDB: Buffer pool(s) load completed at 230603  9:52:23
    | 2023-06-03T09:52:23.957927Z 0 [Note] Auto generated SSL certificates are placed in data directory.
    | 2023-06-03T09:52:23.957986Z 0 [Warning] A deprecated TLS version TLSv1 is enabled. Please use TLSv1.2 or higher.
    | 2023-06-03T09:52:23.957995Z 0 [Warning] A deprecated TLS version TLSv1.1 is enabled. Please use TLSv1.2 or higher.
    | 2023-06-03T09:52:23.958874Z 0 [Warning] CA certificate ca.pem is self signed.
    | 2023-06-03T09:52:23.958955Z 0 [Note] Skipping generation of RSA key pair as key files are present in data directory.
    | 2023-06-03T09:52:23.982623Z 0 [Note] Event Scheduler: Loaded 0 events
    | 2023-06-03T09:52:23.983683Z 0 [Note] mysqld: ready for connections.
    | Version: '5.7.41-44-57'  socket: '/tmp/mysql.sock'  port: 0  Percona XtraDB Cluster (GPL), Release rel44, Revision b46a1c3, WSREP version 31.65, wsrep_31.65
    | 2023-06-03T09:52:23.983867Z 2 [Note] WSREP: Initialized wsrep sidno 2
    | 2023-06-03T09:52:23.983880Z 2 [Note] WSREP: Auto Increment Offset/Increment re-align with cluster membership change (Offset: 1 -> 1) (Increment: 1 -> 1)
    | 2023-06-03T09:52:23.983928Z 2 [Note] WSREP: wsrep_notify_cmd is not defined, skipping notification.
    | 2023-06-03T09:52:23.983967Z 2 [Note] WSREP: Assign initial position for certification: 0, protocol version: 4
    | 2023-06-03T09:52:23.984140Z 0 [Note] WSREP: Service thread queue flushed.
    | 2023-06-03T09:52:23.984858Z 2 [Note] WSREP: GCache history reset: 00000000-0000-0000-0000-000000000000:0 -> 56cedc56-01f4-11ee-b577-47a2f3cad270:0
    | 2023-06-03T09:52:23.985508Z 2 [Note] WSREP: Synchronized with group, ready for connections
    | 2023-06-03T09:52:23.985541Z 2 [Note] WSREP: Setting wsrep_ready to true
    | 2023-06-03T09:52:23.985549Z 2 [Note] WSREP: wsrep_notify_cmd is not defined, skipping notification.
    | + for i in {120..0}
    | ++ echo 'SELECT variable_value FROM performance_schema.global_status WHERE variable_name='\''wsrep_local_state_comment'\'''
    | ++ mysql --protocol=socket -uroot -hlocalhost --socket=/tmp/mysql.sock --password= -s
    | + wsrep_local_state=Synced
    | + '[' Synced = Synced ']'
    | + break
    | + '[' 119 = 0 ']'
    | + '[' -z '' ']'
    | + mysql_tzinfo_to_sql /usr/share/zoneinfo
    | + sed 's/Local time zone must be set--see zic manual page/FCTY/'
    | + mysql --protocol=socket -uroot -hlocalhost --socket=/tmp/mysql.sock --password= mysql
    | mysql: [Warning] Using a password on the command line interface can be insecure.
    | 2023-06-03T09:52:24.325354Z 6 [Note] InnoDB: Stopping purge
    | 2023-06-03T09:52:24.327865Z 6 [Note] InnoDB: Resuming purge
    | 2023-06-03T09:52:24.330510Z 6 [Note] InnoDB: Stopping purge
    | 2023-06-03T09:52:24.331789Z 6 [Note] InnoDB: Resuming purge
    | 2023-06-03T09:52:24.334457Z 6 [Note] InnoDB: Stopping purge
    | 2023-06-03T09:52:24.335752Z 6 [Note] InnoDB: Resuming purge
    | 2023-06-03T09:52:24.338754Z 6 [Note] InnoDB: Stopping purge
    | 2023-06-03T09:52:24.340038Z 6 [Note] InnoDB: Resuming purge
    | Warning: Unable to load '/usr/share/zoneinfo/iso3166.tab' as time zone. Skipping it.
    | Warning: Unable to load '/usr/share/zoneinfo/leap-seconds.list' as time zone. Skipping it.
    | Warning: Unable to load '/usr/share/zoneinfo/leapseconds' as time zone. Skipping it.
    | 2023-06-03T09:52:25.418704Z 7 [Note] Access denied for user 'root'@'localhost' (using password: YES)
    | 2023-06-03T09:52:25.569940Z 8 [Note] Access denied for user 'clustercheck'@'localhost' (using password: YES)
    | Warning: Unable to load '/usr/share/zoneinfo/tzdata.zi' as time zone. Skipping it.
    | Warning: Unable to load '/usr/share/zoneinfo/zone.tab' as time zone. Skipping it.
    | Warning: Unable to load '/usr/share/zoneinfo/zone1970.tab' as time zone. Skipping it.
    | + '[' '!' -z '' ']'
    | + rootCreate=
    | + file_env MYSQL_ROOT_HOST %
    | + set +o xtrace
    | + '[' '!' -z % -a % '!=' localhost ']'
    | + read -r -d '' rootCreate
    | + true
    | + file_env MONITOR_HOST localhost
    | + set +o xtrace
    | + file_env MONITOR_PASSWORD monitor
    | + set +o xtrace
    | + mysql --protocol=socket -uroot -hlocalhost --socket=/tmp/mysql.sock --password=
    | mysql: [Warning] Using a password on the command line interface can be insecure.
    | + '[' '!' -z '123456' ']'
    | + mysql+=(-p"${MYSQL_ROOT_PASSWORD}")
    | + file_env MYSQL_DATABASE
    | + set +o xtrace
    | + '[' '' ']'
    | + file_env MYSQL_USER
    | + set +o xtrace
    | + file_env MYSQL_PASSWORD
    | + set +o xtrace
    | + '[' '' -a '' ']'
    | + echo
    | + ls /docker-entrypoint-initdb.d/
    |
    | + '[' '!' -z '' ']'
    | + kill -s TERM 104
    | + wait 104
    | 2023-06-03T09:52:28.142209Z 0 [Note] WSREP: Received shutdown signal. Will sleep for 10 secs before initiating shutdown. pxc_maint_mode switched to SHUTDOWN
    | 2023-06-03T09:52:38.143205Z 0 [Note] WSREP: Stop replication
    | 2023-06-03T09:52:38.143256Z 0 [Note] WSREP: Closing send monitor...
    | 2023-06-03T09:52:38.143274Z 0 [Note] WSREP: Closed send monitor.
    | 2023-06-03T09:52:38.143295Z 0 [Note] WSREP: gcomm: terminating thread
    | 2023-06-03T09:52:38.143331Z 0 [Note] WSREP: gcomm: joining thread
    | 2023-06-03T09:52:38.143443Z 0 [Note] WSREP: gcomm: closing backend
    | 2023-06-03T09:52:38.143527Z 0 [Note] WSREP: Current view of cluster as seen by this node
    | view ((empty))
    | 2023-06-03T09:52:38.143798Z 0 [Note] WSREP: gcomm: closed
    | 2023-06-03T09:52:38.143828Z 0 [Note] WSREP: Received self-leave message.
    | 2023-06-03T09:52:38.143841Z 0 [Note] WSREP: Flow-control interval: [0, 0]
    | 2023-06-03T09:52:38.143845Z 0 [Note] WSREP: Received SELF-LEAVE. Closing connection.
    | 2023-06-03T09:52:38.143850Z 0 [Note] WSREP: Shifting SYNCED -> CLOSED (TO: 18)
    | 2023-06-03T09:52:38.143866Z 0 [Note] WSREP: RECV thread exiting 0: Success
    | 2023-06-03T09:52:38.143943Z 4 [Note] WSREP: New cluster view: global state: 56cedc56-01f4-11ee-b577-47a2f3cad270:18, view# -1: non-Primary, number of nodes: 0, my index: -1, protocol version 3
    | 2023-06-03T09:52:38.143991Z 4 [Note] WSREP: Setting wsrep_ready to false
    | 2023-06-03T09:52:38.143955Z 0 [Note] WSREP: recv_thread() joined.
    | 2023-06-03T09:52:38.144020Z 0 [Note] WSREP: Closing replication queue.
    | 2023-06-03T09:52:38.144026Z 0 [Note] WSREP: Closing slave action queue.
    | 2023-06-03T09:52:38.144031Z 0 [Note] WSREP: Waiting for active wsrep applier to exit
    | 2023-06-03T09:52:38.144045Z 4 [Note] WSREP: wsrep_notify_cmd is not defined, skipping notification.
    | 2023-06-03T09:52:38.145041Z 4 [Note] WSREP: applier thread exiting (code:0)
    | 2023-06-03T09:52:38.154610Z 2 [Note] WSREP: applier thread exiting (code:6)
    | 2023-06-03T09:52:38.154831Z 1 [Note] WSREP: rollbacker thread exiting
    | 2023-06-03T09:52:38.155367Z 0 [Note] Giving 0 client threads a chance to die gracefully
    | 2023-06-03T09:52:38.155398Z 0 [Note] Shutting down slave threads
    | 2023-06-03T09:52:38.155406Z 0 [Note] Forcefully disconnecting 0 remaining clients
    | 2023-06-03T09:52:38.155413Z 0 [Note] Event Scheduler: Purging the queue. 0 events
    | 2023-06-03T09:52:38.185458Z 0 [Note] WSREP: Service thread queue flushed.
    | 2023-06-03T09:52:38.188748Z 0 [Note] WSREP: MemPool(SlaveTrxHandle): hit ratio: 0, misses: 0, in use: 0, in pool: 0
    | 2023-06-03T09:52:38.198261Z 0 [Note] Binlog end
    | 2023-06-03T09:52:38.198994Z 0 [Note] Shutting down plugin 'ngram'
    | 2023-06-03T09:52:38.199028Z 0 [Note] Shutting down plugin 'partition'
    | 2023-06-03T09:52:38.199036Z 0 [Note] Shutting down plugin 'BLACKHOLE'
    | 2023-06-03T09:52:38.199042Z 0 [Note] Shutting down plugin 'ARCHIVE'
    | 2023-06-03T09:52:38.199045Z 0 [Note] Shutting down plugin 'MRG_MYISAM'
    | 2023-06-03T09:52:38.199050Z 0 [Note] Shutting down plugin 'MyISAM'
    | 2023-06-03T09:52:38.199060Z 0 [Note] Shutting down plugin 'INNODB_TABLESPACES_SCRUBBING'
    | 2023-06-03T09:52:38.199065Z 0 [Note] Shutting down plugin 'INNODB_TABLESPACES_ENCRYPTION'
    | 2023-06-03T09:52:38.199069Z 0 [Note] Shutting down plugin 'INNODB_SYS_VIRTUAL'
    | 2023-06-03T09:52:38.199072Z 0 [Note] Shutting down plugin 'INNODB_CHANGED_PAGES'
    | 2023-06-03T09:52:38.199076Z 0 [Note] Shutting down plugin 'INNODB_SYS_DATAFILES'
    | 2023-06-03T09:52:38.199080Z 0 [Note] Shutting down plugin 'INNODB_SYS_TABLESPACES'
    | 2023-06-03T09:52:38.199083Z 0 [Note] Shutting down plugin 'INNODB_SYS_FOREIGN_COLS'
    | 2023-06-03T09:52:38.199087Z 0 [Note] Shutting down plugin 'INNODB_SYS_FOREIGN'
    | 2023-06-03T09:52:38.199091Z 0 [Note] Shutting down plugin 'INNODB_SYS_FIELDS'
    | 2023-06-03T09:52:38.199095Z 0 [Note] Shutting down plugin 'INNODB_SYS_COLUMNS'
    | 2023-06-03T09:52:38.199098Z 0 [Note] Shutting down plugin 'INNODB_SYS_INDEXES'
    | 2023-06-03T09:52:38.199102Z 0 [Note] Shutting down plugin 'INNODB_SYS_TABLESTATS'
    | 2023-06-03T09:52:38.199105Z 0 [Note] Shutting down plugin 'INNODB_SYS_TABLES'
    | 2023-06-03T09:52:38.199109Z 0 [Note] Shutting down plugin 'INNODB_FT_INDEX_TABLE'
    | 2023-06-03T09:52:38.199113Z 0 [Note] Shutting down plugin 'INNODB_FT_INDEX_CACHE'
    | 2023-06-03T09:52:38.199116Z 0 [Note] Shutting down plugin 'INNODB_FT_CONFIG'
    | 2023-06-03T09:52:38.199120Z 0 [Note] Shutting down plugin 'INNODB_FT_BEING_DELETED'
    | 2023-06-03T09:52:38.199124Z 0 [Note] Shutting down plugin 'INNODB_FT_DELETED'
    | 2023-06-03T09:52:38.199128Z 0 [Note] Shutting down plugin 'INNODB_FT_DEFAULT_STOPWORD'
    | 2023-06-03T09:52:38.199132Z 0 [Note] Shutting down plugin 'INNODB_METRICS'
    | 2023-06-03T09:52:38.199135Z 0 [Note] Shutting down plugin 'INNODB_TEMP_TABLE_INFO'
    | 2023-06-03T09:52:38.199139Z 0 [Note] Shutting down plugin 'INNODB_BUFFER_POOL_STATS'
    | 2023-06-03T09:52:38.199142Z 0 [Note] Shutting down plugin 'INNODB_BUFFER_PAGE_LRU'
    | 2023-06-03T09:52:38.199146Z 0 [Note] Shutting down plugin 'INNODB_BUFFER_PAGE'
    | 2023-06-03T09:52:38.199150Z 0 [Note] Shutting down plugin 'INNODB_CMP_PER_INDEX_RESET'
    | 2023-06-03T09:52:38.199154Z 0 [Note] Shutting down plugin 'INNODB_CMP_PER_INDEX'
    | 2023-06-03T09:52:38.199157Z 0 [Note] Shutting down plugin 'INNODB_CMPMEM_RESET'
    | 2023-06-03T09:52:38.199161Z 0 [Note] Shutting down plugin 'INNODB_CMPMEM'
    | 2023-06-03T09:52:38.199165Z 0 [Note] Shutting down plugin 'INNODB_CMP_RESET'
    | 2023-06-03T09:52:38.199232Z 0 [Note] Shutting down plugin 'INNODB_CMP'
    | 2023-06-03T09:52:38.199237Z 0 [Note] Shutting down plugin 'INNODB_LOCK_WAITS'
    | 2023-06-03T09:52:38.199241Z 0 [Note] Shutting down plugin 'INNODB_LOCKS'
    | 2023-06-03T09:52:38.199245Z 0 [Note] Shutting down plugin 'INNODB_TRX'
    | 2023-06-03T09:52:38.199249Z 0 [Note] Shutting down plugin 'XTRADB_ZIP_DICT_COLS'
    | 2023-06-03T09:52:38.199253Z 0 [Note] Shutting down plugin 'XTRADB_ZIP_DICT'
    | 2023-06-03T09:52:38.199256Z 0 [Note] Shutting down plugin 'XTRADB_RSEG'
    | 2023-06-03T09:52:38.199260Z 0 [Note] Shutting down plugin 'XTRADB_INTERNAL_HASH_TABLES'
    | 2023-06-03T09:52:38.199264Z 0 [Note] Shutting down plugin 'XTRADB_READ_VIEW'
    | 2023-06-03T09:52:38.199268Z 0 [Note] Shutting down plugin 'InnoDB'
    | 2023-06-03T09:52:38.199327Z 0 [Note] InnoDB: FTS optimize thread exiting.
    | 2023-06-03T09:52:38.199866Z 0 [Note] InnoDB: Starting shutdown...
    | 2023-06-03T09:52:38.300498Z 0 [Note] InnoDB: Dumping buffer pool(s) to /var/lib/mysql/ib_buffer_pool
    | 2023-06-03T09:52:38.301163Z 0 [Note] InnoDB: Buffer pool(s) dump completed at 230603  9:52:38
    | 2023-06-03T09:52:38.802677Z 0 [Note] InnoDB: Waiting for page_cleaner to finish flushing of buffer pool
    | 2023-06-03T09:52:40.214642Z 0 [Note] InnoDB: Shutdown completed; log sequence number 12438866
    | 2023-06-03T09:52:40.214813Z 0 [Note] InnoDB: Removed temporary tablespace data file: "ibtmp1"
    | 2023-06-03T09:52:40.214835Z 0 [Note] Shutting down plugin 'MEMORY'
    | 2023-06-03T09:52:40.214843Z 0 [Note] Shutting down plugin 'CSV'
    | 2023-06-03T09:52:40.214849Z 0 [Note] Shutting down plugin 'PERFORMANCE_SCHEMA'
    | 2023-06-03T09:52:40.214877Z 0 [Note] Shutting down plugin 'sha256_password'
    | 2023-06-03T09:52:40.214904Z 0 [Note] Shutting down plugin 'mysql_native_password'
    | 2023-06-03T09:52:40.214912Z 0 [Note] Shutting down plugin 'wsrep'
    | 2023-06-03T09:52:40.215079Z 0 [Note] Shutting down plugin 'binlog'
    | 2023-06-03T09:52:40.216274Z 0 [Note] mysqld: Shutdown complete
3 
   | MySQL init process done. Ready for start up.
    |
    | + echo
    | + echo 'MySQL init process done. Ready for start up.'
    | + echo
    | + '[' '!' -z '' ']'
    | + '[' mysqld = mysqld -a -z '' ']'
    | + mysqld --version
    | + tee /tmp/version_info
    | mysqld  Ver 5.7.41-44-57 for Linux on x86_64 (Percona XtraDB Cluster (GPL), Release rel44, Revision b46a1c3, WSREP version 31.65, wsrep_31.65)
    | ++ _get_config datadir mysqld
    | ++ local conf=datadir
    | ++ shift
    | ++ awk '$1 == "datadir" && /^[^ \t]/ { sub(/^[^ \t]+[ \t]+/, ""); print; exit }'
    | +++ mktemp -u
    | ++ mysqld --verbose --help --wsrep-provider=none --log-bin-index=/tmp/tmp.kbaQ5jEkdb
    | + DATADIR=/var/lib/mysql/
    | + '[' -f /var/lib/mysql//version_info ']'
    | + mysqld --version
    | + grep -v wsrep_sst_auth /etc/mysql/node.cnf
    | [mysqld]
    |
    | ignore-db-dir=lost+found
    | datadir=/var/lib/mysql
    | socket=/tmp/mysql.sock
    | skip-host-cache
    |
    | #coredumper
    | #server_id=0
    | binlog_format=ROW
    | default_storage_engine=InnoDB
    |
    | innodb_flush_log_at_trx_commit  = 0
    | innodb_flush_method             = O_DIRECT
    | innodb_file_per_table           = 1
    | innodb_autoinc_lock_mode=2
    |
    | bind_address = 0.0.0.0
    |
    | wsrep_slave_threads=2
    | wsrep_cluster_address=gcomm://
    | wsrep_provider=/usr/lib64/galera3/libgalera_smm.so
    |
    | wsrep_cluster_name=mysql_cluster01
    | wsrep_node_address=10.180.0.181
    | wsrep_node_incoming_address=fe9efafd2f8e:3306
    |
    | wsrep_sst_method=xtrabackup-v2
    |
    | [client]
    | socket=/tmp/mysql.sock
    |
    | [sst]
    | progress=/var/lib/mysql/sst_in_progress
    | + wsrep_start_position_opt=
    | + '[' mysqld = mysqld -a -z '' ']'
    | ++ _get_config datadir mysqld
    | ++ local conf=datadir
    | ++ shift
    | ++ awk '$1 == "datadir" && /^[^ \t]/ { sub(/^[^ \t]+[ \t]+/, ""); print; exit }'
    | +++ mktemp -u
    | ++ mysqld --verbose --help --wsrep-provider=none --log-bin-index=/tmp/tmp.u5izyj9UMY
    | + DATADIR=/var/lib/mysql/
    | + grastate_loc=/var/lib/mysql//grastate.dat
    | + '[' -s /var/lib/mysql//grastate.dat -a -d /var/lib/mysql//mysql ']'
    | ++ grep uuid: /var/lib/mysql//grastate.dat
    | ++ cut -d: -f2
    | ++ tr -d ' '
    | + uuid=56cedc56-01f4-11ee-b577-47a2f3cad270
    | ++ cut -d: -f2
    | ++ grep seqno: /var/lib/mysql//grastate.dat
    | ++ tr -d ' '
    | + seqno=18
    | + '[' -n 18 ']'
    | + '[' 18 -ne -1 ']'
    | + echo 'Skipping wsrep-recover for 56cedc56-01f4-11ee-b577-47a2f3cad270:18 pair'
    | + echo 'Assigning 56cedc56-01f4-11ee-b577-47a2f3cad270:18 to wsrep_start_position'
    | + wsrep_start_position_opt=--wsrep_start_position=56cedc56-01f4-11ee-b577-47a2f3cad270:18
    | + '[' -z --wsrep_start_position=56cedc56-01f4-11ee-b577-47a2f3cad270:18 -a -d /var/lib/mysql//mysql ']'
    | Skipping wsrep-recover for 56cedc56-01f4-11ee-b577-47a2f3cad270:18 pair
    | Assigning 56cedc56-01f4-11ee-b577-47a2f3cad270:18 to wsrep_start_position
    | + exec mysqld --wsrep_start_position=56cedc56-01f4-11ee-b577-47a2f3cad270:18
    | 2023-06-03T09:52:40.560187Z 0 [Warning] TIMESTAMP with implicit DEFAULT value is deprecated. Please use --explicit_defaults_for_timestamp server option (see documentation for more details).
    | 2023-06-03T09:52:40.563004Z 0 [Note] mysqld (mysqld 5.7.41-44-57) starting as process 1 ...
    | 2023-06-03T09:52:40.568255Z 0 [Note] WSREP: Setting wsrep_ready to false
    | 2023-06-03T09:52:40.568303Z 0 [Note] WSREP: No pre-stored wsrep-start position found. Skipping position initialization.
    | 2023-06-03T09:52:40.568314Z 0 [Note] WSREP: wsrep_load(): loading provider library '/usr/lib64/galera3/libgalera_smm.so'
    | 2023-06-03T09:52:40.569091Z 0 [Note] WSREP: wsrep_load(): Galera 3.65(rf47405c) by Codership Oy <info@codership.com> loaded successfully.
    | 2023-06-03T09:52:40.569199Z 0 [Note] WSREP: CRC-32C: using 64-bit x86 acceleration.
    | 2023-06-03T09:52:40.569577Z 0 [Note] WSREP: Found saved state: 56cedc56-01f4-11ee-b577-47a2f3cad270:18, safe_to_bootstrap: 1
    | 2023-06-03T09:52:40.571482Z 0 [Note] WSREP: Passing config to GCS: base_dir = /var/lib/mysql/; base_host = 10.180.0.181; base_port = 4567; cert.log_conflicts = no; cert.optimistic_pa = yes; debug = no; evs.auto_evict = 0; evs.delay_margin = PT1S; evs.delayed_keep_period = PT30S; evs.inactive_check_period = PT0.5S; evs.inactive_timeout = PT15S; evs.join_retrans_period = PT1S; evs.max_install_timeouts = 3; evs.send_window = 10; evs.stats_report_period = PT1M; evs.suspect_timeout = PT5S; evs.user_send_window = 4; evs.view_forget_timeout = PT24H; gcache.dir = /var/lib/mysql/; gcache.freeze_purge_at_seqno = -1; gcache.keep_pages_count = 0; gcache.keep_pages_size = 0; gcache.mem_size = 0; gcache.name = /var/lib/mysql//galera.cache; gcache.page_size = 128M; gcache.recover = no; gcache.size = 128M; gcomm.thread_prio = ; gcs.fc_debug = 0; gcs.fc_factor = 1; gcs.fc_limit = 100; gcs.fc_master_slave = no; gcs.max_packet_size = 64500; gcs.max_throttle = 0.25; gcs.recv_q_hard_limit = 9223372036854775807; gcs.recv_q_soft_limit = 0.25; gcs.sync_donor = no; gmcast.segment = 0; gmcast.version = 0; pc.announce_timeout = PT3S; pc.checksum = false; pc.ignore_quorum = false; pc.ignore_sb = false; pc.npvo = false; pc.recovery = true; pc.version = 0; pc.wait_prim = true; pc.wait_prim_timeout = PT30S; pc.weight = 1; protonet.backend = asio; protonet.version = 0; repl.causal_read_timeout = PT30S; repl.commit_order = 3; repl.key_format = FLAT8; repl.max_ws_size = 2147483647; repl.proto_max = 9; socket.checksum = 2; socket.recv_buf_size = auto; socket.send_buf_size = auto;
    | 2023-06-03T09:52:40.583043Z 0 [Note] WSREP: GCache history reset: 56cedc56-01f4-11ee-b577-47a2f3cad270:0 -> 56cedc56-01f4-11ee-b577-47a2f3cad270:18
    | 2023-06-03T09:52:40.583464Z 0 [Note] WSREP: Assign initial position for certification: 18, protocol version: -1
    | 2023-06-03T09:52:40.583514Z 0 [Note] WSREP: Preparing to initiate SST/IST
    | 2023-06-03T09:52:40.583523Z 0 [Note] WSREP: Starting replication
    | 2023-06-03T09:52:40.583535Z 0 [Note] WSREP: Setting initial position to 56cedc56-01f4-11ee-b577-47a2f3cad270:18
    | 2023-06-03T09:52:40.584049Z 0 [Note] WSREP: Using CRC-32C for message checksums.
    | 2023-06-03T09:52:40.584307Z 0 [Note] WSREP: gcomm thread scheduling priority set to other:0
    | 2023-06-03T09:52:40.584446Z 0 [Note] WSREP: Fail to access the file (/var/lib/mysql//gvwstate.dat) error (No such file or directory). It is possible if node is booting for first time or re-booting after a graceful shutdown
    | 2023-06-03T09:52:40.584462Z 0 [Note] WSREP: Restoring primary-component from disk failed. Either node is booting for first time or re-booting after a graceful shutdown
    | 2023-06-03T09:52:40.584645Z 0 [Note] WSREP: GMCast version 0
    | 2023-06-03T09:52:40.584754Z 0 [Note] WSREP: (60fc1367, 'tcp://0.0.0.0:4567') listening at tcp://0.0.0.0:4567
    | 2023-06-03T09:52:40.584767Z 0 [Note] WSREP: (60fc1367, 'tcp://0.0.0.0:4567') multicast: , ttl: 1
    | 2023-06-03T09:52:40.585093Z 0 [Note] WSREP: EVS version 0
    | 2023-06-03T09:52:40.585267Z 0 [Note] WSREP: gcomm: connecting to group 'mysql_cluster01', peer ''
    | 2023-06-03T09:52:40.585307Z 0 [Note] WSREP: start_prim is enabled, turn off pc_recovery
    | 2023-06-03T09:52:40.585538Z 0 [Note] WSREP: Node 60fc1367 state primary
    | 2023-06-03T09:52:40.585563Z 0 [Note] WSREP: Current view of cluster as seen by this node
    | view (view_id(PRIM,60fc1367,1)
    | memb {
    |  60fc1367,0
    |  }
    | joined {
    |  }
    | left {
    |  }
    | partitioned {
    |  }
    | )
    | 2023-06-03T09:52:40.585573Z 0 [Note] WSREP: Save the discovered primary-component to disk
    | 2023-06-03T09:52:40.586446Z 0 [Note] WSREP: gcomm: connected
    | 2023-06-03T09:52:40.586586Z 0 [Note] WSREP: Shifting CLOSED -> OPEN (TO: 0)
    | 2023-06-03T09:52:40.586723Z 0 [Note] WSREP: New COMPONENT: primary = yes, bootstrap = no, my_idx = 0, memb_num = 1
    | 2023-06-03T09:52:40.586905Z 0 [Note] WSREP: Waiting for SST/IST to complete.
    | 2023-06-03T09:52:40.586945Z 0 [Note] WSREP: STATE_EXCHANGE: sent state UUID: 60fc70be-01f4-11ee-a2a8-23123eeb547e
    | 2023-06-03T09:52:40.586962Z 0 [Note] WSREP: STATE EXCHANGE: sent state msg: 60fc70be-01f4-11ee-a2a8-23123eeb547e
    | 2023-06-03T09:52:40.586971Z 0 [Note] WSREP: STATE EXCHANGE: got state msg: 60fc70be-01f4-11ee-a2a8-23123eeb547e from 0 (fe9efafd2f8e)
    | 2023-06-03T09:52:40.586980Z 0 [Note] WSREP: Quorum results:
    |  version    = 6,
    |  component  = PRIMARY,
    |  conf_id    = 0,
    |  members    = 1/1 (primary/total),
    |  act_id     = 18,
    |  last_appl. = -1,
    |  protocols  = 0/9/3 (gcs/repl/appl),
    |  group UUID = 56cedc56-01f4-11ee-b577-47a2f3cad270
    | 2023-06-03T09:52:40.586987Z 0 [Note] WSREP: Flow-control interval: [100, 100]
    | 2023-06-03T09:52:40.586992Z 0 [Note] WSREP: Restored state OPEN -> JOINED (18)
    | 2023-06-03T09:52:40.587009Z 0 [Note] WSREP: Member 0.0 (fe9efafd2f8e) synced with group.
    | 2023-06-03T09:52:40.587017Z 0 [Note] WSREP: Shifting JOINED -> SYNCED (TO: 18)
    | 2023-06-03T09:52:40.587553Z 2 [Note] WSREP: REPL Protocols: 9 (4, 2)
    | 2023-06-03T09:52:40.587601Z 2 [Note] WSREP: REPL Protocols: 9 (4, 2)
    | 2023-06-03T09:52:40.587659Z 2 [Note] WSREP: New cluster view: global state: 56cedc56-01f4-11ee-b577-47a2f3cad270:18, view# 1: Primary, number of nodes: 1, my index: 0, protocol version 3
    | 2023-06-03T09:52:40.587680Z 2 [Note] WSREP: Setting wsrep_ready to true
    | 2023-06-03T09:52:40.587786Z 0 [Note] WSREP: SST complete, seqno: 18
    | 2023-06-03T09:52:40.590011Z 0 [Note] InnoDB: PUNCH HOLE support available
    | 2023-06-03T09:52:40.590055Z 0 [Note] InnoDB: Mutexes and rw_locks use GCC atomic builtins
    | 2023-06-03T09:52:40.590064Z 0 [Note] InnoDB: Uses event mutexes
    | 2023-06-03T09:52:40.590070Z 0 [Note] InnoDB: GCC builtin __atomic_thread_fence() is used for memory barrier
    | 2023-06-03T09:52:40.590076Z 0 [Note] InnoDB: Compressed tables use zlib 1.2.12
    | 2023-06-03T09:52:40.590082Z 0 [Note] InnoDB: Using Linux native AIO
    | 2023-06-03T09:52:40.590474Z 0 [Note] InnoDB: Number of pools: 1
    | 2023-06-03T09:52:40.590665Z 0 [Note] InnoDB: Using CPU crc32 instructions
    | 2023-06-03T09:52:40.595067Z 0 [Note] InnoDB: Initializing buffer pool, total size = 128M, instances = 1, chunk size = 128M
    | 2023-06-03T09:52:40.600319Z 0 [Note] InnoDB: Completed initialization of buffer pool
    | 2023-06-03T09:52:40.604997Z 0 [Note] InnoDB: If the mysqld execution user is authorized, page cleaner thread priority can be changed. See the man page of setpriority().
    | 2023-06-03T09:52:40.619611Z 0 [Note] InnoDB: Crash recovery did not find the parallel doublewrite buffer at /var/lib/mysql/xb_doublewrite
    | 2023-06-03T09:52:40.621223Z 0 [Note] InnoDB: Highest supported file format is Barracuda.
    | 2023-06-03T09:52:40.632738Z 0 [Note] InnoDB: Created parallel doublewrite buffer at /var/lib/mysql/xb_doublewrite, size 3932160 bytes
    | 2023-06-03T09:52:40.666944Z 0 [Note] InnoDB: Creating shared tablespace for temporary tables
    | 2023-06-03T09:52:40.667119Z 0 [Note] InnoDB: Setting file './ibtmp1' size to 12 MB. Physically writing the file full; Please wait ...
    | 2023-06-03T09:52:40.682923Z 0 [Note] InnoDB: File './ibtmp1' size is now 12 MB.
    | 2023-06-03T09:52:40.683605Z 0 [Note] InnoDB: 96 redo rollback segment(s) found. 96 redo rollback segment(s) are active.
    | 2023-06-03T09:52:40.683624Z 0 [Note] InnoDB: 32 non-redo rollback segment(s) are active.
    | 2023-06-03T09:52:40.685289Z 0 [Note] InnoDB: Percona XtraDB (http://www.percona.com) 5.7.41-44 started; log sequence number 12438866
    | 2023-06-03T09:52:40.685797Z 0 [Note] InnoDB: Loading buffer pool(s) from /var/lib/mysql/ib_buffer_pool
    | 2023-06-03T09:52:40.686197Z 0 [Note] Plugin 'FEDERATED' is disabled.
    | 2023-06-03T09:52:40.694121Z 0 [Note] Found ca.pem, server-cert.pem and server-key.pem in data directory. Trying to enable SSL support using them.
    | 2023-06-03T09:52:40.694195Z 0 [Note] Skipping generation of SSL certificates as certificate files are present in data directory.
    | 2023-06-03T09:52:40.694213Z 0 [Warning] A deprecated TLS version TLSv1 is enabled. Please use TLSv1.2 or higher.
    | 2023-06-03T09:52:40.694224Z 0 [Warning] A deprecated TLS version TLSv1.1 is enabled. Please use TLSv1.2 or higher.
    | 2023-06-03T09:52:40.695969Z 0 [Warning] CA certificate ca.pem is self signed.
    | 2023-06-03T09:52:40.696253Z 0 [Note] Skipping generation of RSA key pair as key files are present in data directory.
    | 2023-06-03T09:52:40.696589Z 0 [Note] Server hostname (bind-address): '0.0.0.0'; port: 3306
    | 2023-06-03T09:52:40.696640Z 0 [Note]   - '0.0.0.0' resolves to '0.0.0.0';
    | 2023-06-03T09:52:40.696762Z 0 [Note] Server socket created on IP: '0.0.0.0'.
    | 2023-06-03T09:52:40.724258Z 0 [Note] Event Scheduler: Loaded 0 events
    | 2023-06-03T09:52:40.725704Z 0 [Note] mysqld: ready for connections.
    | Version: '5.7.41-44-57'  socket: '/tmp/mysql.sock'  port: 3306  Percona XtraDB Cluster (GPL), Release rel44, Revision b46a1c3, WSREP version 31.65, wsrep_31.65
    | 2023-06-03T09:52:40.726012Z 2 [Note] WSREP: Initialized wsrep sidno 2
    | 2023-06-03T09:52:40.726060Z 2 [Note] WSREP: Auto Increment Offset/Increment re-align with cluster membership change (Offset: 1 -> 1) (Increment: 1 -> 1)
    | 2023-06-03T09:52:40.726147Z 2 [Note] WSREP: wsrep_notify_cmd is not defined, skipping notification.
    | 2023-06-03T09:52:40.726187Z 2 [Note] WSREP: Assign initial position for certification: 18, protocol version: 4
    | 2023-06-03T09:52:40.726251Z 0 [Note] WSREP: Service thread queue flushed.
    | 2023-06-03T09:52:40.726348Z 2 [Note] WSREP: GCache history reset: 56cedc56-01f4-11ee-b577-47a2f3cad270:0 -> 56cedc56-01f4-11ee-b577-47a2f3cad270:18
    | 2023-06-03T09:52:40.727268Z 2 [Note] WSREP: Synchronized with group, ready for connections
    | 2023-06-03T09:52:40.727285Z 2 [Note] WSREP: Setting wsrep_ready to true
    | 2023-06-03T09:52:40.727291Z 2 [Note] WSREP: wsrep_notify_cmd is not defined, skipping notification.
    | 2023-06-03T09:52:40.733608Z 0 [Note] InnoDB: Buffer pool(s) load completed at 230603  9:52:40
------------------------------------------------
above is the first node successfully booted
6

the second node boot time 2

    | ++ hostname -f
    | + NODE_NAME=fb7e1fdcb1e5
    | + NODE_PORT=3306
    | Registering in the discovery service
    | + '[' -n '' ']'
    | + '[' -n etcdnode:2379 ']'
    | + echo 'Registering in the discovery service'
    | ++ hostname -I
    | ++ awk ' { print $1 } '
    | + NODE_IP=10.180.0.190
    | + '[' etcd '!=' http ']'
    | + DISCOVERY_SERVICE=http://etcdnode:2379
    | + curl http://etcdnode:2379/v2/keys/pxc-cluster/queue/mysql_cluster01 -XPOST -d value=10.180.0.190 -d ttl=60
    |   % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
    |                                  Dload  Upload   Total   Spent    Left  Speed
    | {"action":"create","node":{"key":"/pxc-cluster/queue/mysql_cluster01/00000000000000000153","value":"10.180.0.190","expiration":"2023-06-03T09:57:18.870876689Z","ttl":60,"modifiedIndex":153,"createdIndex":153}}
100   235  100   210  100    25  52500   6250 --:--:-- --:--:-- --:--:-- 58750
    | ++ curl http://etcdnode:2379/v2/keys/pxc-cluster/queue/mysql_cluster01
    | ++ jq -r '.node.nodes[].value'
    |   % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
    |                                  Dload  Upload   Total   Spent    Left  Speed
100   490  100   490    0     0   159k      0 --:--:-- --:--:-- --:--:--  239k
    | + i='10.180.0.189
    | 10.180.0.190'
    | + i1='10.180.0.189
    | '
    | + curl http://etcdnode:2379/v2/keys/pxc-cluster/mysql_cluster01/10.180.0.190/ipaddr -XPUT -d value=10.180.0.190 -d ttl=30
    |   % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
    |                                  Dload  Upload   Total   Spent    Left  Speed
100   225  100   200  100    25  66666   8333 --:--:-- --:--:-- --:--:-- 75000
    | {"action":"set","node":{"key":"/pxc-cluster/mysql_cluster01/10.180.0.190/ipaddr","value":"10.180.0.190","expiration":"2023-06-03T09:56:48.900739854Z","ttl":30,"modifiedIndex":154,"createdIndex":154}}
    | + curl http://etcdnode:2379/v2/keys/pxc-cluster/mysql_cluster01/10.180.0.190/hostname -XPUT -d value=fb7e1fdcb1e5 -d ttl=30
    |   % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
    |                                  Dload  Upload   Total   Spent    Left  Speed
    | {"action":"set","node":{"key":"/pxc-cluster/mysql_cluster01/10.180.0.190/hostname","value":"fb7e1fdcb1e5","expiration":"2023-06-03T09:56:48.915279874Z","ttl":30,"modifiedIndex":155,"createdIndex":155}}
100   227  100   202  100    25  67333   8333 --:--:-- --:--:-- --:--:-- 75666
    | + curl http://etcdnode:2379/v2/keys/pxc-cluster/mysql_cluster01/10.180.0.190 -XPUT -d ttl=30 -d dir=true -d prevExist=true
    |   % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
    |                                  Dload  Upload   Total   Spent    Left  Speed
100   327  100   297  100    30  99000  10000 --:--:-- --:--:-- --:--:--  106k
    | {"action":"update","node":{"key":"/pxc-cluster/mysql_cluster01/10.180.0.190","dir":true,"expiration":"2023-06-03T09:56:48.929061194Z","ttl":30,"modifiedIndex":156,"createdIndex":154},"prevNode":{"key":"/pxc-cluster/mysql_cluster01/10.180.0.190","dir":true,"modifiedIndex":154,"createdIndex":154}}
    | ++ curl 'http://etcdnode:2379/v2/keys/pxc-cluster/mysql_cluster01/?quorum=true'
    | ++ jq -r '.node.nodes[]?.key'
    | ++ awk -F/ '{print $(NF)}'
    |   % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
    |                                  Dload  Upload   Total   Spent    Left  Speed
100   429  100   429    0     0   104k      0 --:--:-- --:--:-- --:--:--  139k
    | + i='10.180.0.181
    | 10.180.0.190'
    | + i2='10.180.0.181
    | '
    | ++ join , 10.180.0.189 10.180.0.181
    | ++ local IFS=,
    | ++ shift
    | +++ tr , '\n'
    | +++ tr '\n' ,
    | +++ sort -u
    | ++ joined=10.180.0.181,10.180.0.189,
    | ++ echo 10.180.0.181,10.180.0.189
    | + CLUSTER_JOIN=10.180.0.181,10.180.0.189
    | + sed -r 's|^[#]?wsrep_node_address=.*$|wsrep_node_address=10.180.0.190|' /etc/mysql/node.cnf
    | + sed -r 's|^[#]?wsrep_cluster_name=.*$|wsrep_cluster_name=mysql_cluster01|' /etc/mysql/node.cnf
    | + sed -r 's|^[#]?wsrep_cluster_address=.*$|wsrep_cluster_address=gcomm://10.180.0.181,10.180.0.189|' /etc/mysql/node.cnf
    | + sed -r 's|^[#]?wsrep_node_incoming_address=.*$|wsrep_node_incoming_address=fb7e1fdcb1e5:3306|' /etc/mysql/node.cnf
    | + sed -r 's|^[#]?wsrep_sst_auth=.*$|wsrep_sst_auth='\''xtrabackup:xtrabackup'\''|' /etc/mysql/node.cnf
    | + /usr/bin/clustercheckcron clustercheck clustercheck 1 /var/lib/mysql/clustercheck.log 1
    | + '[' -z 10.180.0.181,10.180.0.189 ']'
    | + '[' mysqld = mysqld -a -z '' ']'
    | + mysqld --version
    | + tee /tmp/version_info
    | mysqld  Ver 5.7.41-44-57 for Linux on x86_64 (Percona XtraDB Cluster (GPL), Release rel44, Revision b46a1c3, WSREP version 31.65, wsrep_31.65)
    | ++ _get_config datadir mysqld
    | ++ local conf=datadir
    | ++ shift
    | +++ mktemp -u
    | ++ awk '$1 == "datadir" && /^[^ \t]/ { sub(/^[^ \t]+[ \t]+/, ""); print; exit }'
    | ++ mysqld --verbose --help --wsrep-provider=none --log-bin-index=/tmp/tmp.KjD73FQdMa
    | + DATADIR=/var/lib/mysql/
    | + '[' -f /var/lib/mysql//version_info ']'
    | + mysqld --version
    | + grep -v wsrep_sst_auth /etc/mysql/node.cnf
    | [mysqld]
    |
    | ignore-db-dir=lost+found
    | datadir=/var/lib/mysql
    | socket=/tmp/mysql.sock
    | skip-host-cache
    |
    | #coredumper
    | #server_id=0
    | binlog_format=ROW
    | default_storage_engine=InnoDB
    |
    | innodb_flush_log_at_trx_commit  = 0
    | innodb_flush_method             = O_DIRECT
    | innodb_file_per_table           = 1
    | innodb_autoinc_lock_mode=2
    |
    | bind_address = 0.0.0.0
    |
    | wsrep_slave_threads=2
    | wsrep_cluster_address=gcomm://10.180.0.181,10.180.0.189
    | wsrep_provider=/usr/lib64/galera3/libgalera_smm.so
    |
    | wsrep_cluster_name=mysql_cluster01
    | wsrep_node_address=10.180.0.190
    | wsrep_node_incoming_address=fb7e1fdcb1e5:3306
    |
    | wsrep_sst_method=xtrabackup-v2
    |
    | [client]
    | socket=/tmp/mysql.sock
    |
    | [sst]
    | progress=/var/lib/mysql/sst_in_progress
    | + wsrep_start_position_opt=
    | + '[' mysqld = mysqld -a -z '' ']'
    | ++ _get_config datadir mysqld
    | ++ local conf=datadir
    | ++ shift
    | ++ awk '$1 == "datadir" && /^[^ \t]/ { sub(/^[^ \t]+[ \t]+/, ""); print; exit }'
    | +++ mktemp -u
    | ++ mysqld --verbose --help --wsrep-provider=none --log-bin-index=/tmp/tmp.ztwWJpkdBo
    | + DATADIR=/var/lib/mysql/
    | + grastate_loc=/var/lib/mysql//grastate.dat
    | + '[' -s /var/lib/mysql//grastate.dat -a -d /var/lib/mysql//mysql ']'
    | + '[' -z '' -a -d /var/lib/mysql//mysql ']'
    | + exec mysqld
    | 2023-06-03T09:56:19.287942Z 0 [Warning] TIMESTAMP with implicit DEFAULT value is deprecated. Please use --explicit_defaults_for_timestamp server option (see documentation for more details).
    | 2023-06-03T09:56:19.290680Z 0 [Note] mysqld (mysqld 5.7.41-44-57) starting as process 1 ...
    | 2023-06-03T09:56:19.295615Z 0 [Note] WSREP: Setting wsrep_ready to false
    | 2023-06-03T09:56:19.295645Z 0 [Note] WSREP: No pre-stored wsrep-start position found. Skipping position initialization.
    | 2023-06-03T09:56:19.295654Z 0 [Note] WSREP: wsrep_load(): loading provider library '/usr/lib64/galera3/libgalera_smm.so'
    | 2023-06-03T09:56:19.296511Z 0 [Note] WSREP: wsrep_load(): Galera 3.65(rf47405c) by Codership Oy <info@codership.com> loaded successfully.
    | 2023-06-03T09:56:19.296592Z 0 [Note] WSREP: CRC-32C: using 64-bit x86 acceleration.
    | 2023-06-03T09:56:19.296903Z 0 [Warning] WSREP: Could not open state file for reading: '/var/lib/mysql//grastate.dat'
    | 2023-06-03T09:56:19.296959Z 0 [Warning] WSREP: No persistent state found. Bootstraping with default state
    | 2023-06-03T09:56:19.297043Z 0 [Note] WSREP: Found saved state: 00000000-0000-0000-0000-000000000000:-1, safe_to_bootstrap: 1
    | 2023-06-03T09:56:19.301509Z 0 [Note] WSREP: Passing config to GCS: base_dir = /var/lib/mysql/; base_host = 10.180.0.190; base_port = 4567; cert.log_conflicts = no; cert.optimistic_pa = yes; debug = no; evs.auto_evict = 0; evs.delay_margin = PT1S; evs.delayed_keep_period = PT30S; evs.inactive_check_period = PT0.5S; evs.inactive_timeout = PT15S; evs.join_retrans_period = PT1S; evs.max_install_timeouts = 3; evs.send_window = 10; evs.stats_report_period = PT1M; evs.suspect_timeout = PT5S; evs.user_send_window = 4; evs.view_forget_timeout = PT24H; gcache.dir = /var/lib/mysql/; gcache.freeze_purge_at_seqno = -1; gcache.keep_pages_count = 0; gcache.keep_pages_size = 0; gcache.mem_size = 0; gcache.name = /var/lib/mysql//galera.cache; gcache.page_size = 128M; gcache.recover = no; gcache.size = 128M; gcomm.thread_prio = ; gcs.fc_debug = 0; gcs.fc_factor = 1; gcs.fc_limit = 100; gcs.fc_master_slave = no; gcs.max_packet_size = 64500; gcs.max_throttle = 0.25; gcs.recv_q_hard_limit = 9223372036854775807; gcs.recv_q_soft_limit = 0.25; gcs.sync_donor = no; gmcast.segment = 0; gmcast.version = 0; pc.announce_timeout = PT3S; pc.checksum = false; pc.ignore_quorum = false; pc.ignore_sb = false; pc.npvo = false; pc.recovery = true; pc.version = 0; pc.wait_prim = true; pc.wait_prim_timeout = PT30S; pc.weight = 1; protonet.backend = asio; protonet.version = 0; repl.causal_read_timeout = PT30S; repl.commit_order = 3; repl.key_format = FLAT8; repl.max_ws_size = 2147483647; repl.proto_max = 9; socket.checksum = 2; socket.recv_buf_size = auto; socket.send_buf_size = auto;
    | 2023-06-03T09:56:19.313084Z 0 [Note] WSREP: GCache history reset: 00000000-0000-0000-0000-000000000000:0 -> 00000000-0000-0000-0000-000000000000:-1
    | 2023-06-03T09:56:19.313452Z 0 [Note] WSREP: Assign initial position for certification: -1, protocol version: -1
    | 2023-06-03T09:56:19.313501Z 0 [Note] WSREP: Preparing to initiate SST/IST
    | 2023-06-03T09:56:19.313510Z 0 [Note] WSREP: Starting replication
    | 2023-06-03T09:56:19.313523Z 0 [Note] WSREP: Setting initial position to 00000000-0000-0000-0000-000000000000:-1
    | 2023-06-03T09:56:19.314051Z 0 [Note] WSREP: Using CRC-32C for message checksums.
    | 2023-06-03T09:56:19.314260Z 0 [Note] WSREP: gcomm thread scheduling priority set to other:0
    | 2023-06-03T09:56:19.314382Z 0 [Note] WSREP: Fail to access the file (/var/lib/mysql//gvwstate.dat) error (No such file or directory). It is possible if node is booting for first time or re-booting after a graceful shutdown
    | 2023-06-03T09:56:19.314397Z 0 [Note] WSREP: Restoring primary-component from disk failed. Either node is booting for first time or re-booting after a graceful shutdown
    | 2023-06-03T09:56:19.314615Z 0 [Note] WSREP: GMCast version 0
    | 2023-06-03T09:56:19.314784Z 0 [Note] WSREP: (e35b9df6, 'tcp://0.0.0.0:4567') listening at tcp://0.0.0.0:4567
    | 2023-06-03T09:56:19.314795Z 0 [Note] WSREP: (e35b9df6, 'tcp://0.0.0.0:4567') multicast: , ttl: 1
    | 2023-06-03T09:56:19.315097Z 0 [Note] WSREP: EVS version 0
    | 2023-06-03T09:56:19.315229Z 0 [Note] WSREP: gcomm: connecting to group 'mysql_cluster01', peer '10.180.0.181:,10.180.0.189:'
    | 2023-06-03T09:56:22.317564Z 0 [Note] WSREP: (e35b9df6, 'tcp://0.0.0.0:4567') connection to peer 00000000 with addr tcp://10.180.0.181:4567 timed out, no messages seen in PT3S (gmcast.peer_timeout), socket stats: rtt: 0 rttvar: 250000 rto: 2000000 lost: 1 last_data_recv: 799777432 cwnd: 1 last_queued_since: 800077431840533 last_delivered_since: 800077431840533 send_queue_length: 0 send_queue_bytes: 0
    | 2023-06-03T09:56:22.317706Z 0 [Note] WSREP: (e35b9df6, 'tcp://0.0.0.0:4567') connection to peer 00000000 with addr tcp://10.180.0.189:4567 timed out, no messages seen in PT3S (gmcast.peer_timeout), socket stats: rtt: 0 rttvar: 250000 rto: 2000000 lost: 1 last_data_recv: 799777433 cwnd: 1 last_queued_since: 800077432036472 last_delivered_since: 800077432036472 send_queue_length: 0 send_queue_bytes: 0
    | 2023-06-03T09:56:22.317752Z 0 [Note] WSREP: announce period timed out (pc.announce_timeout)
    | 2023-06-03T09:56:22.317895Z 0 [Warning] WSREP: no nodes coming from prim view, prim not possible
    | 2023-06-03T09:56:22.317917Z 0 [Note] WSREP: Current view of cluster as seen by this node
    | view (view_id(NON_PRIM,e35b9df6,1)
    | memb {
    |  e35b9df6,0
    |  }
    | joined {
    |  }
    | left {
    |  }
    | partitioned {
    |  }
    | )
    | 2023-06-03T09:56:22.818024Z 0 [Warning] WSREP: last inactive check more than PT1.5S (3*evs.inactive_check_period) ago (PT3.50293S), skipping check
    | 2023-06-03T09:56:26.320114Z 0 [Note] WSREP: (e35b9df6, 'tcp://0.0.0.0:4567') connection to peer 00000000 with addr tcp://10.180.0.181:4567 timed out, no messages seen in PT3S (gmcast.peer_timeout), socket stats: rtt: 0 rttvar: 250000 rto: 2000000 lost: 1 last_data_recv: 799781435 cwnd: 1 last_queued_since: 800081434413530 last_delivered_since: 800081434413530 send_queue_length: 0 send_queue_bytes: 0
    | 2023-06-03T09:56:29.321435Z 0 [Note] WSREP: (e35b9df6, 'tcp://0.0.0.0:4567') connection to peer 00000000 with addr tcp://10.180.0.189:4567 timed out, no messages seen in PT3S (gmcast.peer_timeout), socket stats: rtt: 0 rttvar: 250000 rto: 2000000 lost: 1 last_data_recv: 799784436 cwnd: 1 last_queued_since: 800084435733227 last_delivered_since: 800084435733227 send_queue_length: 0 send_queue_bytes: 0
    | 2023-06-03T09:56:32.324989Z 0 [Note] WSREP: (e35b9df6, 'tcp://0.0.0.0:4567') connection to peer 00000000 with addr tcp://10.180.0.181:4567 timed out, no messages seen in PT3S (gmcast.peer_timeout), socket stats: rtt: 0 rttvar: 250000 rto: 2000000 lost: 1 last_data_recv: 799787440 cwnd: 1 last_queued_since: 800087439289949 last_delivered_since: 800087439289949 send_queue_length: 0 send_queue_bytes: 0
    | 2023-06-03T09:56:35.329268Z 0 [Note] WSREP: (e35b9df6, 'tcp://0.0.0.0:4567') connection to peer 00000000 with addr tcp://10.180.0.189:4567 timed out, no messages seen in PT3S (gmcast.peer_timeout), socket stats: rtt: 0 rttvar: 250000 rto: 2000000 lost: 1 last_data_recv: 799790444 cwnd: 1 last_queued_since: 800090443569990 last_delivered_since: 800090443569990 send_queue_length: 0 send_queue_bytes: 0
    | 2023-06-03T09:56:38.333020Z 0 [Note] WSREP: (e35b9df6, 'tcp://0.0.0.0:4567') connection to peer 00000000 with addr tcp://10.180.0.181:4567 timed out, no messages seen in PT3S (gmcast.peer_timeout), socket stats: rtt: 0 rttvar: 250000 rto: 2000000 lost: 1 last_data_recv: 799793448 cwnd: 1 last_queued_since: 800093447320452 last_delivered_since: 800093447320452 send_queue_length: 0 send_queue_bytes: 0
    | 2023-06-03T09:56:41.336601Z 0 [Note] WSREP: (e35b9df6, 'tcp://0.0.0.0:4567') connection to peer 00000000 with addr tcp://10.180.0.189:4567 timed out, no messages seen in PT3S (gmcast.peer_timeout), socket stats: rtt: 0 rttvar: 250000 rto: 2000000 lost: 1 last_data_recv: 799796451 cwnd: 1 last_queued_since: 800096450903395 last_delivered_since: 800096450903395 send_queue_length: 0 send_queue_bytes: 0
    | 2023-06-03T09:56:44.340521Z 0 [Note] WSREP: (e35b9df6, 'tcp://0.0.0.0:4567') connection to peer 00000000 with addr tcp://10.180.0.181:4567 timed out, no messages seen in PT3S (gmcast.peer_timeout), socket stats: rtt: 0 rttvar: 250000 rto: 2000000 lost: 1 last_data_recv: 799799455 cwnd: 1 last_queued_since: 800099454823278 last_delivered_since: 800099454823278 send_queue_length: 0 send_queue_bytes: 0
    | 2023-06-03T09:56:47.345374Z 0 [Note] WSREP: (e35b9df6, 'tcp://0.0.0.0:4567') connection to peer 00000000 with addr tcp://10.180.0.189:4567 timed out, no messages seen in PT3S (gmcast.peer_timeout), socket stats: rtt: 0 rttvar: 250000 rto: 2000000 lost: 1 last_data_recv: 799802460 cwnd: 1 last_queued_since: 800102459675135 last_delivered_since: 800102459675135 send_queue_length: 0 send_queue_bytes: 0
    | 2023-06-03T09:56:50.349677Z 0 [Note] WSREP: (e35b9df6, 'tcp://0.0.0.0:4567') connection to peer 00000000 with addr tcp://10.180.0.181:4567 timed out, no messages seen in PT3S (gmcast.peer_timeout), socket stats: rtt: 0 rttvar: 250000 rto: 2000000 lost: 1 last_data_recv: 799805464 cwnd: 1 last_queued_since: 800105463978814 last_delivered_since: 800105463978814 send_queue_length: 0 send_queue_bytes: 0
    | 2023-06-03T09:56:52.352834Z 0 [Note] WSREP: Current view of cluster as seen by this node
    | view ((empty))
    | 2023-06-03T09:56:52.353034Z 0 [ERROR] WSREP: failed to open gcomm backend connection: 110: failed to reach primary view (pc.wait_prim_timeout): 110 (Connection timed out)
    |   at gcomm/src/pc.cpp:connect():161
    | 2023-06-03T09:56:52.353052Z 0 [ERROR] WSREP: gcs/src/gcs_core.cpp:gcs_core_open():209: Failed to open backend connection: -110 (Connection timed out)
    | 2023-06-03T09:56:53.353270Z 0 [Note] WSREP: gcomm: terminating thread
    | 2023-06-03T09:56:53.353330Z 0 [Note] WSREP: gcomm: joining thread
    | 2023-06-03T09:56:53.353444Z 0 [ERROR] WSREP: gcs/src/gcs.cpp:gcs_open():1578: Failed to open channel 'mysql_cluster01' at 'gcomm://10.180.0.181,10.180.0.189': -110 (Connection timed out)
    | 2023-06-03T09:56:53.353504Z 0 [ERROR] WSREP: gcs connect failed: Connection timed out
    | 2023-06-03T09:56:53.353515Z 0 [ERROR] WSREP: Provider/Node (gcomm://10.180.0.181,10.180.0.189) failed to establish connection with cluster (reason: 7)
    | 2023-06-03T09:56:53.353521Z 0 [ERROR] Aborting
    |
    | 2023-06-03T09:56:53.353526Z 0 [Note] Giving 0 client threads a chance to die gracefully
    | 2023-06-03T09:56:53.353533Z 0 [Note] WSREP: Waiting for active wsrep applier to exit
    | 2023-06-03T09:56:53.353538Z 0 [Note] WSREP: Service disconnected.
    | 2023-06-03T09:56:53.353543Z 0 [Note] WSREP: Waiting to close threads......
    | 2023-06-03T09:56:58.353745Z 0 [Note] WSREP: Some threads may fail to exit.
    | 2023-06-03T09:56:58.361621Z 0 [Note] WSREP: Service thread queue flushed.
    | 2023-06-03T09:56:58.361893Z 0 [Note] WSREP: MemPool(SlaveTrxHandle): hit ratio: 0, misses: 0, in use: 0, in pool: 0
    | 2023-06-03T09:56:58.362104Z 0 [Note] WSREP: Shifting CLOSED -> DESTROYED (TO: 0)
    | 2023-06-03T09:56:58.363727Z 0 [Note] Binlog end
    | 2023-06-03T09:56:58.364380Z 0 [Note] mysqld: Shutdown complete
    |
7

could someone give me a hand?

8

Hi,

I don’t see any meaningful messages from the outputs you sent, so it’s not possible to know why it’s failing to bring up the second node. Have you throughly checked the logs? I would first check the MySQL-side logs, and then XtraBackup logs (which should be in the datadir).

Also, what blog post did you mean?