Hi All,
My bootstrap server is starting up nicely except for not loading the provider for a percona cluster 
in the config:
root@percona-01-blue-prod:/etc/mysql# cat my.cnf
# The MariaDB configuration file
#
# The MariaDB/MySQL tools read configuration files in the following order:
# 1. "/etc/mysql/mariadb.cnf" (this file) to set global defaults,
# 2. "/etc/mysql/conf.d/*.cnf" to set global options.
# 3. "/etc/mysql/mariadb.conf.d/*.cnf" to set MariaDB-only options.
# 4. "~/.my.cnf" to set user-specific options.
#
# If the same option is defined multiple times, the last one will apply.
#
# One can use all long options that the program supports.
# Run program with --help to get a list of available options and with
# --print-defaults to see which it would actually understand and use.
#
# This group is read both both by the client and the server
# use it for options that affect everything
#
[client-server]
# Import all .cnf files from configuration directory
!includedir /etc/mysql/conf.d/
!includedir /etc/mysql/mariadb.conf.d/
root@percona-01-blue-prod:/etc/mysql/conf.d# cat mysqld.cnf
[mysqld]
wsrep_provider=/usr/lib/libgalera_smm.so
wsrep_cluster_name=pxc-cluster
wsrep_cluster_address=gcomm://10.0.12.11,10.0.12.12,10.0.12.13
wsrep_node_name="percona-01-blue-prod"
wsrep_node_address="10.0.12.11"
pxc_strict_mode=ENFORCING
pxc-encrypt-cluster-traffic=ON
wsrep_provider_options=”socket.ssl_key=/etc/mysql/certs/server-key.pem;socket.ssl_cert=/etc/mysql/certs/server-cert.pem;socket.ssl_ca=/etc/mysql/certs/ca.pem”
ssl-ca=/etc/mysql/certs/ca.pem
ssl-cert=/etc/mysql/certs/server-cert.pem
ssl-key=/etc/mysql/certs/server-key.pem
[sst]
encrypt=4
ssl-key=/etc/mysql/certs/server-key.pem
ssl-ca=/etc/mysql/certs/ca.pem
ssl-cert=/etc/mysql/certs/server-cert.pem
When starting the first server with systemctl start mysql@bootstrap.service I get this in the status:
root@percona-01-blue-prod:/etc/mysql# systemctl status mysql@bootstrap.service
● mysql@bootstrap.service - Percona XtraDB Cluster with config /etc/default/mysql.bootstrap
Loaded: loaded (/lib/systemd/system/mysql@.service; disabled; vendor preset: enabled)
Active: active (running) since Mon 2021-04-05 15:07:39 CEST; 16min ago
Process: 1257 ExecStartPre=/usr/bin/mysql-systemd start-pre (code=exited, status=0/SUCCESS)
Process: 1313 ExecStartPre=/bin/sh -c systemctl unset-environment _WSREP_START_POSITION (code=exited, status=0/SUCCESS)
Process: 1315 ExecStartPre=/bin/sh -c VAR=`bash /usr/bin/mysql-systemd galera-recovery`; [ $? -eq 0 ] && systemctl set-environment _WSREP_START_POSITION=$VAR || exit 1 (code=exited, status=0/SUCCESS)
Process: 1426 ExecStartPost=/bin/sh -c systemctl unset-environment _WSREP_START_POSITION (code=exited, status=0/SUCCESS)
Process: 1428 ExecStartPost=/usr/bin/mysql-systemd start-post $MAINPID (code=exited, status=0/SUCCESS)
Main PID: 1381 (mysqld)
Status: "Server is operational"
Tasks: 41 (limit: 4580)
Memory: 357.8M
CGroup: /system.slice/system-mysql.slice/mysql@bootstrap.service
└─1381 /usr/sbin/mysqld --wsrep-new-cluster --wsrep_start_position=18119fd1-5cb2-11eb-8c4e-933a73faae5e:73
apr 05 15:07:39 percona-01-blue-prod mysqld[1381]: 2021-04-05T13:07:39.204171Z 1 [System] [MY-000000] [WSREP] PXC upgrade completed successfully
apr 05 15:07:39 percona-01-blue-prod mysqld[1381]: 2021-04-05T13:07:39.212471Z 0 [System] [MY-011323] [Server] X Plugin ready for connections. Bind-address: '::' port: 33060, socket: /var/run/mysqld/mysqlx.sock
apr 05 15:07:39 percona-01-blue-prod mysqld[1381]: 2021-04-05T13:07:39.296472Z 0 [Warning] [MY-010068] [Server] CA certificate ca.pem is self signed.
apr 05 15:07:39 percona-01-blue-prod mysqld[1381]: 2021-04-05T13:07:39.297190Z 0 [System] [MY-013602] [Server] Channel mysql_main configured to support TLS. Encrypted connections are now supported for this channe
apr 05 15:07:39 percona-01-blue-prod mysqld[1381]: 2021-04-05T13:07:39.335224Z 0 [Note] [MY-000000] [WSREP] Initialized wsrep sidno 2
apr 05 15:07:39 percona-01-blue-prod mysqld[1381]: 2021-04-05T13:07:39.335291Z 0 [Note] [MY-000000] [Galera] Loading provider none initial position: 18119fd1-5cb2-11eb-8c4e-933a73faae5e:73
apr 05 15:07:39 percona-01-blue-prod mysqld[1381]: 2021-04-05T13:07:39.335301Z 0 [Note] [MY-000000] [Galera] wsrep_load(): loading provider library 'none'
apr 05 15:07:39 percona-01-blue-prod mysqld[1381]: 2021-04-05T13:07:39.336295Z 0 [System] [MY-010931] [Server] /usr/sbin/mysqld: ready for connections. Version: '8.0.21-12.1' socket: '/var/run/mysqld/mysqld.sock
apr 05 15:07:39 percona-01-blue-prod mysql-systemd[1428]: SUCCESS!
apr 05 15:07:39 percona-01-blue-prod systemd[1]: Started Percona XtraDB Cluster with config /etc/default/mysql.bootstrap.
But no provider is loaded ?
In mysql:
mysql> SHOW STATUS LIKE 'wsrep_%';
+-----------------------------+-----------------------------------+
| Variable_name | Value |
+-----------------------------+-----------------------------------+
| wsrep_cluster_capabilities | |
| wsrep_cluster_conf_id | 18446744073709551615 |
| wsrep_cluster_size | 0 |
| wsrep_cluster_state_uuid | |
| wsrep_cluster_status | Disconnected |
| wsrep_connected | OFF |
| wsrep_local_bf_aborts | 0 |
| wsrep_local_index | 18446744073709551615 |
| wsrep_provider_capabilities | |
| wsrep_provider_name | none |
| wsrep_provider_vendor | Codership Oy <info@codership.com> |
| wsrep_provider_version | 26 |
| wsrep_ready | OFF |
| wsrep_thread_count | 0 |
+-----------------------------+-----------------------------------+
14 rows in set (0.01 sec)
I’m racking my brain but cannot understand what is going on. /usr/lib/libgalera_smm.so exists and is readable, the same error on all three nodes when trying them as a bootstrap.
OS: Debian GNU/Linux 10
Percona: 8.0.21-12.1