Hello team,
I am trying to set up crunchydata postgresql cluster, operator starts successfully but I am having troubles to set up the pg cluster.
I am using this version of operator 2.4.0 and this version of pg cluster 2.4.1
I have configured pgbackrest to backup on a local pvc but this seems to be causing an issue as following:
The configmap pgback-rest-config is not being created and this seems to be blocking the init container.
Has someone dealt with this issue previously, how can I solve this. Should the configmap normally be created when I deploy the cluster?
kubelet MountVolume.SetUp failed for volume "pgbackrest-config" : configmap "percona-db-pg-db-pgbackrest-config" not found
Statefulset content:
apiVersion: apps/v1
kind: StatefulSet
metadata:
annotations:
policies.kyverno.io/last-applied-patches: |
mutate-postgresql-sts.percona-resources.kyverno.io: replaced /spec/template/spec/containers/3/resources/limits/memory
creationTimestamp: "2024-07-22T12:40:08Z"
generation: 28747
labels:
app.kubernetes.io/component: pg
app.kubernetes.io/instance: percona-db-pg-db
app.kubernetes.io/managed-by: percona-postgresql-operator
app.kubernetes.io/name: percona-postgresql
app.kubernetes.io/part-of: percona-postgresql
pgv2.percona.com/version: 2.4.0
postgres-operator.crunchydata.com/cluster: percona-db-pg-db
postgres-operator.crunchydata.com/data: postgres
postgres-operator.crunchydata.com/instance: percona-db-pg-db-instance1-mrsr
postgres-operator.crunchydata.com/instance-set: instance1
name: percona-db-pg-db-instance1-mrsr
namespace: percona-operator
ownerReferences:
- apiVersion: postgres-operator.crunchydata.com/v1beta1
blockOwnerDeletion: true
controller: true
kind: PostgresCluster
name: percona-db-pg-db
uid: 21903927-fd44-4238-8eac-24f04caaecfe
resourceVersion: "1257109456"
uid: e53607f7-71e9-4e03-a8a3-a45b17665c66
spec:
podManagementPolicy: OrderedReady
replicas: 1
revisionHistoryLimit: 0
selector:
matchLabels:
postgres-operator.crunchydata.com/cluster: percona-db-pg-db
postgres-operator.crunchydata.com/instance: percona-db-pg-db-instance1-mrsr
postgres-operator.crunchydata.com/instance-set: instance1
serviceName: percona-db-pg-db-pods
template:
metadata:
creationTimestamp: null
labels:
app.kubernetes.io/component: pg
app.kubernetes.io/instance: percona-db-pg-db
app.kubernetes.io/managed-by: percona-postgresql-operator
app.kubernetes.io/name: percona-postgresql
app.kubernetes.io/part-of: percona-postgresql
pgv2.percona.com/version: 2.4.0
postgres-operator.crunchydata.com/cluster: percona-db-pg-db
postgres-operator.crunchydata.com/data: postgres
postgres-operator.crunchydata.com/instance: percona-db-pg-db-instance1-mrsr
postgres-operator.crunchydata.com/instance-set: instance1
postgres-operator.crunchydata.com/patroni: percona-db-pg-db-ha
spec:
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- podAffinityTerm:
labelSelector:
matchLabels:
postgres-operator.crunchydata.com/data: postgres
topologyKey: kubernetes.io/hostname
weight: 1
containers:
- command:
- patroni
- /etc/patroni
env:
- name: PGDATA
value: /pgdata/pg16
- name: PGHOST
value: /tmp/postgres
- name: PGPORT
value: "5432"
- name: KRB5_CONFIG
value: /etc/postgres/krb5.conf
- name: KRB5RCACHEDIR
value: /tmp
- name: PATRONI_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
- name: PATRONI_KUBERNETES_POD_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.podIP
- name: PATRONI_KUBERNETES_PORTS
value: |
- name: postgres
port: 5432
protocol: TCP
- name: PATRONI_POSTGRESQL_CONNECT_ADDRESS
value: $(PATRONI_NAME).percona-db-pg-db-pods:5432
- name: PATRONI_POSTGRESQL_LISTEN
value: '*:5432'
- name: PATRONI_POSTGRESQL_CONFIG_DIR
value: /pgdata/pg16
- name: PATRONI_POSTGRESQL_DATA_DIR
value: /pgdata/pg16
- name: PATRONI_RESTAPI_CONNECT_ADDRESS
value: $(PATRONI_NAME).percona-db-pg-db-pods:8008
- name: PATRONI_RESTAPI_LISTEN
value: '*:8008'
- name: PATRONICTL_CONFIG_FILE
value: /etc/patroni
- name: LD_PRELOAD
value: /usr/lib64/libnss_wrapper.so
- name: NSS_WRAPPER_PASSWD
value: /tmp/nss_wrapper/postgres/passwd
- name: NSS_WRAPPER_GROUP
value: /tmp/nss_wrapper/postgres/group
image: percona/percona-postgresql-operator:2.4.0-ppg16.3-postgres
imagePullPolicy: Always
livenessProbe:
failureThreshold: 3
httpGet:
path: /liveness
port: 8008
scheme: HTTPS
initialDelaySeconds: 3
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 5
name: database
ports:
- containerPort: 5432
name: postgres
protocol: TCP
readinessProbe:
failureThreshold: 3
httpGet:
path: /readiness
port: 8008
scheme: HTTPS
initialDelaySeconds: 3
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 5
resources:
limits:
cpu: 200m
memory: 256Mi
requests:
cpu: 100m
memory: 128Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
readOnlyRootFilesystem: true
runAsNonRoot: true
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /pgconf/tls
name: cert-volume
readOnly: true
- mountPath: /pgdata
name: postgres-data
- mountPath: /etc/database-containerinfo
name: database-containerinfo
readOnly: true
- mountPath: /etc/pgbackrest/conf.d
name: pgbackrest-config
readOnly: true
- mountPath: /etc/patroni
name: patroni-config
readOnly: true
- mountPath: /tmp
name: tmp
- mountPath: /dev/shm
name: dshm
- command:
- bash
- -ceu
- --
- |-
monitor() {
declare -r directory="/pgconf/tls"
exec {fd}<> <(:)
while read -r -t 5 -u "${fd}" || true; do
if [ "${directory}" -nt "/proc/self/fd/${fd}" ] &&
install -D --mode=0600 -t "/tmp/replication" "${directory}"/{replication/tls.crt,replication/tls.key,replication/ca.crt} &&
pkill -HUP --exact --parent=1 postgres
then
exec {fd}>&- && exec {fd}<> <(:)
stat --format='Loaded certificates dated %y' "${directory}"
fi
done
}; export -f monitor; exec -a "$0" bash -ceu monitor
- replication-cert-copy
image: percona/percona-postgresql-operator:2.4.0-ppg16.3-postgres
imagePullPolicy: Always
name: replication-cert-copy
resources:
limits:
cpu: 200m
memory: 256Mi
requests:
cpu: 100m
memory: 128Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
readOnlyRootFilesystem: true
runAsNonRoot: true
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /pgconf/tls
name: cert-volume
readOnly: true
- mountPath: /tmp
name: tmp
- command:
- pgbackrest
- server
env:
- name: LD_PRELOAD
value: /usr/lib64/libnss_wrapper.so
- name: NSS_WRAPPER_PASSWD
value: /tmp/nss_wrapper/postgres/passwd
- name: NSS_WRAPPER_GROUP
value: /tmp/nss_wrapper/postgres/group
image: percona/percona-postgresql-operator:2.4.0-ppg16.3-pgbackrest2.51-1
imagePullPolicy: Always
livenessProbe:
exec:
command:
- pgbackrest
- server-ping
failureThreshold: 3
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
name: pgbackrest
resources:
limits:
cpu: 200m
memory: 256Mi
requests:
cpu: 100m
memory: 128Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
readOnlyRootFilesystem: true
runAsNonRoot: true
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /etc/pgbackrest/server
name: pgbackrest-server
readOnly: true
- mountPath: /pgdata
name: postgres-data
- mountPath: /etc/pgbackrest/conf.d
name: pgbackrest-config
readOnly: true
- mountPath: /tmp
name: tmp
- command:
- bash
- -ceu
- --
- |-
monitor() {
exec {fd}<> <(:)
until read -r -t 5 -u "${fd}"; do
if
[ "${filename}" -nt "/proc/self/fd/${fd}" ] &&
pkill -HUP --exact --parent=0 pgbackrest
then
exec {fd}>&- && exec {fd}<> <(:)
stat --dereference --format='Loaded configuration dated %y' "${filename}"
elif
{ [ "${directory}" -nt "/proc/self/fd/${fd}" ] ||
[ "${authority}" -nt "/proc/self/fd/${fd}" ]
} &&
pkill -HUP --exact --parent=0 pgbackrest
then
exec {fd}>&- && exec {fd}<> <(:)
stat --format='Loaded certificates dated %y' "${directory}"
fi
done
}; export directory="$1" authority="$2" filename="$3"; export -f monitor; exec -a "$0" bash -ceu monitor
- pgbackrest-config
- /etc/pgbackrest/server
- /etc/pgbackrest/conf.d/~postgres-operator/tls-ca.crt
- /etc/pgbackrest/conf.d/~postgres-operator_server.conf
image: percona/percona-postgresql-operator:2.4.0-ppg16.3-pgbackrest2.51-1
imagePullPolicy: Always
name: pgbackrest-config
resources:
limits:
cpu: 200m
memory: 256Mi
requests:
cpu: 100m
memory: 128Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
readOnlyRootFilesystem: true
runAsNonRoot: true
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /etc/pgbackrest/server
name: pgbackrest-server
readOnly: true
- mountPath: /etc/pgbackrest/conf.d
name: pgbackrest-config
readOnly: true
- mountPath: /tmp
name: tmp
dnsPolicy: ClusterFirst
enableServiceLinks: false
initContainers:
- command:
- bash
- -ceu
- --
- |-
declare -r expected_major_version="$1" pgwal_directory="$2" pgbrLog_directory="$3"
permissions() { while [[ -n "$1" ]]; do set "${1%/*}" "$@"; done; shift; stat -Lc '%A %4u %4g %n' "$@"; }
halt() { local rc=$?; >&2 echo "$@"; exit "${rc/#0/1}"; }
results() { printf '::postgres-operator: %s::%s\n' "$@"; }
recreate() (
local tmp; tmp=$(mktemp -d -p "${1%/*}"); GLOBIGNORE='.:..'; set -x
chmod "$2" "${tmp}"; mv "$1"/* "${tmp}"; rmdir "$1"; mv "${tmp}" "$1"
)
safelink() (
local desired="$1" name="$2" current
current=$(realpath "${name}")
if [ "${current}" = "${desired}" ]; then return; fi
set -x; mv --no-target-directory "${current}" "${desired}"
ln --no-dereference --force --symbolic "${desired}" "${name}"
)
echo Initializing ...
results 'uid' "$(id -u)" 'gid' "$(id -G)"
results 'postgres path' "$(command -v postgres)"
results 'postgres version' "${postgres_version:=$(postgres --version)}"
[[ "${postgres_version}" =~ ") ${expected_major_version}"($|[^0-9]) ]] ||
halt Expected PostgreSQL version "${expected_major_version}"
results 'config directory' "${PGDATA:?}"
postgres_data_directory=$([ -d "${PGDATA}" ] && postgres -C data_directory || echo "${PGDATA}")
results 'data directory' "${postgres_data_directory}"
[[ "${postgres_data_directory}" == "${PGDATA}" ]] ||
halt Expected matching config and data directories
bootstrap_dir="${postgres_data_directory}_bootstrap"
[ -d "${bootstrap_dir}" ] && results 'bootstrap directory' "${bootstrap_dir}"
[ -d "${bootstrap_dir}" ] && postgres_data_directory="${bootstrap_dir}"
if [[ ! -e "${postgres_data_directory}" || -O "${postgres_data_directory}" ]]; then
install --directory --mode=0700 "${postgres_data_directory}"
elif [[ -w "${postgres_data_directory}" && -g "${postgres_data_directory}" ]]; then
recreate "${postgres_data_directory}" '0700'
else (halt Permissions!); fi ||
halt "$(permissions "${postgres_data_directory}" ||:)"
results 'pgBackRest log directory' "${pgbrLog_directory}"
install --directory --mode=0775 "${pgbrLog_directory}" ||
halt "$(permissions "${pgbrLog_directory}" ||:)"
install -D --mode=0600 -t "/tmp/replication" "/pgconf/tls/replication"/{tls.crt,tls.key,ca.crt}
[ -f "${postgres_data_directory}/PG_VERSION" ] || exit 0
results 'data version' "${postgres_data_version:=$(< "${postgres_data_directory}/PG_VERSION")}"
[[ "${postgres_data_version}" == "${expected_major_version}" ]] ||
halt Expected PostgreSQL data version "${expected_major_version}"
[[ ! -f "${postgres_data_directory}/postgresql.conf" ]] &&
touch "${postgres_data_directory}/postgresql.conf"
safelink "${pgwal_directory}" "${postgres_data_directory}/pg_wal"
results 'wal directory' "$(realpath "${postgres_data_directory}/pg_wal")"
rm -f "${postgres_data_directory}/recovery.signal"
- startup
- "16"
- /pgdata/pg16_wal
- /pgdata/pgbackrest/log
env:
- name: PGDATA
value: /pgdata/pg16
- name: PGHOST
value: /tmp/postgres
- name: PGPORT
value: "5432"
- name: KRB5_CONFIG
value: /etc/postgres/krb5.conf
- name: KRB5RCACHEDIR
value: /tmp
image: percona/percona-postgresql-operator:2.4.0-ppg16.3-postgres
imagePullPolicy: Always
name: postgres-startup
resources:
limits:
cpu: "2"
memory: 4Gi
requests:
cpu: "2"
memory: 4Gi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
readOnlyRootFilesystem: true
runAsNonRoot: true
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /pgconf/tls
name: cert-volume
readOnly: true
- mountPath: /pgdata
name: postgres-data
- mountPath: /tmp
name: tmp
- command:
- bash
- -c
- "export NSS_WRAPPER_SUBDIR=postgres CRUNCHY_NSS_USERNAME=postgres CRUNCHY_NSS_USER_DESC=\"postgres\"
\n# Define nss_wrapper directory and passwd & group files that will be utilized
by nss_wrapper. The\n# nss_wrapper_env.sh script (which also sets these
vars) isn't sourced here since the nss_wrapper\n# has not yet been setup,
and we therefore don't yet want the nss_wrapper vars in the environment.\nmkdir
-p /tmp/nss_wrapper\nchmod g+rwx /tmp/nss_wrapper\n\nNSS_WRAPPER_DIR=\"/tmp/nss_wrapper/${NSS_WRAPPER_SUBDIR}\"\nNSS_WRAPPER_PASSWD=\"${NSS_WRAPPER_DIR}/passwd\"\nNSS_WRAPPER_GROUP=\"${NSS_WRAPPER_DIR}/group\"\n\n#
create the nss_wrapper directory\nmkdir -p \"${NSS_WRAPPER_DIR}\"\n\n# grab
the current user ID and group ID\nUSER_ID=$(id -u)\nexport USER_ID\nGROUP_ID=$(id
-g)\nexport GROUP_ID\n\n# get copies of the passwd and group files\n[[ -f
\"${NSS_WRAPPER_PASSWD}\" ]] || cp \"/etc/passwd\" \"${NSS_WRAPPER_PASSWD}\"\n[[
-f \"${NSS_WRAPPER_GROUP}\" ]] || cp \"/etc/group\" \"${NSS_WRAPPER_GROUP}\"\n\n#
if the username is missing from the passwd file, then add it\nif [[ ! $(cat
\"${NSS_WRAPPER_PASSWD}\") =~ ${CRUNCHY_NSS_USERNAME}:x:${USER_ID} ]]; then\n
\ echo \"nss_wrapper: adding user\"\n passwd_tmp=\"${NSS_WRAPPER_DIR}/passwd_tmp\"\n
\ cp \"${NSS_WRAPPER_PASSWD}\" \"${passwd_tmp}\"\n sed -i \"/${CRUNCHY_NSS_USERNAME}:x:/d\"
\"${passwd_tmp}\"\n # needed for OCP 4.x because crio updates /etc/passwd
with an entry for USER_ID\n sed -i \"/${USER_ID}:x:/d\" \"${passwd_tmp}\"\n
\ printf '${CRUNCHY_NSS_USERNAME}:x:${USER_ID}:${GROUP_ID}:${CRUNCHY_NSS_USER_DESC}:${HOME}:/bin/bash\\n'
>> \"${passwd_tmp}\"\n envsubst < \"${passwd_tmp}\" > \"${NSS_WRAPPER_PASSWD}\"\n
\ rm \"${passwd_tmp}\"\nelse\n echo \"nss_wrapper: user exists\"\nfi\n\n#
if the username (which will be the same as the group name) is missing from
group file, then add it\nif [[ ! $(cat \"${NSS_WRAPPER_GROUP}\") =~ ${CRUNCHY_NSS_USERNAME}:x:${USER_ID}
]]; then\n echo \"nss_wrapper: adding group\"\n group_tmp=\"${NSS_WRAPPER_DIR}/group_tmp\"\n
\ cp \"${NSS_WRAPPER_GROUP}\" \"${group_tmp}\"\n sed -i \"/${CRUNCHY_NSS_USERNAME}:x:/d\"
\"${group_tmp}\"\n printf '${CRUNCHY_NSS_USERNAME}:x:${USER_ID}:${CRUNCHY_NSS_USERNAME}\\n'
>> \"${group_tmp}\"\n envsubst < \"${group_tmp}\" > \"${NSS_WRAPPER_GROUP}\"\n
\ rm \"${group_tmp}\"\nelse\n echo \"nss_wrapper: group exists\"\nfi\n\n#
export the nss_wrapper env vars\n# define nss_wrapper directory and passwd
& group files that will be utilized by nss_wrapper\nNSS_WRAPPER_DIR=\"/tmp/nss_wrapper/${NSS_WRAPPER_SUBDIR}\"\nNSS_WRAPPER_PASSWD=\"${NSS_WRAPPER_DIR}/passwd\"\nNSS_WRAPPER_GROUP=\"${NSS_WRAPPER_DIR}/group\"\n\nexport
LD_PRELOAD=/usr/lib64/libnss_wrapper.so\nexport NSS_WRAPPER_PASSWD=\"${NSS_WRAPPER_PASSWD}\"\nexport
NSS_WRAPPER_GROUP=\"${NSS_WRAPPER_GROUP}\"\n\necho \"nss_wrapper: environment
configured\"\n"
image: percona/percona-postgresql-operator:2.4.0-ppg16.3-postgres
imagePullPolicy: Always
name: nss-wrapper-init
resources:
limits:
cpu: "2"
memory: 4Gi
requests:
cpu: "2"
memory: 4Gi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
readOnlyRootFilesystem: true
runAsNonRoot: true
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /tmp
name: tmp
restartPolicy: Always
schedulerName: default-scheduler
securityContext:
fsGroup: 1001
runAsGroup: 1001
runAsNonRoot: true
runAsUser: 1001
serviceAccount: percona-db-pg-db-instance
serviceAccountName: percona-db-pg-db-instance
shareProcessNamespace: true
terminationGracePeriodSeconds: 30
topologySpreadConstraints:
- labelSelector:
matchExpressions:
- key: postgres-operator.crunchydata.com/data
operator: In
values:
- postgres
- pgbackrest
matchLabels:
postgres-operator.crunchydata.com/cluster: percona-db-pg-db
maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
- labelSelector:
matchExpressions:
- key: postgres-operator.crunchydata.com/data
operator: In
values:
- postgres
- pgbackrest
matchLabels:
postgres-operator.crunchydata.com/cluster: percona-db-pg-db
maxSkew: 1
topologyKey: topology.kubernetes.io/zone
whenUnsatisfiable: ScheduleAnyway
volumes:
- name: cert-volume
projected:
defaultMode: 384
sources:
- secret:
items:
- key: tls.crt
path: tls.crt
- key: tls.key
path: tls.key
- key: ca.crt
path: ca.crt
name: percona-db-pg-db-cluster-cert
- secret:
items:
- key: tls.crt
path: replication/tls.crt
- key: tls.key
path: replication/tls.key
- key: ca.crt
path: replication/ca.crt
name: percona-db-pg-db-replication-cert
- name: postgres-data
persistentVolumeClaim:
claimName: percona-db-pg-db-instance1-mrsr-pgdata
- downwardAPI:
defaultMode: 420
items:
- path: cpu_limit
resourceFieldRef:
containerName: database
divisor: 1m
resource: limits.cpu
- path: cpu_request
resourceFieldRef:
containerName: database
divisor: 1m
resource: requests.cpu
- path: mem_limit
resourceFieldRef:
containerName: database
divisor: 1Mi
resource: limits.memory
- path: mem_request
resourceFieldRef:
containerName: database
divisor: 1Mi
resource: requests.memory
- fieldRef:
apiVersion: v1
fieldPath: metadata.labels
path: labels
- fieldRef:
apiVersion: v1
fieldPath: metadata.annotations
path: annotations
name: database-containerinfo
- name: pgbackrest-server
projected:
defaultMode: 420
sources:
- secret:
items:
- key: pgbackrest-server.crt
path: server-tls.crt
- key: pgbackrest-server.key
mode: 384
path: server-tls.key
name: percona-db-pg-db-instance1-mrsr-certs
- name: pgbackrest-config
projected:
defaultMode: 420
sources:
- configMap:
items:
- key: pgbackrest_instance.conf
path: pgbackrest_instance.conf
- key: config-hash
path: config-hash
- key: pgbackrest-server.conf
path: ~postgres-operator_server.conf
name: percona-db-pg-db-pgbackrest-config
- secret:
items:
- key: pgbackrest.ca-roots
path: ~postgres-operator/tls-ca.crt
- key: pgbackrest-client.crt
path: ~postgres-operator/client-tls.crt
- key: pgbackrest-client.key
mode: 384
path: ~postgres-operator/client-tls.key
name: percona-db-pg-db-pgbackrest
optional: true
- name: patroni-config
projected:
defaultMode: 420
sources:
- configMap:
items:
- key: patroni.yaml
path: ~postgres-operator_cluster.yaml
name: percona-db-pg-db-config
- configMap:
items:
- key: patroni.yaml
path: ~postgres-operator_instance.yaml
name: percona-db-pg-db-instance1-mrsr-config
- secret:
items:
- key: patroni.ca-roots
path: ~postgres-operator/patroni.ca-roots
- key: patroni.crt-combined
path: ~postgres-operator/patroni.crt+key
name: percona-db-pg-db-instance1-mrsr-certs
- emptyDir:
sizeLimit: 2Gi
name: tmp
- emptyDir:
medium: Memory
name: dshm
updateStrategy:
type: OnDelete
status:
availableReplicas: 0
collisionCount: 0
currentReplicas: 1
currentRevision: percona-db-pg-db-instance1-mrsr-c798b7464
observedGeneration: 28747
replicas: 1
updateRevision: percona-db-pg-db-instance1-mrsr-c798b7464
updatedReplicas: 1