diff --git a/api/v1alpha1/perconaservermysql_types.go b/api/v1alpha1/perconaservermysql_types.go index 564fd8d35..7361d6a8a 100644 --- a/api/v1alpha1/perconaservermysql_types.go +++ b/api/v1alpha1/perconaservermysql_types.go @@ -119,6 +119,8 @@ type MySQLSpec struct { SidecarVolumes []corev1.Volume `json:"sidecarVolumes,omitempty"` SidecarPVCs []SidecarPVC `json:"sidecarPVCs,omitempty"` + VaultSecretName string `json:"vaultSecretName,omitempty"` + PodSpec `json:",inline"` } @@ -572,6 +574,16 @@ func (cr *PerconaServerMySQL) SetVersion() { cr.Spec.CRVersion = version.Version() } +func (cr *PerconaServerMySQL) Version() *v.Version { + return v.Must(v.NewVersion(cr.Spec.CRVersion)) +} + +// CompareVersion compares given version to current version. +// Returns -1, 0, or 1 if given version is smaller, equal, or larger than the current version, respectively. +func (cr *PerconaServerMySQL) CompareVersion(ver string) int { + return cr.Version().Compare(v.Must(v.NewVersion(ver))) +} + // CheckNSetDefaults validates and sets default values for the PerconaServerMySQL custom resource. func (cr *PerconaServerMySQL) CheckNSetDefaults(_ context.Context, serverVersion *platform.ServerVersion) error { if len(cr.Spec.MySQL.ClusterType) == 0 { @@ -862,6 +874,10 @@ func (cr *PerconaServerMySQL) CheckNSetDefaults(_ context.Context, serverVersion cr.Spec.SSLSecretName = cr.Name + "-ssl" } + if cr.Spec.MySQL.VaultSecretName == "" { + cr.Spec.MySQL.VaultSecretName = cr.Name + "-vault" + } + return nil } diff --git a/build/ps-entrypoint.sh b/build/ps-entrypoint.sh index c609d8135..525e12b0e 100755 --- a/build/ps-entrypoint.sh +++ b/build/ps-entrypoint.sh @@ -167,6 +167,26 @@ create_default_cnf() { sed -i "/\[mysqld\]/a ssl_key=${TLS_DIR}/tls.key" $CFG fi + # if vault secret file exists we assume we need to turn on encryption + vault_secret="/etc/mysql/vault-keyring-secret/keyring_vault.conf" + if [[ -f "${vault_secret}" ]]; then + sed -i "/\[mysqld\]/a early-plugin-load=keyring_vault.so" $CFG + sed -i "/\[mysqld\]/a keyring_vault_config=${vault_secret}" $CFG + + if [[ ${MYSQL_VERSION} =~ ^(8\.0|8\.4)$ ]]; then + sed -i "/\[mysqld\]/a default_table_encryption=ON" $CFG + sed -i "/\[mysqld\]/a table_encryption_privilege_check=ON" $CFG + sed -i "/\[mysqld\]/a innodb_undo_log_encrypt=ON" $CFG + sed -i "/\[mysqld\]/a innodb_redo_log_encrypt=ON" $CFG + sed -i "/\[mysqld\]/a binlog_encryption=ON" $CFG + sed -i "/\[mysqld\]/a binlog_rotate_encryption_master_key_at_startup=ON" $CFG + sed -i "/\[mysqld\]/a innodb_temp_tablespace_encrypt=ON" $CFG + sed -i "/\[mysqld\]/a innodb_parallel_dblwr_encrypt=ON" $CFG + sed -i "/\[mysqld\]/a innodb_encrypt_online_alter_logs=ON" $CFG + sed -i "/\[mysqld\]/a encrypt_tmp_files=ON" $CFG + fi + fi + for f in "${CUSTOM_CONFIG_FILES[@]}"; do echo "${f}" if [ -f "${f}" ]; then diff --git a/build/run-restore.sh b/build/run-restore.sh index 94f126876..1b70dae9d 100755 --- a/build/run-restore.sh +++ b/build/run-restore.sh @@ -41,7 +41,13 @@ main() { "azure") run_azure | extract "${tmpdir}" ;; esac - xtrabackup --prepare --rollback-prepared-trx --target-dir="${tmpdir}" + local keyring="" + if [[ -f ${KEYRING_VAULT_PATH} ]]; then + echo "Using keyring vault config: ${KEYRING_VAULT_PATH}" + keyring="--keyring-vault-config=${KEYRING_VAULT_PATH}" + fi + + xtrabackup --prepare --rollback-prepared-trx --target-dir="${tmpdir}" ${keyring} xtrabackup --datadir="${DATADIR}" --move-back --force-non-empty-directories --target-dir="${tmpdir}" rm -rf "${tmpdir}" diff --git a/config/crd/bases/ps.percona.com_perconaservermysqls.yaml b/config/crd/bases/ps.percona.com_perconaservermysqls.yaml index 3b592d8b1..35d0abf1b 100644 --- a/config/crd/bases/ps.percona.com_perconaservermysqls.yaml +++ b/config/crd/bases/ps.percona.com_perconaservermysqls.yaml @@ -5001,6 +5001,8 @@ spec: - whenUnsatisfiable type: object type: array + vaultSecretName: + type: string volumeSpec: properties: emptyDir: diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml index 6d5520cbd..56241657a 100644 --- a/deploy/bundle.yaml +++ b/deploy/bundle.yaml @@ -6924,6 +6924,8 @@ spec: - whenUnsatisfiable type: object type: array + vaultSecretName: + type: string volumeSpec: properties: emptyDir: diff --git a/deploy/cr.yaml b/deploy/cr.yaml index e31f0ba9d..465a35386 100644 --- a/deploy/cr.yaml +++ b/deploy/cr.yaml @@ -42,6 +42,7 @@ spec: image: perconalab/percona-server-mysql-operator:main-psmysql imagePullPolicy: Always # initImage: perconalab/percona-server-mysql-operator:main +# vaultSecretName: cluster1-vault size: 3 # env: diff --git a/deploy/crd.yaml b/deploy/crd.yaml index 26afa1edb..d0d294e2d 100644 --- a/deploy/crd.yaml +++ b/deploy/crd.yaml @@ -6924,6 +6924,8 @@ spec: - whenUnsatisfiable type: object type: array + vaultSecretName: + type: string volumeSpec: properties: emptyDir: diff --git a/deploy/cw-bundle.yaml b/deploy/cw-bundle.yaml index d3d73a029..86d37eb67 100644 --- a/deploy/cw-bundle.yaml +++ b/deploy/cw-bundle.yaml @@ -6924,6 +6924,8 @@ spec: - whenUnsatisfiable type: object type: array + vaultSecretName: + type: string volumeSpec: properties: emptyDir: diff --git a/deploy/vault-secret.yaml b/deploy/vault-secret.yaml new file mode 100644 index 000000000..7a23da0a2 --- /dev/null +++ b/deploy/vault-secret.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: cluster1-vault +type: Opaque +stringData: + keyring_vault.conf: |- + token = + vault_url = http://vault-service.vault-service.svc.cluster.local:8200 + secret_mount_point = secret diff --git a/e2e-tests/conf/vault-secret.yaml b/e2e-tests/conf/vault-secret.yaml new file mode 100644 index 000000000..71032f6d2 --- /dev/null +++ b/e2e-tests/conf/vault-secret.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Secret +metadata: + name: vault-keyring-secret +type: Opaque +stringData: + keyring_vault.conf: |- + token = #token + vault_url = #vault_url + secret_mount_point = #secret + #vault_ca = /etc/mysql/vault-keyring-secret/ca.cert + ca.cert: |- + #certVal diff --git a/e2e-tests/functions b/e2e-tests/functions index 2f13a9d07..20a3cdf88 100755 --- a/e2e-tests/functions +++ b/e2e-tests/functions @@ -100,12 +100,16 @@ deploy_tls_cluster_secrets() { } deploy_client() { - yq eval "$(printf '.spec.containers[0].image="%s"' "${IMAGE_MYSQL}")" "${TESTS_CONFIG_DIR}/client.yaml" | \ - kubectl -n "${NAMESPACE}" apply -f - + yq eval "$(printf '.spec.containers[0].image="%s"' "${IMAGE_MYSQL}")" "${TESTS_CONFIG_DIR}/client.yaml" \ + | kubectl -n "${NAMESPACE}" apply -f - } -apply_s3_storage_secrets() { +apply_minio_secret() { kubectl -n "${NAMESPACE}" apply -f "${TESTS_CONFIG_DIR}/minio-secret.yml" +} + +apply_s3_storage_secrets() { + apply_minio_secret kubectl -n "${NAMESPACE}" apply -f "${TESTS_CONFIG_DIR}/cloud-secret.yml" } @@ -310,6 +314,217 @@ deploy_minio() { /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing" } +prepare_vault_tls() { + local name=$1 + local namespace=$2 + local csr_name=vault-csr-${RANDOM} + local csr_api_ver="v1" + local csr_signer + + if [[ ${platform} == "eks" ]]; then + csr_signer=" signerName: beta.eks.amazonaws.com/app-serving" + else + csr_signer=" signerName: kubernetes.io/kubelet-serving" + fi + + openssl genrsa -out ${tmp_dir}/vault.key 2048 + cat <${tmp_dir}/csr.conf +[req] +req_extensions = v3_req +distinguished_name = req_distinguished_name +[req_distinguished_name] +[ v3_req ] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +extendedKeyUsage = serverAuth +subjectAltName = @alt_names +[alt_names] +DNS.1 = ${name} +DNS.2 = ${name}.${namespace} +DNS.3 = ${name}.${namespace}.svc +DNS.4 = ${name}.${namespace}.svc.cluster.local +IP.1 = 127.0.0.1 +EOF + + openssl req -new \ + -key ${tmp_dir}/vault.key \ + -subj "/CN=system:node:${name}.${namespace}.svc;/O=system:nodes" \ + -out ${tmp_dir}/server.csr \ + -config ${tmp_dir}/csr.conf + + cat <${tmp_dir}/csr.yaml +apiVersion: certificates.k8s.io/${csr_api_ver} +kind: CertificateSigningRequest +metadata: + name: ${csr_name} +spec: + groups: + - system:authenticated + request: $(cat ${tmp_dir}/server.csr | base64 | tr -d '\n') +${csr_signer} + usages: + - digital signature + - key encipherment + - server auth +EOF + + kubectl create -n ${namespace} -f ${tmp_dir}/csr.yaml + sleep 10 + kubectl certificate approve ${csr_name} + kubectl get csr ${csr_name} -o jsonpath='{.status.certificate}' >${tmp_dir}/serverCert + openssl base64 -in ${tmp_dir}/serverCert -d -A -out ${tmp_dir}/vault.crt + kubectl config view --raw --minify --flatten -o jsonpath='{.clusters[].cluster.certificate-authority-data}' | base64 -d >${tmp_dir}/vault.ca + if [[ ${platform} == "openshift" ]]; then + if [[ "x$(kubectl get namespaces | awk '{print $1}' | grep openshift-kube-controller-manager-operator)" != "x" ]]; then + #Detecting openshift 4+ + kubectl -n openshift-kube-controller-manager-operator get secret csr-signer -o jsonpath='{.data.tls\.crt}' \ + | base64 -d >${tmp_dir}/vault.ca + else + ca_secret_name=$(kubectl -n default get secrets \ + | grep default \ + | grep service-account-token \ + | head -n 1 \ + | awk {'print $1'}) + kubectl -n default get secret ${ca_secret_name} -o jsonpath='{.data.ca\.crt}' \ + | base64 -d >${tmp_dir}/vault.ca + fi + fi + + kubectl create secret generic ${name} \ + --namespace ${namespace} \ + --from-file=vault.key=${tmp_dir}/vault.key \ + --from-file=vault.crt=${tmp_dir}/vault.crt \ + --from-file=vault.ca=${tmp_dir}/vault.ca + +} + +_deploy_vault() { + local name=$1 + local protocol=$2 + local platform=$3 + local namespace=$4 + local tmp_dir=$5 + + if kubectl get ns | grep ${namespace}; then + echo "${namespace} is already exist, assuming Vault is running there." + return + fi + + rm -rf ${tmp_dir} + mkdir -p ${tmp_dir} + + helm repo add hashicorp https://helm.releases.hashicorp.com + helm repo update + + kubectl create namespace "${namespace}" + helm uninstall "$name" || : + + echo "install Vault $name" + + if [[ ${platform} == "openshift" ]]; then + oc patch clusterrole system:auth-delegator --type='json' -p '[{"op":"add","path":"/rules/-", "value":{"apiGroups":["security.openshift.io"], "attributeRestrictions":null, "resourceNames": ["privileged"], "resources":["securitycontextconstraints"],"verbs":["use"]}}]' + fi + + if [ $protocol == "https" ]; then + prepare_vault_tls ${name} ${namespace} + helm install $name hashicorp/vault \ + --disable-openapi-validation \ + --version ${VAULT_VER} \ + --namespace "${namespace}" \ + --set dataStorage.enabled=false \ + --set global.tlsDisable=false \ + --set global.logLevel="trace" \ + --set global.platform="${platform}" \ + --set server.extraVolumes[0].type=secret \ + --set server.extraVolumes[0].name=$name \ + --set server.extraEnvironmentVars.VAULT_CACERT=/vault/userconfig/$name/vault.ca \ + --set server.standalone.config=" +listener \"tcp\" { + address = \"[::]:8200\" + cluster_address = \"[::]:8201\" + tls_cert_file = \"/vault/userconfig/$name/vault.crt\" + tls_key_file = \"/vault/userconfig/$name/vault.key\" + tls_client_ca_file = \"/vault/userconfig/$name/vault.ca\" +} + +storage \"file\" { + path = \"/vault/data\" +}" + else + helm install $name hashicorp/vault \ + --disable-openapi-validation \ + --version ${VAULT_VER} \ + --namespace "${namespace}" \ + --set dataStorage.enabled=false \ + --set global.logLevel="trace" \ + --set global.platform="${platform}" + fi + + if [[ ${platform} == "openshift" ]]; then + oc patch clusterrole $name-agent-injector-clusterrole --type='json' -p '[{"op":"add","path":"/rules/-", "value":{"apiGroups":["security.openshift.io"], "attributeRestrictions":null, "resourceNames": ["privileged"], "resources":["securitycontextconstraints"],"verbs":["use"]}}]' + oc adm policy add-scc-to-user privileged $name-agent-injector + fi + + set +o xtrace + retry=0 + echo -n "waiting for pod/$name-0 to be ready" + until kubectl get -n ${namespace} pod/$name-0 -o 'jsonpath={.status.containerStatuses[0].state}' 2>/dev/null | grep 'running'; do + echo -n . + sleep 1 + let retry+=1 + if [[ $retry -ge 480 ]]; then + kubectl -n ${namespace} describe pod/$name-0 + kubectl -n ${namespace} logs $name-0 + echo max retry count "$retry" reached. something went wrong with vault + exit 1 + fi + done + set -o xtrace + + kubectl -n ${namespace} exec -it $name-0 -- vault operator init -tls-skip-verify -key-shares=1 -key-threshold=1 -format=json >"$tmp_dir/$name" + local unsealKey=$(jq -r ".unseal_keys_b64[]" <"$tmp_dir/$name") + local token=$(jq -r ".root_token" <"$tmp_dir/$name") + sleep 10 + + kubectl -n ${namespace} exec -it $name-0 -- vault operator unseal -tls-skip-verify "$unsealKey" + kubectl -n ${namespace} exec -it $name-0 -- \ + sh -c "export VAULT_TOKEN=$token && export VAULT_LOG_LEVEL=trace \ + && vault secrets enable --version=1 -tls-skip-verify -path=secret kv \ + && vault audit enable file file_path=/vault/vault-audit.log" + sleep 10 +} + +deploy_vault() { + local name=${1:-vault-service} + local protocol=${2:-http} + local platform=${3:-kubernetes} + local namespace=${4:-${name}} + + local tmp_dir=/tmp/vault + echo "Using tmp dir: ${tmp_dir}" + + _deploy_vault ${name} ${protocol} ${platform} ${namespace} ${tmp_dir} + + local unsealKey=$(jq -r ".unseal_keys_b64[]" <"$tmp_dir/$name") + local token=$(jq -r ".root_token" <"$tmp_dir/$name") + + cat ${ROOT_REPO}/e2e-tests/conf/vault-secret.yaml \ + | sed -e "s/#token/$token/" \ + | sed -e "s/#vault_url/$protocol:\/\/$name.$name.svc.cluster.local:8200/" \ + | sed -e "s/#secret/secret/" >"${tmp_dir}/vault-secret.yaml" + + if [ $protocol == "https" ]; then + sed -e 's/^/ /' ${tmp_dir}/vault.ca >${tmp_dir}/vault.new.ca + sed -i "s/#vault_ca/vault_ca/" "${tmp_dir}/vault-secret.yaml" + sed -i "/#certVal/r ${tmp_dir}/vault.new.ca" "${tmp_dir}/vault-secret.yaml" + sed -i "/#certVal/d" "${tmp_dir}/vault-secret.yaml" + else + sed -i "/#vault_ca/d" "${tmp_dir}/vault-secret.yaml" + fi + + kubectl apply -n "${NAMESPACE}" -f ${tmp_dir}/vault-secret.yaml +} + retry() { local max=$1 local delay=$2 @@ -418,8 +633,7 @@ get_innodb_cluster_name() { } get_mysqlsh_uri_for_pod() { - local pod=$1 - + local pod=$1 echo "root:root_password@${pod}.$(get_cluster_name)-mysql.${NAMESPACE}" } @@ -814,7 +1028,12 @@ deploy_chaos_mesh() { if [[ -n $OPENSHIFT ]]; then oc adm policy add-scc-to-user privileged -z chaos-daemon --namespace=${NAMESPACE} fi - sleep 10 + + echo "Waiting for chaos-mesh DaemonSet to be ready..." + until [ "$(kubectl get daemonset chaos-daemon -n ${NAMESPACE} -o jsonpath='{.status.numberReady}')" = "$(kubectl get daemonset chaos-daemon -n ${NAMESPACE} -o jsonpath='{.status.desiredNumberScheduled}')" ]; do + echo "Waiting for DaemonSet chaos-daemon..." + sleep 5 + done } destroy_chaos_mesh() { @@ -892,120 +1111,120 @@ network_loss() { } wait_until_chaos_applied() { - local chaos_type=$1 - local chaos_name=$2 - - local resource - case ${chaos_type} in - "kill"|"failure"|"full-cluster-crash") - resource=podchaos/${chaos_name} - ;; - "network") - resource=networkchaos/${chaos_name} - ;; - esac - - local retry=0 - until [[ ${retry} == 30 ]]; do - sleep 10 - retry=$((retry + 1)) - - succeeded=$(kubectl -n ${NAMESPACE} get ${resource} -o yaml \ - | yq '.status.experiment.containerRecords[].events[] + local chaos_type=$1 + local chaos_name=$2 + + local resource + case ${chaos_type} in + "kill" | "failure" | "full-cluster-crash") + resource=podchaos/${chaos_name} + ;; + "network") + resource=networkchaos/${chaos_name} + ;; + esac + + local retry=0 + until [[ ${retry} == 30 ]]; do + sleep 10 + retry=$((retry + 1)) + + succeeded=$(kubectl -n ${NAMESPACE} get ${resource} -o yaml \ + | yq '.status.experiment.containerRecords[].events[] | select(.operation == "Apply" and .type == "Succeeded")') - if [[ -n ${succeeded} ]]; then - return - fi - done + if [[ -n ${succeeded} ]]; then + return + fi + done - echo "Timeout (300s) exceeded while waiting for chaos to be applied" - exit 1 + echo "Timeout (300s) exceeded while waiting for chaos to be applied" + exit 1 } wait_until_chaos_recovered() { - local chaos_type=$1 - local chaos_name=$2 - - local resource - case ${chaos_type} in - "kill"|"failure") - resource=podchaos/${chaos_name} - ;; - "network") - resource=networkchaos/${chaos_name} - ;; - esac - - local retry=0 - until [[ ${retry} == 30 ]]; do - sleep 10 - retry=$((retry + 1)) - - succeeded=$(kubectl -n ${NAMESPACE} get ${resource} -o yaml \ - | yq '.status.experiment.containerRecords[].events[] + local chaos_type=$1 + local chaos_name=$2 + + local resource + case ${chaos_type} in + "kill" | "failure") + resource=podchaos/${chaos_name} + ;; + "network") + resource=networkchaos/${chaos_name} + ;; + esac + + local retry=0 + until [[ ${retry} == 30 ]]; do + sleep 10 + retry=$((retry + 1)) + + succeeded=$(kubectl -n ${NAMESPACE} get ${resource} -o yaml \ + | yq '.status.experiment.containerRecords[].events[] | select(.operation == "Recover" and .type == "Succeeded")') - if [[ -n ${succeeded} ]]; then - return - fi - done + if [[ -n ${succeeded} ]]; then + return + fi + done - echo "Timeout (300s) exceeded while waiting for chaos to be recovered" - exit 1 + echo "Timeout (300s) exceeded while waiting for chaos to be recovered" + exit 1 } check_primary_chaos() { - local chaos_type=$1 - local ns=$2 - local primary_before_failure=$3 - - local chaos_name - case ${chaos_type} in - "kill") - chaos_name="chaos-pod-kill-primary" - kill_pods "${ns}" "pod" "${primary_before_failure}" "" "${chaos_name}" - ;; - "full-cluster-crash") - chaos_name="chaos-kill-label-cluster-crash" - kill_pods "${ns}" "label" "app.kubernetes.io/instance" "gr-self-healing" "${chaos_name}" - ;; - "failure") - chaos_name="chaos-pod-failure-primary" - failure_pod "${ns}" "${primary_before_failure}" "${chaos_name}" - ;; - "network") - chaos_name="chaos-pod-network-loss-primary" - network_loss "${ns}" "${primary_before_failure}" "${chaos_name}" - ;; - esac - - wait_until_chaos_applied ${chaos_type} ${chaos_name} - if [[ ${chaos_type} == "failure" || ${chaos_type} == "network" ]]; then - wait_until_chaos_recovered ${chaos_type} ${chaos_name} - fi - - wait_cluster_consistency_gr "$(get_cluster_name)" 3 3 - - primary_after_failure=$(get_primary_from_group_replication) - uri=$(get_mysqlsh_uri_for_pod ${primary_after_failure}) - wait_until_innodb_ok ${uri} - - if [[ "${primary_before_failure}" == "${primary_after_failure}" ]]; then - echo "primary pod was not killed! something went wrong." - exit 1 - fi - - uri=$(get_mysqlsh_uri_for_pod $(get_primary_from_group_replication)) - online_members=$(get_innodb_cluster_status ${uri} \ - | jq .defaultReplicaSet.topology[].status \ - | grep ONLINE \ - | wc -l) - - if [[ ${online_members} != 3 ]]; then - echo "expected 3 online members, got ${online_members}" - exit 1 - fi + local chaos_type=$1 + local ns=$2 + local primary_before_failure=$3 + + local chaos_name + case ${chaos_type} in + "kill") + chaos_name="chaos-pod-kill-primary" + kill_pods "${ns}" "pod" "${primary_before_failure}" "" "${chaos_name}" + ;; + "full-cluster-crash") + chaos_name="chaos-kill-label-cluster-crash" + kill_pods "${ns}" "label" "app.kubernetes.io/instance" "gr-self-healing" "${chaos_name}" + ;; + "failure") + chaos_name="chaos-pod-failure-primary" + failure_pod "${ns}" "${primary_before_failure}" "${chaos_name}" + ;; + "network") + chaos_name="chaos-pod-network-loss-primary" + network_loss "${ns}" "${primary_before_failure}" "${chaos_name}" + ;; + esac + + wait_until_chaos_applied ${chaos_type} ${chaos_name} + if [[ ${chaos_type} == "failure" || ${chaos_type} == "network" ]]; then + wait_until_chaos_recovered ${chaos_type} ${chaos_name} + fi + + wait_cluster_consistency_gr "$(get_cluster_name)" 3 3 + + primary_after_failure=$(get_primary_from_group_replication) + uri=$(get_mysqlsh_uri_for_pod ${primary_after_failure}) + wait_until_innodb_ok ${uri} + + if [[ ${primary_before_failure} == "${primary_after_failure}" ]]; then + echo "primary pod was not killed! something went wrong." + exit 1 + fi + + uri=$(get_mysqlsh_uri_for_pod $(get_primary_from_group_replication)) + online_members=$(get_innodb_cluster_status ${uri} \ + | jq .defaultReplicaSet.topology[].status \ + | grep ONLINE \ + | wc -l) + + if [[ ${online_members} != 3 ]]; then + echo "expected 3 online members, got ${online_members}" + exit 1 + fi } renew_certificate() { diff --git a/e2e-tests/run-pr.csv b/e2e-tests/run-pr.csv index 801e0d733..2e3263cab 100644 --- a/e2e-tests/run-pr.csv +++ b/e2e-tests/run-pr.csv @@ -4,6 +4,8 @@ auto-config config config-router demand-backup +async-data-at-rest-encryption +gr-data-at-rest-encryption gr-demand-backup gr-demand-backup-haproxy gr-finalizer diff --git a/e2e-tests/run-release.csv b/e2e-tests/run-release.csv index 801e0d733..2e3263cab 100644 --- a/e2e-tests/run-release.csv +++ b/e2e-tests/run-release.csv @@ -4,6 +4,8 @@ auto-config config config-router demand-backup +async-data-at-rest-encryption +gr-data-at-rest-encryption gr-demand-backup gr-demand-backup-haproxy gr-finalizer diff --git a/e2e-tests/tests/async-data-at-rest-encryption/00-assert.yaml b/e2e-tests/tests/async-data-at-rest-encryption/00-assert.yaml new file mode 100644 index 000000000..458108d98 --- /dev/null +++ b/e2e-tests/tests/async-data-at-rest-encryption/00-assert.yaml @@ -0,0 +1,25 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 120 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: percona-server-mysql-operator +status: + availableReplicas: 1 + observedGeneration: 1 + readyReplicas: 1 + replicas: 1 + updatedReplicas: 1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio-service +status: + availableReplicas: 1 + observedGeneration: 1 + readyReplicas: 1 + replicas: 1 + updatedReplicas: 1 diff --git a/e2e-tests/tests/async-data-at-rest-encryption/00-deploy-operator.yaml b/e2e-tests/tests/async-data-at-rest-encryption/00-deploy-operator.yaml new file mode 100644 index 000000000..364a1f51d --- /dev/null +++ b/e2e-tests/tests/async-data-at-rest-encryption/00-deploy-operator.yaml @@ -0,0 +1,19 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - timeout: 120 + script: |- + set -o errexit + set -o xtrace + + source ../../functions + init_temp_dir # do this only in the first TestStep + + deploy_client + deploy_operator + + deploy_non_tls_cluster_secrets + deploy_tls_cluster_secrets + + apply_minio_secret + deploy_minio diff --git a/e2e-tests/tests/async-data-at-rest-encryption/01-assert.yaml b/e2e-tests/tests/async-data-at-rest-encryption/01-assert.yaml new file mode 100644 index 000000000..4f641c581 --- /dev/null +++ b/e2e-tests/tests/async-data-at-rest-encryption/01-assert.yaml @@ -0,0 +1,9 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 60 +--- +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: vault-keyring-secret diff --git a/e2e-tests/tests/async-data-at-rest-encryption/01-deploy-vault.yaml b/e2e-tests/tests/async-data-at-rest-encryption/01-deploy-vault.yaml new file mode 100644 index 000000000..b1876a522 --- /dev/null +++ b/e2e-tests/tests/async-data-at-rest-encryption/01-deploy-vault.yaml @@ -0,0 +1,11 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - timeout: 180 + script: |- + set -o errexit + set -o xtrace + + source ../../functions + + deploy_vault diff --git a/e2e-tests/tests/async-data-at-rest-encryption/02-assert.yaml b/e2e-tests/tests/async-data-at-rest-encryption/02-assert.yaml new file mode 100644 index 000000000..46feb9f17 --- /dev/null +++ b/e2e-tests/tests/async-data-at-rest-encryption/02-assert.yaml @@ -0,0 +1,54 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 420 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: async-data-at-rest-encryption-mysql +status: + observedGeneration: 1 + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + updatedReplicas: 3 + collisionCount: 0 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: async-data-at-rest-encryption-haproxy +status: + observedGeneration: 1 + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + updatedReplicas: 3 + collisionCount: 0 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: async-data-at-rest-encryption-orc +status: + observedGeneration: 1 + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + updatedReplicas: 3 + collisionCount: 0 +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: async-data-at-rest-encryption +status: + mysql: + ready: 3 + size: 3 + state: ready + haproxy: + ready: 3 + size: 3 + state: ready + state: ready diff --git a/e2e-tests/tests/async-data-at-rest-encryption/02-create-cluster.yaml b/e2e-tests/tests/async-data-at-rest-encryption/02-create-cluster.yaml new file mode 100644 index 000000000..c71e6613d --- /dev/null +++ b/e2e-tests/tests/async-data-at-rest-encryption/02-create-cluster.yaml @@ -0,0 +1,22 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 10 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + get_cr \ + | yq eval '.spec.mysql.clusterType="async"' - \ + | yq eval '.spec.proxy.haproxy.enabled=true' - \ + | yq eval '.spec.mysql.size=3' - \ + | yq eval '.spec.mysql.affinity.antiAffinityTopologyKey="none"' - \ + | yq eval '.spec.mysql.vaultSecretName="vault-keyring-secret"' - \ + | yq eval '.spec.backup.storages.minio.type="s3"' - \ + | yq eval '.spec.backup.storages.minio.s3.bucket="operator-testing"' - \ + | yq eval '.spec.backup.storages.minio.s3.credentialsSecret="minio-secret"' - \ + | yq eval ".spec.backup.storages.minio.s3.endpointUrl=\"http://minio-service.${NAMESPACE}:9000\"" - \ + | yq eval '.spec.backup.storages.minio.s3.region="us-east-1"' - \ + | kubectl -n "${NAMESPACE}" apply -f - diff --git a/e2e-tests/tests/async-data-at-rest-encryption/03-write-data.yaml b/e2e-tests/tests/async-data-at-rest-encryption/03-write-data.yaml new file mode 100644 index 000000000..69aa4debc --- /dev/null +++ b/e2e-tests/tests/async-data-at-rest-encryption/03-write-data.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + run_mysql \ + "CREATE DATABASE IF NOT EXISTS myDB; CREATE TABLE IF NOT EXISTS myDB.myTable (id int PRIMARY KEY)" \ + "-h $(get_haproxy_svc $(get_cluster_name)) -uroot -proot_password" + + run_mysql \ + "INSERT myDB.myTable (id) VALUES (100500)" \ + "-h $(get_haproxy_svc $(get_cluster_name)) -uroot -proot_password" diff --git a/e2e-tests/tests/async-data-at-rest-encryption/04-assert.yaml b/e2e-tests/tests/async-data-at-rest-encryption/04-assert.yaml new file mode 100644 index 000000000..7033b2b75 --- /dev/null +++ b/e2e-tests/tests/async-data-at-rest-encryption/04-assert.yaml @@ -0,0 +1,10 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 60 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: 04-check-encryption +data: + myTable: ENCRYPTION='Y' diff --git a/e2e-tests/tests/async-data-at-rest-encryption/04-check-encryption.yaml b/e2e-tests/tests/async-data-at-rest-encryption/04-check-encryption.yaml new file mode 100644 index 000000000..5746737b2 --- /dev/null +++ b/e2e-tests/tests/async-data-at-rest-encryption/04-check-encryption.yaml @@ -0,0 +1,15 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + # Check if table is encrypted by examining create options in information schema + data=$(run_mysql \ + "SELECT CREATE_OPTIONS FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA='myDB' AND TABLE_NAME='myTable'" \ + "-h $(get_haproxy_svc $(get_cluster_name)) -uroot -proot_password") + + kubectl create configmap -n "${NAMESPACE}" 04-check-encryption --from-literal=myTable="${data}" diff --git a/e2e-tests/tests/async-data-at-rest-encryption/05-assert.yaml b/e2e-tests/tests/async-data-at-rest-encryption/05-assert.yaml new file mode 100644 index 000000000..5ae7abd9b --- /dev/null +++ b/e2e-tests/tests/async-data-at-rest-encryption/05-assert.yaml @@ -0,0 +1,13 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + name: backup1 +spec: + clusterName: async-data-at-rest-encryption + storageName: minio +status: + state: Succeeded diff --git a/e2e-tests/tests/async-data-at-rest-encryption/05-create-backup.yaml b/e2e-tests/tests/async-data-at-rest-encryption/05-create-backup.yaml new file mode 100644 index 000000000..61c25cf1b --- /dev/null +++ b/e2e-tests/tests/async-data-at-rest-encryption/05-create-backup.yaml @@ -0,0 +1,9 @@ +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + name: backup1 + finalizers: + - percona.com/delete-backup +spec: + clusterName: async-data-at-rest-encryption + storageName: minio diff --git a/e2e-tests/tests/async-data-at-rest-encryption/06-delete-data.yaml b/e2e-tests/tests/async-data-at-rest-encryption/06-delete-data.yaml new file mode 100644 index 000000000..bc5fc55eb --- /dev/null +++ b/e2e-tests/tests/async-data-at-rest-encryption/06-delete-data.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + run_mysql \ + "DROP TABLE myDB.myTable; DROP DATABASE myDB" \ + "-h $(get_haproxy_svc $(get_cluster_name)) -uroot -proot_password" diff --git a/e2e-tests/tests/async-data-at-rest-encryption/07-assert.yaml b/e2e-tests/tests/async-data-at-rest-encryption/07-assert.yaml new file mode 100644 index 000000000..17b45f99c --- /dev/null +++ b/e2e-tests/tests/async-data-at-rest-encryption/07-assert.yaml @@ -0,0 +1,61 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLRestore +metadata: + name: restore1 +spec: + backupName: backup1 + clusterName: async-data-at-rest-encryption +status: + state: Succeeded +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: async-data-at-rest-encryption-mysql +status: + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + updatedReplicas: 3 + collisionCount: 0 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: async-data-at-rest-encryption-haproxy +status: + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + updatedReplicas: 3 + collisionCount: 0 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: async-data-at-rest-encryption-orc +status: + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + updatedReplicas: 3 + collisionCount: 0 +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: async-data-at-rest-encryption +status: + mysql: + ready: 3 + size: 3 + state: ready + router: + ready: 3 + size: 3 + state: ready + state: ready diff --git a/e2e-tests/tests/async-data-at-rest-encryption/07-restore-data.yaml b/e2e-tests/tests/async-data-at-rest-encryption/07-restore-data.yaml new file mode 100644 index 000000000..1c2b705dd --- /dev/null +++ b/e2e-tests/tests/async-data-at-rest-encryption/07-restore-data.yaml @@ -0,0 +1,7 @@ +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLRestore +metadata: + name: restore1 +spec: + clusterName: async-data-at-rest-encryption + backupName: backup1 diff --git a/e2e-tests/tests/async-data-at-rest-encryption/08-assert.yaml b/e2e-tests/tests/async-data-at-rest-encryption/08-assert.yaml new file mode 100644 index 000000000..370498619 --- /dev/null +++ b/e2e-tests/tests/async-data-at-rest-encryption/08-assert.yaml @@ -0,0 +1,11 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: 08-read-data +data: + data: |- + 100500 diff --git a/e2e-tests/tests/async-data-at-rest-encryption/08-read-data.yaml b/e2e-tests/tests/async-data-at-rest-encryption/08-read-data.yaml new file mode 100644 index 000000000..86fd90060 --- /dev/null +++ b/e2e-tests/tests/async-data-at-rest-encryption/08-read-data.yaml @@ -0,0 +1,14 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + data=$(run_mysql \ + "SELECT * FROM myDB.myTable" \ + "-h $(get_haproxy_svc $(get_cluster_name)) -uroot -proot_password") + + kubectl create configmap -n "${NAMESPACE}" 08-read-data --from-literal=data="${data}" diff --git a/e2e-tests/tests/async-data-at-rest-encryption/09-assert.yaml b/e2e-tests/tests/async-data-at-rest-encryption/09-assert.yaml new file mode 100644 index 000000000..724af4dd5 --- /dev/null +++ b/e2e-tests/tests/async-data-at-rest-encryption/09-assert.yaml @@ -0,0 +1,28 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: async-data-at-rest-encryption-mysql +status: + replicas: 5 + readyReplicas: 5 + currentReplicas: 5 + collisionCount: 0 +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: async-data-at-rest-encryption +status: + mysql: + ready: 5 + size: 5 + state: ready + haproxy: + ready: 3 + size: 3 + state: ready + state: ready diff --git a/e2e-tests/tests/async-data-at-rest-encryption/09-scale-up.yaml b/e2e-tests/tests/async-data-at-rest-encryption/09-scale-up.yaml new file mode 100644 index 000000000..177a5016c --- /dev/null +++ b/e2e-tests/tests/async-data-at-rest-encryption/09-scale-up.yaml @@ -0,0 +1,7 @@ +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: async-data-at-rest-encryption +spec: + mysql: + size: 5 diff --git a/e2e-tests/tests/async-data-at-rest-encryption/10-assert.yaml b/e2e-tests/tests/async-data-at-rest-encryption/10-assert.yaml new file mode 100644 index 000000000..1b97b3346 --- /dev/null +++ b/e2e-tests/tests/async-data-at-rest-encryption/10-assert.yaml @@ -0,0 +1,28 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: async-data-at-rest-encryption-mysql +status: + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + collisionCount: 0 +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: async-data-at-rest-encryption +status: + mysql: + ready: 3 + size: 3 + state: ready + haproxy: + ready: 3 + size: 3 + state: ready + state: ready diff --git a/e2e-tests/tests/async-data-at-rest-encryption/10-scale-down.yaml b/e2e-tests/tests/async-data-at-rest-encryption/10-scale-down.yaml new file mode 100644 index 000000000..c69d76f28 --- /dev/null +++ b/e2e-tests/tests/async-data-at-rest-encryption/10-scale-down.yaml @@ -0,0 +1,7 @@ +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: async-data-at-rest-encryption +spec: + mysql: + size: 3 diff --git a/e2e-tests/tests/async-data-at-rest-encryption/98-drop-finalizer.yaml b/e2e-tests/tests/async-data-at-rest-encryption/98-drop-finalizer.yaml new file mode 100644 index 000000000..7048f591b --- /dev/null +++ b/e2e-tests/tests/async-data-at-rest-encryption/98-drop-finalizer.yaml @@ -0,0 +1,5 @@ +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: async-data-at-rest-encryption + finalizers: [] diff --git a/e2e-tests/tests/async-data-at-rest-encryption/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/async-data-at-rest-encryption/99-remove-cluster-gracefully.yaml new file mode 100644 index 000000000..62f8762d8 --- /dev/null +++ b/e2e-tests/tests/async-data-at-rest-encryption/99-remove-cluster-gracefully.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: ps.percona.com/v1alpha1 + kind: PerconaServerMySQL + metadata: + name: async-data-at-rest-encryption +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + destroy_operator + timeout: 60 diff --git a/e2e-tests/tests/gr-data-at-rest-encryption/00-assert.yaml b/e2e-tests/tests/gr-data-at-rest-encryption/00-assert.yaml new file mode 100644 index 000000000..458108d98 --- /dev/null +++ b/e2e-tests/tests/gr-data-at-rest-encryption/00-assert.yaml @@ -0,0 +1,25 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 120 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: percona-server-mysql-operator +status: + availableReplicas: 1 + observedGeneration: 1 + readyReplicas: 1 + replicas: 1 + updatedReplicas: 1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio-service +status: + availableReplicas: 1 + observedGeneration: 1 + readyReplicas: 1 + replicas: 1 + updatedReplicas: 1 diff --git a/e2e-tests/tests/gr-data-at-rest-encryption/00-deploy-operator.yaml b/e2e-tests/tests/gr-data-at-rest-encryption/00-deploy-operator.yaml new file mode 100644 index 000000000..364a1f51d --- /dev/null +++ b/e2e-tests/tests/gr-data-at-rest-encryption/00-deploy-operator.yaml @@ -0,0 +1,19 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - timeout: 120 + script: |- + set -o errexit + set -o xtrace + + source ../../functions + init_temp_dir # do this only in the first TestStep + + deploy_client + deploy_operator + + deploy_non_tls_cluster_secrets + deploy_tls_cluster_secrets + + apply_minio_secret + deploy_minio diff --git a/e2e-tests/tests/gr-data-at-rest-encryption/01-assert.yaml b/e2e-tests/tests/gr-data-at-rest-encryption/01-assert.yaml new file mode 100644 index 000000000..4f641c581 --- /dev/null +++ b/e2e-tests/tests/gr-data-at-rest-encryption/01-assert.yaml @@ -0,0 +1,9 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 60 +--- +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: vault-keyring-secret diff --git a/e2e-tests/tests/gr-data-at-rest-encryption/01-deploy-vault.yaml b/e2e-tests/tests/gr-data-at-rest-encryption/01-deploy-vault.yaml new file mode 100644 index 000000000..b1876a522 --- /dev/null +++ b/e2e-tests/tests/gr-data-at-rest-encryption/01-deploy-vault.yaml @@ -0,0 +1,11 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - timeout: 180 + script: |- + set -o errexit + set -o xtrace + + source ../../functions + + deploy_vault diff --git a/e2e-tests/tests/gr-data-at-rest-encryption/02-assert.yaml b/e2e-tests/tests/gr-data-at-rest-encryption/02-assert.yaml new file mode 100644 index 000000000..6ff41f3e0 --- /dev/null +++ b/e2e-tests/tests/gr-data-at-rest-encryption/02-assert.yaml @@ -0,0 +1,41 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 420 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: gr-data-at-rest-encryption-mysql +status: + observedGeneration: 1 + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + updatedReplicas: 3 + collisionCount: 0 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + generation: 1 + name: gr-data-at-rest-encryption-router +status: + observedGeneration: 1 + readyReplicas: 3 + replicas: 3 + updatedReplicas: 3 +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: gr-data-at-rest-encryption +status: + mysql: + ready: 3 + size: 3 + state: ready + router: + ready: 3 + size: 3 + state: ready + state: ready diff --git a/e2e-tests/tests/gr-data-at-rest-encryption/02-create-cluster.yaml b/e2e-tests/tests/gr-data-at-rest-encryption/02-create-cluster.yaml new file mode 100644 index 000000000..8afd7a06f --- /dev/null +++ b/e2e-tests/tests/gr-data-at-rest-encryption/02-create-cluster.yaml @@ -0,0 +1,23 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 10 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + get_cr \ + | yq eval '.spec.mysql.clusterType="group-replication"' - \ + | yq eval '.spec.proxy.router.enabled=true' - \ + | yq eval '.spec.proxy.haproxy.enabled=false' - \ + | yq eval '.spec.mysql.size=3' - \ + | yq eval '.spec.mysql.affinity.antiAffinityTopologyKey="none"' - \ + | yq eval '.spec.mysql.vaultSecretName="vault-keyring-secret"' - \ + | yq eval '.spec.backup.storages.minio.type="s3"' - \ + | yq eval '.spec.backup.storages.minio.s3.bucket="operator-testing"' - \ + | yq eval '.spec.backup.storages.minio.s3.credentialsSecret="minio-secret"' - \ + | yq eval ".spec.backup.storages.minio.s3.endpointUrl=\"http://minio-service.${NAMESPACE}:9000\"" - \ + | yq eval '.spec.backup.storages.minio.s3.region="us-east-1"' - \ + | kubectl -n "${NAMESPACE}" apply -f - diff --git a/e2e-tests/tests/gr-data-at-rest-encryption/03-write-data.yaml b/e2e-tests/tests/gr-data-at-rest-encryption/03-write-data.yaml new file mode 100644 index 000000000..259c21a73 --- /dev/null +++ b/e2e-tests/tests/gr-data-at-rest-encryption/03-write-data.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + run_mysql \ + "CREATE DATABASE IF NOT EXISTS myDB; CREATE TABLE IF NOT EXISTS myDB.myTable (id int PRIMARY KEY)" \ + "-h $(get_mysql_router_service $(get_cluster_name)) -P 6446 -uroot -proot_password" + + run_mysql \ + "INSERT myDB.myTable (id) VALUES (100500)" \ + "-h $(get_mysql_router_service $(get_cluster_name)) -P 6446 -uroot -proot_password" \ No newline at end of file diff --git a/e2e-tests/tests/gr-data-at-rest-encryption/04-assert.yaml b/e2e-tests/tests/gr-data-at-rest-encryption/04-assert.yaml new file mode 100644 index 000000000..7033b2b75 --- /dev/null +++ b/e2e-tests/tests/gr-data-at-rest-encryption/04-assert.yaml @@ -0,0 +1,10 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 60 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: 04-check-encryption +data: + myTable: ENCRYPTION='Y' diff --git a/e2e-tests/tests/gr-data-at-rest-encryption/04-check-encryption.yaml b/e2e-tests/tests/gr-data-at-rest-encryption/04-check-encryption.yaml new file mode 100644 index 000000000..5243561d4 --- /dev/null +++ b/e2e-tests/tests/gr-data-at-rest-encryption/04-check-encryption.yaml @@ -0,0 +1,15 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + # Check if table is encrypted by examining create options in information schema + data=$(run_mysql \ + "SELECT CREATE_OPTIONS FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA='myDB' AND TABLE_NAME='myTable'" \ + "-h $(get_mysql_router_service $(get_cluster_name)) -P 6446 -uroot -proot_password") + + kubectl create configmap -n "${NAMESPACE}" 04-check-encryption --from-literal=myTable="${data}" diff --git a/e2e-tests/tests/gr-data-at-rest-encryption/05-assert.yaml b/e2e-tests/tests/gr-data-at-rest-encryption/05-assert.yaml new file mode 100644 index 000000000..690f1c484 --- /dev/null +++ b/e2e-tests/tests/gr-data-at-rest-encryption/05-assert.yaml @@ -0,0 +1,13 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + name: backup1 +spec: + clusterName: gr-data-at-rest-encryption + storageName: minio +status: + state: Succeeded diff --git a/e2e-tests/tests/gr-data-at-rest-encryption/05-create-backup.yaml b/e2e-tests/tests/gr-data-at-rest-encryption/05-create-backup.yaml new file mode 100644 index 000000000..956645856 --- /dev/null +++ b/e2e-tests/tests/gr-data-at-rest-encryption/05-create-backup.yaml @@ -0,0 +1,9 @@ +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + name: backup1 + finalizers: + - percona.com/delete-backup +spec: + clusterName: gr-data-at-rest-encryption + storageName: minio diff --git a/e2e-tests/tests/gr-data-at-rest-encryption/06-delete-data.yaml b/e2e-tests/tests/gr-data-at-rest-encryption/06-delete-data.yaml new file mode 100644 index 000000000..b89bb6090 --- /dev/null +++ b/e2e-tests/tests/gr-data-at-rest-encryption/06-delete-data.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + run_mysql \ + "DROP TABLE myDB.myTable; DROP DATABASE myDB" \ + "-h $(get_mysql_router_service $(get_cluster_name)) -P 6446 -uroot -proot_password" \ No newline at end of file diff --git a/e2e-tests/tests/gr-data-at-rest-encryption/07-assert.yaml b/e2e-tests/tests/gr-data-at-rest-encryption/07-assert.yaml new file mode 100644 index 000000000..bd531f6c8 --- /dev/null +++ b/e2e-tests/tests/gr-data-at-rest-encryption/07-assert.yaml @@ -0,0 +1,48 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLRestore +metadata: + name: restore1 +spec: + backupName: backup1 + clusterName: gr-data-at-rest-encryption +status: + state: Succeeded +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: gr-data-at-rest-encryption-mysql +status: + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + updatedReplicas: 3 + collisionCount: 0 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gr-data-at-rest-encryption-router +status: + readyReplicas: 3 + replicas: 3 + updatedReplicas: 3 +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: gr-data-at-rest-encryption +status: + mysql: + ready: 3 + size: 3 + state: ready + router: + ready: 3 + size: 3 + state: ready + state: ready diff --git a/e2e-tests/tests/gr-data-at-rest-encryption/07-restore-data.yaml b/e2e-tests/tests/gr-data-at-rest-encryption/07-restore-data.yaml new file mode 100644 index 000000000..32f22579f --- /dev/null +++ b/e2e-tests/tests/gr-data-at-rest-encryption/07-restore-data.yaml @@ -0,0 +1,7 @@ +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLRestore +metadata: + name: restore1 +spec: + clusterName: gr-data-at-rest-encryption + backupName: backup1 diff --git a/e2e-tests/tests/gr-data-at-rest-encryption/08-assert.yaml b/e2e-tests/tests/gr-data-at-rest-encryption/08-assert.yaml new file mode 100644 index 000000000..6c5085714 --- /dev/null +++ b/e2e-tests/tests/gr-data-at-rest-encryption/08-assert.yaml @@ -0,0 +1,11 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 60 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: 08-read-data +data: + data: |- + 100500 diff --git a/e2e-tests/tests/gr-data-at-rest-encryption/08-read-data.yaml b/e2e-tests/tests/gr-data-at-rest-encryption/08-read-data.yaml new file mode 100644 index 000000000..8fcfd6d05 --- /dev/null +++ b/e2e-tests/tests/gr-data-at-rest-encryption/08-read-data.yaml @@ -0,0 +1,14 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + data=$(run_mysql \ + "SELECT * FROM myDB.myTable" \ + "-h $(get_mysql_router_service $(get_cluster_name)) -P 6446 -uroot -proot_password") + + kubectl create configmap -n "${NAMESPACE}" 08-read-data --from-literal=data="${data}" diff --git a/e2e-tests/tests/gr-data-at-rest-encryption/09-assert.yaml b/e2e-tests/tests/gr-data-at-rest-encryption/09-assert.yaml new file mode 100644 index 000000000..5bbec4a8c --- /dev/null +++ b/e2e-tests/tests/gr-data-at-rest-encryption/09-assert.yaml @@ -0,0 +1,28 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: gr-data-at-rest-encryption-mysql +status: + replicas: 5 + readyReplicas: 5 + currentReplicas: 5 + collisionCount: 0 +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: gr-data-at-rest-encryption +status: + mysql: + ready: 5 + size: 5 + state: ready + router: + ready: 3 + size: 3 + state: ready + state: ready diff --git a/e2e-tests/tests/gr-data-at-rest-encryption/09-scale-up.yaml b/e2e-tests/tests/gr-data-at-rest-encryption/09-scale-up.yaml new file mode 100644 index 000000000..8de40b574 --- /dev/null +++ b/e2e-tests/tests/gr-data-at-rest-encryption/09-scale-up.yaml @@ -0,0 +1,9 @@ +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: gr-data-at-rest-encryption + finalizers: + - percona.com/delete-mysql-pods-in-order +spec: + mysql: + size: 5 diff --git a/e2e-tests/tests/gr-data-at-rest-encryption/10-assert.yaml b/e2e-tests/tests/gr-data-at-rest-encryption/10-assert.yaml new file mode 100644 index 000000000..fc19c59e0 --- /dev/null +++ b/e2e-tests/tests/gr-data-at-rest-encryption/10-assert.yaml @@ -0,0 +1,28 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: gr-data-at-rest-encryption-mysql +status: + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + collisionCount: 0 +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: gr-data-at-rest-encryption +status: + mysql: + ready: 3 + size: 3 + state: ready + router: + ready: 3 + size: 3 + state: ready + state: ready diff --git a/e2e-tests/tests/gr-data-at-rest-encryption/10-scale-down.yaml b/e2e-tests/tests/gr-data-at-rest-encryption/10-scale-down.yaml new file mode 100644 index 000000000..8f2da5744 --- /dev/null +++ b/e2e-tests/tests/gr-data-at-rest-encryption/10-scale-down.yaml @@ -0,0 +1,9 @@ +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: gr-data-at-rest-encryption + finalizers: + - percona.com/delete-mysql-pods-in-order +spec: + mysql: + size: 3 diff --git a/e2e-tests/tests/gr-data-at-rest-encryption/98-drop-finalizer.yaml b/e2e-tests/tests/gr-data-at-rest-encryption/98-drop-finalizer.yaml new file mode 100644 index 000000000..077262be2 --- /dev/null +++ b/e2e-tests/tests/gr-data-at-rest-encryption/98-drop-finalizer.yaml @@ -0,0 +1,5 @@ +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: gr-data-at-rest-encryption + finalizers: [] diff --git a/e2e-tests/tests/gr-data-at-rest-encryption/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/gr-data-at-rest-encryption/99-remove-cluster-gracefully.yaml new file mode 100644 index 000000000..d6649eeaf --- /dev/null +++ b/e2e-tests/tests/gr-data-at-rest-encryption/99-remove-cluster-gracefully.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: ps.percona.com/v1alpha1 + kind: PerconaServerMySQL + metadata: + name: gr-data-at-rest-encryption +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + destroy_operator + timeout: 60 diff --git a/e2e-tests/vars.sh b/e2e-tests/vars.sh index 7818bceca..d6e0eec66 100644 --- a/e2e-tests/vars.sh +++ b/e2e-tests/vars.sh @@ -23,6 +23,7 @@ export IMAGE_PMM_SERVER=${IMAGE_PMM_SERVER:-"perconalab/pmm-server:3-dev-latest" export CERT_MANAGER_VER="1.17.2" export MINIO_VER="5.4.0" export CHAOS_MESH_VER="2.7.2" +export VAULT_VER="0.16.1" date=$(which gdate || which date) diff --git a/go.mod b/go.mod index fca8d66ab..81d74d22e 100644 --- a/go.mod +++ b/go.mod @@ -42,6 +42,7 @@ require ( github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/google/btree v1.1.3 // indirect + github.com/hashicorp/go-version v1.7.0 // indirect github.com/minio/crc64nvme v1.0.1 // indirect github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c // indirect github.com/tinylib/msgp v1.3.0 // indirect diff --git a/pkg/mysql/mysql.go b/pkg/mysql/mysql.go index b404d4724..fa940d583 100644 --- a/pkg/mysql/mysql.go +++ b/pkg/mysql/mysql.go @@ -8,6 +8,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" apiv1alpha1 "github.com/percona/percona-server-mysql-operator/api/v1alpha1" "github.com/percona/percona-server-mysql-operator/pkg/k8s" @@ -17,19 +18,21 @@ import ( ) const ( - ComponentName = "mysql" - DataVolumeName = "datadir" - DataMountPath = "/var/lib/mysql" - CustomConfigKey = "my.cnf" - configVolumeName = "config" - configMountPath = "/etc/mysql/config" - credsVolumeName = "users" - CredsMountPath = "/etc/mysql/mysql-users-secret" - mysqlshVolumeName = "mysqlsh" - mysqlshMountPath = "/.mysqlsh" - tlsVolumeName = "tls" - tlsMountPath = "/etc/mysql/mysql-tls-secret" - BackupLogDir = "/var/log/xtrabackup" + ComponentName = "mysql" + DataVolumeName = "datadir" + DataMountPath = "/var/lib/mysql" + CustomConfigKey = "my.cnf" + configVolumeName = "config" + configMountPath = "/etc/mysql/config" + credsVolumeName = "users" + CredsMountPath = "/etc/mysql/mysql-users-secret" + mysqlshVolumeName = "mysqlsh" + mysqlshMountPath = "/.mysqlsh" + tlsVolumeName = "tls" + tlsMountPath = "/etc/mysql/mysql-tls-secret" + BackupLogDir = "/var/log/xtrabackup" + vaultSecretVolumeName = "vault-keyring-secret" + vaultSecretMountPath = "/etc/mysql/vault-keyring-secret" ) const ( @@ -122,11 +125,119 @@ func MatchLabels(cr *apiv1alpha1.PerconaServerMySQL) map[string]string { cr.Labels()) } +func statefulSetVolumes(cr *apiv1alpha1.PerconaServerMySQL) []corev1.Volume { + spec := cr.MySQLSpec() + + volumes := []corev1.Volume{ + { + Name: apiv1alpha1.BinVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: mysqlshVolumeName, // In OpenShift, we should use emptyDir for ./mysqlsh to avoid permission issues. + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: credsVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: cr.InternalSecretName(), + }, + }, + }, + { + Name: tlsVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: cr.Spec.SSLSecretName, + }, + }, + }, + { + Name: configVolumeName, + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{ + { + ConfigMap: &corev1.ConfigMapProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: ConfigMapName(cr), + }, + Items: []corev1.KeyToPath{ + { + Key: CustomConfigKey, + Path: "my-config.cnf", + }, + }, + Optional: ptr.To(true), + }, + }, + { + ConfigMap: &corev1.ConfigMapProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: AutoConfigMapName(cr), + }, + Items: []corev1.KeyToPath{ + { + Key: CustomConfigKey, + Path: "auto-config.cnf", + }, + }, + Optional: ptr.To(true), + }, + }, + { + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: ConfigMapName(cr), + }, + Items: []corev1.KeyToPath{ + { + Key: CustomConfigKey, + Path: "my-secret.cnf", + }, + }, + Optional: ptr.To(true), + }, + }, + }, + }, + }, + }, + { + Name: "backup-logs", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + } + + if cr.CompareVersion("0.11.0") >= 0 { + volumes = append(volumes, corev1.Volume{ + Name: vaultSecretVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: spec.VaultSecretName, + Optional: ptr.To(true), + }, + }, + }) + } + + volumes = append(volumes, spec.SidecarVolumes...) + + return volumes + +} + func StatefulSet(cr *apiv1alpha1.PerconaServerMySQL, initImage, configHash, tlsHash string, secret *corev1.Secret) *appsv1.StatefulSet { labels := MatchLabels(cr) spec := cr.MySQLSpec() replicas := spec.Size - t := true annotations := make(map[string]string) if configHash != "" { @@ -182,97 +293,8 @@ func StatefulSet(cr *apiv1alpha1.PerconaServerMySQL, initImage, configHash, tlsH RestartPolicy: corev1.RestartPolicyAlways, SchedulerName: spec.SchedulerName, DNSPolicy: corev1.DNSClusterFirst, - Volumes: append( - []corev1.Volume{ - { - Name: apiv1alpha1.BinVolumeName, - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - { - Name: mysqlshVolumeName, // In OpenShift, we should use emptyDir for ./mysqlsh to avoid permission issues. - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - { - Name: credsVolumeName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: cr.InternalSecretName(), - }, - }, - }, - { - Name: tlsVolumeName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: cr.Spec.SSLSecretName, - }, - }, - }, - { - Name: configVolumeName, - VolumeSource: corev1.VolumeSource{ - Projected: &corev1.ProjectedVolumeSource{ - Sources: []corev1.VolumeProjection{ - { - ConfigMap: &corev1.ConfigMapProjection{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: ConfigMapName(cr), - }, - Items: []corev1.KeyToPath{ - { - Key: CustomConfigKey, - Path: "my-config.cnf", - }, - }, - Optional: &t, - }, - }, - { - ConfigMap: &corev1.ConfigMapProjection{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: AutoConfigMapName(cr), - }, - Items: []corev1.KeyToPath{ - { - Key: CustomConfigKey, - Path: "auto-config.cnf", - }, - }, - Optional: &t, - }, - }, - { - Secret: &corev1.SecretProjection{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: ConfigMapName(cr), - }, - Items: []corev1.KeyToPath{ - { - Key: CustomConfigKey, - Path: "my-secret.cnf", - }, - }, - Optional: &t, - }, - }, - }, - }, - }, - }, - { - Name: "backup-logs", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - }, - spec.SidecarVolumes..., - ), - SecurityContext: spec.PodSecurityContext, + SecurityContext: spec.PodSecurityContext, + Volumes: statefulSetVolumes(cr), }, }, }, @@ -523,6 +545,44 @@ func containers(cr *apiv1alpha1.PerconaServerMySQL, secret *corev1.Secret) []cor return appendUniqueContainers(containers, cr.Spec.MySQL.Sidecars...) } +func mysqldVolumeMounts(cr *apiv1alpha1.PerconaServerMySQL) []corev1.VolumeMount { + mounts := []corev1.VolumeMount{ + { + Name: apiv1alpha1.BinVolumeName, + MountPath: apiv1alpha1.BinVolumePath, + }, + { + Name: DataVolumeName, + MountPath: DataMountPath, + }, + { + Name: mysqlshVolumeName, + MountPath: mysqlshMountPath, + }, + { + Name: credsVolumeName, + MountPath: CredsMountPath, + }, + { + Name: tlsVolumeName, + MountPath: tlsMountPath, + }, + { + Name: configVolumeName, + MountPath: configMountPath, + }, + } + + if cr.CompareVersion("0.11.0") >= 0 { + mounts = append(mounts, corev1.VolumeMount{ + Name: vaultSecretVolumeName, + MountPath: vaultSecretMountPath, + }) + } + + return mounts +} + func mysqldContainer(cr *apiv1alpha1.PerconaServerMySQL) corev1.Container { spec := cr.MySQLSpec() @@ -567,40 +627,15 @@ func mysqldContainer(cr *apiv1alpha1.PerconaServerMySQL) corev1.Container { env = append(env, spec.Env...) container := corev1.Container{ - Name: ComponentName, - Image: spec.Image, - ImagePullPolicy: spec.ImagePullPolicy, - Resources: spec.Resources, - Ports: containerPorts(cr), - Env: env, - EnvFrom: spec.EnvFrom, - VolumeMounts: []corev1.VolumeMount{ - { - Name: apiv1alpha1.BinVolumeName, - MountPath: apiv1alpha1.BinVolumePath, - }, - { - Name: DataVolumeName, - MountPath: DataMountPath, - }, - { - Name: mysqlshVolumeName, - MountPath: mysqlshMountPath, - }, - { - Name: credsVolumeName, - MountPath: CredsMountPath, - }, - { - Name: tlsVolumeName, - MountPath: tlsMountPath, - }, - { - Name: configVolumeName, - MountPath: configMountPath, - }, - }, + Name: ComponentName, + Image: spec.Image, + ImagePullPolicy: spec.ImagePullPolicy, + Resources: spec.Resources, + Ports: containerPorts(cr), + Env: env, + EnvFrom: spec.EnvFrom, Command: []string{"/opt/percona/ps-entrypoint.sh"}, + VolumeMounts: mysqldVolumeMounts(cr), Args: []string{"mysqld"}, TerminationMessagePath: "/dev/termination-log", TerminationMessagePolicy: corev1.TerminationMessageReadFile, @@ -620,6 +655,29 @@ func mysqldContainer(cr *apiv1alpha1.PerconaServerMySQL) corev1.Container { return container } +func backupVolumeMounts(cr *apiv1alpha1.PerconaServerMySQL) []corev1.VolumeMount { + mounts := []corev1.VolumeMount{ + { + Name: apiv1alpha1.BinVolumeName, + MountPath: apiv1alpha1.BinVolumePath, + }, + { + Name: DataVolumeName, + MountPath: DataMountPath, + }, + { + Name: credsVolumeName, + MountPath: CredsMountPath, + }, + { + Name: "backup-logs", + MountPath: BackupLogDir, + }, + } + + return mounts +} + func backupContainer(cr *apiv1alpha1.PerconaServerMySQL) corev1.Container { return corev1.Container{ Name: "xtrabackup", @@ -632,24 +690,7 @@ func backupContainer(cr *apiv1alpha1.PerconaServerMySQL) corev1.Container { ContainerPort: SidecarHTTPPort, }, }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: apiv1alpha1.BinVolumeName, - MountPath: apiv1alpha1.BinVolumePath, - }, - { - Name: DataVolumeName, - MountPath: DataMountPath, - }, - { - Name: credsVolumeName, - MountPath: CredsMountPath, - }, - { - Name: "backup-logs", - MountPath: BackupLogDir, - }, - }, + VolumeMounts: backupVolumeMounts(cr), Command: []string{"/opt/percona/sidecar"}, TerminationMessagePath: "/dev/termination-log", TerminationMessagePolicy: corev1.TerminationMessageReadFile, diff --git a/pkg/xtrabackup/xtrabackup.go b/pkg/xtrabackup/xtrabackup.go index 2ae7c0ceb..5b5e8775d 100644 --- a/pkg/xtrabackup/xtrabackup.go +++ b/pkg/xtrabackup/xtrabackup.go @@ -10,6 +10,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/utils/ptr" apiv1alpha1 "github.com/percona/percona-server-mysql-operator/api/v1alpha1" "github.com/percona/percona-server-mysql-operator/pkg/k8s" @@ -19,16 +20,18 @@ import ( ) const ( - componentName = "xtrabackup" - componentShortName = "xb" - dataVolumeName = "datadir" - dataMountPath = "/var/lib/mysql" - credsVolumeName = "users" - credsMountPath = "/etc/mysql/mysql-users-secret" - tlsVolumeName = "tls" - tlsMountPath = "/etc/mysql/mysql-tls-secret" - backupVolumeName = componentName - backupMountPath = "/backup" + componentName = "xtrabackup" + componentShortName = "xb" + dataVolumeName = "datadir" + dataMountPath = "/var/lib/mysql" + credsVolumeName = "users" + credsMountPath = "/etc/mysql/mysql-users-secret" + tlsVolumeName = "tls" + tlsMountPath = "/etc/mysql/mysql-tls-secret" + backupVolumeName = componentName + backupMountPath = "/backup" + vaultSecretVolumeName = "vault-keyring-secret" + vaultSecretMountPath = "/etc/mysql/vault-keyring-secret" ) func Name(cr *apiv1alpha1.PerconaServerMySQLBackup) string { @@ -419,6 +422,15 @@ func RestoreJob( }, }, }, + { + Name: vaultSecretVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: cluster.Spec.MySQL.VaultSecretName, + Optional: ptr.To(true), + }, + }, + }, }, }, }, @@ -507,6 +519,10 @@ func restoreContainer(cluster *apiv1alpha1.PerconaServerMySQL, restore *apiv1alp Name: "VERIFY_TLS", Value: strconv.FormatBool(verifyTLS), }, + { + Name: "KEYRING_VAULT_PATH", + Value: fmt.Sprintf("%s/keyring_vault.conf", vaultSecretMountPath), + }, }, VolumeMounts: []corev1.VolumeMount{ { @@ -521,6 +537,10 @@ func restoreContainer(cluster *apiv1alpha1.PerconaServerMySQL, restore *apiv1alp Name: tlsVolumeName, MountPath: tlsMountPath, }, + { + Name: vaultSecretVolumeName, + MountPath: vaultSecretMountPath, + }, }, Command: []string{"/opt/percona/run-restore.sh"}, TerminationMessagePath: "/dev/termination-log",