K8SPS-421: Add keyring vault support

api/v1alpha1/perconaservermysql_types.go

@@ -119,6 +119,8 @@ type MySQLSpec struct {
 	SidecarVolumes []corev1.Volume    `json:"sidecarVolumes,omitempty"`
 	SidecarPVCs    []SidecarPVC       `json:"sidecarPVCs,omitempty"`
 
+	VaultSecretName string `json:"vaultSecretName,omitempty"`
+
 	PodSpec `json:",inline"`
 }
 
@@ -572,6 +574,16 @@ func (cr *PerconaServerMySQL) SetVersion() {
 	cr.Spec.CRVersion = version.Version()
 }
 
+func (cr *PerconaServerMySQL) Version() *v.Version {
+	return v.Must(v.NewVersion(cr.Spec.CRVersion))
+}
+
+// CompareVersion compares given version to current version.
+// Returns -1, 0, or 1 if given version is smaller, equal, or larger than the current version, respectively.
+func (cr *PerconaServerMySQL) CompareVersion(ver string) int {
+	return cr.Version().Compare(v.Must(v.NewVersion(ver)))
+}
+
 // CheckNSetDefaults validates and sets default values for the PerconaServerMySQL custom resource.
 func (cr *PerconaServerMySQL) CheckNSetDefaults(_ context.Context, serverVersion *platform.ServerVersion) error {
 	if len(cr.Spec.MySQL.ClusterType) == 0 {
@@ -862,6 +874,10 @@ func (cr *PerconaServerMySQL) CheckNSetDefaults(_ context.Context, serverVersion
 		cr.Spec.SSLSecretName = cr.Name + "-ssl"
 	}
 
+	if cr.Spec.MySQL.VaultSecretName == "" {
+		cr.Spec.MySQL.VaultSecretName = cr.Name + "-vault"
+	}
+
 	return nil
 }
 

build/ps-entrypoint.sh

@@ -58,7 +58,7 @@
 	case "$f" in
 		*.sh)
 			echo "$0: running $f"
 			. "$f"
 			;;
 		*.sql)
 			echo "$0: running $f"
@@ -167,6 +167,26 @@
 		sed -i "/\[mysqld\]/a ssl_key=${TLS_DIR}/tls.key" $CFG
 	fi
 
+	# if vault secret file exists we assume we need to turn on encryption
+	vault_secret="/etc/mysql/vault-keyring-secret/keyring_vault.conf"
+	if [[ -f "${vault_secret}" ]]; then
+		sed -i "/\[mysqld\]/a early-plugin-load=keyring_vault.so" $CFG
+		sed -i "/\[mysqld\]/a keyring_vault_config=${vault_secret}" $CFG
+
+		if [[ ${MYSQL_VERSION} =~ ^(8\.0|8\.4)$ ]]; then
+			sed -i "/\[mysqld\]/a default_table_encryption=ON" $CFG
+			sed -i "/\[mysqld\]/a table_encryption_privilege_check=ON" $CFG
+			sed -i "/\[mysqld\]/a innodb_undo_log_encrypt=ON" $CFG
+			sed -i "/\[mysqld\]/a innodb_redo_log_encrypt=ON" $CFG
+			sed -i "/\[mysqld\]/a binlog_encryption=ON" $CFG
+			sed -i "/\[mysqld\]/a binlog_rotate_encryption_master_key_at_startup=ON" $CFG
+			sed -i "/\[mysqld\]/a innodb_temp_tablespace_encrypt=ON" $CFG
+			sed -i "/\[mysqld\]/a innodb_parallel_dblwr_encrypt=ON" $CFG
+			sed -i "/\[mysqld\]/a innodb_encrypt_online_alter_logs=ON" $CFG
+			sed -i "/\[mysqld\]/a encrypt_tmp_files=ON" $CFG
+		fi
+	fi
+
 	for f in "${CUSTOM_CONFIG_FILES[@]}"; do
 		echo "${f}"
 		if [ -f "${f}" ]; then
@@ -197,7 +217,7 @@
 	exit 1
 fi
 
 if [ "$1" = 'mysqld' -a -z "$wantHelp" ]; then
 	# still need to check config, container may have started with --user
 	_check_config "$@"
 
@@ -213,7 +233,7 @@
 		touch /var/lib/mysql/bootstrap.lock
 		file_env 'MYSQL_ROOT_PASSWORD' '' 'root'
 		{ set +x; } 2>/dev/null
 		if [ -z "$MYSQL_ROOT_PASSWORD" -a -z "$MYSQL_ALLOW_EMPTY_PASSWORD" -a -z "$MYSQL_RANDOM_ROOT_PASSWORD" ]; then
 			echo >&2 'error: database is uninitialized and password option is not specified '
 			echo >&2 '  You need to specify one of MYSQL_ROOT_PASSWORD, MYSQL_ALLOW_EMPTY_PASSWORD and MYSQL_RANDOM_ROOT_PASSWORD'
 			exit 1
@@ -271,7 +291,7 @@
 		rootCreate=
 		# default root to listen for connections from anywhere
 		file_env 'MYSQL_ROOT_HOST' '%'
 		if [ -n "$MYSQL_ROOT_HOST" -a "$MYSQL_ROOT_HOST" != 'localhost' ]; then
 			# no, we don't care if read finds a terminating character in this heredoc
 			# https://unix.stackexchange.com/questions/265149/why-is-set-o-errexit-breaking-this-read-heredoc-expression/265151#265151
 			read -r -d '' rootCreate <<-EOSQL || true
@@ -353,7 +373,7 @@
 		file_env 'MYSQL_USER'
 		file_env 'MYSQL_PASSWORD'
 		{ set +x; } 2>/dev/null
 		if [ "$MYSQL_USER" -a "$MYSQL_PASSWORD" ]; then
 			echo "CREATE USER '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD' ;" | "${mysql[@]}"
 
 			if [ "$MYSQL_DATABASE" ]; then
@@ -401,7 +421,7 @@
 if [[ -f /var/lib/mysql/full-cluster-crash ]]; then
 	set +o xtrace
 	node_name=$(hostname -f)
 	cluster_name=$(hostname | cut -d '-' -f1) # TODO: This won't work if CR has `-` in its name.
 	gtid_executed=$(</var/lib/mysql/full-cluster-crash)
 	namespace=$(</var/run/secrets/kubernetes.io/serviceaccount/namespace)
 

build/run-restore.sh

@@ -41,7 +41,13 @@
 		"azure") run_azure | extract "${tmpdir}" ;;
 	esac
 
-	xtrabackup --prepare --rollback-prepared-trx --target-dir="${tmpdir}"
+	local keyring=""
+	if [[ -f ${KEYRING_VAULT_PATH} ]]; then
+		echo "Using keyring vault config: ${KEYRING_VAULT_PATH}"
+		keyring="--keyring-vault-config=${KEYRING_VAULT_PATH}"
+	fi
+
+	xtrabackup --prepare --rollback-prepared-trx --target-dir="${tmpdir}" ${keyring}
 	xtrabackup --datadir="${DATADIR}" --move-back --force-non-empty-directories --target-dir="${tmpdir}"
 
 	rm -rf "${tmpdir}"

config/crd/bases/ps.percona.com_perconaservermysqls.yaml

@@ -5001,6 +5001,8 @@ spec:
                       - whenUnsatisfiable
                       type: object
                     type: array
+                  vaultSecretName:
+                    type: string
                   volumeSpec:
                     properties:
                       emptyDir:

deploy/bundle.yaml

@@ -6924,6 +6924,8 @@ spec:
                       - whenUnsatisfiable
                       type: object
                     type: array
+                  vaultSecretName:
+                    type: string
                   volumeSpec:
                     properties:
                       emptyDir:

deploy/cr.yaml

@@ -42,6 +42,7 @@ spec:
     image: perconalab/percona-server-mysql-operator:main-psmysql
     imagePullPolicy: Always
 #    initImage: perconalab/percona-server-mysql-operator:main
+#    vaultSecretName: cluster1-vault
     size: 3
 
 #    env:

deploy/crd.yaml

@@ -6924,6 +6924,8 @@ spec:
                       - whenUnsatisfiable
                       type: object
                     type: array
+                  vaultSecretName:
+                    type: string
                   volumeSpec:
                     properties:
                       emptyDir:

deploy/cw-bundle.yaml

@@ -6924,6 +6924,8 @@ spec:
                       - whenUnsatisfiable
                       type: object
                     type: array
+                  vaultSecretName:
+                    type: string
                   volumeSpec:
                     properties:
                       emptyDir:

deploy/vault-secret.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: cluster1-vault
+type: Opaque
+stringData:
+  keyring_vault.conf: |-
+    token = <secret>
+    vault_url = http://vault-service.vault-service.svc.cluster.local:8200
+    secret_mount_point = secret

e2e-tests/conf/vault-secret.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: vault-keyring-secret
+type: Opaque
+stringData:
+  keyring_vault.conf: |-
+    token = #token
+    vault_url = #vault_url
+    secret_mount_point = #secret
+    #vault_ca = /etc/mysql/vault-keyring-secret/ca.cert
+  ca.cert: |-
+    #certVal