fix: move metastore db credentials into a secret (#452)

* fix: move metastore db credentials into a secret

* changelog

* changed references in documentation/examples

* changelog

* Update rust/operator-binary/src/controller.rs

Co-authored-by: Malte Sander <contact@maltesander.com>

* Update CHANGELOG.md

Co-authored-by: Malte Sander <contact@maltesander.com>

* consistent ordering

---------

Co-authored-by: Malte Sander <contact@maltesander.com>

Author: adwk67

CHANGELOG.md

@@ -8,6 +8,10 @@ All notable changes to this project will be documented in this file.
 
 - Added documentation/tutorial on using external database drivers ([#449]).
 
+### Fixed
+
+- [BREAKING] Move the metastore `user` and `password` DB credentials out of the CRD into a Secret containing the keys `username` and `password` ([#452]).
+
 ### Changed
 
 - BREAKING: Switch to new image that only contains HMS.
@@ -17,6 +21,7 @@ All notable changes to this project will be documented in this file.
 
 [#447]: https://github.com/stackabletech/hive-operator/pull/447
 [#449]: https://github.com/stackabletech/hive-operator/pull/449
+[#452]: https://github.com/stackabletech/hive-operator/pull/452
 
 ## [24.3.0] - 2024-03-20
 

deploy/helm/hive-operator/crds/crds.yaml

@@ -50,6 +50,9 @@ spec:
                         connString:
                           description: 'A connection string for the database. For example: `jdbc:postgresql://hivehdfs-postgresql:5432/hivehdfs`'
                           type: string
+                        credentialsSecret:
+                          description: A reference to a Secret containing the database credentials. The Secret needs to contain the keys `username` and `password`.
+                          type: string
                         dbType:
                           description: 'The type of database to connect to. Supported are: `postgres`, `mysql`, `oracle`, `mssql` and `derby`. This value is used to configure the jdbc driver class.'
                           enum:
@@ -59,17 +62,10 @@ spec:
                             - oracle
                             - mssql
                           type: string
-                        password:
-                          description: The password for the database user.
-                          type: string
-                        user:
-                          description: The database user.
-                          type: string
                       required:
                         - connString
+                        - credentialsSecret
                         - dbType
-                        - password
-                        - user
                       type: object
                     hdfs:
                       description: HDFS connection specification.

docs/modules/hive/examples/getting_started/hive-postgres-s3.yaml

@@ -9,12 +9,20 @@ spec:
   clusterConfig:
     database:
       connString: jdbc:postgresql://postgresql:5432/hive
-      user: hive
-      password: hive
+      credentialsSecret: hive-credentials
       dbType: postgres
     s3:
       reference: minio
   metastore:
     roleGroups:
       default:
         replicas: 1
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: hive-credentials
+type: Opaque
+stringData:
+  username: hive
+  password: hive

docs/modules/hive/examples/getting_started/hive-postgres-s3.yaml.j2

@@ -9,12 +9,20 @@ spec:
   clusterConfig:
     database:
       connString: jdbc:postgresql://postgresql:5432/hive
-      user: hive
-      password: hive
+      credentialsSecret: hive-credentials
       dbType: postgres
     s3:
       reference: minio
   metastore:
     roleGroups:
       default:
         replicas: 1
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: hive-credentials
+type: Opaque
+stringData:
+  username: hive
+  password: hive

docs/modules/hive/pages/reference/discovery.adoc

@@ -25,13 +25,21 @@ spec:
   clusterConfig:
     database:
       connString: jdbc:postgresql://postgresql:5432/hive
-      user: hive
-      password: hive
+      credentialsSecret: hive-credentials
       dbType: postgres
   metastore:
     roleGroups:
       default: # <3>
         replicas: 2
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: hive-credentials
+type: Opaque
+stringData:
+  username: hive
+  password: hive
 ----
 <1> The name of the Hive cluster, which is also the name of the created discovery ConfigMap.
 <2> The namespace of the discovery ConfigMap.

docs/modules/hive/pages/usage-guide/database-driver.adoc

@@ -145,8 +145,7 @@ spec:
   clusterConfig:
     database:
       connString: jdbc:mysql://mysql:3306/hive  # <1>
-      user: hive  # <2>
-      password: hive
+      credentialsSecret: hive-credentials  # <2>
       dbType: mysql
     s3:
       reference: minio  # <3>
@@ -167,10 +166,19 @@ spec:
                 persistentVolumeClaim:
                   claimName: pvc-hive-drivers
         replicas: 1
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: hive-credentials  # <2>
+type: Opaque
+stringData:
+  username: hive
+  password: hive
 ----
 
 <1> The database connection details matching those given when deploying the MySQL Helm chart
-<2> Plain-text Hive credentials will be replaced in an upcoming release!
+<2> Hive credentials are retrieved from a Secret
 <3> A reference to the file store using S3 (this has been omitted from this article for the sake of brevity, but is described in e.g. the xref:getting_started/first_steps.adoc[] guide)
 <4> Use `envOverrides` to set the driver path
 <5> Use `podOverrides` to mount the driver

docs/modules/hive/pages/usage-guide/derby-example.adoc

@@ -20,13 +20,21 @@ spec:
   clusterConfig:
     database:
       connString: jdbc:derby:;databaseName=/tmp/metastore_db;create=true
-      user: APP
-      password: mine
+      credentialsSecret: hive-credentials
       dbType: derby
   metastore:
     roleGroups:
       default:
         replicas: 1
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: hive-credentials
+type: Opaque
+stringData:
+  username: APP
+  password: mine
 ----
 
 WARNING: You should not use the `Derby` database in production. Derby stores data locally which does not work in high availability setups (multiple replicas) and all data is lost after Pod restarts.
@@ -62,8 +70,7 @@ spec:
   clusterConfig:
     database:
       connString: jdbc:derby:;databaseName=/stackable/metastore_db;create=true
-      user: APP
-      password: mine
+      credentialsSecret: hive-credentials
       dbType: derby
     s3:
       inline:
@@ -96,6 +103,15 @@ metadata:
 stringData:
   accessKey: minio-access-key
   secretKey: minio-secret-key
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: hive-credentials
+type: Opaque
+stringData:
+  username: APP
+  password: mine
 ----
 
 
@@ -131,11 +147,19 @@ spec:
   clusterConfig:
     database:
       connString: jdbc:postgresql://hive-postgresql.default.svc.cluster.local:5432/hive
-      user: hive
-      password: hive
+      credentialsSecret: hive-credentials
       dbType: postgres
   metastore:
     roleGroups:
       default:
         replicas: 1
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: hive-credentials
+type: Opaque
+stringData:
+  username: hive
+  password: hive
 ----

examples/simple-hive-cluster-postgres-s3.yaml

@@ -22,8 +22,7 @@ spec:
   clusterConfig:
     database:
       connString: jdbc:derby:;databaseName=/tmp/hive;create=true
-      user: APP
-      password: mine
+      credentialsSecret: hive-credentials
       dbType: derby
     s3:
       inline:
@@ -56,3 +55,12 @@ metadata:
 stringData:
   accessKey: minio-access-key
   secretKey: minio-secret-key
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: hive-credentials
+type: Opaque
+stringData:
+  username: APP
+  password: mine

examples/simple-hive-cluster.yaml

@@ -10,8 +10,7 @@ spec:
   clusterConfig:
     database:
       connString: jdbc:derby:;databaseName=/tmp/hive;create=true
-      user: APP
-      password: mine
+      credentialsSecret: hive-credentials
       dbType: derby
   metastore:
     roleGroups:
@@ -24,3 +23,12 @@ spec:
               max: "2"
             memory:
               limit: 5Gi
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: hive-credentials
+type: Opaque
+stringData:
+  username: APP
+  password: mine

rust/crd/src/affinity.rs

@@ -49,9 +49,8 @@ mod tests {
           clusterConfig:
             database:
               connString: jdbc:derby:;databaseName=/tmp/hive;create=true
-              user: APP
-              password: mine
               dbType: derby
+              credentialsSecret: mySecret
           metastore:
             roleGroups:
               default: