From 7f8e76bb78d81a621fc2e70f6dd4bf8085f9780b Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Wed, 28 Aug 2024 16:31:36 +0200 Subject: [PATCH 1/2] feat: Add support for Hive 4.0.0 --- .../kuttl/kerberos-s3/70-install-access-hive.yaml.j2 | 2 +- tests/templates/kuttl/smoke/test_metastore.py | 2 +- tests/test-definition.yaml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/templates/kuttl/kerberos-s3/70-install-access-hive.yaml.j2 b/tests/templates/kuttl/kerberos-s3/70-install-access-hive.yaml.j2 index b935859c..4651a178 100644 --- a/tests/templates/kuttl/kerberos-s3/70-install-access-hive.yaml.j2 +++ b/tests/templates/kuttl/kerberos-s3/70-install-access-hive.yaml.j2 @@ -165,7 +165,7 @@ data: # S3 access try: - hive_client.create_table(table(database_name, s3_test_table_name, "s3a://hive/")) + hive_client.create_table(table(database_name, s3_test_table_name, "s3a://hive/s3_one_column_table/")) except AlreadyExistsException: print(f"[INFO]: Table {s3_test_table_name} already existed") print("[DEBUG]: Fetching schema") diff --git a/tests/templates/kuttl/smoke/test_metastore.py b/tests/templates/kuttl/smoke/test_metastore.py index 5af1690e..f54cce06 100755 --- a/tests/templates/kuttl/smoke/test_metastore.py +++ b/tests/templates/kuttl/smoke/test_metastore.py @@ -88,7 +88,7 @@ def table(db_name, table_name, location): # S3 access try: hive_client.create_table( - table(database_name, s3_test_table_name, "s3a://hive/") + table(database_name, s3_test_table_name, "s3a://hive/s3_one_column_table/") ) except AlreadyExistsException: print(f"[INFO]: Table {s3_test_table_name} already existed") diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 6ed5e3f6..ec8a3b6c 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -19,13 +19,13 @@ dimensions: - 3.1.3 # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version # as in the example below. - # - 3.1.3,docker.stackable.tech/sandbox/hive:3.1.3-stackable0.0.0-dev + - 4.0.0,docker.stackable.tech/sandbox/hive:4.0.0-stackable0.0.0-dev - name: hive-latest values: - 3.1.3 # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version # as in the example below. - # - 3.1.3,docker.stackable.tech/sandbox/hive:3.1.3-stackable0.0.0-dev + - 4.0.0,docker.stackable.tech/sandbox/hive:4.0.0-stackable0.0.0-dev - name: hdfs-latest values: - 3.3.6 From 5eb7120c0c6f4b9f205651b6ab04de545cd47293 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 3 Sep 2024 11:23:30 +0200 Subject: [PATCH 2/2] Update docs and tests --- CHANGELOG.md | 5 +++++ .../examples/getting_started/hive-postgres-s3.yaml | 2 +- .../examples/getting_started/hive-postgres-s3.yaml.j2 | 2 +- .../hive/pages/usage-guide/database-driver.adoc | 10 +++++----- docs/modules/hive/pages/usage-guide/derby-example.adoc | 8 ++++---- docs/modules/hive/partials/supported-versions.adoc | 3 ++- examples/simple-hive-cluster-postgres-s3.yaml | 2 +- examples/simple-hive-cluster.yaml | 2 +- rust/crd/src/affinity.rs | 2 +- tests/test-definition.yaml | 7 ++++--- 10 files changed, 25 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b37681c..ea51f878 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +### Added + +- Add support for Hive `4.0.0` ([#508]). + ### Changed - Reduce CRD size from `487KB` to `60KB` by accepting arbitrary YAML input instead of the underlying schema for the following fields ([#505]): @@ -11,6 +15,7 @@ All notable changes to this project will be documented in this file. - `affinity` [#505]: https://github.com/stackabletech/hive-operator/pull/505 +[#508]: https://github.com/stackabletech/hive-operator/pull/508 ## [24.7.0] - 2024-07-24 diff --git a/docs/modules/hive/examples/getting_started/hive-postgres-s3.yaml b/docs/modules/hive/examples/getting_started/hive-postgres-s3.yaml index b578b2a5..9ee5f617 100644 --- a/docs/modules/hive/examples/getting_started/hive-postgres-s3.yaml +++ b/docs/modules/hive/examples/getting_started/hive-postgres-s3.yaml @@ -5,7 +5,7 @@ metadata: name: hive-postgres-s3 spec: image: - productVersion: 3.1.3 + productVersion: 4.0.0 clusterConfig: database: connString: jdbc:postgresql://postgresql:5432/hive diff --git a/docs/modules/hive/examples/getting_started/hive-postgres-s3.yaml.j2 b/docs/modules/hive/examples/getting_started/hive-postgres-s3.yaml.j2 index b578b2a5..9ee5f617 100644 --- a/docs/modules/hive/examples/getting_started/hive-postgres-s3.yaml.j2 +++ b/docs/modules/hive/examples/getting_started/hive-postgres-s3.yaml.j2 @@ -5,7 +5,7 @@ metadata: name: hive-postgres-s3 spec: image: - productVersion: 3.1.3 + productVersion: 4.0.0 clusterConfig: database: connString: jdbc:postgresql://postgresql:5432/hive diff --git a/docs/modules/hive/pages/usage-guide/database-driver.adoc b/docs/modules/hive/pages/usage-guide/database-driver.adoc index 0ec2ee62..9b8d4cfb 100644 --- a/docs/modules/hive/pages/usage-guide/database-driver.adoc +++ b/docs/modules/hive/pages/usage-guide/database-driver.adoc @@ -141,7 +141,7 @@ metadata: name: hive-with-drivers spec: image: - productVersion: 3.1.3 + productVersion: 4.0.0 clusterConfig: database: connString: jdbc:mysql://mysql:3306/hive # <1> @@ -194,7 +194,7 @@ If you have access to a registry to store custom images, another approach is to .Copy the driver [source] ---- -FROM docker.stackable.tech/stackable/hive:3.1.3-stackable0.0.0-dev +FROM docker.stackable.tech/stackable/hive:4.0.0-stackable0.0.0-dev RUN curl --fail -L https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.31/mysql-connector-j-8.0.31.jar -o /stackable/mysql-connector-j-8.0.31.jar ---- @@ -202,7 +202,7 @@ RUN curl --fail -L https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8. .Build and tag the image [source] ---- -docker build -f ./Dockerfile -t docker.stackable.tech/stackable/hive:3.1.3-stackable0.0.0-dev-mysql . +docker build -f ./Dockerfile -t docker.stackable.tech/stackable/hive:4.0.0-stackable0.0.0-dev-mysql . ---- .Reference the new path to the driver without the need for using a volume mounted from a PVC @@ -215,8 +215,8 @@ metadata: name: hive spec: image: - custom: docker.stackable.tech/stackable/hive:3.1.3-stackable0.0.0-dev-mysql # <1> - productVersion: 3.1.3 + custom: docker.stackable.tech/stackable/hive:4.0.0-stackable0.0.0-dev-mysql # <1> + productVersion: 4.0.0 clusterConfig: database: ... diff --git a/docs/modules/hive/pages/usage-guide/derby-example.adoc b/docs/modules/hive/pages/usage-guide/derby-example.adoc index 38c8f959..56bee6bc 100644 --- a/docs/modules/hive/pages/usage-guide/derby-example.adoc +++ b/docs/modules/hive/pages/usage-guide/derby-example.adoc @@ -16,7 +16,7 @@ metadata: name: simple-hive-derby spec: image: - productVersion: 3.1.3 + productVersion: 4.0.0 clusterConfig: database: connString: jdbc:derby:;databaseName=/tmp/metastore_db;create=true @@ -39,7 +39,7 @@ stringData: WARNING: You should not use the `Derby` database in production. Derby stores data locally which does not work in high availability setups (multiple replicas) and all data is lost after Pod restarts. -To create a single node Apache Hive Metastore (v3.1.3) cluster with derby and S3 access, deploy a minio (or use any available S3 bucket): +To create a single node Apache Hive Metastore (v4.0.0) cluster with derby and S3 access, deploy a minio (or use any available S3 bucket): [source,bash] ---- helm install minio \ @@ -66,7 +66,7 @@ metadata: name: simple-hive-derby spec: image: - productVersion: 3.1.3 + productVersion: 4.0.0 clusterConfig: database: connString: jdbc:derby:;databaseName=/stackable/metastore_db;create=true @@ -143,7 +143,7 @@ metadata: name: simple-hive-postgres spec: image: - productVersion: 3.1.3 + productVersion: 4.0.0 clusterConfig: database: connString: jdbc:postgresql://hive-postgresql.default.svc.cluster.local:5432/hive diff --git a/docs/modules/hive/partials/supported-versions.adoc b/docs/modules/hive/partials/supported-versions.adoc index 8e1903bf..ec56e788 100644 --- a/docs/modules/hive/partials/supported-versions.adoc +++ b/docs/modules/hive/partials/supported-versions.adoc @@ -2,4 +2,5 @@ // This is a separate file, since it is used by both the direct Hive-Operator documentation, and the overarching // Stackable Platform documentation. -- 3.1.3 +- 3.1.3 (LTS) +- 4.0.0 (experimental) diff --git a/examples/simple-hive-cluster-postgres-s3.yaml b/examples/simple-hive-cluster-postgres-s3.yaml index 60e7039e..685cf3a0 100644 --- a/examples/simple-hive-cluster-postgres-s3.yaml +++ b/examples/simple-hive-cluster-postgres-s3.yaml @@ -17,7 +17,7 @@ metadata: name: simple-hive-postgres spec: image: - productVersion: 3.1.3 + productVersion: 4.0.0 stackableVersion: 0.0.0-dev clusterConfig: database: diff --git a/examples/simple-hive-cluster.yaml b/examples/simple-hive-cluster.yaml index 9c9655c1..d2df618b 100644 --- a/examples/simple-hive-cluster.yaml +++ b/examples/simple-hive-cluster.yaml @@ -5,7 +5,7 @@ metadata: name: simple-hive-derby spec: image: - productVersion: 3.1.3 + productVersion: 4.0.0 stackableVersion: 0.0.0-dev clusterConfig: database: diff --git a/rust/crd/src/affinity.rs b/rust/crd/src/affinity.rs index a2d6b794..ecaf78f9 100644 --- a/rust/crd/src/affinity.rs +++ b/rust/crd/src/affinity.rs @@ -45,7 +45,7 @@ mod tests { name: simple-hive spec: image: - productVersion: 3.1.3 + productVersion: 4.0.0 clusterConfig: database: connString: jdbc:derby:;databaseName=/tmp/hive;create=true diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index ec8a3b6c..06488c05 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -17,15 +17,16 @@ dimensions: - name: hive values: - 3.1.3 + - 4.0.0 # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version # as in the example below. - - 4.0.0,docker.stackable.tech/sandbox/hive:4.0.0-stackable0.0.0-dev + # - 4.0.0,docker.stackable.tech/sandbox/hive:4.0.0-stackable0.0.0-dev - name: hive-latest values: - - 3.1.3 + - 4.0.0 # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version # as in the example below. - - 4.0.0,docker.stackable.tech/sandbox/hive:4.0.0-stackable0.0.0-dev + # - 4.0.0,docker.stackable.tech/sandbox/hive:4.0.0-stackable0.0.0-dev - name: hdfs-latest values: - 3.3.6