chore: Merge branch 'main' into ci/tweak-integration-test-config

Author: Techassi

.github/workflows/build.yml

@@ -25,7 +25,7 @@ env:
   CARGO_TERM_COLOR: always
   CARGO_INCREMENTAL: '0'
   CARGO_PROFILE_DEV_DEBUG: '0'
-  RUST_TOOLCHAIN_VERSION: "1.81.0"
+  RUST_TOOLCHAIN_VERSION: "1.82.0"
   RUSTFLAGS: "-D warnings"
   RUSTDOCFLAGS: "-D warnings"
   RUST_LOG: "info"

.github/workflows/integration-test.yml

@@ -14,19 +14,23 @@ env:
   TEST_PARAMETER: ${{ inputs.test-parameter }}
 
 on:
-  schedule:
+  # schedule:
     # At 00:00 on Sunday. See: https://crontab.guru/#0_0_*_*_0
-    - cron: "0 0 * * 0"
+    # - cron: "0 0 * * 0"
   workflow_dispatch:
     inputs:
       test-platform:
         description: |
-          The test platform to run on (kind doesn't support `arm64`)
+          The test platform to run on
         required: true
         type: choice
         options:
-          - kind-1.31.0
-          - kind-1.30.3
+          - kind-1.31.2
+          - kind-1.30.6
+          - rke2-1.31.2
+          - rke2-1.30.6
+          - k3s-1.31.2
+          - k3s-1.30.6
           - aks-1.29
           - aks-1.28
           - aks-1.27
@@ -41,7 +45,8 @@ on:
           - okd-4.13
       test-architecture:
         description: |
-          The architecture the tests will run on
+          The architecture the tests will run on. Consult the run-integration-test action README for
+          more details on supported architectures for each distribution
         required: true
         type: choice
         options:
@@ -81,8 +86,7 @@ jobs:
 
       - name: Run Integration Test
         id: test
-        # NOTE (@Techassi): Use tagged version instead, changed only for testing
-        uses: stackabletech/actions/run-integration-test@99211a06f470cda1b2bcf7b4054ea1ccfe4ba071
+        uses: stackabletech/actions/run-integration-test@5901c3b1455488820c4be367531e07c3c3e82538 # v0.4.0
         with:
           test-platform: ${{ env.TEST_PLATFORM }}-${{ env.TEST_ARCHITECTURE }}
           test-run: ${{ env.TEST_RUN }}

.github/workflows/pr_pre-commit.yaml

@@ -6,7 +6,7 @@ on:
 
 env:
   CARGO_TERM_COLOR: always
-  RUST_TOOLCHAIN_VERSION: "1.81.0"
+  RUST_TOOLCHAIN_VERSION: "1.82.0"
   HADOLINT_VERSION: "v2.12.0"
   PYTHON_VERSION: "3.12"
 

CHANGELOG.md

@@ -2,14 +2,26 @@
 
 ## [Unreleased]
 
+### Fixed
+
+- BREAKING: Use distinct ServiceAccounts for the Stacklets, so that multiple Stacklets can be
+  deployed in one namespace. Existing Stacklets will use the newly created ServiceAccounts after
+  restart ([#545]).
+- Fix OIDC endpoint construction in case the `rootPath` does not have a trailing slash ([#547]).
+
+[#545]: https://github.com/stackabletech/airflow-operator/pull/545
+[#547]: https://github.com/stackabletech/airflow-operator/pull/547
+
+## [24.11.0] - 2024-11-18
+
 ### Added
 
 - Allowing arbitrary python code as `EXPERIMENTAL_FILE_HEADER` and `EXPERIMENTAL_FILE_FOOTER` in `webserver_config.py` ([#493]).
 - The operator can now run on Kubernetes clusters using a non-default cluster domain.
   Use the env var `KUBERNETES_CLUSTER_DOMAIN` or the operator Helm chart property `kubernetesClusterDomain` to set a non-default cluster domain ([#518]).
 - Support for `2.9.3` ([#494]).
 - Experimental Support for `2.10.2` ([#512]).
-- Add support for OpenID Connect ([#524])
+- Add support for OpenID Connect ([#524], [#530])
 
 ### Changed
 
@@ -32,6 +44,7 @@
 [#518]: https://github.com/stackabletech/airflow-operator/pull/518
 [#520]: https://github.com/stackabletech/airflow-operator/pull/520
 [#524]: https://github.com/stackabletech/airflow-operator/pull/524
+[#530]: https://github.com/stackabletech/airflow-operator/pull/530
 
 ## [24.7.0] - 2024-07-24
 

Cargo.lock

@@ -23,10 +23,11 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 serde_yaml = "0.9"
 snafu = "0.8"
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.80.0" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.82.0" }
 strum = { version = "0.26", features = ["derive"] }
 tokio = { version = "1.40", features = ["full"] }
 tracing = "0.1"
 
 # [patch."https://github.com/stackabletech/operator-rs.git"]
 # stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" }
+# stackable-operator = { path = "../operator-rs/crates/stackable-operator" }

Cargo.nix

@@ -29,6 +29,9 @@ SHELL=/usr/bin/env bash -euo pipefail
 render-readme:
 	scripts/render_readme.sh
 
+render-docs:
+	scripts/docs_templating.sh
+
 ## Docker related targets
 docker-build:
 	docker build --force-rm --build-arg VERSION=${VERSION} -t "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}:${VERSION}-${ARCH}" -f docker/Dockerfile .
@@ -48,7 +51,7 @@ docker-publish:
 	# Uses the keyless signing flow with Github Actions as identity provider\
 	cosign sign -y "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}@$$REPO_DIGEST_OF_IMAGE";\
 	# Generate the SBOM for the operator image, this leverages the already generated SBOM for the operator binary by cargo-cyclonedx\
-	syft scan --output cyclonedx-json=sbom.json --select-catalogers "-cargo-auditable-binary-cataloger" --scope all-layers --source-name "${OPERATOR_NAME}" --source-version "${VERSION}-${ARCH}" "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}@$$REPO_DIGEST_OF_IMAGE";\
+	syft scan --output cyclonedx-json@1.5=sbom.json --select-catalogers "-cargo-auditable-binary-cataloger,+sbom-cataloger" --scope all-layers --source-name "${OPERATOR_NAME}" --source-version "${VERSION}-${ARCH}" "${DOCKER_REPO}/${ORGANIZATION}/${OPERATOR_NAME}@$$REPO_DIGEST_OF_IMAGE";\
 	# Determine the PURL for the container image\
 	URLENCODED_REPO_DIGEST_OF_IMAGE=$$(echo "$$REPO_DIGEST_OF_IMAGE" | sed 's/:/%3A/g');\
 	PURL="pkg:oci/${OPERATOR_NAME}@$$URLENCODED_REPO_DIGEST_OF_IMAGE?arch=${ARCH}&repository_url=${DOCKER_REPO}%2F${ORGANIZATION}%2F${OPERATOR_NAME}";\
@@ -74,7 +77,7 @@ docker-publish:
 	# Uses the keyless signing flow with Github Actions as identity provider\
 	cosign sign -y "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}@$$REPO_DIGEST_OF_IMAGE";\
 	# Generate the SBOM for the operator image, this leverages the already generated SBOM for the operator binary by cargo-cyclonedx\
-	syft scan --output cyclonedx-json=sbom.json --select-catalogers "-cargo-auditable-binary-cataloger" --scope all-layers --source-name "${OPERATOR_NAME}" --source-version "${VERSION}-${ARCH}" "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}@$$REPO_DIGEST_OF_IMAGE";\
+	syft scan --output cyclonedx-json@1.5=sbom.json --select-catalogers "-cargo-auditable-binary-cataloger,+sbom-cataloger" --scope all-layers --source-name "${OPERATOR_NAME}" --source-version "${VERSION}-${ARCH}" "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}@$$REPO_DIGEST_OF_IMAGE";\
 	# Determine the PURL for the container image\
 	URLENCODED_REPO_DIGEST_OF_IMAGE=$$(echo "$$REPO_DIGEST_OF_IMAGE" | sed 's/:/%3A/g');\
 	PURL="pkg:oci/${OPERATOR_NAME}@$$URLENCODED_REPO_DIGEST_OF_IMAGE?arch=${ARCH}&repository_url=${OCI_REGISTRY_HOSTNAME}%2F${OCI_REGISTRY_PROJECT_IMAGES}%2F${OPERATOR_NAME}";\

Cargo.toml

@@ -9,6 +9,27 @@ targets = [
 
 [advisories]
 yanked = "deny"
+ignore = [
+    # https://rustsec.org/advisories/RUSTSEC-2023-0071
+    # "rsa" crate: Marvin Attack: potential key recovery through timing sidechannel
+    #
+    # No patch is yet available, however work is underway to migrate to a fully constant-time implementation
+    # So we need to accept this, as of SDP 24.11 we are not using the rsa crate to create certificates used in production
+    # setups.
+    #
+    # TODO: Remove after https://github.com/RustCrypto/RSA/pull/394 is merged
+    "RUSTSEC-2023-0071",
+
+    # https://rustsec.org/advisories/RUSTSEC-2024-0384
+    # "instant" is unmaintained
+    #
+    # The upstream "kube" crate also silenced this in https://github.com/kube-rs/kube/commit/4f1e889f265da8f19f03f60683569cae1a154fda
+    # They/we are actively working on migrating kube from backoff to backon, which removes the transitive dependency on
+    # instant, in https://github.com/kube-rs/kube/pull/1652.
+    #
+    # TODO: Remove after https://github.com/kube-rs/kube/pull/1652 is merged
+    "RUSTSEC-2024-0384",
+]
 
 [bans]
 multiple-versions = "allow"
@@ -26,6 +47,7 @@ allow = [
     "LicenseRef-webpki",
     "MIT",
     "MPL-2.0",
+    "OpenSSL", # Needed for the ring and/or aws-lc-sys crate. See https://github.com/stackabletech/operator-templating/pull/464 for details
     "Unicode-3.0",
     "Unicode-DFS-2016",
     "Zlib",

Makefile

@@ -25,7 +25,9 @@
 
 class SparkKubernetesSensor(BaseSensorOperator):  # <3>
     template_fields = ("application_name", "namespace")
-    FAILURE_STATES = ("Failed", "Unknown")
+    # See https://github.com/stackabletech/spark-k8s-operator/pull/460/files#diff-d737837121132af6b60f50279a78464b05dcfd06c05d1d090f4198a5e962b5f6R371
+    # Unknown is set immediately so it must be excluded from the failed states.
+    FAILURE_STATES = "Failed"
     SUCCESS_STATES = "Succeeded"
 
     def __init__(

crate-hashes.json

@@ -2,19 +2,7 @@
 set -euo pipefail
 
 # DO NOT EDIT THE SCRIPT
-# Instead, update the j2 template, and regenerate it for dev:
-# cat <<EOF | jinja2 --format yaml getting_started.sh.j2 -o getting_started.sh
-# helm:
-#   repo_name: stackable-dev
-#   repo_url: https://repo.stackable.tech/repository/helm-dev/
-# versions:
-#   commons: 0.0.0-dev
-#   secret: 0.0.0-dev
-#   listener: 0.0.0-dev
-#   airflow: 0.0.0-dev
-#   postgresql: 12.1.5
-#   redis: 17.3.7
-# EOF
+# Instead, update the j2 template, and regenerate it for dev with `make render-docs`.
 
 # This script contains all the code snippets from the guide, as well as some assert tests
 # to test if the instructions in the guide work. The user *could* use it, but it is intended

deny.toml

@@ -2,19 +2,7 @@
 set -euo pipefail
 
 # DO NOT EDIT THE SCRIPT
-# Instead, update the j2 template, and regenerate it for dev:
-# cat <<EOF | jinja2 --format yaml getting_started.sh.j2 -o getting_started.sh
-# helm:
-#   repo_name: stackable-dev
-#   repo_url: https://repo.stackable.tech/repository/helm-dev/
-# versions:
-#   commons: 0.0.0-dev
-#   secret: 0.0.0-dev
-#   listener: 0.0.0-dev
-#   airflow: 0.0.0-dev
-#   postgresql: 12.1.5
-#   redis: 17.3.7
-# EOF
+# Instead, update the j2 template, and regenerate it for dev with `make render-docs`.
 
 # This script contains all the code snippets from the guide, as well as some assert tests
 # to test if the instructions in the guide work. The user *could* use it, but it is intended