From de76d11be712310ec77ab012e148fe2b1323e72f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Wed, 11 Jun 2025 12:42:32 +0200 Subject: [PATCH 1/6] Slim down GH action to use the same code path for dry and wet runs --- .github/workflows/generate_prs.yml | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/.github/workflows/generate_prs.yml b/.github/workflows/generate_prs.yml index d51dd140..894723fb 100644 --- a/.github/workflows/generate_prs.yml +++ b/.github/workflows/generate_prs.yml @@ -113,10 +113,10 @@ jobs: env: DEBIAN_FRONTEND: noninteractive run: | - sudo apt update && \ - sudo apt install -y software-properties-common && \ - sudo apt-add-repository ppa:ansible/ansible -y && \ - sudo apt install -y ansible + sudo apt update && \ + sudo apt install -y software-properties-common && \ + sudo apt-add-repository ppa:ansible/ansible -y && \ + sudo apt install -y ansible # NOTE (@NickLarsenNZ): This could be removed in favor of nix-shell and rrbutani/use-nix-shell-action - name: Install deps for operators @@ -159,25 +159,13 @@ jobs: # For debugging cat config/repositories.yaml - # Generate PRs - name: Run playbook - if: ${{ !inputs.dry-run }} env: CUSTOM_PR_TITLE: ${{ inputs.custom-pr-title }} GH_ACCESS_TOKEN: ${{ secrets.gh_access_token }} + # --tags local excludes all actions that actually generate PRs + DRY_RUN_FLAGS: ${{ inputs.dry-run && "--tags local" || "" }} run: | # Funnel via JSON to ensure that values are escaped properly - # Check if custom PR title is set. If so, insert it into the extra vars - if [ -n "$CUSTOM_PR_TITLE" ]; then - echo '{}' | jq '{commit_hash: $ENV.GITHUB_SHA, author: $ENV.AUTHOR, reason: $ENV.REASON, custom_pr_title: $ENV.CUSTOM_PR_TITLE, base_dir: $pwd, gh_access_token: $ENV.GH_ACCESS_TOKEN}' --arg pwd "$(pwd)" > vars.json - else - echo '{}' | jq '{commit_hash: $ENV.GITHUB_SHA, author: $ENV.AUTHOR, reason: $ENV.REASON, base_dir: $pwd, gh_access_token: $ENV.GH_ACCESS_TOKEN}' --arg pwd "$(pwd)" > vars.json - fi - ansible-playbook playbook/playbook.yaml --extra-vars "@vars.json" - - # Do Not Generate PRs - - name: Run playbook (dry-run) - if: ${{ inputs.dry-run }} - run: ./test.sh - env: - GH_ACCESS_TOKEN: "" + echo '{}' | jq '{commit_hash: $ENV.GITHUB_SHA, author: $ENV.AUTHOR, reason: $ENV.REASON, custom_pr_title: $ENV.CUSTOM_PR_TITLE, base_dir: $pwd, gh_access_token: $ENV.GH_ACCESS_TOKEN} | with_entries(select(.value != null and .value != ""))' --arg pwd "$(pwd)" > vars.json + ansible-playbook playbook/playbook.yaml --extra-vars "@vars.json" $DRY_RUN_FLAGS From 47070e49cd837adedfc20ab3735b8baf065c8709 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Wed, 11 Jun 2025 12:46:25 +0200 Subject: [PATCH 2/6] Let Ansible handle repository filtering This fixes repository indexing (fixes #411), and makes config/repositories.yaml authoritative (again) on the operator configuration. --- .github/workflows/generate_prs.yml | 109 ++++++----------------------- config/repositories.yaml | 6 +- playbook/playbook.yaml | 1 + test.sh | 2 +- 4 files changed, 25 insertions(+), 93 deletions(-) diff --git a/.github/workflows/generate_prs.yml b/.github/workflows/generate_prs.yml index 894723fb..63dc9b93 100644 --- a/.github/workflows/generate_prs.yml +++ b/.github/workflows/generate_prs.yml @@ -27,83 +27,26 @@ jobs: runs-on: ubuntu-latest strategy: fail-fast: ${{ inputs.fail-fast }} - # If you add a new repository here, also add it to config/repositories.yaml matrix: + # This should be the list of repository names from config/repositories.yaml: + # yq '.repositories | map(.name)' config/repositories.yaml repository: - - name: airflow-operator - pretty_string: Apache Airflow - product_string: airflow - url: stackabletech/airflow-operator.git - - name: commons-operator - include_productconfig: false - has_product: false - pretty_string: Stackable Commons - product_string: commons - url: stackabletech/commons-operator.git - - name: druid-operator - pretty_string: Apache Druid - product_string: druid - url: stackabletech/druid-operator.git - - name: hbase-operator - pretty_string: Apache HBase - product_string: hbase - url: stackabletech/hbase-operator.git - - name: hdfs-operator - pretty_string: Apache HDFS - product_string: hdfs - url: stackabletech/hdfs-operator.git - - name: hive-operator - pretty_string: Apache Hive - product_string: hive - url: stackabletech/hive-operator.git - - name: kafka-operator - pretty_string: Apache Kafka - product_string: kafka - url: stackabletech/kafka-operator.git - - name: nifi-operator - pretty_string: Apache NiFi - product_string: nifi - url: stackabletech/nifi-operator.git - - name: listener-operator - include_productconfig: false - has_product: false - pretty_string: Stackable Listener Operator - product_string: listener-operator - run_as: custom - url: stackabletech/listener-operator.git - - name: opa-operator - extra_crates: - - stackable-opa-bundle-builder - pretty_string: OpenPolicyAgent - product_string: opa - url: stackabletech/opa-operator.git - - name: opensearch-operator - pretty_string: OpenSearch - product_string: opensearch - url: stackabletech/opensearch-operator.git - - name: secret-operator - include_productconfig: false - has_product: false - pretty_string: Stackable Secret Operator - product_string: secret-operator - run_as: custom - url: stackabletech/secret-operator.git - - name: spark-k8s-operator - pretty_string: Apache Spark-on-Kubernetes - product_string: spark-k8s - url: stackabletech/spark-k8s-operator.git - - name: superset-operator - pretty_string: Apache Superset - product_string: superset - url: stackabletech/superset-operator.git - - name: trino-operator - pretty_string: Trino - product_string: trino - url: stackabletech/trino-operator.git - - name: zookeeper-operator - pretty_string: Apache ZooKeeper - product_string: zookeeper - url: stackabletech/zookeeper-operator.git + - airflow-operator + - commons-operator + - druid-operator + - hbase-operator + - hdfs-operator + - hive-operator + - kafka-operator + - nifi-operator + - listener-operator + - opa-operator + - opensearch-operator + - secret-operator + - spark-k8s-operator + - superset-operator + - trino-operator + - zookeeper-operator steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -148,24 +91,14 @@ jobs: echo "AUTHOR=stackabletech/developers" echo "REASON=Daily run triggered" >> "$GITHUB_ENV" - - name: Overwrite repositories.yaml for ${{ matrix.repository.name }} - run: | - cat >config/repositories.yaml < vars.json + echo '{}' | jq '{commit_hash: $ENV.GITHUB_SHA, author: $ENV.AUTHOR, reason: $ENV.REASON, custom_pr_title: $ENV.CUSTOM_PR_TITLE, base_dir: $pwd, gh_access_token: $ENV.GH_ACCESS_TOKEN, shard_repositories: [$ENV.REPOSITORY]} | with_entries(select(.value != null and .value != ""))' --arg pwd "$(pwd)" > vars.json ansible-playbook playbook/playbook.yaml --extra-vars "@vars.json" $DRY_RUN_FLAGS diff --git a/config/repositories.yaml b/config/repositories.yaml index 0a6e3228..879b81f9 100644 --- a/config/repositories.yaml +++ b/config/repositories.yaml @@ -1,7 +1,5 @@ -# This file exists just for the purpose of running test.sh locally. -# The GH workflow generate_prs.yml will overwrite it with the contents if the -# matrix run. -# If you add a new repository here, you also need to add it to that workflow file. +# If you add a new repository here, you also need to add it to the list of +# repositories in ../.github/workflows/generate_prs.yaml. --- repositories: - name: airflow-operator diff --git a/playbook/playbook.yaml b/playbook/playbook.yaml index 49449f63..1e98e590 100644 --- a/playbook/playbook.yaml +++ b/playbook/playbook.yaml @@ -24,6 +24,7 @@ - name: Update repositories from templates include_tasks: "update_repo.yaml" + when: shard_repositories is undefined or operator.name in shard_repositories with_items: "{{ repositories }}" loop_control: loop_var: operator diff --git a/test.sh b/test.sh index e350439e..75769c2d 100755 --- a/test.sh +++ b/test.sh @@ -2,4 +2,4 @@ rm -fr work || true mkdir -p work -ansible-playbook playbook/playbook.yaml --tags "local" --extra-vars "gh_access_token=unneeded base_dir=$(pwd) commit_hash=12345 reason='original message'" +ansible-playbook playbook/playbook.yaml --tags "local" --extra-vars "gh_access_token=unneeded base_dir=$(pwd) commit_hash=12345 reason='original message'" "$@" From 2c11d823e70797e192bd25efe82bd1a25b4f77b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Wed, 11 Jun 2025 12:49:22 +0200 Subject: [PATCH 3/6] Try to fix GH workflow syntax, maybe? --- .github/workflows/generate_prs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generate_prs.yml b/.github/workflows/generate_prs.yml index 63dc9b93..ba2cf034 100644 --- a/.github/workflows/generate_prs.yml +++ b/.github/workflows/generate_prs.yml @@ -97,7 +97,7 @@ jobs: GH_ACCESS_TOKEN: ${{ secrets.gh_access_token }} REPOSITORY: ${{ matrix.repository }} # tags local excludes all actions that actually generate PRs - DRY_RUN_FLAGS: ${{ inputs.dry-run && "--tags local" || "" }} + DRY_RUN_FLAGS: ${{ inputs.dry-run && '--tags local' || '' }} run: | # Funnel via JSON to ensure that values are escaped properly echo '{}' | jq '{commit_hash: $ENV.GITHUB_SHA, author: $ENV.AUTHOR, reason: $ENV.REASON, custom_pr_title: $ENV.CUSTOM_PR_TITLE, base_dir: $pwd, gh_access_token: $ENV.GH_ACCESS_TOKEN, shard_repositories: [$ENV.REPOSITORY]} | with_entries(select(.value != null and .value != ""))' --arg pwd "$(pwd)" > vars.json From ba7b0057964e3db446743ea172142d98a2bbfd1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Wed, 11 Jun 2025 12:55:53 +0200 Subject: [PATCH 4/6] Document sharding for the local test setup --- README.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.adoc b/README.adoc index 5cf3b4d4..10163049 100644 --- a/README.adoc +++ b/README.adoc @@ -85,7 +85,7 @@ Consult the section above to learn more about the structure of the template. === Test changes locally -1. Run the `test.sh` script. +1. Run the `test.sh` script. To limit the run to a single operator, add `--extra-vars "shard_repositories=['nifi-operator']"` (or your choice of operator). It will automatically delete and recreate a `work` directory. 2. The changes can be examined with `git status`. When the pull request is later merged into the `main` branch then pull requests with these changes will be created automatically. From 49591344dd093e4c60106785b24778db0370fcdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Wed, 11 Jun 2025 12:57:30 +0200 Subject: [PATCH 5/6] .yml is not .yaml --- config/repositories.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/repositories.yaml b/config/repositories.yaml index 879b81f9..ddea14ca 100644 --- a/config/repositories.yaml +++ b/config/repositories.yaml @@ -1,5 +1,5 @@ # If you add a new repository here, you also need to add it to the list of -# repositories in ../.github/workflows/generate_prs.yaml. +# repositories in ../.github/workflows/generate_prs.yml. --- repositories: - name: airflow-operator From 3a907662c088b22c500934a08c570db495430223 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Wed, 11 Jun 2025 13:10:44 +0200 Subject: [PATCH 6/6] Satisfy pre-commit --- .github/workflows/generate_prs.yml | 8 ++++++-- test.sh | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/generate_prs.yml b/.github/workflows/generate_prs.yml index ba2cf034..60b7aee9 100644 --- a/.github/workflows/generate_prs.yml +++ b/.github/workflows/generate_prs.yml @@ -97,8 +97,12 @@ jobs: GH_ACCESS_TOKEN: ${{ secrets.gh_access_token }} REPOSITORY: ${{ matrix.repository }} # tags local excludes all actions that actually generate PRs - DRY_RUN_FLAGS: ${{ inputs.dry-run && '--tags local' || '' }} + DRY_RUN_FLAGS: ${{ inputs.dry-run && '--tags=local' || '' }} + # shellsheck disable=SC2086 run: | - # Funnel via JSON to ensure that values are escaped properly echo '{}' | jq '{commit_hash: $ENV.GITHUB_SHA, author: $ENV.AUTHOR, reason: $ENV.REASON, custom_pr_title: $ENV.CUSTOM_PR_TITLE, base_dir: $pwd, gh_access_token: $ENV.GH_ACCESS_TOKEN, shard_repositories: [$ENV.REPOSITORY]} | with_entries(select(.value != null and .value != ""))' --arg pwd "$(pwd)" > vars.json + # $DRY_RUN_FLAGS is intentionally not quoted, since we need to avoid + # passing an argument if we're not in dry mode. + # This is safe, since it always has one of two hard-coded values. + # shellcheck disable=SC2086 ansible-playbook playbook/playbook.yaml --extra-vars "@vars.json" $DRY_RUN_FLAGS diff --git a/test.sh b/test.sh index 75769c2d..8f7e5ec1 100755 --- a/test.sh +++ b/test.sh @@ -2,4 +2,4 @@ rm -fr work || true mkdir -p work -ansible-playbook playbook/playbook.yaml --tags "local" --extra-vars "gh_access_token=unneeded base_dir=$(pwd) commit_hash=12345 reason='original message'" "$@" +ansible-playbook playbook/playbook.yaml --tags="local" --extra-vars "gh_access_token=unneeded base_dir=$(pwd) commit_hash=12345 reason='original message'" "$@"