diff --git a/.github/renovate.json5 b/.github/renovate.json5 index f43c59f..1bc481a 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -1,42 +1,99 @@ { - "$schema": "https://docs.renovatebot.com/renovate-schema.json", - "extends": [ - "config:recommended", - "helpers:pinGitHubActionDigests", - ":semanticCommits" + $schema: 'https://docs.renovatebot.com/renovate-schema.json', + extends: [ + 'config:recommended', + 'helpers:pinGitHubActionDigests', + ':semanticCommits', ], - "rebaseWhen": "conflicted", - "prConcurrentLimit": 5, - "baseBranches": ["main"], - "labels": ["automated"], - "customManagers": [ + rebaseWhen: 'auto', + rebaseLabel: 'rebase', + prConcurrentLimit: 5, + autoApprove: true, + automerge: true, + automergeType: 'pr', + baseBranches: [ + 'main', + ], + labels: [ + 'automated', + 'run-e2e-tests', + ], + ignorePaths: [ // default renovate ignorePaths without '**/examples/**' + '**/node_modules/**', + '**/bower_components/**', + '**/vendor/**', + '**/__tests__/**', + '**/test/**', + '**/tests/**', + '**/__fixtures__/**' + ], + crossplane: { + fileMatch: ['(^|/)examples/.*\\.ya?ml$'] + }, + packageRules: [ + { + matchFileNames: [ + '.github/**', + ], + groupName: 'github-actions dependencies', + }, { - "customType": "regex", - "description": "Bump up version in the Makefile", - "fileMatch": ["^Makefile$"], - "matchStrings": [ - "UP_VERSION = (?.*?)\\n" - ], - "datasourceTemplate": "github-releases", - "depNameTemplate": "upbound/up", - }, { - "customType": "regex", - "description": "Bump uptest version in the Makefile", - "fileMatch": ["^Makefile$"], - "matchStrings": [ - "UPTEST_VERSION = (?.*?)\\n" - ], - "datasourceTemplate": "github-releases", - "depNameTemplate": "upbound/uptest", - }, { - "customType": "regex", - "description": "Bump providers/functions/configurations in crossplane.yaml", - "fileMatch": ["crossplane.yaml"], - "matchStrings": [ - "#\\s*renovate:\\s*datasource=(?[^\\s]+)\\s+depName=(?[^\\s]+)\\s*\\n\\s*version:\\s*\"(?[^\"]+)\"" - ], - "datasourceTemplate": "{{{datasource}}}", - "depNameTemplate": "{{{depName}}}", - } + matchFileNames: [ + 'crossplane.yaml', + ], + groupName: 'crossplane dependencies', + }, + { + matchFileNames: [ + 'Makefile', + ], + groupName: 'Makefile dependencies', + }, + { + matchManagers: ['crossplane'], + matchFileNames: ['examples/**'], + groupName: 'examples' + }, + ], + customManagers: [ + { + customType: 'regex', + description: 'Bump up version in the Makefile', + fileMatch: [ + '^Makefile$', + ], + matchStrings: [ + 'UP_VERSION = (?.*?)\\n', + ], + datasourceTemplate: 'github-releases', + depNameTemplate: 'upbound/up', + }, + { + customType: 'regex', + description: 'Bump uptest version in the Makefile', + fileMatch: [ + '^Makefile$', + ], + matchStrings: [ + 'UPTEST_VERSION = (?.*?)\\n', + ], + datasourceTemplate: 'github-releases', + depNameTemplate: 'upbound/uptest', + }, + { + customType: 'regex', + description: 'Bump providers/functions/configurations in crossplane.yaml', + fileMatch: [ + 'crossplane.yaml', + ], + matchStrings: [ + '#\\s*renovate:\\s*datasource=(?[^\\s]+)\\s+depName=(?[^\\s]+)\\s*\\n\\s*version:\\s*"(?[^"]+)"', + ], + datasourceTemplate: '{{{datasource}}}', + depNameTemplate: '{{{depName}}}', + }, ], + "git-submodules": { + "enabled": true + } } diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c117684..1915b2e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -14,37 +14,32 @@ env: jobs: detect-noop: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 outputs: noop: ${{ steps.noop.outputs.should_skip }} steps: - name: Detect No-op Changes id: noop - uses: fkirc/skip-duplicate-actions@0c0fd7dfd27f1de7871e98791077b9ae5f1d8757 # v2.2.0 + uses: fkirc/skip-duplicate-actions@f75f66ce1886f00957d99748a42c724f4330bdcf # v5.3.1 with: github_token: ${{ secrets.GITHUB_TOKEN }} paths_ignore: '["**.md", "**.png", "**.jpg"]' do_not_skip: '["workflow_dispatch", "schedule", "push"]' publish-artifacts: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: detect-noop if: needs.detect-noop.outputs.noop != 'true' steps: - - name: Setup QEMU - uses: docker/setup-qemu-action@v1 - with: - platforms: all - - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3 with: version: ${{ env.DOCKER_BUILDX_VERSION }} install: true - name: Checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 with: submodules: true @@ -59,13 +54,13 @@ jobs: BUILD_ARGS: "--load" - name: Publish Artifacts to GitHub - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: output path: _output/** - name: Login to Upbound - uses: docker/login-action@v1 + uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3 if: env.XPKG_ACCESS_ID != '' with: registry: xpkg.upbound.io diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index c1ccfd4..885d97a 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -3,10 +3,11 @@ name: End to End Testing on: issue_comment: types: [created] - + pull_request: + types: [labeled] jobs: e2e: - uses: upbound/uptest/.github/workflows/pr-comment-trigger.yml@main + uses: upbound/official-providers-ci/.github/workflows/pr-comment-trigger.yml@main with: package-type: configuration secrets: diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml index 48d7764..1d2de5a 100644 --- a/.github/workflows/tag.yml +++ b/.github/workflows/tag.yml @@ -12,14 +12,14 @@ on: jobs: create-tag: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 - name: Create Tag - uses: negz/create-tag@v1 + uses: negz/create-tag@39bae1e0932567a58c20dea5a1a0d18358503320 # v1 with: version: ${{ github.event.inputs.version }} message: ${{ github.event.inputs.message }} diff --git a/.github/workflows/yamllint.yaml b/.github/workflows/yamllint.yaml index 16a78d3..a796551 100644 --- a/.github/workflows/yamllint.yaml +++ b/.github/workflows/yamllint.yaml @@ -5,9 +5,9 @@ jobs: name: runner / yamllint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 - name: yamllint - uses: reviewdog/action-yamllint@v1.9.0 + uses: reviewdog/action-yamllint@f01d8a48fd8d89f89895499fca2cff09f9e9e8c0 # v1.21.0 with: reporter: github-pr-review filter_mode: nofilter diff --git a/.gitmodules b/.gitmodules index c2fad47..394e1ee 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "build"] path = build - url = https://github.com/upbound/build + url = https://github.com/crossplane/build.git diff --git a/Makefile b/Makefile index 1ba5d07..2e23f27 100644 --- a/Makefile +++ b/Makefile @@ -1,80 +1,250 @@ -# Project Setup -PROJECT_NAME := platform-ref-aws-castai -PROJECT_REPO := github.com/upbound/$(PROJECT_NAME) +# ==================================================================================== +# Crossplane Configuration Package Makefile +# ==================================================================================== -# NOTE(hasheddan): the platform is insignificant here as Configuration package -# images are not architecture-specific. We constrain to one platform to avoid -# needlessly pushing a multi-arch image. -PLATFORMS ?= linux_amd64 --include build/makelib/common.mk +# USAGE DOCUMENTATION +# ==================================================================================== +# +# This is a generic Makefile to be used across repositories building Crossplane +# configuration packages. It provides a comprehensive set of targets for development, +# testing, and deployment. +# +# PROJECT CONFIGURATION +# ------------------- +# Create a project.mk file in your repository to configure project-specific settings. +# Required variables: +# - PROJECT_NAME: Name of your Crossplane configuration package +# +# Example project.mk: +# PROJECT_NAME = custom-config +# UPTEST_DEFAULT_TIMEOUT = 3600s +# UPTEST_SKIP_IMPORT = true +# +# PRIMARY TARGETS +# -------------- +# +# Development Tools: +# ----------------- +# - `yamllint` +# Runs yamllint recursively on all files in the `api` folder to ensure YAML +# quality and consistency +# +# - `check-examples` +# Validates consistency between example configurations and dependencies: +# - Compares Function package versions in examples/ against crossplane.yaml +# - Ensures all Function versions in examples match dependency declarations +# - Helps prevent version mismatches that could cause deployment issues +# Example errors: +# - Example using function-foo:v1.2.0 while crossplane.yaml specifies v1.1.0 +# - Missing Function dependencies in crossplane.yaml that are used in examples +# Usage: Run before committing changes to ensure example validity +# +# Rendering and Validation: +# ----------------- +# - `render` +# Renders the composition output for rapid feedback during template development. +# Requirements: +# - Claims must have these annotations: +# render.crossplane.io/composition-path: apis/pat/composition.yaml +# render.crossplane.io/function-path: examples/functions.yaml +# Note: This only populates the cache. Use `render.show` to view output. +# +# - `render.show` +# Displays the rendered YAML output. Useful for: +# - Manual validation +# - Piping to validation tools, e.g.: +# make render.show | crossplane beta validate crossplane.yaml - +# +# Testing: +# ----------------- +# - `render.test` +# Executes kcl-unit tests on rendered manifests. Tests should be: +# - Located in the `test` folder +# - Written as standard kcl-tests +# This ensures the rendered output meets expected specifications. +# +# - `e2e` +# Comprehensive end-to-end testing, including: +# - Cluster creation +# - Configuration setup +# - Testing create, import, and delete operations +# +# Cloud Provider Requirements: +# For configurations creating cloud provider resources, set: +# UPTEST_CLOUD_CREDENTIALS - Provider-specific credentials: +# - AWS: export UPTEST_CLOUD_CREDENTIALS=$(cat ~/.aws/credentials) +# - GCP: export UPTEST_CLOUD_CREDENTIALS=$(cat gcp-sa.json) +# - Azure: export UPTEST_CLOUD_CREDENTIALS=$(cat azure.json) +# +# Configuration Options: +# - UPTEST_SKIP_DELETE (default: false) +# Skip deletion testing of created resources +# - UPTEST_SKIP_UPDATE (default: false) +# Skip testing of claim updates +# - UPTEST_SKIP_IMPORT (default: false) +# Skip testing of resource imports +# +# Example Usage: +# make e2e UPTEST_SKIP_DELETE=true +# +# LANGUAGE-SPECIFIC OPTIONS +# ------------------------ +# +# KCL Support: +# - KCL_COMPOSITION_PATH +# Path to the KCL file generating composition.yaml +# Default: apis/kcl/generate.k +# +# NOTE: The platform setting is constrained to linux_amd64 as Configuration package +# images are not architecture-specific. This avoids unnecessary multi-arch image +# generation. # ==================================================================================== -# Setup Kubernetes tools +# Project Configuration +# ==================================================================================== + +# Include project.mk for project specific settings +include project.mk + +ifndef PROJECT_NAME + $(error PROJECT_NAME is not set. Please create `project.mk` and set it there.) +endif -UP_VERSION = v0.21.0 +# Project Configuration +# ------------------ +PROJECT_REPO := github.com/upbound/$(PROJECT_NAME) +PLATFORMS ?= linux_amd64 + +# Tool Versions +# ------------------ +UP_VERSION = v0.41.0 UP_CHANNEL = stable -UPTEST_VERSION = v0.9.0 +CROSSPLANE_CLI_VERSION = v1.18.0 +CROSSPLANE_VERSION = v1.18.0-up.1 +UPTEST_VERSION = v1.2.0 + +# Crossplane Configuration +# ------------------ +CROSSPLANE_CHART_REPO = https://charts.upbound.io/stable +CROSSPLANE_CHART_NAME = universal-crossplane +CROSSPLANE_NAMESPACE = upbound-system +CROSSPLANE_ARGS = "--enable-usages" +KIND_CLUSTER_NAME ?= uptest-$(PROJECT_NAME) --include build/makelib/k8s_tools.mk -# ==================================================================================== -# Setup XPKG +# XPKG Configuration +# ------------------ XPKG_DIR = $(shell pwd) -XPKG_IGNORE = .github/workflows/*.yaml,.github/workflows/*.yml,examples/*.yaml,.work/uptest-datasource.yaml +XPKG_IGNORE ?= .github/workflows/*.yaml,.github/workflows/*.yml,examples/*.yaml,.work/uptest-datasource.yaml,.cache/render/* XPKG_REG_ORGS ?= xpkg.upbound.io/upbound -# NOTE(hasheddan): skip promoting on xpkg.upbound.io as channel tags are -# inferred. +# NOTE: Skip promoting on xpkg.upbound.io as channel tags are inferred XPKG_REG_ORGS_NO_PROMOTE ?= xpkg.upbound.io/upbound XPKGS = $(PROJECT_NAME) --include build/makelib/xpkg.mk -CROSSPLANE_NAMESPACE = upbound-system -CROSSPLANE_ARGS = "--enable-usages" +# Testing Configuration +# ------------------ +UPTEST_LOCAL_DEPLOY_TARGET = local.xpkg.deploy.configuration.$(PROJECT_NAME) +UPTEST_DEFAULT_TIMEOUT ?= 2400s + +# KCL Configuration +# ------------------ +KCL_COMPOSITION_PATH ?= apis/kcl/generate.k +LANG_KCL := $(shell find ./apis -type f -name '*.k') + +# Overwrite example list if it is set by CI +# For example with comment `/test-examples="path/to/example.yaml"` +ifdef UPTEST_EXAMPLE_LIST + UPTEST_INPUT_MANIFESTS=$(UPTEST_EXAMPLE_LIST) +endif + +# Include makelib files +# ------------------ +-include build/makelib/common.mk +-include build/makelib/k8s_tools.mk +-include build/makelib/xpkg.mk -include build/makelib/local.xpkg.mk -include build/makelib/controlplane.mk +-include build/makelib/uptest.mk # ==================================================================================== # Targets +# ==================================================================================== -# run `make help` to see the targets and options - +# Initial Setup +# ------------------ # We want submodules to be set up the first time `make` is run. # We manage the build/ folder and its Makefiles as a submodule. # The first time `make` is run, the includes of build/*.mk files will # all fail, and this target will be run. The next time, the default as defined # by the includes will be run instead. -fallthrough: submodules +fallthrough: submodules ## Initial setup and submodule initialization @echo Initial setup complete. Running make again . . . @make -# Update the submodules, such as the common build scripts. -submodules: +submodules: ## Update the submodules, including common build scripts @git submodule sync @git submodule update --init --recursive -# We must ensure up is installed in tool cache prior to build as including the k8s_tools machinery prior to the xpkg -# machinery sets UP to point to tool cache. -build.init: $(UP) - -# ==================================================================================== -# End to End Testing - -# This target requires the following environment variables to be set: -# $ export UPTEST_CLOUD_CREDENTIALS=$(echo "AWS='$(cat ~/.aws/credentials)'\nAZURE='$(cat ~/.azure/credentials.json)'\nGCP='$(cat ~/.gcloud/credentials.json)") -uptest: $(UPTEST) $(KUBECTL) $(KUTTL) - @$(INFO) running automated tests - @KUBECTL=$(KUBECTL) KUTTL=$(KUTTL) CROSSPLANE_NAMESPACE=$(CROSSPLANE_NAMESPACE) $(UPTEST) e2e examples/cluster.yaml --data-source="${UPTEST_DATASOURCE_PATH}" --setup-script=test/setup.sh --default-timeout=4800 || $(FAIL) - @$(OK) running automated tests - -# This target requires the following environment variables to be set: -# $ export UPTEST_CLOUD_CREDENTIALS=$(echo "AWS='$(cat ~/.aws/credentials)'\nAZURE='$(cat ~/.azure/credentials.json)'\nGCP='$(cat ~/.gcloud/credentials.json)") -e2e: build controlplane.up local.xpkg.deploy.configuration.$(PROJECT_NAME) uptest - - -render: - crossplane beta render examples/cluster.yaml apis/composition.yaml examples/functions.yaml -r -yamllint: - @$(INFO) running yamllint - @yamllint ./apis || $(FAIL) - @$(OK) running yamllint - -.PHONY: uptest e2e render yamllint +# Build Targets +# ------------------ +# We must ensure up is installed in tool cache prior to build as including the k8s_tools +# machinery prior to the xpkg machinery sets UP to point to tool cache. +build.init: $(UP) ## Initialize build requirements + +# KCL Targets +# ------------------ +ifdef LANG_KCL +kcl: $(KCL) ## Generate KCL-based Composition + @$(INFO) Generating kcl composition + @$(KCL) $(KCL_COMPOSITION_PATH) 1>/dev/null + @$(OK) Generated kcl composition + +render: kcl ## Render the composition output +build.init: kcl +.PHONY: kcl +endif + +# Testing Targets +# ------------------ +render.test: $(CROSSPLANE_CLI) $(KCL) render ## Test rendered compositions + @for RENDERED_COMPOSITION in $$(find .cache/render -maxdepth 1 -type f -name '*.yaml'); do \ + $(INFO) "Testing $${RENDERED_COMPOSITION}"; \ + export RENDERED_COMPOSITION; \ + $(KCL) test test/ && \ + $(OK) "Success testing \"$${RENDERED_COMPOSITION}\"!" || \ + ($(ERR) "Failure testing \"$${RENDERED_COMPOSITION}\"!" && exit 1); \ + done + +check-examples: ## Validate package versions in examples match dependencies + @$(INFO) Checking if package versions in dependencies match examples + @FN_EXAMPLES=$$( \ + find examples -type f -name "*.yaml" | \ + xargs $(YQ) -r -o=json 'select(.kind == "Function" and (.apiVersion | test("^pkg.crossplane.io/"))) | .spec.package' | \ + sort -u); \ + FN_DEPS=$$( \ + $(YQ) '.spec.dependsOn[] | select(.function != null) | (.function + ":" + .version)' crossplane.yaml | \ + sort -u \ + ); \ + if [ "$$FN_EXAMPLES" != "$$FN_DEPS" ]; then \ + echo "Function package versions in examples and in crossplane.yaml don't match!"; \ + echo "" ; \ + echo "Versions in dependencies:"; \ + echo "---" ; \ + echo "$$FN_DEPS"; \ + echo "" ; \ + echo "Versions in examples:"; \ + echo "---" ; \ + echo "$$FN_EXAMPLES"; \ + exit 1; \ + fi; + @$(OK) Package versions are sane + +# Help Targets +# ------------------ +help: help.local ## Display this help message + +help.local: + @echo "Available targets:" + @echo + @grep -E '^[a-zA-Z_-]+.*:.*?## .*$$' Makefile | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + +.PHONY: uptest e2e render yamllint help help.local check-examples render.test