Test Fleet in Rancher #267
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Test fleet in Rancher with MC tests | |
| name: Test Fleet in Rancher | |
| on: | |
| schedule: | |
| # Run everyday at 1:00 PM | |
| - cron: '0 13 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| charts_repo: | |
| description: Repository from which to source Fleet charts | |
| required: true | |
| default: "fleetrepoci/charts" | |
| type: string | |
| charts_branch: | |
| description: Branch from which to source Fleet charts | |
| required: true | |
| default: "dev-v2.12" | |
| type: string | |
| push: | |
| tags: [ 'v*' ] | |
| paths-ignore: | |
| - '*.md' | |
| env: | |
| GOARCH: amd64 | |
| CGO_ENABLED: 0 | |
| SETUP_K3D_VERSION: 'v5.8.3' | |
| SETUP_K3S_VERSION: 'v1.33.4-k3s1' | |
| jobs: | |
| release-against-test-charts: | |
| uses: ./.github/workflows/release-against-test-charts.yml | |
| with: | |
| charts_repo: ${{ github.event.inputs.charts_repo }} | |
| charts_base_branch: ${{ github.event.inputs.charts_branch }} | |
| secrets: inherit # needed to enable access to repo secrets from the called workflow | |
| rancher-integration: | |
| runs-on: runs-on,runner=16cpu-linux-x64,run-id=${{ github.run_id }} | |
| needs: release-against-test-charts | |
| steps: | |
| - | |
| name: Add paths into PATH | |
| run: | | |
| export PATH=~/.local/bin:$PATH | |
| mkdir -p ~/.local/bin | |
| - | |
| name: Checkout | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 | |
| with: | |
| fetch-depth: 0 | |
| - | |
| name: Install Go | |
| uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6 | |
| with: | |
| go-version-file: 'go.mod' | |
| check-latest: true | |
| - | |
| name: Install Ginkgo CLI | |
| run: go install github.com/onsi/ginkgo/v2/ginkgo | |
| - | |
| name: Determine cache key | |
| id: cache-key | |
| run: | | |
| DAY_OF_YEAR=$(date +%j) | |
| if [ $(($DAY_OF_YEAR % 28)) -eq 0 ]; then | |
| echo "value=$(date +%Y-%m-%d)" >> $GITHUB_OUTPUT | |
| else | |
| echo "value=latest" >> $GITHUB_OUTPUT | |
| fi | |
| - | |
| name: Cache crust-gather CLI | |
| id: cache-crust | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | |
| with: | |
| path: ~/.local/bin/crust-gather | |
| key: ${{ runner.os }}-crust-gather-${{ steps.cache-key.outputs.value }} | |
| restore-keys: | | |
| ${{ runner.os }}-crust-gather- | |
| - | |
| name: Install crust-gather CLI | |
| run: | | |
| if [ "${{ steps.cache-crust.outputs.cache-hit }}" != "true" ]; then | |
| echo "Cache not found, downloading from source" | |
| mkdir -p ~/.local/bin | |
| if curl -sSfL https://github.com/crust-gather/crust-gather/raw/main/install.sh | sh -s -- --yes; then | |
| # Cache the binary for future runs | |
| if [ ! -f ~/.local/bin/crust-gather ]; then | |
| which crust-gather && cp $(which crust-gather) ~/.local/bin/ | |
| fi | |
| else | |
| echo "Failed to download crust-gather" | |
| exit 1 | |
| fi | |
| else | |
| echo "Using cached crust-gather CLI" | |
| chmod +x ~/.local/bin/crust-gather | |
| sudo ln -sf ~/.local/bin/crust-gather /usr/local/bin/ | |
| fi | |
| - | |
| name: Set up build cache | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | |
| id: rancher-cli-cache | |
| with: | |
| path: ~/.local/bin | |
| key: ${{ runner.os }}-rancher-cli | |
| - | |
| name: Install Dependencies | |
| run: | | |
| curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" | |
| mv kubectl ~/.local/bin/ | |
| chmod +x ~/.local/bin/kubectl | |
| kubectl version --client | |
| curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | |
| chmod 700 get_helm.sh | |
| HELM_INSTALL_DIR=~/.local/bin ./get_helm.sh | |
| helm version | |
| sudo apt-get update && sudo apt install -y wget | |
| - | |
| name: Install Rancher CLI | |
| if: steps.rancher-cli-cache.outputs.cache-hit != 'true' | |
| run: | | |
| latest_version=$(curl -ILs -o /dev/null -w %{url_effective} github.com/rancher/cli/releases/latest | sed 's,.*/,,') | |
| wget -q https://github.com/rancher/cli/releases/download/$latest_version/rancher-linux-amd64-$latest_version.tar.gz | |
| tar -xz --strip-components=2 -f rancher-linux-amd64-$latest_version.tar.gz -C ~/.local/bin | |
| export PATH=~/.local/bin:$PATH | |
| rancher --version | |
| - | |
| name: Install k3d | |
| run: curl --silent --fail https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG=${{ env.SETUP_K3D_VERSION }} bash | |
| - | |
| name: Set up k3d control-plane cluster | |
| run: | | |
| k3d cluster create upstream --wait \ | |
| -p "80:80@agent:0:direct" \ | |
| -p "443:443@agent:0:direct" \ | |
| --api-port 6443 \ | |
| --agents 1 \ | |
| --k3s-arg '--kubelet-arg=eviction-hard=imagefs.available<1%,nodefs.available<1%@agent:*' \ | |
| --k3s-arg '--kubelet-arg=eviction-minimum-reclaim=imagefs.available=1%,nodefs.available=1%@agent:*' \ | |
| --network "nw01" \ | |
| --image docker.io/rancher/k3s:${{ env.SETUP_K3S_VERSION }} | |
| - | |
| name: Set up k3d downstream cluster | |
| run: | | |
| k3d cluster create downstream --wait \ | |
| -p "81:80@agent:0:direct" \ | |
| -p "444:443@agent:0:direct" \ | |
| --api-port 6644 \ | |
| --agents 1 \ | |
| --k3s-arg '--kubelet-arg=eviction-hard=imagefs.available<1%,nodefs.available<1%@agent:*' \ | |
| --k3s-arg '--kubelet-arg=eviction-minimum-reclaim=imagefs.available=1%,nodefs.available=1%@agent:*' \ | |
| --network "nw01" \ | |
| --image docker.io/rancher/k3s:${{ env.SETUP_K3S_VERSION }} | |
| - | |
| name: Set up latest Rancher | |
| run: | | |
| set -x | |
| charts_repo=${{ needs.release-against-test-charts.outputs.charts_repo }} | |
| charts_branch=${{ needs.release-against-test-charts.outputs.target_branch }} | |
| fleet_version=999.9.9+up9.9.9 | |
| kubectl config use-context k3d-upstream | |
| until kubectl get service -n kube-system traefik -o jsonpath='{.status.loadBalancer.ingress[0].ip}'; do sleep 3; done | |
| ip=$(kubectl get service -n kube-system traefik -o jsonpath='{.status.loadBalancer.ingress[0].ip}') | |
| helm repo add jetstack https://charts.jetstack.io | |
| helm install cert-manager jetstack/cert-manager \ | |
| --namespace cert-manager \ | |
| --create-namespace \ | |
| --set crds.enabled=true \ | |
| --set extraArgs[0]=--enable-certificate-owner-ref=true | |
| kubectl wait --for=condition=Available deployment --timeout=2m -n cert-manager --all | |
| helm repo add rancher-latest https://releases.rancher.com/server-charts/latest | |
| # set CATTLE_SERVER_URL and CATTLE_BOOTSTRAP_PASSWORD to get rancher out of "bootstrap" mode | |
| helm upgrade rancher rancher-latest/rancher \ | |
| --install --wait \ | |
| --devel \ | |
| --create-namespace \ | |
| --namespace cattle-system \ | |
| --set replicas=1 \ | |
| --set hostname=$ip.sslip.io \ | |
| --set bootstrapPassword=admin \ | |
| --set agentTLSMode=system-store \ | |
| --set "extraEnv[0].name=CATTLE_CHART_DEFAULT_URL" \ | |
| --set "extraEnv[0].value=https://github.com/${charts_repo}" \ | |
| --set "extraEnv[1].name=CATTLE_CHART_DEFAULT_BRANCH" \ | |
| --set "extraEnv[1].value=${charts_branch}" \ | |
| --set "extraEnv[2].name=CATTLE_FLEET_VERSION" \ | |
| --set "extraEnv[2].value=${fleet_version}" \ | |
| --set "extraEnv[3].name=CATTLE_SERVER_URL" \ | |
| --set "extraEnv[3].value=https://$ip.sslip.io" \ | |
| # wait for deployment of rancher | |
| kubectl -n cattle-system rollout status deploy/rancher | |
| # wait for rancher to create fleet namespace, deployment and controller | |
| until kubectl get deployments -n cattle-fleet-system | grep -q "fleet"; do sleep 3; done | |
| kubectl -n cattle-fleet-system rollout status deploy/fleet-controller | |
| until kubectl get bundles -n fleet-local | grep -q "fleet-agent-local.*1/1"; do sleep 3; done | |
| # wait for deployment of webhook, needed by tests | |
| until kubectl get deployments -n cattle-system | grep -q "rancher-webhook"; do sleep 3; done | |
| kubectl -n cattle-system rollout status deploy/rancher-webhook | |
| helm list -A | |
| - | |
| name: Checkout rancher/rancher | |
| uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 | |
| with: | |
| fetch-depth: 0 | |
| repository: rancher/rancher | |
| ref: main | |
| path: rancher | |
| - | |
| name: Rancher Provisioning Tests | |
| run: | | |
| cd rancher | |
| k8s_version=${{ env.SETUP_K3S_VERSION }} | |
| export SOME_K8S_VERSION=${k8s_version/-/+} | |
| go test -v -count=1 ./tests/v2prov/tests/fleet | |
| - | |
| name: Register Rancher's downstream clusters | |
| run: | | |
| ip=$(kubectl get service -n kube-system traefik -o jsonpath='{.status.loadBalancer.ingress[0].ip}') | |
| export PATH=~/.local/bin:$PATH | |
| export public_hostname=$ip.sslip.io | |
| ./.github/scripts/wait-for-loadbalancer.sh | |
| ./.github/scripts/register-downstream-clusters.sh | |
| # wait for cluster to settle | |
| sleep 30 | |
| ./.github/scripts/label-downstream-cluster.sh | |
| - | |
| name: E2E tests for examples | |
| env: | |
| FLEET_E2E_NS: fleet-local | |
| FLEET_E2E_NS_DOWNSTREAM: fleet-default | |
| run: | | |
| kubectl config use-context k3d-upstream | |
| ginkgo --github-output --trace e2e/acceptance/single-cluster-examples | |
| export CI_REGISTERED_CLUSTER=$(kubectl get clusters.fleet.cattle.io -n $FLEET_E2E_NS_DOWNSTREAM -o jsonpath='{..name}') | |
| ginkgo --github-output --trace e2e/multi-cluster | |
| - | |
| name: Dump Failed Downstream Environment | |
| if: failure() | |
| run: | | |
| kubectl config use-context k3d-downstream | |
| crust-gather collect --exclude-namespace=kube-system --exclude-kind=Lease --duration=5s -f tmp/downstream | |
| - | |
| name: Upload logs | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| if: failure() | |
| with: | |
| name: gha-fleet-rancher-logs-${{ github.sha }}-${{ github.run_id }} | |
| path: | | |
| tmp/downstream | |
| tmp/upstream | |
| retention-days: 2 |