From 738aaceff1f3f3862f3d2021977bd16110facfed Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Wed, 14 May 2025 08:48:05 +0100 Subject: [PATCH 01/11] v4 worker multi-arch builds --- .github/workflows/publish-worker-v4.yml | 53 +++++++++++-------------- 1 file changed, 23 insertions(+), 30 deletions(-) diff --git a/.github/workflows/publish-worker-v4.yml b/.github/workflows/publish-worker-v4.yml index ee27e6f862..fedc5f5fbe 100644 --- a/.github/workflows/publish-worker-v4.yml +++ b/.github/workflows/publish-worker-v4.yml @@ -35,6 +35,9 @@ jobs: env: DOCKER_BUILDKIT: "1" steps: + - name: 🏭 Setup Depot CLI + uses: depot/setup-action@v1 + - name: ⬇️ Checkout git repo uses: actions/checkout@v4 @@ -49,26 +52,26 @@ jobs: fi echo "repo=${repo}" >> "$GITHUB_OUTPUT" - - id: get_tag + - name: "#️⃣ Get image tag" + id: get_tag uses: ./.github/actions/get-image-tag with: tag: ${{ inputs.image_tag }} - - name: 🐋 Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + - name: 📛 Set tags to push + id: set_tags + run: | + ref_without_tag=ghcr.io/triggerdotdev/${{ steps.get_repository.outputs.repo }} + image_tags=$ref_without_tag:${{ steps.get_tag.outputs.tag }} - # ..to avoid rate limits when pulling images - - name: 🐳 Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + # if tag is a semver, also tag it as v4 + if [[ "${{ steps.get_tag.outputs.is_semver }}" == true ]]; then + # TODO: switch to v4 tag on GA + image_tags=$image_tags,$ref_without_tag:v4-beta + fi - - name: 🚢 Build Container Image - run: | - docker build -t infra_image -f ./apps/${{ matrix.package }}/Containerfile . + echo "image_tags=${image_tags}" >> "$GITHUB_OUTPUT" - # ..to push image - name: 🐙 Login to GitHub Container Registry uses: docker/login-action@v3 with: @@ -76,20 +79,10 @@ jobs: username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: 🐙 Push to GitHub Container Registry - run: | - docker tag infra_image "$REGISTRY/$REPOSITORY:$IMAGE_TAG" - docker push "$REGISTRY/$REPOSITORY:$IMAGE_TAG" - env: - REGISTRY: ghcr.io/triggerdotdev - REPOSITORY: ${{ steps.get_repository.outputs.repo }} - IMAGE_TAG: ${{ steps.get_tag.outputs.tag }} - - # - name: 🐙 Push 'v3' tag to GitHub Container Registry - # if: steps.get_tag.outputs.is_semver == 'true' - # run: | - # docker tag infra_image "$REGISTRY/$REPOSITORY:v3" - # docker push "$REGISTRY/$REPOSITORY:v3" - # env: - # REGISTRY: ghcr.io/triggerdotdev - # REPOSITORY: ${{ steps.get_repository.outputs.repo }} + - name: 🐳 Build image and push to GitHub Container Registry + uses: depot/build-push-action@v1 + with: + file: ./apps/${{ matrix.package }}/Containerfile + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.get_tag.outputs.tag }} + # push: true From cf0ada2d762d0a199d502cfde73cba729be847cd Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Wed, 14 May 2025 08:48:51 +0100 Subject: [PATCH 02/11] v4-beta tag for webapp --- .github/workflows/publish-webapp.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-webapp.yml b/.github/workflows/publish-webapp.yml index b977ef0a19..6ba6652ab2 100644 --- a/.github/workflows/publish-webapp.yml +++ b/.github/workflows/publish-webapp.yml @@ -43,9 +43,10 @@ jobs: ref_without_tag=ghcr.io/triggerdotdev/trigger.dev image_tags=$ref_without_tag:${{ steps.get_tag.outputs.tag }} - # if tag is a semver, also tag it as v3 + # if tag is a semver, also tag it as v4 if [[ "${{ steps.get_tag.outputs.is_semver }}" == true ]]; then - image_tags=$image_tags,$ref_without_tag:v3 + # TODO: switch to v4 tag on GA + image_tags=$image_tags,$ref_without_tag:v4-beta fi echo "image_tags=${image_tags}" >> "$GITHUB_OUTPUT" From 18c4f53e168a8ef166056d6583b9461dfbabe5fb Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Wed, 14 May 2025 09:12:52 +0100 Subject: [PATCH 03/11] add oidc permission --- .github/workflows/publish-worker-v4.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish-worker-v4.yml b/.github/workflows/publish-worker-v4.yml index fedc5f5fbe..5e95f27181 100644 --- a/.github/workflows/publish-worker-v4.yml +++ b/.github/workflows/publish-worker-v4.yml @@ -14,6 +14,7 @@ on: - "re2-prod-*" permissions: + id-token: write packages: write contents: read From c017bbb04d41d1716d073907388fae8bcbf2be34 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Wed, 14 May 2025 09:38:46 +0100 Subject: [PATCH 04/11] deps fetcher needs python --- apps/supervisor/Containerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/supervisor/Containerfile b/apps/supervisor/Containerfile index d1a8e567cc..5c8754b3c4 100644 --- a/apps/supervisor/Containerfile +++ b/apps/supervisor/Containerfile @@ -19,6 +19,7 @@ COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspa RUN corepack enable && corepack prepare --activate FROM base AS deps-fetcher +RUN apk add --no-cache python3 RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --frozen-lockfile FROM deps-fetcher AS dev-deps From ec46d5849fb212efe5cad34bf81d15465fd5cc1c Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Wed, 14 May 2025 10:09:43 +0100 Subject: [PATCH 05/11] node-gyp needs complete toolchain --- apps/supervisor/Containerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/supervisor/Containerfile b/apps/supervisor/Containerfile index 5c8754b3c4..27887d0c87 100644 --- a/apps/supervisor/Containerfile +++ b/apps/supervisor/Containerfile @@ -19,7 +19,7 @@ COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspa RUN corepack enable && corepack prepare --activate FROM base AS deps-fetcher -RUN apk add --no-cache python3 +RUN apk add --no-cache python3-dev make g++ gcc linx-headers RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --frozen-lockfile FROM deps-fetcher AS dev-deps From c1d509359145d18552b3161ffdb067ee559acc7c Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Wed, 14 May 2025 10:10:16 +0100 Subject: [PATCH 06/11] fix headers --- apps/supervisor/Containerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/supervisor/Containerfile b/apps/supervisor/Containerfile index 27887d0c87..abf697b589 100644 --- a/apps/supervisor/Containerfile +++ b/apps/supervisor/Containerfile @@ -19,7 +19,7 @@ COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspa RUN corepack enable && corepack prepare --activate FROM base AS deps-fetcher -RUN apk add --no-cache python3-dev make g++ gcc linx-headers +RUN apk add --no-cache python3-dev make g++ gcc linux-headers RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --frozen-lockfile FROM deps-fetcher AS dev-deps From a8ba8190038d642c6fb70b749f312aecbe31c080 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Wed, 14 May 2025 10:20:41 +0100 Subject: [PATCH 07/11] bust pnpm cache --- apps/supervisor/Containerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/supervisor/Containerfile b/apps/supervisor/Containerfile index abf697b589..a7f9b0f289 100644 --- a/apps/supervisor/Containerfile +++ b/apps/supervisor/Containerfile @@ -20,12 +20,12 @@ RUN corepack enable && corepack prepare --activate FROM base AS deps-fetcher RUN apk add --no-cache python3-dev make g++ gcc linux-headers -RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --frozen-lockfile +RUN --mount=type=cache,id=pnpm-2,target=/root/.local/share/pnpm/store pnpm fetch --frozen-lockfile FROM deps-fetcher AS dev-deps ENV NODE_ENV development -RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --frozen-lockfile --offline --ignore-scripts +RUN --mount=type=cache,id=pnpm-2,target=/root/.local/share/pnpm/store pnpm install --frozen-lockfile --offline --ignore-scripts FROM base AS builder From 285f14f13bcf1702877d4419ba7a580c6ce497b2 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Wed, 14 May 2025 10:28:36 +0100 Subject: [PATCH 08/11] add setuptools --- apps/supervisor/Containerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/supervisor/Containerfile b/apps/supervisor/Containerfile index a7f9b0f289..5092bcdaa1 100644 --- a/apps/supervisor/Containerfile +++ b/apps/supervisor/Containerfile @@ -19,7 +19,7 @@ COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspa RUN corepack enable && corepack prepare --activate FROM base AS deps-fetcher -RUN apk add --no-cache python3-dev make g++ gcc linux-headers +RUN apk add --no-cache python3-dev py3-setuptools make g++ gcc linux-headers RUN --mount=type=cache,id=pnpm-2,target=/root/.local/share/pnpm/store pnpm fetch --frozen-lockfile FROM deps-fetcher AS dev-deps From b5b2be0f6b1c705683dea33166f8cf0c4c082a1d Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Wed, 14 May 2025 10:35:38 +0100 Subject: [PATCH 09/11] switch back to old cache id --- apps/supervisor/Containerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/supervisor/Containerfile b/apps/supervisor/Containerfile index 5092bcdaa1..b0b1ba9271 100644 --- a/apps/supervisor/Containerfile +++ b/apps/supervisor/Containerfile @@ -20,12 +20,12 @@ RUN corepack enable && corepack prepare --activate FROM base AS deps-fetcher RUN apk add --no-cache python3-dev py3-setuptools make g++ gcc linux-headers -RUN --mount=type=cache,id=pnpm-2,target=/root/.local/share/pnpm/store pnpm fetch --frozen-lockfile +RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --frozen-lockfile FROM deps-fetcher AS dev-deps ENV NODE_ENV development -RUN --mount=type=cache,id=pnpm-2,target=/root/.local/share/pnpm/store pnpm install --frozen-lockfile --offline --ignore-scripts +RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --frozen-lockfile --offline --ignore-scripts FROM base AS builder From c91b8a381d7ac4252d6263a49b010508316a9999 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Wed, 14 May 2025 10:36:02 +0100 Subject: [PATCH 10/11] push worker images --- .github/workflows/publish-worker-v4.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-worker-v4.yml b/.github/workflows/publish-worker-v4.yml index 5e95f27181..5d797c39a8 100644 --- a/.github/workflows/publish-worker-v4.yml +++ b/.github/workflows/publish-worker-v4.yml @@ -86,4 +86,4 @@ jobs: file: ./apps/${{ matrix.package }}/Containerfile platforms: linux/amd64,linux/arm64 tags: ${{ steps.get_tag.outputs.tag }} - # push: true + push: true From 0691f98a6103c5b7eb92634fc8e7be67366be2ae Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Wed, 14 May 2025 11:01:01 +0100 Subject: [PATCH 11/11] pass the correct tags --- .github/workflows/publish-worker-v4.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-worker-v4.yml b/.github/workflows/publish-worker-v4.yml index 5d797c39a8..4a2853da08 100644 --- a/.github/workflows/publish-worker-v4.yml +++ b/.github/workflows/publish-worker-v4.yml @@ -85,5 +85,5 @@ jobs: with: file: ./apps/${{ matrix.package }}/Containerfile platforms: linux/amd64,linux/arm64 - tags: ${{ steps.get_tag.outputs.tag }} + tags: ${{ steps.set_tags.outputs.image_tags }} push: true