From 34a2ee9330add3190d693d53544a5b5c57a0b30a Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Sun, 27 Apr 2025 19:15:31 +0200 Subject: [PATCH 1/4] deploy ./api to stage --- .github/workflows/cd.deploy.stg.yml | 2 +- api.Dockerfile | 25 +++++++++++++ api/package.json | 3 +- docker-compose.stage.yml | 52 ++++++++++++++++++++++++++++ packages/tooling/write-dockerfile.ts | 9 +++-- packages/utils/src/config/index.ts | 2 +- scripts/deploy.rs | 46 ++++++++++++++++++++---- web-server.Dockerfile | 4 ++- web-server/package.json | 2 +- 9 files changed, 130 insertions(+), 15 deletions(-) create mode 100644 api.Dockerfile diff --git a/.github/workflows/cd.deploy.stg.yml b/.github/workflows/cd.deploy.stg.yml index 103f9523a..289fb0523 100644 --- a/.github/workflows/cd.deploy.stg.yml +++ b/.github/workflows/cd.deploy.stg.yml @@ -80,7 +80,7 @@ jobs: - name: "Bundle ./web for deployment" run: | cd ./web - npm run bundle + npm run bundle:alone npm run pre-deploy - name: "Write ./web-server deps into Dockerfile..." diff --git a/api.Dockerfile b/api.Dockerfile new file mode 100644 index 000000000..4ec1e4dcc --- /dev/null +++ b/api.Dockerfile @@ -0,0 +1,25 @@ +FROM --platform=linux/amd64 node:22 + +WORKDIR /usr/src/repo + +COPY ./package.json ./package.json + +# AUTO_GEN +COPY ./api/dist ./api/dist +COPY ./api/package.json ./api/package.json +COPY ./api/db ./api/db +COPY ./packages/tooling/package.json ./packages/tooling/package.json +COPY ./data/dist ./data/dist +COPY ./data/package.json ./data/package.json +COPY ./data/models ./data/models +COPY ./packages/models/dist ./packages/models/dist +COPY ./packages/models/package.json ./packages/models/package.json +COPY ./packages/utils/dist ./packages/utils/dist +COPY ./packages/utils/package.json ./packages/utils/package.json +# AUTO_GEN_END + +RUN npm install --omit=dev --workspace=@dzcode.io/api + +ENV PORT=80 +WORKDIR /usr/src/repo/api +CMD [ "npm", "run", "start" ] diff --git a/api/package.json b/api/package.json index fd862625f..ee6345d38 100644 --- a/api/package.json +++ b/api/package.json @@ -78,8 +78,9 @@ "lint:prettier": "prettier --config ../packages/tooling/.prettierrc --ignore-path ../packages/tooling/.prettierignore --log-level warn", "lint:ts-prune": "tsx ../packages/tooling/setup-ts-prune.ts && ts-prune --error", "lint:tsc": "tspc --noEmit", - "start": "wait-port postgres:5432 && wait-port meilisearch:7700 && delay 2 && node dist/app/index.js", + "start": "node dist/app/index.js", "start:dev": "tsx ../packages/tooling/nodemon.ts \"@dzcode.io/api\" && npm-run-all --parallel start:nodemon db:server", + "prepare-dockerfile": "tsx ../packages/tooling/write-dockerfile.ts @dzcode.io/api", "start:nodemon": "wait-port localhost:5432 && delay 2 && nodemon dist/app/index.js", "test": "npm run build && npm run test:alone", "test:alone": "jest --config ../packages/tooling/jest.config.ts --rootDir .", diff --git a/docker-compose.stage.yml b/docker-compose.stage.yml index 0b0dbf1d5..0a554253d 100644 --- a/docker-compose.stage.yml +++ b/docker-compose.stage.yml @@ -1,3 +1,5 @@ +name: dzcode-stage + services: web: image: ghcr.io/dzcode-io/stage-dot-dzcode-dot-io-server:latest @@ -10,6 +12,56 @@ services: networks: - main-infra-network + api: + image: ghcr.io/dzcode-io/api-dot-stage-dot-dzcode-dot-io-server:latest + pull_policy: always + restart: unless-stopped + depends_on: + postgres: + condition: service_started + meilisearch: + condition: service_started + environment: + - VIRTUAL_HOST=api.stage.dzcode.io + - LETSENCRYPT_HOST=api.stage.dzcode.io + - NODE_ENV=staging + - OPENAI_KEY={{OPENAI_KEY}} + - GITHUB_TOKEN={{GITHUB_TOKEN}} + volumes: + - fetch_cache:/usr/src/repo/api/fetch_cache + - sqlite_db:/usr/src/repo/api/sqlite_db + networks: + - main-infra-network + - internal-network + + postgres: + image: postgres + volumes: + - postgres_db:/var/lib/postgresql/data + environment: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_DB: db + networks: + - internal-network + + meilisearch: + image: getmeili/meilisearch:v1.11.3 # database schema is different between versions + volumes: + - meilisearch_db:/meili_data + environment: + MEILI_NO_ANALYTICS: true + MEILI_MASTER_KEY: "default" # we only access it through `./api` + networks: + - internal-network + networks: main-infra-network: external: true + internal-network: + internal: true + +volumes: + postgres_db: + meilisearch_db: + fetch_cache: + sqlite_db: diff --git a/packages/tooling/write-dockerfile.ts b/packages/tooling/write-dockerfile.ts index c40a62c2e..efce26e48 100644 --- a/packages/tooling/write-dockerfile.ts +++ b/packages/tooling/write-dockerfile.ts @@ -2,16 +2,19 @@ import { execSync } from "child_process"; import { existsSync, readFileSync, writeFileSync } from "fs"; import { join } from "path"; +const scope = process.argv[2]; +if (!scope) throw new Error("Please provide a scope"); + const workspaceRoot = join(__dirname, "../.."); -const dockerFilePath = join(workspaceRoot, "web-server.Dockerfile"); +const dockerFilePath = join(workspaceRoot, `${scope.replace("@dzcode.io/", "")}.Dockerfile`); console.log(`writing ${dockerFilePath} ...`); const stdout = execSync( - `lerna list --include-dependencies --json --all --loglevel silent --scope @dzcode.io/web-server`, + `lerna list --include-dependencies --json --all --loglevel silent --scope ${scope}`, ); const dependencies = JSON.parse(stdout.toString()) as Array<{ name: string; location: string }>; -const subPaths = ["dist", "bundle", "package.json"]; +const subPaths = ["dist", "bundle", "package.json", "db", "models"]; const directoriesToCopy = dependencies .map(({ location }) => location) .reduce>( diff --git a/packages/utils/src/config/index.ts b/packages/utils/src/config/index.ts index 397d327de..ac1747e6b 100644 --- a/packages/utils/src/config/index.ts +++ b/packages/utils/src/config/index.ts @@ -13,7 +13,7 @@ export const fsConfig = (env: Environment, extra?: Record) => { port: apiPort, url: [ `http://${hostname}:${apiPort}`, - "https://api-stage.dzcode.io", + "https://api.stage.dzcode.io", "https://api.dzcode.io", ][e], }, diff --git a/scripts/deploy.rs b/scripts/deploy.rs index 075a7badf..aeb15b35c 100755 --- a/scripts/deploy.rs +++ b/scripts/deploy.rs @@ -22,6 +22,10 @@ struct Args { /// Environment to deploy to #[arg(short, long, value_enum)] env: Env, + + /// Clean build + #[arg(short, long, default_value_t = false)] + clean: bool, } fn main() { @@ -32,17 +36,26 @@ fn main() { println!("Ensuring docker is running ..."); cli_run::cli_run("docker", vec!["ps"]); - println!("Building ./web-server ..."); - cli_run::cli_run("npm", vec!["run", "clean"]); - cli_run::cli_run("npm", vec!["run", "bundle", "--workspace=@dzcode.io/web"]); - cli_run::cli_run("npm", vec!["run", "pre-deploy", "--workspace=@dzcode.io/web"]); - cli_run::cli_run("npm", vec!["run", "build", "--workspace=@dzcode.io/web-server"]); + if args.clean { + println!("clean..."); + cli_run::cli_run("npm", vec!["run", "clean"]); + } + + println!("Build..."); + cli_run::cli_run("npm", vec!["run", "build"]); - println!("Writing ./web-server deps into Dockerfile..."); + println!("Preparing ./web-server ..."); + cli_run::cli_run("npm", vec!["run", "bundle:alone", "--workspace=@dzcode.io/web"]); + cli_run::cli_run("npm", vec!["run", "pre-deploy", "--workspace=@dzcode.io/web"]); cli_run::cli_run("npm", vec!["run", "prepare-dockerfile", "--workspace=@dzcode.io/web-server"]); - println!("Building docker image ..."); + println!("Preparing ./api ..."); + cli_run::cli_run("npm", vec!["run", "prepare-dockerfile", "--workspace=@dzcode.io/api"]); + + let env_str = format!("{:?}", env).to_lowercase(); + + println!("Building ./web-server docker image ..."); cli_run::cli_run( "docker", vec![ @@ -56,6 +69,21 @@ fn main() { ], ); + println!("Building ./api docker image ..."); + cli_run::cli_run( + "docker", + vec![ + "buildx", + "build", + "-f", + "api.Dockerfile", + ".", + "-t", + &format!("ghcr.io/dzcode-io/api-dot-{}-dot-dzcode-dot-io-server:latest", env_str), + ], + ); + + println!("Logging in to GitHub Container Registry ..."); let gh_token = std::env::var("DOCKER_REGISTRY_PASSWORD") .expect("DOCKER_REGISTRY_PASSWORD environment variable not set"); @@ -76,6 +104,10 @@ fn main() { "docker", vec!["push", &format!("ghcr.io/dzcode-io/{}-dot-dzcode-dot-io-server:latest", env_str)], ); + cli_run::cli_run( + "docker", + vec!["push", &format!("ghcr.io/dzcode-io/api-dot-{}-dot-dzcode-dot-io-server:latest", env_str)], + ); println!("Deploying to zcluster ..."); cli_run::cli_run( diff --git a/web-server.Dockerfile b/web-server.Dockerfile index ae5fc9a5b..81821b25a 100644 --- a/web-server.Dockerfile +++ b/web-server.Dockerfile @@ -9,6 +9,7 @@ COPY ./web-server/dist ./web-server/dist COPY ./web-server/package.json ./web-server/package.json COPY ./api/dist ./api/dist COPY ./api/package.json ./api/package.json +COPY ./api/db ./api/db COPY ./packages/models/dist ./packages/models/dist COPY ./packages/models/package.json ./packages/models/package.json COPY ./packages/utils/dist ./packages/utils/dist @@ -19,9 +20,10 @@ COPY ./web/package.json ./web/package.json COPY ./packages/tooling/package.json ./packages/tooling/package.json COPY ./data/dist ./data/dist COPY ./data/package.json ./data/package.json +COPY ./data/models ./data/models # AUTO_GEN_END -RUN npm install --install-strategy=nested --omit=dev --workspace=@dzcode.io/web-server +RUN npm install --omit=dev --workspace=@dzcode.io/web-server ENV PORT=80 WORKDIR /usr/src/repo/web-server diff --git a/web-server/package.json b/web-server/package.json index bb99b77c4..019cf9517 100644 --- a/web-server/package.json +++ b/web-server/package.json @@ -48,7 +48,7 @@ "lint:prettier": "prettier --config ../packages/tooling/.prettierrc --ignore-path ../packages/tooling/.prettierignore --log-level warn", "lint:ts-prune": "tsx ../packages/tooling/setup-ts-prune.ts && ts-prune --error", "lint:tsc": "tsc --noEmit", - "prepare-dockerfile": "tsx ../packages/tooling/write-dockerfile.ts", + "prepare-dockerfile": "tsx ../packages/tooling/write-dockerfile.ts @dzcode.io/web-server", "prepare-web": "lerna run bundle --scope=@dzcode.io/web --include-dependencies --stream && lerna run pre-deploy --scope=@dzcode.io/web --include-dependencies --stream", "start": "node dist/app/index.js", "start:dev": "npm run prepare-web && tsx ../packages/tooling/nodemon.ts \"@dzcode.io/web-server\" && npm run start:nodemon", From 6052da69939de2ac872e62121ec8ed514c4175d3 Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Sun, 27 Apr 2025 19:23:53 +0200 Subject: [PATCH 2/4] update api deployment job in stage cd action --- .github/workflows/cd.deploy.stg.yml | 44 ++++++++++++++++++----------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/.github/workflows/cd.deploy.stg.yml b/.github/workflows/cd.deploy.stg.yml index 289fb0523..8a3f10e54 100644 --- a/.github/workflows/cd.deploy.stg.yml +++ b/.github/workflows/cd.deploy.stg.yml @@ -19,7 +19,6 @@ jobs: env: CI: true STAGE: staging - SSH_ADDRESS_STG: ${{ secrets.SSH_ADDRESS_STG }} DEPLOY_VERSION: ${{ github.ref_type == 'tag' && github.ref_name || format('stg-0.0.0-{0}-{1}-{2}', github.ref_name, github.run_number, github.run_attempt) }} steps: @@ -35,18 +34,31 @@ jobs: uses: actions/download-artifact@v4 with: name: build output (ubuntu-latest, 20) - - name: "SSH" - uses: shimataro/ssh-key-action@v2 - with: - key: ${{ secrets.SSH_KEY }} - known_hosts: ${{ secrets.SSH_KNOWN_HOSTS }} + - name: "Generate Bundle info" run: npm run generate:bundle-info $DEPLOY_VERSION stage + - name: "Sentry Release" # todo-zm: remove sentry entirely run: cd ./api && npm run generate:sentry-release $DEPLOY_VERSION stage ${{ secrets.SENTRY_AUTH_TOKEN }} - - name: "Deploy" - run: cd ./api && npm run deploy:stg + + - name: "Write ./api deps into Dockerfile..." + run: | + cd ./api + npm run prepare-dockerfile + + - name: Build docker image + run: | + docker buildx build -f api.Dockerfile . -t ghcr.io/api-dot-stage-dot-dzcode-dot-io-server:latest:latest + env: + DOCKER_BUILDKIT: 1 + + - name: Push docker image + run: | + echo $CR_PAT | docker login ghcr.io -u dzcode-io --password-stdin + docker push ghcr.io/api-dot-stage-dot-dzcode-dot-io-server:latest:latest + env: + CR_PAT: ${{ secrets.CR_PAT }} deploy-web-server: needs: build @@ -101,12 +113,12 @@ jobs: env: CR_PAT: ${{ secrets.CR_PAT }} - - name: install zcluster - run: | - curl -fsSL https://infra.zak-man.com/install.sh | sh - echo "/home/runner/.zcluster/bin" >> $GITHUB_PATH + # - name: install zcluster + # run: | + # curl -fsSL https://infra.zak-man.com/install.sh | sh + # echo "/home/runner/.zcluster/bin" >> $GITHUB_PATH - - name: Deploy to zcluster - run: zcluster deploy -p stage-dzcode ./docker-compose.stage.yml - env: - ADMIN_AUTH_TOKEN: ${{ secrets.ADMIN_AUTH_TOKEN }} + # - name: Deploy to zcluster + # run: zcluster deploy -p stage-dzcode ./docker-compose.stage.yml + # env: + # ADMIN_AUTH_TOKEN: ${{ secrets.ADMIN_AUTH_TOKEN }} From df38c56f93a800cfe68643cbe662125013e616fb Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Sun, 27 Apr 2025 19:33:40 +0200 Subject: [PATCH 3/4] deploy once both web-server and api images are built and pushed --- .github/workflows/cd.deploy.stg.yml | 36 +++++++++++++++++++---------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/.github/workflows/cd.deploy.stg.yml b/.github/workflows/cd.deploy.stg.yml index 8a3f10e54..557a21712 100644 --- a/.github/workflows/cd.deploy.stg.yml +++ b/.github/workflows/cd.deploy.stg.yml @@ -13,7 +13,7 @@ jobs: node-version: "20" os: "ubuntu-latest" - deploy-api: + docker-build-push-api: needs: build runs-on: ubuntu-latest env: @@ -49,18 +49,18 @@ jobs: - name: Build docker image run: | - docker buildx build -f api.Dockerfile . -t ghcr.io/api-dot-stage-dot-dzcode-dot-io-server:latest:latest + docker buildx build -f api.Dockerfile . -t ghcr.io/dzcode-io/api-dot-stage-dot-dzcode-dot-io-server:latest:latest env: DOCKER_BUILDKIT: 1 - name: Push docker image run: | echo $CR_PAT | docker login ghcr.io -u dzcode-io --password-stdin - docker push ghcr.io/api-dot-stage-dot-dzcode-dot-io-server:latest:latest + docker push ghcr.io/dzcode-io/api-dot-stage-dot-dzcode-dot-io-server:latest:latest env: CR_PAT: ${{ secrets.CR_PAT }} - deploy-web-server: + docker-build-push-web-server: needs: build runs-on: ubuntu-latest env: @@ -113,12 +113,24 @@ jobs: env: CR_PAT: ${{ secrets.CR_PAT }} - # - name: install zcluster - # run: | - # curl -fsSL https://infra.zak-man.com/install.sh | sh - # echo "/home/runner/.zcluster/bin" >> $GITHUB_PATH + deploy-to-zcluster: + needs: [docker-build-push-api, docker-build-push-web-server] + runs-on: ubuntu-latest + env: + CI: true + + steps: + - name: "Git" + uses: actions/checkout@v4 - # - name: Deploy to zcluster - # run: zcluster deploy -p stage-dzcode ./docker-compose.stage.yml - # env: - # ADMIN_AUTH_TOKEN: ${{ secrets.ADMIN_AUTH_TOKEN }} + - name: install zcluster + run: | + curl -fsSL https://infra.zak-man.com/install.sh | sh + echo "/home/runner/.zcluster/bin" >> $GITHUB_PATH + + - name: Deploy to zcluster + run: zcluster deploy -p stage-dzcode ./docker-compose.stage.yml + env: + ADMIN_AUTH_TOKEN: ${{ secrets.ADMIN_AUTH_TOKEN }} + OPENAI_KEY: ${{ secrets.OPENAI_KEY }} + GITHUB_TOKEN: ${{ secrets.API_GITHUB_TOKEN }} From 88e636810151391a97aec8398f9d24b0ea03fc13 Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Sun, 27 Apr 2025 19:36:21 +0200 Subject: [PATCH 4/4] fix: remove redundant tag from docker image and push commands --- .github/workflows/cd.deploy.stg.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cd.deploy.stg.yml b/.github/workflows/cd.deploy.stg.yml index 557a21712..f2f4e45eb 100644 --- a/.github/workflows/cd.deploy.stg.yml +++ b/.github/workflows/cd.deploy.stg.yml @@ -49,14 +49,14 @@ jobs: - name: Build docker image run: | - docker buildx build -f api.Dockerfile . -t ghcr.io/dzcode-io/api-dot-stage-dot-dzcode-dot-io-server:latest:latest + docker buildx build -f api.Dockerfile . -t ghcr.io/dzcode-io/api-dot-stage-dot-dzcode-dot-io-server:latest env: DOCKER_BUILDKIT: 1 - name: Push docker image run: | echo $CR_PAT | docker login ghcr.io -u dzcode-io --password-stdin - docker push ghcr.io/dzcode-io/api-dot-stage-dot-dzcode-dot-io-server:latest:latest + docker push ghcr.io/dzcode-io/api-dot-stage-dot-dzcode-dot-io-server:latest env: CR_PAT: ${{ secrets.CR_PAT }}