Merge pull request #655 from dzcode-io/chore/deploy--api-stage-to-zcluster

chore: P2 deploy  `./api` stage to zcluster

Author: ZibanPirate

.github/workflows/cd.deploy.stg.yml

@@ -13,13 +13,12 @@ jobs:
       node-version: "20"
       os: "ubuntu-latest"
 
-  deploy-api:
+  docker-build-push-api:
     needs: build
     runs-on: ubuntu-latest
     env:
       CI: true
       STAGE: staging
-      SSH_ADDRESS_STG: ${{ secrets.SSH_ADDRESS_STG }}
       DEPLOY_VERSION: ${{ github.ref_type == 'tag' && github.ref_name || format('stg-0.0.0-{0}-{1}-{2}', github.ref_name, github.run_number, github.run_attempt) }}
 
     steps:
@@ -35,20 +34,33 @@ jobs:
         uses: actions/download-artifact@v4
         with:
           name: build output (ubuntu-latest, 20)
-      - name: "SSH"
-        uses: shimataro/ssh-key-action@v2
-        with:
-          key: ${{ secrets.SSH_KEY }}
-          known_hosts: ${{ secrets.SSH_KNOWN_HOSTS }}
+
       - name: "Generate Bundle info"
         run: npm run generate:bundle-info $DEPLOY_VERSION stage
+
       - name: "Sentry Release"
         # todo-zm: remove sentry entirely
         run: cd ./api && npm run generate:sentry-release $DEPLOY_VERSION stage ${{ secrets.SENTRY_AUTH_TOKEN }}
-      - name: "Deploy"
-        run: cd ./api && npm run deploy:stg
 
-  deploy-web-server:
+      - name: "Write ./api deps into Dockerfile..."
+        run: |
+          cd ./api
+          npm run prepare-dockerfile
+
+      - name: Build docker image
+        run: |
+          docker buildx build -f api.Dockerfile . -t ghcr.io/dzcode-io/api-dot-stage-dot-dzcode-dot-io-server:latest
+        env:
+          DOCKER_BUILDKIT: 1
+
+      - name: Push docker image
+        run: |
+          echo $CR_PAT | docker login ghcr.io -u dzcode-io --password-stdin
+          docker push ghcr.io/dzcode-io/api-dot-stage-dot-dzcode-dot-io-server:latest
+        env:
+          CR_PAT: ${{ secrets.CR_PAT }}
+
+  docker-build-push-web-server:
     needs: build
     runs-on: ubuntu-latest
     env:
@@ -80,7 +92,7 @@ jobs:
       - name: "Bundle ./web for deployment"
         run: |
           cd ./web
-          npm run bundle
+          npm run bundle:alone
           npm run pre-deploy
 
       - name: "Write ./web-server deps into Dockerfile..."
@@ -101,6 +113,16 @@ jobs:
         env:
           CR_PAT: ${{ secrets.CR_PAT }}
 
+  deploy-to-zcluster:
+    needs: [docker-build-push-api, docker-build-push-web-server]
+    runs-on: ubuntu-latest
+    env:
+      CI: true
+
+    steps:
+      - name: "Git"
+        uses: actions/checkout@v4
+
       - name: install zcluster
         run: |
           curl -fsSL https://infra.zak-man.com/install.sh | sh
@@ -110,3 +132,5 @@ jobs:
         run: zcluster deploy -p stage-dzcode ./docker-compose.stage.yml
         env:
           ADMIN_AUTH_TOKEN: ${{ secrets.ADMIN_AUTH_TOKEN }}
+          OPENAI_KEY: ${{ secrets.OPENAI_KEY }}
+          GITHUB_TOKEN: ${{ secrets.API_GITHUB_TOKEN }}

api.Dockerfile

@@ -0,0 +1,25 @@
+FROM --platform=linux/amd64 node:22
+
+WORKDIR /usr/src/repo
+
+COPY ./package.json ./package.json
+
+# AUTO_GEN
+COPY ./api/dist ./api/dist
+COPY ./api/package.json ./api/package.json
+COPY ./api/db ./api/db
+COPY ./packages/tooling/package.json ./packages/tooling/package.json
+COPY ./data/dist ./data/dist
+COPY ./data/package.json ./data/package.json
+COPY ./data/models ./data/models
+COPY ./packages/models/dist ./packages/models/dist
+COPY ./packages/models/package.json ./packages/models/package.json
+COPY ./packages/utils/dist ./packages/utils/dist
+COPY ./packages/utils/package.json ./packages/utils/package.json
+# AUTO_GEN_END
+
+RUN npm install --omit=dev --workspace=@dzcode.io/api
+
+ENV PORT=80
+WORKDIR /usr/src/repo/api
+CMD [ "npm", "run", "start" ]

api/package.json

@@ -78,8 +78,9 @@
     "lint:prettier": "prettier  --config ../packages/tooling/.prettierrc --ignore-path ../packages/tooling/.prettierignore --log-level warn",
     "lint:ts-prune": "tsx ../packages/tooling/setup-ts-prune.ts && ts-prune --error",
     "lint:tsc": "tspc --noEmit",
-    "start": "wait-port postgres:5432 && wait-port meilisearch:7700 && delay 2 && node dist/app/index.js",
+    "start": "node dist/app/index.js",
     "start:dev": "tsx ../packages/tooling/nodemon.ts \"@dzcode.io/api\" && npm-run-all --parallel start:nodemon db:server",
+    "prepare-dockerfile": "tsx ../packages/tooling/write-dockerfile.ts @dzcode.io/api",
     "start:nodemon": "wait-port localhost:5432 && delay 2 && nodemon dist/app/index.js",
     "test": "npm run build && npm run test:alone",
     "test:alone": "jest --config ../packages/tooling/jest.config.ts --rootDir .",

docker-compose.stage.yml

@@ -1,3 +1,5 @@
+name: dzcode-stage
+
 services:
   web:
     image: ghcr.io/dzcode-io/stage-dot-dzcode-dot-io-server:latest
@@ -10,6 +12,56 @@ services:
     networks:
       - main-infra-network
 
+  api:
+    image: ghcr.io/dzcode-io/api-dot-stage-dot-dzcode-dot-io-server:latest
+    pull_policy: always
+    restart: unless-stopped
+    depends_on:
+      postgres:
+        condition: service_started
+      meilisearch:
+        condition: service_started
+    environment:
+      - VIRTUAL_HOST=api.stage.dzcode.io
+      - LETSENCRYPT_HOST=api.stage.dzcode.io
+      - NODE_ENV=staging
+      - OPENAI_KEY={{OPENAI_KEY}}
+      - GITHUB_TOKEN={{GITHUB_TOKEN}}
+    volumes:
+      - fetch_cache:/usr/src/repo/api/fetch_cache
+      - sqlite_db:/usr/src/repo/api/sqlite_db
+    networks:
+      - main-infra-network
+      - internal-network
+
+  postgres:
+    image: postgres
+    volumes:
+      - postgres_db:/var/lib/postgresql/data
+    environment:
+      POSTGRES_HOST_AUTH_METHOD: trust
+      POSTGRES_DB: db
+    networks:
+      - internal-network
+
+  meilisearch:
+    image: getmeili/meilisearch:v1.11.3 # database schema is different between versions
+    volumes:
+      - meilisearch_db:/meili_data
+    environment:
+      MEILI_NO_ANALYTICS: true
+      MEILI_MASTER_KEY: "default" # we only access it through `./api`
+    networks:
+      - internal-network
+
 networks:
   main-infra-network:
     external: true
+  internal-network:
+    internal: true
+
+volumes:
+  postgres_db:
+  meilisearch_db:
+  fetch_cache:
+  sqlite_db:

packages/tooling/write-dockerfile.ts

@@ -2,16 +2,19 @@ import { execSync } from "child_process";
 import { existsSync, readFileSync, writeFileSync } from "fs";
 import { join } from "path";
 
+const scope = process.argv[2];
+if (!scope) throw new Error("Please provide a scope");
+
 const workspaceRoot = join(__dirname, "../..");
-const dockerFilePath = join(workspaceRoot, "web-server.Dockerfile");
+const dockerFilePath = join(workspaceRoot, `${scope.replace("@dzcode.io/", "")}.Dockerfile`);
 
 console.log(`writing ${dockerFilePath} ...`);
 
 const stdout = execSync(
-  `lerna list --include-dependencies --json --all --loglevel silent --scope @dzcode.io/web-server`,
+  `lerna list --include-dependencies --json --all --loglevel silent --scope ${scope}`,
 );
 const dependencies = JSON.parse(stdout.toString()) as Array<{ name: string; location: string }>;
-const subPaths = ["dist", "bundle", "package.json"];
+const subPaths = ["dist", "bundle", "package.json", "db", "models"];
 const directoriesToCopy = dependencies
   .map<string>(({ location }) => location)
   .reduce<Array<string>>(

packages/utils/src/config/index.ts

@@ -13,7 +13,7 @@ export const fsConfig = (env: Environment, extra?: Record<string, unknown>) => {
       port: apiPort,
       url: [
         `http://${hostname}:${apiPort}`,
-        "https://api-stage.dzcode.io",
+        "https://api.stage.dzcode.io",
         "https://api.dzcode.io",
       ][e],
     },

scripts/deploy.rs

@@ -22,6 +22,10 @@ struct Args {
     /// Environment to deploy to
     #[arg(short, long, value_enum)]
     env: Env,
+
+    /// Clean build
+    #[arg(short, long, default_value_t = false)]
+    clean: bool,
 }
 
 fn main() {
@@ -32,17 +36,26 @@ fn main() {
     println!("Ensuring docker is running ...");
     cli_run::cli_run("docker", vec!["ps"]);
 
-    println!("Building ./web-server ...");
-    cli_run::cli_run("npm", vec!["run", "clean"]);
-    cli_run::cli_run("npm", vec!["run", "bundle", "--workspace=@dzcode.io/web"]);
-    cli_run::cli_run("npm", vec!["run", "pre-deploy", "--workspace=@dzcode.io/web"]);
-    cli_run::cli_run("npm", vec!["run", "build", "--workspace=@dzcode.io/web-server"]);
+    if args.clean {
+        println!("clean...");
+        cli_run::cli_run("npm", vec!["run", "clean"]);
+      }
+
+    println!("Build...");
+    cli_run::cli_run("npm", vec!["run", "build"]);
 
-    println!("Writing ./web-server deps into Dockerfile...");
+    println!("Preparing ./web-server ...");
+    cli_run::cli_run("npm", vec!["run", "bundle:alone", "--workspace=@dzcode.io/web"]);
+    cli_run::cli_run("npm", vec!["run", "pre-deploy", "--workspace=@dzcode.io/web"]);
     cli_run::cli_run("npm", vec!["run", "prepare-dockerfile", "--workspace=@dzcode.io/web-server"]);
 
-    println!("Building docker image ...");
+    println!("Preparing ./api ...");
+    cli_run::cli_run("npm", vec!["run", "prepare-dockerfile", "--workspace=@dzcode.io/api"]);
+
+
     let env_str = format!("{:?}", env).to_lowercase();
+
+    println!("Building ./web-server docker image ...");
     cli_run::cli_run(
         "docker",
         vec![
@@ -56,6 +69,21 @@ fn main() {
         ],
     );
 
+    println!("Building ./api docker image ...");
+    cli_run::cli_run(
+        "docker",
+        vec![
+            "buildx",
+            "build",
+            "-f",
+            "api.Dockerfile",
+            ".",
+            "-t",
+            &format!("ghcr.io/dzcode-io/api-dot-{}-dot-dzcode-dot-io-server:latest", env_str),
+        ],
+    );
+
+
     println!("Logging in to GitHub Container Registry ...");
     let gh_token = std::env::var("DOCKER_REGISTRY_PASSWORD")
         .expect("DOCKER_REGISTRY_PASSWORD environment variable not set");
@@ -76,6 +104,10 @@ fn main() {
         "docker",
         vec!["push", &format!("ghcr.io/dzcode-io/{}-dot-dzcode-dot-io-server:latest", env_str)],
     );
+    cli_run::cli_run(
+        "docker",
+        vec!["push", &format!("ghcr.io/dzcode-io/api-dot-{}-dot-dzcode-dot-io-server:latest", env_str)],
+    );
 
     println!("Deploying to zcluster ...");
     cli_run::cli_run(

web-server.Dockerfile

@@ -9,6 +9,7 @@ COPY ./web-server/dist ./web-server/dist
 COPY ./web-server/package.json ./web-server/package.json
 COPY ./api/dist ./api/dist
 COPY ./api/package.json ./api/package.json
+COPY ./api/db ./api/db
 COPY ./packages/models/dist ./packages/models/dist
 COPY ./packages/models/package.json ./packages/models/package.json
 COPY ./packages/utils/dist ./packages/utils/dist
@@ -19,9 +20,10 @@ COPY ./web/package.json ./web/package.json
 COPY ./packages/tooling/package.json ./packages/tooling/package.json
 COPY ./data/dist ./data/dist
 COPY ./data/package.json ./data/package.json
+COPY ./data/models ./data/models
 # AUTO_GEN_END
 
-RUN npm install --install-strategy=nested --omit=dev --workspace=@dzcode.io/web-server
+RUN npm install --omit=dev --workspace=@dzcode.io/web-server
 
 ENV PORT=80
 WORKDIR /usr/src/repo/web-server

web-server/package.json

@@ -48,7 +48,7 @@
     "lint:prettier": "prettier  --config ../packages/tooling/.prettierrc --ignore-path ../packages/tooling/.prettierignore --log-level warn",
     "lint:ts-prune": "tsx ../packages/tooling/setup-ts-prune.ts && ts-prune --error",
     "lint:tsc": "tsc --noEmit",
-    "prepare-dockerfile": "tsx ../packages/tooling/write-dockerfile.ts",
+    "prepare-dockerfile": "tsx ../packages/tooling/write-dockerfile.ts @dzcode.io/web-server",
     "prepare-web": "lerna run bundle --scope=@dzcode.io/web --include-dependencies --stream && lerna run pre-deploy --scope=@dzcode.io/web --include-dependencies --stream",
     "start": "node dist/app/index.js",
     "start:dev": "npm run prepare-web && tsx ../packages/tooling/nodemon.ts \"@dzcode.io/web-server\" && npm run start:nodemon",