diff --git a/.github/workflows/cd.deploy.yml b/.github/workflows/cd.deploy.yml index 656d1fd03..3dd65d6dd 100644 --- a/.github/workflows/cd.deploy.yml +++ b/.github/workflows/cd.deploy.yml @@ -18,7 +18,6 @@ jobs: env: CI: true STAGE: production - SSH_ADDRESS_PRD: ${{ secrets.SSH_ADDRESS_PRD }} DEPLOY_VERSION: ${{ github.ref_type == 'tag' && github.ref_name || format('0.0.0-{0}-{1}-{2}', github.ref_name, github.run_number, github.run_attempt) }} steps: @@ -34,29 +33,103 @@ jobs: uses: actions/download-artifact@v4 with: name: build output (ubuntu-latest, 20) - - name: "SSH" - uses: shimataro/ssh-key-action@v2 + + - name: "Generate Bundle info" + run: npm run generate:bundle-info $DEPLOY_VERSION production + + - name: "Sentry Release" + # todo-zm: remove sentry entirely + run: cd ./api && npm run generate:sentry-release $DEPLOY_VERSION production ${{ secrets.SENTRY_AUTH_TOKEN }} + + - name: "Write ./api deps into Dockerfile..." + run: | + cd ./api + npm run prepare-dockerfile + + - name: Build docker image + run: | + docker buildx build -f api.Dockerfile . -t ghcr.io/dzcode-io/api-dot-production-dot-dzcode-dot-io-server:latest + env: + DOCKER_BUILDKIT: 1 + + - name: Push docker image + run: | + echo $CR_PAT | docker login ghcr.io -u dzcode-io --password-stdin + docker push ghcr.io/dzcode-io/api-dot-production-dot-dzcode-dot-io-server:latest + env: + CR_PAT: ${{ secrets.CR_PAT }} + + docker-build-push-web-server: + needs: build + runs-on: ubuntu-latest + env: + CI: true + STAGE: production + DEPLOY_VERSION: ${{ github.ref_type == 'tag' && github.ref_name || format('0.0.0-{0}-{1}-{2}', github.ref_name, github.run_number, github.run_attempt) }} + + steps: + - name: "Git" + uses: actions/checkout@v4 + - name: "Nodejs" + uses: actions/setup-node@v4 + with: + node-version: "20" + cache: "npm" + - run: npm ci + - name: Download artifact + uses: actions/download-artifact@v4 with: - key: ${{ secrets.SSH_KEY }} - known_hosts: ${{ secrets.SSH_KNOWN_HOSTS }} - - name: "Bundle info" + name: build output (ubuntu-latest, 20) + + - name: "Generate Bundle info" run: npm run generate:bundle-info $DEPLOY_VERSION production - - name: "Bundle ./web" - run: npx lerna run bundle:alone --scope @dzcode.io/web + - name: "Sentry Release" - run: npm run generate:sentry-release $DEPLOY_VERSION production ${{ secrets.SENTRY_AUTH_TOKEN }} - - name: "Pre-deploy" - run: npm run pre-deploy - - name: "Deploy" - run: npm run deploy - - lighthouse: - needs: bundle-deploy - uses: ./.github/workflows/ci.reusable.lighthouse.yml - with: - serverBaseUrl: "https://lh.dzcode.io" - testBaseUrl: "https://www.dzcode.io" - stage: "production" - secrets: - LH_SERVER_TOKEN_STG: ${{ secrets.LH_SERVER_TOKEN_STG }} - LH_SERVER_TOKEN_PRD: ${{ secrets.LH_SERVER_TOKEN_PRD }} + # todo-zm: remove sentry entirely + run: cd ./web && npm run generate:sentry-release $DEPLOY_VERSION production ${{ secrets.SENTRY_AUTH_TOKEN }} + + - name: "Bundle ./web for deployment" + run: | + cd ./web + npm run bundle:alone + npm run pre-deploy + + - name: "Write ./web-server deps into Dockerfile..." + run: | + cd ./web-server + npm run prepare-dockerfile + + - name: Build docker image + run: | + docker buildx build -f web-server.Dockerfile . -t ghcr.io/dzcode-io/production-dot-dzcode-dot-io-server:latest + env: + DOCKER_BUILDKIT: 1 + + - name: Push docker image + run: | + echo $CR_PAT | docker login ghcr.io -u dzcode-io --password-stdin + docker push ghcr.io/dzcode-io/production-dot-dzcode-dot-io-server:latest + env: + CR_PAT: ${{ secrets.CR_PAT }} + + deploy-to-zcluster: + needs: [docker-build-push-api, docker-build-push-web-server] + runs-on: ubuntu-latest + env: + CI: true + + steps: + - name: "Git" + uses: actions/checkout@v4 + + - name: install zcluster + run: | + curl -fsSL https://infra.zak-man.com/install.sh | sh + echo "/home/runner/.zcluster/bin" >> $GITHUB_PATH + + - name: Deploy to zcluster + run: zcluster deploy -p production-dzcode ./docker-compose.production.yml + env: + ADMIN_AUTH_TOKEN: ${{ secrets.ADMIN_AUTH_TOKEN }} + OPENAI_KEY: ${{ secrets.OPENAI_KEY }} + GITHUB_TOKEN: ${{ secrets.API_GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 9f53a20ce..a54047237 100644 --- a/.gitignore +++ b/.gitignore @@ -16,7 +16,6 @@ coverage .bundle-info.json # api -api/oracle-cloud/build api/fetch_cache api/postgres_db api/meilisearch_db diff --git a/README.md b/README.md index 1c345cfd1..6a521a841 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ The code for [dzcode.io](https://dzcode.io), a website for the Algerian open-sou **Apps:** - [`./web`](./web) ([dzcode.io](https://dzcode.io) or [stage.dzcode.io](https://stage.dzcode.io)) -- [`./api`](./api) ([api.dzcode.io](https://api.dzcode.io) or [api-stage.dzcode.io](https://api-stage.dzcode.io)) +- [`./api`](./api) ([api.dzcode.io](https://api.dzcode.io) or [api.stage.dzcode.io](https://api.stage.dzcode.io)) **Packages:** diff --git a/api.Dockerfile b/api.Dockerfile index 4ec1e4dcc..b4dd01486 100644 --- a/api.Dockerfile +++ b/api.Dockerfile @@ -3,6 +3,7 @@ FROM --platform=linux/amd64 node:22 WORKDIR /usr/src/repo COPY ./package.json ./package.json +COPY ./package-lock.json ./package-lock.json # AUTO_GEN COPY ./api/dist ./api/dist @@ -18,7 +19,7 @@ COPY ./packages/utils/dist ./packages/utils/dist COPY ./packages/utils/package.json ./packages/utils/package.json # AUTO_GEN_END -RUN npm install --omit=dev --workspace=@dzcode.io/api +RUN npm install --omit=dev --workspace=@dzcode.io/api --frozen-lockfile ENV PORT=80 WORKDIR /usr/src/repo/api diff --git a/api/oracle-cloud/Dockerfile b/api/oracle-cloud/Dockerfile deleted file mode 100644 index a94c2be96..000000000 --- a/api/oracle-cloud/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM node:20 -# Create app directory -WORKDIR /usr/src/repo - -# copy app bundle -COPY . ./ - -RUN pwd -RUN ls - -# Install app dependencies -RUN npm install --frozen-lockfile - -# export server port -ENV PORT=7070 -EXPOSE ${PORT} - -# Run the app -WORKDIR /usr/src/repo/api -CMD [ "npm", "run", "start" ] diff --git a/api/oracle-cloud/deploy.ts b/api/oracle-cloud/deploy.ts deleted file mode 100644 index 0d4616d93..000000000 --- a/api/oracle-cloud/deploy.ts +++ /dev/null @@ -1,95 +0,0 @@ -// can be ran locally from ./api: -// SSH_ADDRESS_STG="user@x.x.x.x" SSH_PATH="path/to/private/ssh/key" npm run deploy:stg - -import { execSync } from "child_process"; -import { copySync, existsSync } from "fs-extra"; -import { join } from "path"; - -console.log("🏗 Preparing files ..."); -const stdout = execSync( - "lerna list --include-dependencies --json --all --loglevel silent --scope @dzcode.io/api", -); -const dependencies = JSON.parse(stdout.toString()) as Array<{ name: string; location: string }>; -const subPaths = ["dist", "package.json", "models", "db"]; -const workspaceRoot = join(__dirname, "../.."); -const fromToRecords = dependencies - .map<{ from: string; to: string }>(({ location }) => ({ - from: location, - to: join(workspaceRoot, "api/oracle-cloud/build", location.replace(workspaceRoot, ".")), - })) - .reduce>( - (pV, { from, to }) => [ - ...pV, - ...subPaths.map((subPath) => ({ from: join(from, subPath), to: join(to, subPath) })), - ], - [], - ) - .filter(({ from }) => existsSync(from)); - -fromToRecords.push( - { from: "./oracle-cloud/docker-compose.yml", to: "./oracle-cloud/build/docker-compose.yml" }, - { from: "./oracle-cloud/Dockerfile", to: "./oracle-cloud/build/Dockerfile" }, - { from: "./oracle-cloud/nginx.conf", to: "./oracle-cloud/build/nginx.conf" }, - { from: join(workspaceRoot, "package.json"), to: "./oracle-cloud/build/package.json" }, - { from: join(workspaceRoot, "package-lock.json"), to: "./oracle-cloud/build/package-lock.json" }, -); - -fromToRecords.forEach(({ from, to }) => { - copySync(from, to); - console.log(to); -}); - -console.log("✅ File preparation completed\n"); - -// Deploying with ssh -const isProduction = process.argv.includes("production"); -console.log("⚙️ Deploying to", isProduction ? "Production" : "Staging", "..."); - -let logs: Buffer; -const sshServer = isProduction ? process.env.SSH_ADDRESS_PRD : process.env.SSH_ADDRESS_STG; -const sshKeyPath = process.env.SSH_PATH; -const appPath = "~/app"; -const sshPrefix = - "ssh -o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=60 " + - (sshKeyPath ? `-i ${sshKeyPath} ` : "") + - sshServer + - " "; - -console.log("⚠️ Cleaning up old containers ..."); -logs = execSync(sshPrefix + '"sudo docker container prune --force"'); -console.log(String(logs)); - -console.log("⚠️ Cleaning up old images ..."); -logs = execSync(sshPrefix + '"sudo docker image prune --force"'); -console.log(String(logs)); - -console.log("⚠️ Deleting old code ..."); -logs = execSync(sshPrefix + '"rm -f -r ' + appPath + '"'); -console.log(String(logs)); -logs = execSync(sshPrefix + '"mkdir ' + appPath + '"'); -console.log(String(logs)); - -console.log("⤴️ Uploading new code ..."); -logs = execSync( - "rsync " + - (sshKeyPath ? `-e "ssh -i ${sshKeyPath}" ` : "") + - " -r oracle-cloud/build/* " + - sshServer + - ":" + - appPath, -); -console.log(String(logs)); - -// note-zm: we must take down the containers before starting them up, our weak VPS can not handle -// the load of running containers and building new ones at the same time -// todo-zm: build images in CI and push to private Github registry, so our VPS will only pull the images -console.log("⚙️ Taking down running containers ..."); -logs = execSync(sshPrefix + '"cd ' + appPath + ' && docker compose down --remove-orphans"'); -console.log(String(logs)); - -console.log("\n⚙️ Starting up the app"); -logs = execSync( - sshPrefix + '"cd ' + appPath + ' && docker compose up -d --build --remove-orphans"', -); -console.log(String(logs)); -console.log("✅ Deployment successful."); diff --git a/api/oracle-cloud/docker-compose.yml b/api/oracle-cloud/docker-compose.yml deleted file mode 100644 index 1f113e07a..000000000 --- a/api/oracle-cloud/docker-compose.yml +++ /dev/null @@ -1,43 +0,0 @@ -version: "3" -services: - reverse-proxy: - image: nginx - ports: - - "80:80" - depends_on: - - api - - lhserver - volumes: - - ./nginx.conf:/etc/nginx/nginx.conf - api: - build: "." - ports: - - "7070:7070" - depends_on: - postgres: - condition: service_started - meilisearch: - condition: service_started - env_file: - - /home/ubuntu/app-env/api.env - volumes: - - /home/ubuntu/app-data/api/fetch_cache:/usr/src/repo/api/fetch_cache - - /home/ubuntu/app-data/api/sqlite_db:/usr/src/repo/api/sqlite_db - postgres: - image: postgres - ports: - - "5432:5432" - volumes: - - /home/ubuntu/app-data/api/postgres_db:/var/lib/postgresql/data - environment: - POSTGRES_HOST_AUTH_METHOD: trust - POSTGRES_DB: db - meilisearch: - image: getmeili/meilisearch:v1.11.3 - ports: - - "7700:7700" - volumes: - - /home/ubuntu/app-data/api/meilisearch_db:/meili_data - environment: - MEILI_NO_ANALYTICS: true - MEILI_MASTER_KEY: "default" diff --git a/api/oracle-cloud/nginx.conf b/api/oracle-cloud/nginx.conf deleted file mode 100644 index fbbaf0bdd..000000000 --- a/api/oracle-cloud/nginx.conf +++ /dev/null @@ -1,22 +0,0 @@ -events { -} - - -http { - server { - client_max_body_size 10M; - listen 80; - - location / { - if ($host ~* ^api-stage.dzcode.io$) { - proxy_pass http://api:7070; - } - if ($host ~* ^api_stage.dzcode.io$) { - proxy_pass http://api:7070; - } - if ($host ~* ^api.dzcode.io$) { - proxy_pass http://api:7070; - } - } - } -} diff --git a/api/package.json b/api/package.json index ee6345d38..5ef9ecf5b 100644 --- a/api/package.json +++ b/api/package.json @@ -63,11 +63,9 @@ "build:alone:watch": "tspc --watch --preserveWatchOutput", "build:watch": "lerna run build:alone:watch --scope=@dzcode.io/api --include-dependencies --parallel", "clean": "lerna run clean:alone --scope=@dzcode.io/api --include-dependencies --stream", - "clean:alone": "del dist coverage fetch_cache oracle-cloud/build", + "clean:alone": "del dist coverage fetch_cache", "db:generate-migration": "drizzle-kit generate", "db:server": "docker compose down && docker compose up", - "deploy": "del ./oracle-cloud/build && tsx oracle-cloud/deploy.ts production", - "deploy:stg": "del ./oracle-cloud/build && tsx oracle-cloud/deploy.ts staging", "generate:bundle-info": "tsx ../packages/tooling/bundle-info.ts", "generate:sentry-release": "tsx ../packages/tooling/sentry-release.ts api dist", "lint": "npm run build && npm run lint:alone", @@ -78,9 +76,9 @@ "lint:prettier": "prettier --config ../packages/tooling/.prettierrc --ignore-path ../packages/tooling/.prettierignore --log-level warn", "lint:ts-prune": "tsx ../packages/tooling/setup-ts-prune.ts && ts-prune --error", "lint:tsc": "tspc --noEmit", + "prepare-dockerfile": "tsx ../packages/tooling/write-dockerfile.ts @dzcode.io/api", "start": "node dist/app/index.js", "start:dev": "tsx ../packages/tooling/nodemon.ts \"@dzcode.io/api\" && npm-run-all --parallel start:nodemon db:server", - "prepare-dockerfile": "tsx ../packages/tooling/write-dockerfile.ts @dzcode.io/api", "start:nodemon": "wait-port localhost:5432 && delay 2 && nodemon dist/app/index.js", "test": "npm run build && npm run test:alone", "test:alone": "jest --config ../packages/tooling/jest.config.ts --rootDir .", diff --git a/docker-compose.production.yml b/docker-compose.production.yml new file mode 100644 index 000000000..11050f9b9 --- /dev/null +++ b/docker-compose.production.yml @@ -0,0 +1,67 @@ +name: dzcode-production + +services: + web: + image: ghcr.io/dzcode-io/production-dot-dzcode-dot-io-server:latest + pull_policy: always + restart: unless-stopped + environment: + - VIRTUAL_HOST=www.dzcode.io,dzcode.io + - LETSENCRYPT_HOST=www.dzcode.io,dzcode.io + - STAGE=production + networks: + - main-infra-network + + api: + image: ghcr.io/dzcode-io/api-dot-production-dot-dzcode-dot-io-server:latest + pull_policy: always + restart: unless-stopped + depends_on: + postgres: + condition: service_started + meilisearch: + condition: service_started + environment: + - VIRTUAL_HOST=api.dzcode.io + - LETSENCRYPT_HOST=api.dzcode.io + - NODE_ENV=production + - OPENAI_KEY={{OPENAI_KEY}} + - GITHUB_TOKEN={{GITHUB_TOKEN}} + volumes: + - fetch_cache:/usr/src/repo/api/fetch_cache + - sqlite_db:/usr/src/repo/api/sqlite_db + networks: + - main-infra-network + - internal-network + + postgres: + image: postgres + volumes: + - postgres_db:/var/lib/postgresql/data + environment: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_DB: db + networks: + - internal-network + + meilisearch: + image: getmeili/meilisearch:v1.11.3 # database schema is different between versions + volumes: + - meilisearch_db:/meili_data + environment: + MEILI_NO_ANALYTICS: true + MEILI_MASTER_KEY: "default" # we only access it through `./api` + networks: + - internal-network + +networks: + main-infra-network: + external: true + internal-network: + internal: true + +volumes: + postgres_db: + meilisearch_db: + fetch_cache: + sqlite_db: diff --git a/package.json b/package.json index 52bc01380..63cb023b7 100644 --- a/package.json +++ b/package.json @@ -58,7 +58,6 @@ "build": "lerna run build:alone --stream", "build:watch": "lerna run build:alone:watch --parallel", "clean": "lerna run clean:alone --stream", - "deploy": "lerna run deploy --parallel --stream", "dev": "echo \"Please run one of these commands:\\n\\nnpm run dev:web\\nnpm run dev:api\\nnpm run dev:all\n\"", "dev:all": "npm-run-all \"build --include-dependencies {@}\" --parallel \"build:watch --include-dependencies {@}\" \"start:dev {@}\" --", "dev:api": "npm run dev:all --scope=@dzcode.io/api", diff --git a/packages/tooling/.prettierignore b/packages/tooling/.prettierignore index 44960f697..839a26d47 100644 --- a/packages/tooling/.prettierignore +++ b/packages/tooling/.prettierignore @@ -3,7 +3,6 @@ coverage dist # api -oracle-cloud/build db # web diff --git a/packages/tooling/setup-ts-prune.ts b/packages/tooling/setup-ts-prune.ts index 363d82288..2a567c9c3 100644 --- a/packages/tooling/setup-ts-prune.ts +++ b/packages/tooling/setup-ts-prune.ts @@ -3,7 +3,7 @@ import { join } from "path"; console.log("Setting up .ts-prunerc ..."); -const paths = ["node_modules", "coverage", "dist", "oracle-cloud/build", "bundle"]; +const paths = ["node_modules", "coverage", "dist", "bundle"]; const tsPruneJson = { ignore: paths.join("|") }; writeFileSync(join(process.cwd(), ".ts-prunerc"), JSON.stringify(tsPruneJson, null, 2)); diff --git a/scripts/deploy.rs b/scripts/deploy.rs index aeb15b35c..1f84e849a 100755 --- a/scripts/deploy.rs +++ b/scripts/deploy.rs @@ -13,10 +13,10 @@ use clap::Parser; #[derive(Clone, Debug, clap::ValueEnum)] enum Env { Stage, - Prod, + Production, } -/// Deploy dzcode to stage or prod +/// Deploy dzcode to stage or production #[derive(Parser, Debug)] struct Args { /// Environment to deploy to diff --git a/web-server.Dockerfile b/web-server.Dockerfile index 81821b25a..7d4c6b237 100644 --- a/web-server.Dockerfile +++ b/web-server.Dockerfile @@ -3,6 +3,7 @@ FROM --platform=linux/amd64 node:22 WORKDIR /usr/src/repo COPY ./package.json ./package.json +COPY ./package-lock.json ./package-lock.json # AUTO_GEN COPY ./web-server/dist ./web-server/dist @@ -23,7 +24,7 @@ COPY ./data/package.json ./data/package.json COPY ./data/models ./data/models # AUTO_GEN_END -RUN npm install --omit=dev --workspace=@dzcode.io/web-server +RUN npm install --omit=dev --workspace=@dzcode.io/web-server --frozen-lockfile ENV PORT=80 WORKDIR /usr/src/repo/web-server