refactor: Major refactor to support sync job

Author: tjtanjin

.github/workflows/publish-and-deploy.yml

@@ -11,7 +11,7 @@ on:
 
 jobs:
   publish:
-    name: Publish Docker Image
+    name: Publish Docker Images
     runs-on: ubuntu-latest
     environment: ${{ inputs.target_env }}
     outputs:
@@ -51,44 +51,104 @@ jobs:
             exit 1
           fi
 
-      - name: Backup existing image
+      - name: Detect changed paths
+        id: changes
+        uses: dorny/paths-filter@v3
+        # todo: find a way to detect changes within same branch, for now do not push to dev/prod in parallel
+        with:
+          filters: |
+            api:
+              - 'src/api/**'
+            jobs:
+              - 'src/jobs/**'
+
+      - name: Backup existing API image
         # Pulls the current image using the target tag and backup it with a previous tag if it exists.
+        if: steps.changes.outputs.api == 'true'
         run: |
           OWNER="${{ github.repository_owner }}"
-          BASE="${{ github.event.repository.name }}"
+          BASE="${{ github.event.repository.name }}-api"
           IMAGE="ghcr.io/$OWNER/$BASE:${{ steps.vars.outputs.TARGET_TAG }}"
           BACKUP="ghcr.io/$OWNER/$BASE:${{ steps.vars.outputs.BACKUP_TAG }}"
-          echo "Backing up image if it exists: $IMAGE → $BACKUP"
+          echo "Backing up API image if it exists: $IMAGE → $BACKUP"
           if docker pull "$IMAGE"; then
             docker tag "$IMAGE" "$BACKUP"
             docker push "$BACKUP"
           else
-            echo "No image to back up."
+            echo "No API image to back up."
           fi
 
-      - name: Cleanup Old Image Digests
-        # Cleans up old images
+      - name: Backup existing Jobs image
+        # Pulls the current image using the target tag and backup it with a previous tag if it exists.
+        if: steps.changes.outputs.jobs == 'true'
+        run: |
+          OWNER="${{ github.repository_owner }}"
+          BASE="${{ github.event.repository.name }}-jobs"
+          IMAGE="ghcr.io/$OWNER/$BASE:${{ steps.vars.outputs.TARGET_TAG }}"
+          BACKUP="ghcr.io/$OWNER/$BASE:${{ steps.vars.outputs.BACKUP_TAG }}"
+          echo "Backing up Jobs image if it exists: $IMAGE → $BACKUP"
+          if docker pull "$IMAGE"; then
+            docker tag "$IMAGE" "$BACKUP"
+            docker push "$BACKUP"
+          else
+            echo "No Jobs image to back up."
+          fi
+
+      - name: Cleanup Old API Image Digests
+        # Cleans up old api images
+        if: steps.changes.outputs.api == 'true'
+        uses: actions/delete-package-versions@v5
+        continue-on-error: true
+        with:
+          package-name: ${{ github.event.repository.name }}-api
+          package-type: container
+          owner: ${{ github.repository_owner }}
+          ignore-versions: "^(prod-latest|prod-previous|dev-latest|dev-previous)$" # keeps important tags (latest and previous)
+          token: ${{ secrets.MACHINE_PAT }}
+
+      - name: Cleanup Old Jobs Image Digests
+        # Cleans up old jobs images
+        if: steps.changes.outputs.jobs == 'true'
         uses: actions/delete-package-versions@v5
         continue-on-error: true
         with:
-          package-name: ${{ github.event.repository.name }}
+          package-name: ${{ github.event.repository.name }}-jobs
           package-type: container
           owner: ${{ github.repository_owner }}
-          min-versions-to-keep: 2 # Keeps the 2 most recent versions (latest and previous)
+          ignore-versions: "^(prod-latest|prod-previous|dev-latest|dev-previous)$" # keeps important tags (latest and previous)
           token: ${{ secrets.MACHINE_PAT }}
 
-      - name: Build & push image
+      - name: Build & push API image
+        # Builds a new Docker image with the target tag and pushes it to GHCR.
+        if: steps.changes.outputs.api == 'true'
+        run: |
+          OWNER=${{ github.repository_owner }}
+          BASE=${{ github.event.repository.name }}-api
+          TAG=${{ steps.vars.outputs.TARGET_TAG }}
+          IMAGE=ghcr.io/$OWNER/$BASE:$TAG
+
+          echo "Building image with tag $IMAGE"
+
+          docker build \
+            --file docker/api/Dockerfile \
+            --tag $IMAGE \
+            .
+
+          docker push $IMAGE
+
+      - name: Build & push Jobs image
         # Builds a new Docker image with the target tag and pushes it to GHCR.
+        if: steps.changes.outputs.jobs == 'true'
         run: |
           OWNER=${{ github.repository_owner }}
-          BASE=${{ github.event.repository.name }}
+          BASE=${{ github.event.repository.name }}-jobs
           TAG=${{ steps.vars.outputs.TARGET_TAG }}
           IMAGE=ghcr.io/$OWNER/$BASE:$TAG
 
           echo "Building image with tag $IMAGE"
 
           docker build \
-            --file docker/Dockerfile \
+            --file docker/jobs/Dockerfile \
             --tag $IMAGE \
             .
 
@@ -146,15 +206,17 @@ jobs:
         # Deploys to VPS.
         run: |
           OWNER="${{ vars.GHCR_OWNER }}"
-          APPLICATION_IMAGE="ghcr.io/$OWNER/${{ github.event.repository.name }}:${{ needs.publish.outputs.target_tag }}"
+          APPLICATION_API_IMAGE="ghcr.io/$OWNER/${{ github.event.repository.name }}-api:${{ needs.publish.outputs.target_tag }}"
+          APPLICATION_JOBS_IMAGE="ghcr.io/$OWNER/${{ github.event.repository.name }}-jobs:${{ needs.publish.outputs.target_tag }}"
           echo "Deploying to VPS..."
           ssh -o StrictHostKeyChecking=no ${{ secrets.DEPLOYMENT_SSH_USER }}@${{ secrets.DEPLOYMENT_SERVER }} "\
 
             # exports general variables
             export PROJECT_NAME='${{ vars.PROJECT_NAME }}' && \
             export GHCR_USER='${{ secrets.MACHINE_USER }}' && \
             export GHCR_PAT='${{ secrets.MACHINE_PAT }}' && \
-            export APPLICATION_IMAGE='$APPLICATION_IMAGE' && \
+            export APPLICATION_API_IMAGE='$APPLICATION_API_IMAGE' && \
+            export APPLICATION_JOBS_IMAGE='$APPLICATION_JOBS_IMAGE' && \
 
             # applies only to production for logging
             export HONEYCOMB_API_KEY='${{ secrets.HONEYCOMB_API_KEY }}' && \

README.md

@@ -54,6 +54,10 @@ React ChatBotify RAG API offers the following features:
 **API Documentation:**
 - `/api/v1/docs`: Interactive Swagger UI for exploring and testing all API endpoints.
 
+**API Key Capabilities:**
+- `RAG_MANAGEMENT_API_KEY`: Can perform both management operations (creating, updating, deleting documents) and query operations.
+- `RAG_QUERY_API_KEY`: Can perform query operations only (generating content or streaming content generation). It cannot be used for management operations.
+
 ### Technologies
 Technologies used by React ChatBotify RAG API are as below:
 

config/env/.env.template

@@ -41,4 +41,10 @@ MONGO_INITDB_PORT=27017
 
 # OpenTelemetry Configuration
 OTEL_SERVICE_NAME=otel-collector
-OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
+OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
+
+# GitHub Configuration
+GITHUB_REPO_OWNER=react-chatbotify
+GITHUB_REPO_NAME=core-library-documentation
+GITHUB_DOCS_PATH='docs'
+GITHUB_TOKEN=GITHUB_TOKEN; # optional, unlikely needed

docker/Dockerfile renamed to docker/api/Dockerfile

@@ -14,7 +14,7 @@ COPY ./package.json ./
 RUN npm install
 
 # Copy additional files needed for runtime
-COPY ./nodemon.json ./
+COPY ./nodemon-api.json ./
 
 # Command to run the application with nodemon for development
-CMD ["npm", "run", "start:watch"]
+CMD ["npm", "run", "start:watch-api"]

docker/Dockerfile.local renamed to docker/api/Dockerfile.local

@@ -1,6 +1,9 @@
 # The development environment runs the live application on the development server.
 services:
   api:
-    image: ${APPLICATION_IMAGE}
+    image: ${APPLICATION_API_IMAGE}
     environment:
       - NODE_ENV=development
+
+  jobs:
+    image: ${APPLICATION_JOBS_IMAGE}

docker/docker-compose.dev.yml

@@ -2,8 +2,20 @@ services:
   api:
     build:
       context: ..
-      dockerfile: ./docker/Dockerfile.local
-    command: npm run start:watch
+      dockerfile: ./docker/api/Dockerfile.local
+    command: npm run start:watch-api
+    volumes:
+      - ..:/app
+      - /app/node_modules
+
+  jobs:
+    # build for local dev
+    build:
+      context: ..
+      dockerfile: ./docker/jobs/Dockerfile.local
+    # for hot reloading in local dev
+    command: npm run start:watch-jobs
+    # codebase mounted for local dev
     volumes:
       - ..:/app
       - /app/node_modules

docker/docker-compose.override.yml

@@ -1,10 +1,13 @@
 # The development environment runs the live application on the development server.
 services:
   api:
-    image: ${APPLICATION_IMAGE}
+    image: ${APPLICATION_API_IMAGE}
     environment:
       - NODE_ENV=production
 
+  jobs:
+    image: ${APPLICATION_JOBS_IMAGE}
+
   # add logging in production environment
   otel-collector:
     image: otel/opentelemetry-collector:latest

docker/docker-compose.prod.yml

@@ -1,6 +1,6 @@
 services:
   api:
-    container_name: rag-api
+    container_name: rag-api-core
     environment:
       - NODE_ENV=local
     ports:
@@ -12,6 +12,16 @@ services:
     networks:
       - core-network
 
+  # runs scheduled jobs on the platform
+  jobs:
+    container_name: rag-api-jobs
+    env_file:
+      - ../config/env/.env
+    depends_on:
+      - api
+    networks:
+      - core-network
+
   chromadb:
     container_name: rag-chromadb
     image: chromadb/chroma

docker/docker-compose.yml

@@ -0,0 +1,51 @@
+# Stage 1: Install dependencies
+FROM node:24-slim AS dependencies
+
+# Maintainer information
+LABEL org.opencontainers.image.authors="Tan Jin (tjtanjin)"
+
+# Set the working directory
+WORKDIR /app
+
+# Copy package.json to the container
+COPY ./package.json ./
+
+# Install the dependencies
+RUN npm install
+
+# Stage 2: Build the application
+FROM node:24-slim AS build
+
+# Set the working directory
+WORKDIR /app
+
+# Copy installed dependencies from the previous stage
+COPY --from=dependencies /app/node_modules ./node_modules
+
+# Copy specific necessary files and directories
+COPY ./src ./src
+COPY ./types ./types
+COPY ./config ./config
+COPY ./tsconfig.json ./
+COPY ./package.json ./
+
+# Build application code
+RUN npm run build
+
+# Stage 3: Create the final image
+FROM node:24-slim
+
+# Set the working directory inside the container
+WORKDIR /app
+
+# Copy built application code from the build stage
+COPY --from=build /app/build ./build
+
+# Copy node_modules directory from the build stage
+COPY --from=build /app/node_modules ./node_modules
+
+# Copy additional files needed for runtime
+COPY package.json ./
+
+# Start the application
+CMD ["node", "./build/jobs/index.js"]

docker/jobs/Dockerfile

@@ -0,0 +1,20 @@
+# For development purposes, the dev docker file simply mounts the codebase with hot-reloading
+FROM node:24-slim
+
+# Maintainer information
+LABEL org.opencontainers.image.authors="Tan Jin (tjtanjin)"
+
+# Set the working directory
+WORKDIR /app
+
+# Copy package.json to the container
+COPY ./package.json ./
+
+# Install the application dependencies
+RUN npm install
+
+# Copy additional files needed for runtime
+COPY nodemon-jobs.json ./
+
+# Command to run the application with nodemon for development
+CMD ["npm", "run", "start:watch-jobs"]

docker/jobs/Dockerfile.local

@@ -0,0 +1,12 @@
+{
+	"watch": [
+		"src",
+		"types",
+		"config"
+	],
+	"ext": "ts,js,json",
+	"ignore": [
+		"node_modules"
+	],
+	"exec": "ts-node --transpile-only ./src/api/index.ts"
+}

nodemon-api.json

@@ -8,5 +8,5 @@
 	"ignore": [
 		"node_modules"
 	],
-	"exec": "ts-node --transpile-only ./src/index.ts"
+	"exec": "ts-node ./src/jobs/index.ts"
 }