Adds testing Railway deployment to CI

.github/workflows/test-deploy.yaml

@@ -7,24 +7,35 @@ on:
       - ".github/**"
     branches:
       - main
+      # TODO: remove before merging this PR
+      - miho-railway-deployment-test-ci
     tags:
       - v*
   workflow_dispatch:
 
 env:
-  APP_PREFIX: ci-${{ github.sha }}
+  # We use run_id becuase it's shorter than commit SHA since
+  # the max length for project name in Railway is 25 characters.
+  APP_PREFIX: ci-${{ github.run_id }}
   FLY_API_TOKEN: ${{ secrets.FLY_GITHUB_TESTING_TOKEN }}
   FLY_REGION: mia
+  FLY_ORG: wasp-testing
+  RAILWAY_API_TOKEN: ${{ secrets.RAILWAY_GITHUB_TESTING_TOKEN }}
+  RAILWAY_WASP_WORKSPACE_ID: eb1f9060-40c8-4be0-a372-3a8c9b8bdcf2
   APP_TO_DEPLOY: waspc/examples/todoApp
+  DEPLOY_INFO_DIR: /tmp/deploy-info
+  CACHE_KEY_PREFIX: deploy-info
 
 jobs:
-  fly_deploy_app:
-    name: Deploy Wasp app
-
+  deploy_app:
+    name: Deploy app (${{ matrix.provider }})
     runs-on: ubuntu-latest
-
+    strategy:
+      fail-fast: false
+      matrix:
+        provider: [fly, railway]
     environment:
-      name: fly-deploy-test
+      name: ${{ matrix.provider }}-deploy-test
 
     steps:
       - uses: actions/checkout@v4
@@ -40,15 +51,18 @@ jobs:
         working-directory: waspc
         run: ./run install
 
-        # NOTE: We tell users to install the latest version of Fly CLI,
-        # so we use it here too.
-      - uses: superfly/flyctl-actions/setup-flyctl@v1
+      - name: Install Fly CLI
+        if: matrix.provider == 'fly'
+        uses: superfly/flyctl-actions/setup-flyctl@v1
 
-      - name: Deploy App to Fly.io
+      - name: Install Railway CLI
+        if: matrix.provider == 'railway'
+        run: npm install -g @railway/cli
+
+      - name: Prepare env variables
         working-directory: ${{ env.APP_TO_DEPLOY }}
         run: |
           set -e
-
           # NOTE: This assumes env var values don't contain:
           #   - The character `#`.
           #   - An empty line (i.e., there are no multiline env values).
@@ -58,64 +72,152 @@ jobs:
             awk 'NF {$1=$1;print}' |
             sed -E 's/^/--server-secret\n/'
           )
+          # Save into a file so that subsequent steps can use it.
+          printf '%s\n' "${ENV_VAR_ARGUMENTS[@]}" > .env-args.txt
 
-          echo "Deploying with prefix: $APP_PREFIX"
-          # NOTE:
-          # - The `yes` command is necessary because the `fly launch` command
-          # prompts for confirmation.
-          # - We use a Bash array for `$ENV_VAR_ARGUMENTS` to ensure proper
-          # word splitting (i.e., force Bash to interpret the flags separately
-          # instead of passing it as a single string value).
-          yes | wasp-cli deploy fly launch "$APP_PREFIX" "$FLY_REGION" --org wasp-testing "${ENV_VAR_ARGUMENTS[@]}"
-
-      - name: Save deployed app hostnames
-        id: save_hostnames
+      - name: Deploy app to Fly
+        if: matrix.provider == 'fly'
         working-directory: ${{ env.APP_TO_DEPLOY }}
         run: |
-          echo "server_hostname=$(flyctl status -j -c fly-server.toml | jq -r '.Hostname')" >> "$GITHUB_OUTPUT"
-          echo "client_hostname=$(flyctl status -j -c fly-client.toml | jq -r '.Hostname')" >> "$GITHUB_OUTPUT"
+          set -e
+          echo "Deploying with prefix: $APP_PREFIX (provider: fly)"
+          mapfile -t ENV_VAR_ARGUMENTS < .env-args.txt
+          yes | wasp-cli deploy fly launch "$APP_PREFIX" "$FLY_REGION" --org "$FLY_ORG" "${ENV_VAR_ARGUMENTS[@]}"
 
-    outputs:
-      server_hostname: ${{ steps.save_hostnames.outputs.server_hostname }}
-      client_hostname: ${{ steps.save_hostnames.outputs.client_hostname }}
+      - name: Deploy app to Railway
+        if: matrix.provider == 'railway'
+        working-directory: ${{ env.APP_TO_DEPLOY }}
+        run: |
+          set -e
+          echo "Deploying with prefix: $APP_PREFIX (provider: railway)"
+          mapfile -t ENV_VAR_ARGUMENTS < .env-args.txt
+          wasp-cli deploy railway launch "$APP_PREFIX" --workspace $RAILWAY_WASP_WORKSPACE_ID "${ENV_VAR_ARGUMENTS[@]}"
 
-  smoke_test_app:
-    name: Run smoke tests on deployed app
+      - name: Get deployed app hostnames (Fly)
+        if: matrix.provider == 'fly'
+        working-directory: ${{ env.APP_TO_DEPLOY }}
+        run: |
+          set -e
+          function get_fly_app_hostname() {
+            config_file="$1"
+            flyctl status -j -c "$config_file" | jq -r '.Hostname'
+          }
+          echo "SERVER_HOSTNAME=$(get_fly_app_hostname fly-server.toml)" >> "$GITHUB_ENV"
+          echo "CLIENT_HOSTNAME=$(get_fly_app_hostname fly-client.toml)" >> "$GITHUB_ENV"
+
+      - name: Get deployed app hostnames (Railway)
+        if: matrix.provider == 'railway'
+        working-directory: ${{ env.APP_TO_DEPLOY }}
+        run: |
+          set -e
+          function get_railway_service_hostname() {
+            service_name="$1"
+            railway status --json | jq -r --arg NAME "$service_name" '.services.edges[] | select(.node.name == $NAME) | .node.serviceInstances.edges[0].node.domains.serviceDomains[0].domain'
+          }
+          server_service_name="${APP_PREFIX}-server"
+          client_service_name="${APP_PREFIX}-client"
+          echo "SERVER_HOSTNAME=$(get_railway_service_hostname "$server_service_name")" >> "$GITHUB_ENV"
+          echo "CLIENT_HOSTNAME=$(get_railway_service_hostname "$client_service_name")" >> "$GITHUB_ENV"
+
+      - name: Save deployment info to cache
+        run: |
+          set -e
+          mkdir -p "$DEPLOY_INFO_DIR"
+          echo "$SERVER_HOSTNAME" > "$DEPLOY_INFO_DIR/server-hostname"
+          echo "$CLIENT_HOSTNAME" > "$DEPLOY_INFO_DIR/client-hostname"
+
+      - name: Cache deployment info
+        uses: actions/cache/save@v4
+        with:
+          path: ${{ env.DEPLOY_INFO_DIR }}
+          key: ${{ env.CACHE_KEY_PREFIX }}-${{ matrix.provider }}-${{ github.run_id }}
 
+  smoke_test_app:
+    name: Smoke test app (${{ matrix.provider }})
     runs-on: ubuntu-latest
-    needs: fly_deploy_app
-    env:
-      SERVER_HOSTNAME: ${{ needs.fly_deploy_app.outputs.server_hostname }}
-      CLIENT_HOSTNAME: ${{ needs.fly_deploy_app.outputs.client_hostname }}
+    needs: deploy_app
+    strategy:
+      fail-fast: false
+      matrix:
+        provider: [fly, railway]
 
     steps:
-      - name: Smoke test the server
-        run: |
-          curl --fail --silent -X POST https://$SERVER_HOSTNAME/operations/get-date | jq '.json'
+      - uses: actions/checkout@v4
+
+      - name: Restore deployment info
+        uses: actions/cache/restore@v4
+        with:
+          path: ${{ env.DEPLOY_INFO_DIR }}
+          key: ${{ env.CACHE_KEY_PREFIX }}-${{ matrix.provider }}-${{ github.run_id }}
 
-      - name: Smoke test the client
+      - name: Load hostnames from cache
         run: |
-          curl --fail --silent https://$CLIENT_HOSTNAME | grep 'ToDo App'
+          set -e
+          echo "SERVER_HOSTNAME=$(cat "$DEPLOY_INFO_DIR/server-hostname")" >> "$GITHUB_ENV"
+          echo "CLIENT_HOSTNAME=$(cat "$DEPLOY_INFO_DIR/client-hostname")" >> "$GITHUB_ENV"
+
+      - name: Smoke test deployed app
+        run: ./scripts/smoke-test-deployed-test-app.sh "$SERVER_HOSTNAME" "$CLIENT_HOSTNAME"
 
-  fly_destroy_app:
+  cleanup:
+    name: Cleanup app (${{ matrix.provider }})
     runs-on: ubuntu-latest
-    name: Clean up deployed Fly app
-    needs: [fly_deploy_app, smoke_test_app]
-    # NOTE: Fly deployments can sometimes "fail" but still deploy the apps. We
-    # want to always clean them up.
     if: always()
+    needs: [deploy_app, smoke_test_app]
+    strategy:
+      fail-fast: false
+      matrix:
+        provider: [fly, railway]
 
     steps:
-      - uses: superfly/flyctl-actions/setup-flyctl@v1
+      - uses: actions/checkout@v4
+
+      - name: Prepare Fly CLI for cleanup
+        if: matrix.provider == 'fly'
+        uses: superfly/flyctl-actions/setup-flyctl@v1
         with:
           # NOTE: We pinned the Fly because we don't want the changes in the
           # Fly CLI to affect our cleanup procedure. `fly destroy` isn't a part
           # of Wasp, we're just incidentally using the same tool Wasp uses.
           version: v0.3.164
 
       - name: Clean up testing app from Fly.io
+        if: matrix.provider == 'fly'
         run: |
           # NOTE: We are relying on Wasp's naming conventions here
           flyctl apps destroy -y $APP_PREFIX-server || true
           flyctl apps destroy -y $APP_PREFIX-client || true
           flyctl apps destroy -y $APP_PREFIX-db || true
+
+      - name: Clean up testing app from Railway
+        if: matrix.provider == 'railway'
+        run: |
+          set -e
+
+          function get_railway_project_id() {
+            local project_name="$1"
+            curl --silent --request POST \
+              --url https://backboard.railway.com/graphql/v2 \
+              --header "Authorization: Bearer $RAILWAY_API_TOKEN" \
+              --header 'Content-Type: application/json' \
+              --data "{\"query\":\"query { workspace(workspaceId: \\\"$RAILWAY_WASP_WORKSPACE_ID\\\") { projects { edges { node { id name } } } } }\"}" | \
+              jq -r --arg PROJECT_NAME "$project_name" '.data.workspace.projects.edges[] | select(.node.name == $PROJECT_NAME) | .node.id'
+          }
+
+          function delete_railway_project() {
+            local project_id="$1"
+            curl --request POST \
+              --url https://backboard.railway.com/graphql/v2 \
+              --header "Authorization: Bearer $RAILWAY_API_TOKEN" \
+              --header 'Content-Type: application/json' \
+              --data "{\"query\":\"mutation { projectDelete(id: \\\"$project_id\\\") }\"}"
+          }
+
+          project_id=$(get_railway_project_id "$APP_PREFIX")
+
+          if [ -n "$project_id" ] && [ "$project_id" != "null" ]; then
+            echo "Found project $APP_PREFIX with ID: $project_id"
+            delete_railway_project "$project_id"
+          else
+            echo "Project $APP_PREFIX not found, skipping cleanup"
+          fi

scripts/smoke-test-deployed-test-app.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+MAX_RETRIES=5
+INITIAL_WAIT_SECONDS=10
+TIMEOUT_SECONDS=30
+
+usage() {
+  echo "Usage: $0 <server_hostname> <client_hostname>"
+}
+
+if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
+  usage
+  exit 0
+fi
+
+if [[ $# -ne 2 ]]; then
+  echo "Error: Missing required arguments"
+  usage
+  exit 1
+fi
+
+SERVER_HOSTNAME="$1"
+CLIENT_HOSTNAME="$2"
+SERVER_URL="https://$SERVER_HOSTNAME"
+CLIENT_URL="https://$CLIENT_HOSTNAME"
+
+retry_with_backoff() {
+  local name="$1"
+  shift
+  local attempt=0
+  local wait_time=0
+  while [[ $attempt -lt $MAX_RETRIES ]]; do
+    echo "[$name] Attempt $((attempt + 1))/$MAX_RETRIES at $(date -u +'%H:%M:%S')"
+    if "$@"; then
+      echo "[$name] Success"
+      return 0
+    fi
+    attempt=$((attempt + 1))
+    if [[ $attempt -lt $MAX_RETRIES ]]; then
+      wait_time=$((INITIAL_WAIT_SECONDS * (2 ** (attempt - 1))))
+      echo "[$name] Waiting ${wait_time}s before retry..."
+      sleep "$wait_time"
+    else
+      echo "[$name] Failed after $MAX_RETRIES attempts"
+      return 1
+    fi
+  done
+}
+
+server_check_once() {
+  echo "[Server] Hitting $SERVER_URL/operations/get-date"
+  curl --fail --silent --max-time "$TIMEOUT_SECONDS" -X POST \
+    "$SERVER_URL/operations/get-date" \
+    | jq -e '.json'
+}
+
+client_check_once() {
+  echo "[Client] Hitting $CLIENT_URL"
+  curl --fail --silent --max-time "$TIMEOUT_SECONDS" \
+    "$CLIENT_URL" \
+    | grep 'ToDo App'
+}
+
+echo "Server URL: $SERVER_URL"
+echo "Client URL: $CLIENT_URL"
+
+if ! retry_with_backoff "Server" server_check_once; then
+  echo "Server smoke test failed"
+  exit 1
+fi
+
+if ! retry_with_backoff "Client" client_check_once; then
+  echo "Client smoke test failed"
+  exit 1
+fi
+
+echo "All smoke tests passed"