From 4de699ed301e2421994dba46eda29944ec46e954 Mon Sep 17 00:00:00 2001 From: z Date: Wed, 28 May 2025 16:37:19 +1200 Subject: [PATCH 1/7] added docker-compose with db, pgadmin and app server --- docker-compose.yml | 73 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..8c6c62b --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,73 @@ +services: + db: + image: postgres:latest + container_name: forc-pub-db + env_file: + - .env.local + environment: + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB_NAME} + POSTGRES_USER: ${POSTGRES_USER:-postgres} + ports: + - "${POSTGRES_PORT:-5432}:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + networks: + - forc-pub-net + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-postgres} -d ${POSTGRES_DB_NAME}"] + interval: 10s + timeout: 5s + retries: 5 + + app: + build: + context: . + dockerfile: deployment/Dockerfile + container_name: forc-pub + env_file: + - .env.local + environment: + POSTGRES_USER: ${POSTGRES_USER:-postgres} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_URI: db + POSTGRES_DB_NAME: ${POSTGRES_DB_NAME} + ports: + - "8080:8080" + depends_on: + db: + condition: service_healthy + networks: + - forc-pub-net + + pgadmin: + image: dpage/pgadmin4:latest + container_name: forc-pub-pgadmin + env_file: + - .env.local + environment: + PGADMIN_DEFAULT_EMAIL: ${PGADMIN_EMAIL:-admin@forc.pub} + PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_PASSWORD:-admin} + PGADMIN_CONFIG_SERVER_MODE: 'False' + PGADMIN_CONFIG_MASTER_PASSWORD_REQUIRED: 'False' + PGADMIN_CONFIG_ENHANCED_COOKIE_PROTECTION: 'False' + PGADMIN_CONFIG_WTF_CSRF_ENABLED: 'False' + PGADMIN_SERVER_JSON_FILE: '/pgadmin4/servers.json' + ports: + - "${PGADMIN_PORT:-5050}:80" + volumes: + - pgadmin_data:/var/lib/pgadmin + - ./scripts/pgadmin-servers.json:/pgadmin4/servers.json:ro + depends_on: + db: + condition: service_healthy + networks: + - forc-pub-net + +volumes: + postgres_data: + pgadmin_data: + +networks: + forc-pub-net: + driver: bridge \ No newline at end of file From 1773500266b8bbfcdc221076522f3529acd48d7c Mon Sep 17 00:00:00 2001 From: z Date: Wed, 28 May 2025 16:37:24 +1200 Subject: [PATCH 2/7] readme upd --- README.md | 51 ++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 0361432..0c386cb 100644 --- a/README.md +++ b/README.md @@ -35,9 +35,9 @@ export PATH="${HOME}/.cargo/bin:${PATH}" You will also need to install [Node.js](https://nodejs.org/en/learn/getting-started/how-to-install-nodejs). -To run the Postgres database locally, you will need [Docker](https://docs.docker.com/engine/install/). +To run the application locally, you will need [Docker](https://docs.docker.com/engine/install/) with Docker Compose. -To connect to the database, you will need the [Diesel CLI](https://diesel.rs/guides/getting-started). +To connect to the database for development, you may optionally install the [Diesel CLI](https://diesel.rs/guides/getting-started). Diesel is the Rust ORM used to create and run database migrations. It requires a separate C library called `libpq` to be installed as well. @@ -75,29 +75,57 @@ cargo run --bin forc.pub ### Running the `forc.pub` server -Before starting the server, the local database must be up and running. +The easiest way to run the complete application stack locally is using Docker Compose, which will start the database, backend server, and pgAdmin interface. + +First, set up the environment by creating a `.env.local` file with your configuration. You'll need to add your Pinata test gateway details and Github App env (if testing github login functionality). + +Now start all services with: ```sh -./scripts/start_local_db.sh +./scripts/start-local-server.sh ``` -Next, set up the environment by copying `.env.example` to `.env.local`, and modifying `.env.local` with your Pinata test gateway details and Github App env (if testing github login functionality). +This will start: +- **PostgreSQL database** on port `${POSTGRES_PORT:-5432}` +- **Backend server** on port `8080` +- **pgAdmin** (database admin UI) on port `${PGADMIN_PORT:-5050}` + +You can access: +- Backend API: http://localhost:8080 +- pgAdmin interface: http://localhost:5050 -Now we can run the server with: +To force rebuild the application image: ```sh -cargo run +./scripts/start-local-server.sh -f ``` -Alternatively, the server can be run locally with Docker, as it is in the deployed environment. +To view logs: ```sh -./scripts/start_local_server.sh +docker compose logs -f +``` + +To stop all services: -# Force the server image to be rebuilt -./scripts/start_local_server.sh -f +```sh +docker compose down ``` +### Alternative: Running components separately + +If you prefer to run components separately for development: + +1. Start just the database and pgAdmin: + ```sh + docker compose up -d db pgadmin + ``` + +2. Run the backend server locally: + ```sh + cargo run + ``` + ### Manually trigger the APIs You can manually trigger the APIs with curl, for example: @@ -115,6 +143,7 @@ The frontend requires npm and node to be installed. ```sh cd app npm i +npm run build npm start ``` From c79dfc9ec19c38167697c045692eb68aeeee6f3d Mon Sep 17 00:00:00 2001 From: z Date: Wed, 28 May 2025 16:37:37 +1200 Subject: [PATCH 3/7] pgadmin server registration --- scripts/pgadmin-servers.json | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 scripts/pgadmin-servers.json diff --git a/scripts/pgadmin-servers.json b/scripts/pgadmin-servers.json new file mode 100644 index 0000000..c5651cf --- /dev/null +++ b/scripts/pgadmin-servers.json @@ -0,0 +1,18 @@ +{ + "Servers": { + "1": { + "Name": "forc.pub Database", + "Group": "Servers", + "Host": "db", + "Port": 5432, + "MaintenanceDB": "forc_pub", + "Username": "postgres", + "Password": "localpw", + "SSLMode": "prefer", + "Timeout": 10, + "UseSSHTunnel": 0, + "TunnelPort": "22", + "TunnelAuthentication": 0 + } + } +} \ No newline at end of file From a1406f60b7b6ad319b87677a96eed7651b6a7b1e Mon Sep 17 00:00:00 2001 From: z Date: Wed, 28 May 2025 16:37:55 +1200 Subject: [PATCH 4/7] local server startup script --- scripts/start-local-server.sh | 109 ++++++++++++++++++++++++++++++++++ scripts/start_local_db.sh | 36 ----------- scripts/start_local_server.sh | 72 ---------------------- 3 files changed, 109 insertions(+), 108 deletions(-) create mode 100755 scripts/start-local-server.sh delete mode 100755 scripts/start_local_db.sh delete mode 100755 scripts/start_local_server.sh diff --git a/scripts/start-local-server.sh b/scripts/start-local-server.sh new file mode 100755 index 0000000..07456ed --- /dev/null +++ b/scripts/start-local-server.sh @@ -0,0 +1,109 @@ +#!/bin/bash + +# Script best practices +set -euo pipefail # Exit on error, undefined vars, pipe failures +IFS=$'\n\t' # Secure Internal Field Separator + +# Color codes for logging +readonly RED='\033[0;31m' +readonly GREEN='\033[0;32m' +readonly YELLOW='\033[1;33m' +readonly BLUE='\033[0;34m' +readonly GRAY='\033[0;90m' +readonly LIGHT_GRAY='\033[0;37m' +readonly CYAN='\033[0;36m' +readonly WHITE='\033[1;37m' +readonly NC='\033[0m' # No Color + +# Logging functions +log_info() { + printf "${BLUE}[INFO]${NC} %s\n" "$1" >&2 +} + +log_success() { + printf "${GREEN}[SUCCESS]${NC} %s\n" "$1" >&2 +} + +log_warning() { + printf "${YELLOW}[WARNING]${NC} %s\n" "$1" >&2 +} + +log_error() { + printf "${RED}[ERROR]${NC} %s\n" "$1" >&2 +} + +log_debug() { + if [[ "${DEBUG:-}" == "true" ]]; then + printf "${GRAY}[DEBUG]${NC} ${LIGHT_GRAY}%s${NC}\n" "$1" >&2 + fi +} + +# Error handler +error_handler() { + local line_number=$1 + log_error "Script failed at line ${line_number}" + exit 1 +} + +# Set up error handling +trap 'error_handler ${LINENO}' ERR + +# Initialize variables +FORCE_REBUILD=false +DEBUG=false + +# Parse arguments first +while getopts "fhd" opt; do + case ${opt} in + f) FORCE_REBUILD=true ;; + h) echo "Usage: $0 [-f] [-h] [-d]"; echo " -f: Force rebuild"; echo " -h: Help"; echo " -d: Debug"; exit 0 ;; + d) DEBUG=true ;; + *) log_error "Invalid option: -$OPTARG"; exit 1 ;; + esac +done + +# Check if Docker and Docker Compose are installed +if ! command -v docker &> /dev/null; then + log_error "Docker is not installed" + exit 1 +fi + +if ! docker compose version &> /dev/null 2>&1; then + log_error "Docker Compose is not available. Please ensure you have Docker with Compose plugin installed." + exit 1 +fi + +COMPOSE_CMD=("docker" "compose") +log_debug "Using Docker Compose" + +# Load environment variables +if [[ -f .env.local ]]; then + log_info "Loading environment from .env.local" + source .env.local +else + log_warning ".env.local not found. Create one with required variables" +fi + + + +# Build if needed +if [[ "$FORCE_REBUILD" == "true" ]]; then + log_info "Force rebuilding application image..." + "${COMPOSE_CMD[@]}" build --no-cache app + log_success "Application image rebuilt" +fi + +# Start services +log_info "Starting services with Docker Compose..." +"${COMPOSE_CMD[@]}" up -d + +log_success "Services started!" +log_info "Database: PostgreSQL on port ${POSTGRES_PORT:-5432}" +log_info "Application: forc.pub on port 8080" +log_info "pgAdmin: Database admin UI on port ${PGADMIN_PORT:-5050}" +printf "\n${CYAN}Access URLs:${NC}\n" +printf " backend: http://localhost:8080\n" +printf " pgAdmin: http://localhost:${PGADMIN_PORT:-5050}\n" +printf "\n${CYAN}Commands:${NC}\n" +printf " View logs: docker compose logs -f\n" +printf " Stop services: docker compose down\n" diff --git a/scripts/start_local_db.sh b/scripts/start_local_db.sh deleted file mode 100755 index a9af97e..0000000 --- a/scripts/start_local_db.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -# Source environment variables -source .env -NETWORK_NAME="forc_pub_net" -CONTAINER_NAME="forc_pub_db" - -# Check if Docker is installed -if ! command -v docker &> /dev/null; then - echo "Docker is not installed. Please install Docker to run this script." - exit 1 -fi - -# Check if the PostgreSQL container is already running -if docker ps --format '{{.Names}}' | grep -q ^$CONTAINER_NAME$; then - echo "PostgreSQL container is already running." - exit 0 -fi - -# Create docker network if it does not exist -if [ -z $(docker network ls --filter name=^${NETWORK_NAME}$ --format="{{ .Name }}") ] ; then - echo "Creating docker network ${NETWORK_NAME}." - docker network create $NETWORK_NAME -fi - -# Start PostgreSQL container -docker run \ - --rm -d \ - --name $CONTAINER_NAME \ - --network $NETWORK_NAME \ - -e POSTGRES_PASSWORD=$POSTGRES_PASSWORD \ - -e POSTGRES_DB=$POSTGRES_DB_NAME \ - -p $POSTGRES_PORT:$POSTGRES_PORT \ - postgres - -echo "PostgreSQL container started successfully." \ No newline at end of file diff --git a/scripts/start_local_server.sh b/scripts/start_local_server.sh deleted file mode 100755 index 4b2fc86..0000000 --- a/scripts/start_local_server.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/bash - -# Start the PostgreSQL container -./scripts/start_local_db.sh - -# Source environment variables -source .env - -CONTAINER_NAME="forc_pub_dev" -NETWORK_NAME="forc_pub_net" -DB_CONTAINER_NAME="forc_pub_db" -FORCE_REBUILD=false - -# Check for command line arguments -while getopts "f" opt; do - case ${opt} in - f) - FORCE_REBUILD=true - ;; - \?) - echo "Usage: $0 [-f]" - exit 1 - ;; - esac -done - -# Check if Docker image exists -if $FORCE_REBUILD || [[ "$(docker images -q $DOCKER_IMAGE 2> /dev/null)" == "" ]]; then - echo "Building Docker image $DOCKER_IMAGE..." - - # Build Docker image - docker build -t $CONTAINER_NAME -f deployment/Dockerfile . - - # Check if build was successful - if [ $? -eq 0 ]; then - echo "Docker image $DOCKER_IMAGE built successfully." - else - echo "Failed to build Docker image $DOCKER_IMAGE." - exit 1 - fi -else - echo "Docker image $DOCKER_IMAGE already exists. Use -f flag to force rebuild." -fi - -# Remove the container if it exists -if [[ "$(docker ps -aqf name=$CONTAINER_NAME)" ]]; then - # Stop the container if it's running - if [[ "$(docker ps -q -f name=$CONTAINER_NAME)" ]]; then - echo "Stopping container $CONTAINER_NAME..." - docker stop $CONTAINER_NAME - if [ $? -eq 0 ]; then - echo "Container $CONTAINER_NAME stopped successfully." - else - echo "Failed to stop container $CONTAINER_NAME." - exit 1 - fi - fi -fi - -# Start the Docker container on the same network as the PostgreSQL container -docker run \ - --rm -d \ - --name $CONTAINER_NAME \ - --network $NETWORK_NAME \ - -p 8080:8080 \ - -e POSTGRES_USER=$POSTGRES_USER \ - -e POSTGRES_PASSWORD=$POSTGRES_PASSWORD \ - -e POSTGRES_URI=$DB_CONTAINER_NAME \ - -e POSTGRES_DB_NAME=$POSTGRES_DB_NAME \ - $CONTAINER_NAME - -echo "Server container started successfully." \ No newline at end of file From b17db95939751a7120e65508bac7c9392c0d6ad1 Mon Sep 17 00:00:00 2001 From: z Date: Wed, 28 May 2025 16:48:13 +1200 Subject: [PATCH 5/7] readme upd --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 0c386cb..b64a33d 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,8 @@ You can access: - Backend API: http://localhost:8080 - pgAdmin interface: http://localhost:5050 +**Database Access**: When connecting to the database server through pgAdmin, use password in `.env.local` (default: `localpw`). + To force rebuild the application image: ```sh From 4be4b83bc57946da034e369a47dcfabdf4dadae8 Mon Sep 17 00:00:00 2001 From: z Date: Wed, 28 May 2025 22:26:12 +1200 Subject: [PATCH 6/7] updated ci and script to use docker compose to start up local db --- .github/workflows/backend-ci.yml | 3 ++- tests/db_integration.rs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/backend-ci.yml b/.github/workflows/backend-ci.yml index ba5b4f8..e15d416 100644 --- a/.github/workflows/backend-ci.yml +++ b/.github/workflows/backend-ci.yml @@ -79,7 +79,8 @@ jobs: run: | docker --version docker info - - run: ./scripts/start_local_db.sh + - name: Start local database + run: docker compose up -d db - name: setup binstall uses: taiki-e/install-action@cargo-binstall - name: Install forc for tests diff --git a/tests/db_integration.rs b/tests/db_integration.rs index 880f589..6c978e5 100644 --- a/tests/db_integration.rs +++ b/tests/db_integration.rs @@ -1,5 +1,5 @@ //! Note: Integration tests for the database module assume that the database is running and that the DATABASE_URL environment variable is set. -//! This should be done by running `./scripts/start_local_db.sh` before running the tests. +//! This should be done by running `docker compose up -d db` before running the tests. use std::vec; From 515dda677f402710da67a3fb93e8f0789b84e4af Mon Sep 17 00:00:00 2001 From: z Date: Thu, 29 May 2025 12:48:56 +1200 Subject: [PATCH 7/7] copy local env --- .github/workflows/backend-ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/backend-ci.yml b/.github/workflows/backend-ci.yml index e15d416..19ba9c1 100644 --- a/.github/workflows/backend-ci.yml +++ b/.github/workflows/backend-ci.yml @@ -79,8 +79,10 @@ jobs: run: | docker --version docker info - - name: Start local database - run: docker compose up -d db + - name: Start local database with environment + run: | + cp .env.example .env.local + docker compose up -d db - name: setup binstall uses: taiki-e/install-action@cargo-binstall - name: Install forc for tests