feat(docker): rewrite build and deployment pipeline

.dockerignore

@@ -1,10 +1,64 @@
+# Node modules (will be installed in container)
 node_modules
 **/node_modules
+
+# Environment files
 **/.env
+!frontend/.env
+.env.development
+.env.local
+.env.test
+.env.production
+
+# Credentials files
+deployment/portable/gcp-credentials.json
+
+# Development files
 .git
 .github/
-docs
-.env.development
-server/vespa-data
-server/xyne-data
-server/data
+docs/
+
+# Large data directories (excluded from image, will be restored from sample-data.tar.gz)
+deployment/data/
+deployment/xyne-data*
+deployment/xyne-portable*
+deployment/grafana/grafana-storage/
+deployment/loki/
+server/vespa-data/
+server/xyne-data/
+server/data/
+xyne-data/
+
+# Build artifacts
+dist/
+build/
+**/dist/
+**/build/
+
+# Logs and temporary files
+*.log
+logs/
+**/logs/
+tmp/
+temp/
+**/tmp/
+**/temp/
+
+# OS files
+.DS_Store
+Thumbs.db
+
+# Editor files
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# Test artifacts
+test-results/
+playwright-report/
+coverage/
+
+# Development-specific
+eval-data/
+observability/

.gitignore

@@ -209,3 +209,7 @@ server/.venv/*
 
 deployment/grafana/grafana-storage
 deployment/loki/*
+deployment/xyne-portable-*
+deployment/portable/gcp-credentials.json
+
+data/app-assets/

Dockerfile

@@ -3,48 +3,86 @@ FROM oven/bun:1 AS base
 
 WORKDIR /usr/src/app
 
-# Copy all files into the container
-COPY . .
+# Copy package files first for better layer caching
+COPY server/package.json server/bun.lock* /usr/src/app/server/
+COPY frontend/package.json frontend/bun.lockb /usr/src/app/frontend/
 
 # Switch to server directory and install backend dependencies
 WORKDIR /usr/src/app/server
 RUN bun install
-RUN chmod +x docker-init.sh 
 
-
-# Install dependencies and build the frontend
+# Install frontend dependencies
 WORKDIR /usr/src/app/frontend
 RUN bun install
+
+# Copy server source code and configuration
+WORKDIR /usr/src/app
+COPY server/ /usr/src/app/server/
+COPY frontend/ /usr/src/app/frontend/
+COPY shared/ /usr/src/app/shared/
+
+# Copy other necessary files
+COPY biome.json /usr/src/app/
+
+# Make scripts executable
+WORKDIR /usr/src/app/server
+RUN chmod +x docker-init.sh 2>/dev/null || true
+
+# Build the frontend
+WORKDIR /usr/src/app/frontend
 RUN bun run build
 
 # Set the environment as production
 ENV NODE_ENV=production
 
-# Install required tools and vespa CLI
+# Install required tools, canvas dependencies, and vespa CLI
 USER root
 RUN apt-get update && apt-get install -y \
     wget \
     curl \
     tar \
+    libexpat1 \
+    libexpat1-dev \
+    libcairo2-dev \
+    libpango1.0-dev \
+    libjpeg-dev \
+    libgif-dev \
+    librsvg2-dev \
+    libpixman-1-dev \
+    libfontconfig1-dev \
+    libfreetype6-dev \
     && wget https://github.com/vespa-engine/vespa/releases/download/v8.453.24/vespa-cli_8.453.24_linux_amd64.tar.gz \
     && tar -xzf vespa-cli_8.453.24_linux_amd64.tar.gz \
     && mv vespa-cli_8.453.24_linux_amd64/bin/vespa /usr/local/bin/ \
     && rm -rf vespa-cli_8.453.24_linux_amd64 vespa-cli_8.453.24_linux_amd64.tar.gz \
     && apt-get clean && rm -rf /var/lib/apt/lists/*
 
 
+# Copy data restoration script and make it executable
+#COPY deployment/restore-data.sh /usr/src/app/deployment/restore-data.sh
+#RUN chmod +x /usr/src/app/deployment/restore-data.sh
+
+# Copy sample data archive if it exists (conditional copy during build)
+#COPY deployment/sample-data.tar.gz* /usr/src/app/deployment/
+
 # Set ownership for bun user
 RUN chown -R bun:bun /usr/src/app
 
-# Expose the application port
-EXPOSE 80/tcp
+# Note: Application ports are exposed below
 
 WORKDIR /usr/src/app/server
 
-RUN mkdir -p downloads
+RUN mkdir -p downloads vespa-data vespa-logs uploads migrations
+
+# Copy and setup startup script
+COPY start.sh /usr/src/app/start.sh
+RUN chmod +x /usr/src/app/start.sh
 
 USER bun
 
-## A delay of 20 seconds to wait for the other containers to start running and the migrate changes and deploy schema changes
-CMD ["sh", "-c", "sleep 20 && if [ -f /usr/src/app/server/.env ]; then . /usr/src/app/server/.env; fi && bun run generate && bun run migrate && cd /usr/src/app/server/vespa && EMBEDDING_MODEL=$EMBEDDING_MODEL ./deploy-docker.sh && cd /usr/src/app/server/ && bun run server.ts"]
+# Expose application ports
+EXPOSE 3000
+EXPOSE 3001
+
+CMD ["/usr/src/app/start.sh"]
 

deployment/portable/.env.default

@@ -0,0 +1,20 @@
+DATABASE_HOST=xyne-db
+VESPA_HOST=vespa
+DOCKER_UID=1000
+DOCKER_GID=1000
+HOST=http://localhost:3000
+EMBEDDING_MODEL="bge-small-en-v1.5"
+DATABASE_URL=postgresql://xyne:xyne@${DATABASE_HOST}:5432/xyne
+POSTGRES_PASSWORD=
+REASONING="true"
+GOOGLE_REDIRECT_URI=http://localhost:3000/v1/auth/callback
+GOOGLE_CLIENT_ID=
+GOOGLE_CLIENT_SECRET=
+ENCRYPTION_KEY=
+JWT_SECRET=
+SERVICE_ACCOUNT_ENCRYPTION_KEY=
+AWS_ACCESS_KEY=
+AWS_SECRET_KEY=
+AWS_REGION=
+NODE_ENV=
+USER_SECRET=

deployment/portable/Dockerfile-vespa-gpu

@@ -0,0 +1,9 @@
+FROM vespaengine/vespa
+
+USER root
+
+RUN dnf -y install 'dnf-command(config-manager)'
+RUN dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/cuda-rhel8.repo
+RUN dnf -y install $(rpm -q --queryformat '%{NAME}-cuda-%{VERSION}' vespa-onnxruntime)
+
+USER vespa

deployment/portable/README.md

@@ -0,0 +1,154 @@
+# Xyne Portable Deployment
+
+One-click deployment system for Xyne with separated infrastructure and application management.
+
+## Quick Start
+
+```bash
+# Start all services
+./deploy.sh start
+
+# Access Xyne at http://localhost:3000
+```
+
+## Key Features
+
+- **🚀 One-click deployment** - Complete setup with single command
+- **⚡ Fast app updates** - Update application without touching database/search (~30s vs 3+ min)
+- **🔧 Modular architecture** - Separate infrastructure and application concerns
+- **📊 Built-in monitoring** - Grafana, Prometheus, Loki included
+- **🔄 Export/Import** - Easy transfer between machines
+- **🎯 Auto GPU/CPU detection** - Automatically uses GPU acceleration when available, falls back to CPU-only mode
+
+## Directory Structure
+
+```
+portable/
+├── docker-compose.yml              # Base configuration
+├── docker-compose.infrastructure.yml  # DB, Vespa, monitoring
+├── docker-compose.app.yml          # Xyne application only  
+├── deploy.sh                       # Deployment management
+├── quick-export.sh                 # Export for transfer
+├── prometheus-selfhosted.yml       # Metrics config
+├── loki-config.yaml               # Logging config
+├── promtail-config.yaml           # Log collection config
+└── grafana/                       # Dashboard configs
+```
+
+## Common Commands
+
+### Deployment
+```bash
+./deploy.sh start          # Start all services (auto-detects GPU/CPU)
+./deploy.sh start --force-cpu    # Force CPU-only mode
+./deploy.sh start --force-gpu    # Force GPU mode (if available)
+./deploy.sh stop           # Stop all services  
+./deploy.sh restart        # Restart everything
+./deploy.sh status         # Show service status and GPU/CPU mode
+```
+
+### Updates
+```bash
+./deploy.sh update-app     # Quick app update (30s)
+./deploy.sh update-infra   # Update infrastructure
+```
+
+### Database Management
+```bash
+./deploy.sh db-generate    # Generate migrations (after schema changes)
+./deploy.sh db-migrate     # Apply pending migrations
+./deploy.sh db-studio      # Open Drizzle Studio (localhost:4983)
+```
+
+### Monitoring
+```bash
+./deploy.sh logs           # All service logs
+./deploy.sh logs app       # App logs only
+./deploy.sh cleanup        # Remove old containers
+```
+
+### Export/Import
+```bash
+./quick-export.sh          # Create portable package
+./quick-export.sh --no-export  # Build for same machine
+```
+
+## Access URLs
+
+- **Xyne Application**: http://localhost:3000
+- **Grafana Dashboard**: http://localhost:3002  
+- **Prometheus Metrics**: http://localhost:9090
+- **Loki Logs**: http://localhost:3100
+
+## Requirements
+
+### Essential
+- Docker Engine 20.10+
+- Docker Compose 2.0+
+- 8GB+ RAM (16GB+ recommended)  
+- 50GB+ disk space
+
+### Optional (for GPU acceleration)
+- NVIDIA GPU with CUDA support
+- NVIDIA Container Toolkit
+- **Note**: System automatically detects GPU availability and falls back to CPU-only mode if needed
+
+## Configuration
+
+1. Copy environment template:
+   ```bash
+   cp .env.example .env
+   ```
+
+2. Add your API keys:
+   ```bash
+   nano .env
+   ```
+
+3. Deploy:
+   ```bash
+   ./deploy.sh start
+   ```
+
+## Documentation
+
+📖 **Complete Documentation**: See [Portable Deployment Guide](../../docs/deployment/advanced/portable-deployment.mdx)
+
+## Advantages Over Simple Docker Compose
+
+| Feature | Simple Compose | Portable Deployment |
+|---------|---------------|-------------------|
+| App Updates | Full restart (~3+ min) | App-only restart (~30s) |
+| Infrastructure Management | Manual | Automated with health checks |
+| Monitoring | None | Grafana + Prometheus + Loki |
+| Export/Import | Manual | Built-in scripts |
+| Production Ready | Basic | Advanced with security |
+| Permission Management | Manual | Automated |
+
+## Migration from Simple Compose
+
+1. **Backup current data**:
+   ```bash
+   cp -r ./server/xyne-data ./backup/
+   cp -r ./server/vespa-data ./backup/
+   ```
+
+2. **Deploy portable system**:
+   ```bash
+   cd deployment/portable/
+   ./deploy.sh start
+   ```
+
+3. **Migrate data** (if needed):
+   ```bash
+   ./deploy.sh stop
+   cp -r ../../backup/* ./data/
+   ./deploy.sh start
+   ```
+
+## Support
+
+- 📚 [Full Documentation](../../docs/deployment/advanced/portable-deployment.mdx)
+- 💬 [Slack Community](https://xynerds.slack.com/)
+- 🐛 [GitHub Issues](https://github.com/xynehq/xyne/issues)
+- ✉️ [Email Support](mailto:founders@xynehq.com)