diff --git a/.dockerignore b/.dockerignore index dcd558f..3ee9194 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,6 +13,7 @@ node_modules # Documentation *.md !README.md +docs # IDE .vscode @@ -49,6 +50,7 @@ docker-compose*.yml # CI/CD .github .gitlab-ci.yml +.gitea # Scripts scripts diff --git a/.gitea/workflows/docker-publish-dev.yaml b/.gitea/workflows/docker-publish-dev.yaml new file mode 100644 index 0000000..7cbcee8 --- /dev/null +++ b/.gitea/workflows/docker-publish-dev.yaml @@ -0,0 +1,57 @@ +name: Build and Push Docker Image (Dev) + +on: + push: + branches: + - dev + paths: + - 'src/**' + - 'client/**' + - 'Dockerfile' + - 'docker-compose.dev.yml' + - 'package.json' + - '.gitea/workflows/docker-publish-dev.yaml' + workflow_dispatch: + +env: + REGISTRY: git.app.flexinit.nl + IMAGE_NAME: oussamadouhou/ai-stack-deployer + +jobs: + build-and-push-dev: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Gitea Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: oussamadouhou + password: ${{ secrets.REGISTRY_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=raw,value=dev + type=sha,prefix=dev- + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + file: ./Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.gitea/workflows/docker-publish.yaml b/.gitea/workflows/docker-publish-main.yaml similarity index 80% rename from .gitea/workflows/docker-publish.yaml rename to .gitea/workflows/docker-publish-main.yaml index 3cc7dde..0936782 100644 --- a/.gitea/workflows/docker-publish.yaml +++ b/.gitea/workflows/docker-publish-main.yaml @@ -1,4 +1,4 @@ -name: Build and Push Docker Image +name: Build and Push Docker Image (Production) on: push: @@ -6,8 +6,11 @@ on: - main paths: - 'src/**' + - 'client/**' - 'Dockerfile' - - '.gitea/workflows/**' + - 'docker-compose.prod.yml' + - 'package.json' + - '.gitea/workflows/docker-publish-main.yaml' workflow_dispatch: env: @@ -15,7 +18,7 @@ env: IMAGE_NAME: oussamadouhou/ai-stack-deployer jobs: - build-and-push: + build-and-push-main: runs-on: ubuntu-latest permissions: contents: read @@ -41,8 +44,8 @@ jobs: with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: | - type=raw,value=latest,enable={{is_default_branch}} - type=sha,prefix= + type=raw,value=latest + type=sha,prefix=main- - name: Build and push Docker image uses: docker/build-push-action@v5 diff --git a/.gitea/workflows/docker-publish-staging.yaml b/.gitea/workflows/docker-publish-staging.yaml new file mode 100644 index 0000000..e9fcaf5 --- /dev/null +++ b/.gitea/workflows/docker-publish-staging.yaml @@ -0,0 +1,57 @@ +name: Build and Push Docker Image (Staging) + +on: + push: + branches: + - staging + paths: + - 'src/**' + - 'client/**' + - 'Dockerfile' + - 'docker-compose.staging.yml' + - 'package.json' + - '.gitea/workflows/docker-publish-staging.yaml' + workflow_dispatch: + +env: + REGISTRY: git.app.flexinit.nl + IMAGE_NAME: oussamadouhou/ai-stack-deployer + +jobs: + build-and-push-staging: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Gitea Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: oussamadouhou + password: ${{ secrets.REGISTRY_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=raw,value=staging + type=sha,prefix=staging- + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + file: ./Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 0000000..7afaa70 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,36 @@ +services: + ai-stack-deployer: + image: git.app.flexinit.nl/oussamadouhou/ai-stack-deployer:dev + container_name: ai-stack-deployer-dev + environment: + - NODE_ENV=development + - PORT=3000 + - HOST=0.0.0.0 + - DOKPLOY_URL=${DOKPLOY_URL} + - DOKPLOY_API_TOKEN=${DOKPLOY_API_TOKEN} + - STACK_DOMAIN_SUFFIX=${STACK_DOMAIN_SUFFIX:-ai.flexinit.nl} + - STACK_IMAGE=${STACK_IMAGE:-git.app.flexinit.nl/flexinit/agent-stack:latest} + - RESERVED_NAMES=${RESERVED_NAMES:-admin,api,www,root,system,test,demo,portal} + - SHARED_PROJECT_ID=$${{project.SHARED_PROJECT_ID}} + - SHARED_ENVIRONMENT_ID=$${{project.SHARED_ENVIRONMENT_ID}} + env_file: + - .env + restart: unless-stopped + healthcheck: + test: + [ + "CMD", + "bun", + "--eval", + "fetch('http://localhost:3000/health').then(r => process.exit(r.ok ? 0 : 1)).catch(() => process.exit(1))", + ] + interval: 30s + timeout: 3s + retries: 3 + start_period: 5s + networks: + - ai-stack-network + +networks: + ai-stack-network: + driver: bridge diff --git a/docker-compose.yml b/docker-compose.local.yml similarity index 84% rename from docker-compose.yml rename to docker-compose.local.yml index 73a817b..c1e4692 100644 --- a/docker-compose.yml +++ b/docker-compose.local.yml @@ -1,11 +1,13 @@ +version: "3.8" + services: ai-stack-deployer: build: context: . dockerfile: Dockerfile - container_name: ai-stack-deployer + container_name: ai-stack-deployer-local environment: - - NODE_ENV=production + - NODE_ENV=development - PORT=3000 - HOST=0.0.0.0 - DOKPLOY_URL=${DOKPLOY_URL} @@ -30,6 +32,9 @@ services: start_period: 5s networks: - ai-stack-network + volumes: + - ./src:/app/src:ro + - ./client:/app/client:ro networks: ai-stack-network: diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml new file mode 100644 index 0000000..cef470d --- /dev/null +++ b/docker-compose.prod.yml @@ -0,0 +1,38 @@ +version: "3.8" + +services: + ai-stack-deployer: + image: git.app.flexinit.nl/oussamadouhou/ai-stack-deployer:latest + container_name: ai-stack-deployer + environment: + - NODE_ENV=production + - PORT=3000 + - HOST=0.0.0.0 + - DOKPLOY_URL=${DOKPLOY_URL} + - DOKPLOY_API_TOKEN=${DOKPLOY_API_TOKEN} + - STACK_DOMAIN_SUFFIX=${STACK_DOMAIN_SUFFIX:-ai.flexinit.nl} + - STACK_IMAGE=${STACK_IMAGE:-git.app.flexinit.nl/flexinit/agent-stack:latest} + - RESERVED_NAMES=${RESERVED_NAMES:-admin,api,www,root,system,test,demo,portal} + - SHARED_PROJECT_ID=$${{project.SHARED_PROJECT_ID}} + - SHARED_ENVIRONMENT_ID=$${{project.SHARED_ENVIRONMENT_ID}} + env_file: + - .env + restart: unless-stopped + healthcheck: + test: + [ + "CMD", + "bun", + "--eval", + "fetch('http://localhost:3000/health').then(r => process.exit(r.ok ? 0 : 1)).catch(() => process.exit(1))", + ] + interval: 30s + timeout: 3s + retries: 3 + start_period: 5s + networks: + - ai-stack-network + +networks: + ai-stack-network: + driver: bridge diff --git a/docker-compose.staging.yml b/docker-compose.staging.yml new file mode 100644 index 0000000..20af46f --- /dev/null +++ b/docker-compose.staging.yml @@ -0,0 +1,38 @@ +version: "3.8" + +services: + ai-stack-deployer: + image: git.app.flexinit.nl/oussamadouhou/ai-stack-deployer:staging + container_name: ai-stack-deployer-staging + environment: + - NODE_ENV=staging + - PORT=3000 + - HOST=0.0.0.0 + - DOKPLOY_URL=${DOKPLOY_URL} + - DOKPLOY_API_TOKEN=${DOKPLOY_API_TOKEN} + - STACK_DOMAIN_SUFFIX=${STACK_DOMAIN_SUFFIX:-ai.flexinit.nl} + - STACK_IMAGE=${STACK_IMAGE:-git.app.flexinit.nl/flexinit/agent-stack:latest} + - RESERVED_NAMES=${RESERVED_NAMES:-admin,api,www,root,system,test,demo,portal} + - SHARED_PROJECT_ID=$${{project.SHARED_PROJECT_ID}} + - SHARED_ENVIRONMENT_ID=$${{project.SHARED_ENVIRONMENT_ID}} + env_file: + - .env + restart: unless-stopped + healthcheck: + test: + [ + "CMD", + "bun", + "--eval", + "fetch('http://localhost:3000/health').then(r => process.exit(r.ok ? 0 : 1)).catch(() => process.exit(1))", + ] + interval: 30s + timeout: 3s + retries: 3 + start_period: 5s + networks: + - ai-stack-network + +networks: + ai-stack-network: + driver: bridge diff --git a/docs/DOCKER_BUILD_FIX.md b/docs/DOCKER_BUILD_FIX.md index f0cee83..2533f01 100644 --- a/docs/DOCKER_BUILD_FIX.md +++ b/docs/DOCKER_BUILD_FIX.md @@ -105,7 +105,7 @@ curl http://localhost:3001/ ## Implementation Date **Date**: January 13, 2026 -**Commit**: [To be added after commit] +**Branch**: dev (following Git Flow) **Files Modified**: - `Dockerfile` - Switched build stage from Bun to Node.js - `README.md` - Updated Technology Stack and Troubleshooting sections @@ -173,7 +173,7 @@ If you still encounter AVX errors: 1. **Verify you're using the latest Dockerfile**: ```bash - git pull origin main + git pull origin dev head -10 Dockerfile # Should show: FROM node:20-alpine AS builder ``` diff --git a/docs/DOKPLOY_DEPLOYMENT.md b/docs/DOKPLOY_DEPLOYMENT.md new file mode 100644 index 0000000..142d485 --- /dev/null +++ b/docs/DOKPLOY_DEPLOYMENT.md @@ -0,0 +1,488 @@ +# Dokploy Deployment Guide + +## Overview + +This project uses **Gitea Actions** to build Docker images and **Dokploy** to deploy them. Each branch (dev, staging, main) has its own: +- Docker image tag +- Docker Compose file +- Dokploy application +- Domain + +--- + +## Architecture + +``` +┌─────────────┐ +│ Gitea │ +│ (Source) │ +└──────┬──────┘ + │ push event + ↓ +┌─────────────┐ +│ Gitea │ +│ Actions │ Builds Docker images +│ (CI/CD) │ Tags: dev, staging, latest +└──────┬──────┘ + │ + ↓ +┌─────────────┐ +│ Gitea │ +│ Registry │ git.app.flexinit.nl/oussamadouhou/ai-stack-deployer +└──────┬──────┘ + │ webhook (push event) + ↓ +┌─────────────┐ +│ Dokploy │ Pulls & deploys image +│ (Deploy) │ Uses docker-compose.{env}.yml +└─────────────┘ +``` + +--- + +## Branch Strategy + +| Branch | Image Tag | Compose File | Domain (suggested) | +|-----------|-----------|----------------------------|------------------------------| +| `dev` | `dev` | `docker-compose.dev.yml` | portal-dev.ai.flexinit.nl | +| `staging` | `staging` | `docker-compose.staging.yml` | portal-staging.ai.flexinit.nl | +| `main` | `latest` | `docker-compose.prod.yml` | portal.ai.flexinit.nl | + +--- + +## Gitea Actions Workflow + +**File**: `.gitea/workflows/docker-publish.yaml` + +**Triggers**: Push to `dev`, `staging`, or `main` branches + +**Builds**: +```yaml +dev branch → git.app.flexinit.nl/oussamadouhou/ai-stack-deployer:dev +staging branch → git.app.flexinit.nl/oussamadouhou/ai-stack-deployer:staging +main branch → git.app.flexinit.nl/oussamadouhou/ai-stack-deployer:latest +``` + +**Also creates SHA tags**: `{branch}-{short-sha}` + +--- + +## Docker Compose Files + +### `docker-compose.dev.yml` +- Pulls: `git.app.flexinit.nl/oussamadouhou/ai-stack-deployer:dev` +- Environment: `NODE_ENV=development` +- Container name: `ai-stack-deployer-dev` + +### `docker-compose.staging.yml` +- Pulls: `git.app.flexinit.nl/oussamadouhou/ai-stack-deployer:staging` +- Environment: `NODE_ENV=staging` +- Container name: `ai-stack-deployer-staging` + +### `docker-compose.prod.yml` +- Pulls: `git.app.flexinit.nl/oussamadouhou/ai-stack-deployer:latest` +- Environment: `NODE_ENV=production` +- Container name: `ai-stack-deployer` + +### `docker-compose.local.yml` +- **Builds locally** (doesn't pull from registry) +- For local development only +- Includes volume mounts for hot reload + +--- + +## Shared Project Configuration (IMPORTANT) + +### What is Shared Project Deployment? + +The portal deploys **all user AI stacks as applications within a single shared Dokploy project**, instead of creating a new project for each user. This provides: + +- ✅ Better organization (all stacks in one place) +- ✅ Shared environment variables +- ✅ Centralized monitoring +- ✅ Easier management + +### How It Works + +``` +Dokploy Project: ai-stack-portal +├── Environment: deployments +│ ├── Application: john-dev +│ ├── Application: jane-prod +│ └── Application: alice-test +``` + +### Setting Up the Shared Project + +**Step 1: Create the Shared Project in Dokploy** + +1. In Dokploy UI, create a new project: + - Name: `ai-stack-portal` (or any name you prefer) + - Description: "Shared project for all user AI stacks" + +2. Note the **Project ID** (visible in URL or API response) + - Example: `2y2Glhz5Wy0dBNf6BOR_-` + +3. Get the **Environment ID**: + ```bash + curl -s "http://10.100.0.20:3000/api/project.one?projectId=2y2Glhz5Wy0dBNf6BOR_-" \ + -H "Authorization: Bearer $DOKPLOY_API_TOKEN" | jq -r '.environments[0].id' + ``` + - Example: `RqE9OFMdLwkzN7pif1xN8` + +**Step 2: Configure Project-Level Variables** + +In the shared project (`ai-stack-portal`), add these **project-level environment variables**: + +| Variable Name | Value | Purpose | +|---------------|-------|---------| +| `SHARED_PROJECT_ID` | `2y2Glhz5Wy0dBNf6BOR_-` | The project where user stacks deploy | +| `SHARED_ENVIRONMENT_ID` | `RqE9OFMdLwkzN7pif1xN8` | The environment within that project | + +**Step 3: Reference Variables in Portal Applications** + +The portal's docker-compose files use Dokploy's variable syntax to reference these: + + ```yaml + environment: + - SHARED_PROJECT_ID=$${{project.SHARED_PROJECT_ID}} + - SHARED_ENVIRONMENT_ID=$${{project.SHARED_ENVIRONMENT_ID}} + ``` + +**This syntax `$${{project.VARIABLE}}` tells Dokploy**: "Get this value from the project-level environment variables" + +**Note**: The double `$$` is required to escape the dollar sign in Docker Compose files. + +### Important Notes + +- ⚠️ **Both variables MUST be set** in the shared project for deployment to work +- ⚠️ If not set, portal will fall back to creating separate projects per user (legacy behavior) +- ✅ You can have different shared projects for dev/staging/prod environments +- ✅ All 3 portal deployments (dev/staging/prod) should point to their respective shared projects + +--- + +## Setting Up Dokploy + +### Step 1: Create Dev Application + +1. **In Dokploy UI**, create new application: + - **Name**: `ai-stack-deployer-dev` + - **Type**: Docker Compose + - **Repository**: `ssh://git@git.app.flexinit.nl:22222/oussamadouhou/ai-stack-deployer.git` + - **Branch**: `dev` + - **Compose File**: `docker-compose.dev.yml` + +2. **Configure Domain**: + - Add domain: `portal-dev.ai.flexinit.nl` + - Enable SSL (via Traefik wildcard cert) + +3. **Set Environment Variables**: + + **Important**: The portal application should be deployed **inside the shared project** (e.g., `ai-stack-portal-dev`). + + Then set these **project-level variables** in that shared project: + ```env + SHARED_PROJECT_ID= + SHARED_ENVIRONMENT_ID= + ``` + + And these **application-level variables** in the portal app: + ```env + DOKPLOY_URL=http://10.100.0.20:3000 + DOKPLOY_API_TOKEN= + STACK_DOMAIN_SUFFIX=ai.flexinit.nl + STACK_IMAGE=git.app.flexinit.nl/flexinit/agent-stack:latest + ``` + + The docker-compose file will automatically reference the project-level variables using: + ```yaml + SHARED_PROJECT_ID=${{project.SHARED_PROJECT_ID}} + SHARED_ENVIRONMENT_ID=${{project.SHARED_ENVIRONMENT_ID}} + ``` + +4. **Configure Webhook**: + - Event: **Push** + - Branch: `dev` + - This will auto-deploy when you push to dev branch + +5. **Deploy** + +### Step 2: Create Staging Application + +Repeat Step 1 with these changes: +- **Name**: `ai-stack-deployer-staging` +- **Branch**: `staging` +- **Compose File**: `docker-compose.staging.yml` +- **Domain**: `portal-staging.ai.flexinit.nl` +- **Webhook Branch**: `staging` + +### Step 3: Create Production Application + +Repeat Step 1 with these changes: +- **Name**: `ai-stack-deployer-prod` +- **Branch**: `main` +- **Compose File**: `docker-compose.prod.yml` +- **Domain**: `portal.ai.flexinit.nl` +- **Webhook Branch**: `main` + +--- + +## Deployment Workflow + +### Development Cycle + +```bash +# 1. Make changes on dev branch +git checkout dev +# ... make changes ... +git commit -m "feat: add new feature" +git push origin dev + +# 2. Gitea Actions automatically builds dev image +# 3. Dokploy webhook triggers and deploys to portal-dev.ai.flexinit.nl + +# 4. Test on dev environment +curl https://portal-dev.ai.flexinit.nl/health + +# 5. When ready, merge to staging +git checkout staging +git merge dev +git push origin staging + +# 6. Gitea Actions builds staging image +# 7. Dokploy deploys to portal-staging.ai.flexinit.nl + +# 8. Final testing on staging, then merge to main +git checkout main +git merge staging +git push origin main + +# 9. Gitea Actions builds production image (latest) +# 10. Dokploy deploys to portal.ai.flexinit.nl +``` + +--- + +## Image Tags Explained + +Each push creates multiple tags: + +### Example: Push to `dev` branch (commit `abc1234`) + +Gitea Actions creates: +``` +git.app.flexinit.nl/oussamadouhou/ai-stack-deployer:dev ← Latest dev +git.app.flexinit.nl/oussamadouhou/ai-stack-deployer:dev-abc1234 ← Specific commit +``` + +### Example: Push to `main` branch (commit `xyz5678`) + +Gitea Actions creates: +``` +git.app.flexinit.nl/oussamadouhou/ai-stack-deployer:latest ← Latest production +git.app.flexinit.nl/oussamadouhou/ai-stack-deployer:main-xyz5678 ← Specific commit +``` + +**Why?** +- Branch tags (`dev`, `staging`, `latest`) always point to latest build +- SHA tags allow you to rollback to specific commits if needed + +--- + +## Rollback Strategy + +### Quick Rollback in Dokploy + +If a deployment breaks, you can quickly rollback: + +1. **In Dokploy UI**, go to the application +2. **Edit** the docker-compose file +3. Change the image tag to a previous SHA: + ```yaml + image: git.app.flexinit.nl/oussamadouhou/ai-stack-deployer:main-abc1234 + ``` +4. **Redeploy** + +### Manual Rollback via Git + +```bash +# Find the last working commit +git log --oneline + +# Revert to that commit +git revert HEAD # or git reset --hard + +# Push to trigger rebuild +git push origin main +``` + +--- + +## Local Development + +### Using docker-compose.local.yml + +```bash +# Build and run locally +docker-compose -f docker-compose.local.yml up -d + +# View logs +docker-compose -f docker-compose.local.yml logs -f + +# Stop +docker-compose -f docker-compose.local.yml down +``` + +### Using Bun directly (without Docker) + +```bash +# Install dependencies +bun install + +# Run dev server (API + Vite) +bun run dev + +# Run API only +bun run dev:api + +# Run client only +bun run dev:client +``` + +--- + +## Environment Variables + +### Required in Dokploy + +```env +DOKPLOY_URL=http://10.100.0.20:3000 +DOKPLOY_API_TOKEN= +``` + +### Optional (with defaults) + +```env +PORT=3000 +HOST=0.0.0.0 +STACK_DOMAIN_SUFFIX=ai.flexinit.nl +STACK_IMAGE=git.app.flexinit.nl/flexinit/agent-stack:latest +RESERVED_NAMES=admin,api,www,root,system,test,demo,portal +``` + +### Per-Environment Overrides + +If dev/staging/prod need different configs, set them in Dokploy: + +**Dev**: +```env +STACK_DOMAIN_SUFFIX=dev-ai.flexinit.nl +``` + +**Staging**: +```env +STACK_DOMAIN_SUFFIX=staging-ai.flexinit.nl +``` + +**Prod**: +```env +STACK_DOMAIN_SUFFIX=ai.flexinit.nl +``` + +--- + +## Troubleshooting + +### Build Fails in Gitea Actions + +Check the workflow logs in Gitea: +``` +https://git.app.flexinit.nl/oussamadouhou/ai-stack-deployer/actions +``` + +Common issues: +- **AVX error**: Fixed in Dockerfile (uses Node.js for build) +- **Registry auth**: Check `REGISTRY_TOKEN` secret in Gitea + +### Deployment Fails in Dokploy + +1. **Check Dokploy logs**: Application → Logs +2. **Verify image exists**: + ```bash + docker pull git.app.flexinit.nl/oussamadouhou/ai-stack-deployer:dev + ``` +3. **Check environment variables**: Make sure all required vars are set + +### Health Check Failing + +```bash +# SSH into Dokploy host +ssh user@10.100.0.20 + +# Check container logs +docker logs ai-stack-deployer-dev + +# Test health endpoint +curl http://localhost:3000/health +``` + +### Webhook Not Triggering + +1. **In Dokploy**, check webhook configuration +2. **In Gitea**, go to repo Settings → Webhooks +3. Verify webhook URL and secret match +4. Check recent deliveries for errors + +--- + +## Production Considerations + +### 1. Image Size Optimization + +The Docker image excludes dev files via `.dockerignore`: +- ✅ `docs/` - excluded +- ✅ `scripts/` - excluded +- ✅ `.gitea/` - excluded +- ✅ `*.md` (except README.md) - excluded + +Current image size: ~150MB + +### 2. Security + +- Container runs as non-root user (`nodejs:1001`) +- No secrets in source code (uses `.env`) +- Dokploy API accessible only on internal network + +### 3. Monitoring + +Set up alerts for: +- Container health check failures +- Memory/CPU usage spikes +- Deployment failures + +### 4. Backup Strategy + +- **Database**: This app has no database (stateless) +- **Configuration**: Environment variables stored in Dokploy (backed up) +- **Code**: Stored in Gitea (backed up) + +--- + +## Summary + +| Environment | Domain | Image Tag | Auto-Deploy? | +|-------------|------------------------------|-----------|--------------| +| Dev | portal-dev.ai.flexinit.nl | `dev` | ✅ On push | +| Staging | portal-staging.ai.flexinit.nl | `staging` | ✅ On push | +| Production | portal.ai.flexinit.nl | `latest` | ✅ On push | + +**Next Steps**: +1. ✅ Push changes to `dev` branch +2. ⏳ Create 3 Dokploy applications (dev, staging, prod) +3. ⏳ Configure webhooks for each branch +4. ⏳ Deploy and test each environment + +--- + +**Questions?** Check the main README.md or CLAUDE.md for more details. diff --git a/docs/SHARED_PROJECT_DEPLOYMENT.md b/docs/SHARED_PROJECT_DEPLOYMENT.md new file mode 100644 index 0000000..d17a83a --- /dev/null +++ b/docs/SHARED_PROJECT_DEPLOYMENT.md @@ -0,0 +1,313 @@ +# Shared Project Deployment Architecture + +## Overview + +The AI Stack Deployer portal deploys **all user AI stacks to a single shared Dokploy project** instead of creating a new project for each user. + +--- + +## Architecture Diagram + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Dokploy: ai-stack-portal (Shared Project) │ +│ ID: 2y2Glhz5Wy0dBNf6BOR_- │ +├─────────────────────────────────────────────────────────────────┤ +│ │ +│ 📦 Portal Application: ai-stack-deployer-prod │ +│ ├─ Domain: portal.ai.flexinit.nl │ +│ ├─ Image: git.app.flexinit.nl/.../ai-stack-deployer:latest│ +│ └─ Env: SHARED_PROJECT_ID=$${{project.SHARED_PROJECT_ID}} │ +│ │ +│ ───────────────────────────────────────────────────────────── │ +│ │ +│ 📦 User Stack: john-dev │ +│ ├─ Domain: john-dev.ai.flexinit.nl │ +│ ├─ Image: git.app.flexinit.nl/.../agent-stack:latest │ +│ └─ Deployed by: Portal │ +│ │ +│ 📦 User Stack: jane-prod │ +│ ├─ Domain: jane-prod.ai.flexinit.nl │ +│ ├─ Image: git.app.flexinit.nl/.../agent-stack:latest │ +│ └─ Deployed by: Portal │ +│ │ +│ 📦 User Stack: alice-test │ +│ ├─ Domain: alice-test.ai.flexinit.nl │ +│ ├─ Image: git.app.flexinit.nl/.../agent-stack:latest │ +│ └─ Deployed by: Portal │ +│ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +--- + +## How It Works + +### Step 1: Portal Reads Configuration + +When a user submits a stack name (e.g., "john-dev"), the portal: + +1. **Reads environment variables**: + ```javascript + const sharedProjectId = process.env.SHARED_PROJECT_ID; + const sharedEnvironmentId = process.env.SHARED_ENVIRONMENT_ID; + ``` + +2. **These are set via Dokploy's project-level variables**: + ```yaml + environment: + - SHARED_PROJECT_ID=$${{project.SHARED_PROJECT_ID}} + - SHARED_ENVIRONMENT_ID=$${{project.SHARED_ENVIRONMENT_ID}} + ``` + + **Note**: The double `$$` is required to escape the dollar sign in Docker Compose. + +### Step 2: Portal Deploys to Shared Project + +Instead of creating a new project, the portal: + +```javascript +// OLD BEHAVIOR (legacy): +// createProject(`ai-stack-${username}`) ❌ Creates new project per user + +// NEW BEHAVIOR (current): +// Uses existing shared project ID ✅ +const projectId = sharedProjectId; // From environment variable +const environmentId = sharedEnvironmentId; + +// Creates application IN the shared project +createApplication({ + projectId: projectId, + environmentId: environmentId, + name: `${username}-stack`, + image: 'git.app.flexinit.nl/.../agent-stack:latest', + domain: `${username}.ai.flexinit.nl` +}); +``` + +### Step 3: User Accesses Their Stack + +User visits `https://john-dev.ai.flexinit.nl` → Traefik routes to their application inside the shared project. + +--- + +## Configuration Steps + +### 1. Create Shared Project in Dokploy + +1. In Dokploy UI, create project: + - **Name**: `ai-stack-portal` + - **Description**: "Shared project for all user AI stacks" + +2. Get the **Project ID**: + ```bash + # Via API + curl -s "http://10.100.0.20:3000/api/project.all" \ + -H "Authorization: Bearer $DOKPLOY_API_TOKEN" | \ + jq -r '.[] | select(.name=="ai-stack-portal") | .id' + + # Output: 2y2Glhz5Wy0dBNf6BOR_- + ``` + +3. Get the **Environment ID**: + ```bash + curl -s "http://10.100.0.20:3000/api/project.one?projectId=2y2Glhz5Wy0dBNf6BOR_-" \ + -H "Authorization: Bearer $DOKPLOY_API_TOKEN" | \ + jq -r '.environments[0].id' + + # Output: RqE9OFMdLwkzN7pif1xN8 + ``` + +### 2. Set Project-Level Variables + +In the shared project (`ai-stack-portal`), add these **project-level environment variables**: + +| Variable | Value | Example | +|----------|-------|---------| +| `SHARED_PROJECT_ID` | Your project ID | `2y2Glhz5Wy0dBNf6BOR_-` | +| `SHARED_ENVIRONMENT_ID` | Your environment ID | `RqE9OFMdLwkzN7pif1xN8` | + +**How to set in Dokploy UI**: +- Go to Project → Settings → Environment Variables +- Add variables at **project level** (not application level) + +### 3. Deploy Portal Application + +Deploy the portal **inside the same shared project**: + +1. **Application Details**: + - Name: `ai-stack-deployer-prod` + - Type: Docker Compose + - Compose File: `docker-compose.prod.yml` + - Branch: `main` + +2. **The docker-compose file automatically references project variables**: + ```yaml + environment: + - SHARED_PROJECT_ID=${{project.SHARED_PROJECT_ID}} # ← Magic happens here + - SHARED_ENVIRONMENT_ID=${{project.SHARED_ENVIRONMENT_ID}} + ``` + +3. **Dokploy resolves `${{project.VAR}}`** to the actual value from project-level variables. + +--- + +## Benefits + +### ✅ Centralized Management +All user stacks in one place: +- Easy to list all active stacks +- Shared monitoring dashboard +- Centralized logging + +### ✅ Resource Efficiency +- No overhead of separate projects per user +- Shared network and resources +- Easier to manage quotas + +### ✅ Simplified Configuration +- Project-level environment variables shared by all stacks +- Single source of truth for common configs +- Easy to update STACK_IMAGE for all users + +### ✅ Better Organization +``` +Projects in Dokploy: +├── ai-stack-portal (500 user applications) ✅ Clean +└── NOT: + ├── ai-stack-john + ├── ai-stack-jane + ├── ai-stack-alice + └── ... (500 separate projects) ❌ Messy +``` + +--- + +## Fallback Behavior + +If `SHARED_PROJECT_ID` and `SHARED_ENVIRONMENT_ID` are **not set**, the portal falls back to **legacy behavior**: + +```javascript +// Code in src/orchestrator/production-deployer.ts (lines 187-196) +const sharedProjectId = config.sharedProjectId || process.env.SHARED_PROJECT_ID; +const sharedEnvironmentId = config.sharedEnvironmentId || process.env.SHARED_ENVIRONMENT_ID; + +if (sharedProjectId && sharedEnvironmentId) { + // Use shared project ✅ + state.resources.projectId = sharedProjectId; + state.resources.environmentId = sharedEnvironmentId; + return; +} + +// Fallback: Create separate project per user ⚠️ +const projectName = `ai-stack-${config.stackName}`; +const existingProject = await this.client.findProjectByName(projectName); +// ... +``` + +**This ensures backwards compatibility** but is not recommended. + +--- + +## Troubleshooting + +### Portal Creates Separate Projects Instead of Using Shared Project + +**Cause**: `SHARED_PROJECT_ID` or `SHARED_ENVIRONMENT_ID` not set. + +**Solution**: +1. Check project-level variables in Dokploy: + ```bash + curl -s "http://10.100.0.20:3000/api/project.one?projectId=YOUR_PROJECT_ID" \ + -H "Authorization: Bearer $DOKPLOY_API_TOKEN" | \ + jq '.environmentVariables' + ``` + +2. Ensure the portal application's docker-compose references them: + ```yaml + environment: + - SHARED_PROJECT_ID=${{project.SHARED_PROJECT_ID}} + - SHARED_ENVIRONMENT_ID=${{project.SHARED_ENVIRONMENT_ID}} + ``` + +3. Redeploy the portal application. + +### Variable Reference Not Working + +**Symptom**: Portal logs show `undefined` for `SHARED_PROJECT_ID`. + +**Cause**: Using wrong syntax. + +**Correct syntax**: +```yaml +- SHARED_PROJECT_ID=${{project.SHARED_PROJECT_ID}} ✅ +``` + +**Wrong syntax**: +```yaml +- SHARED_PROJECT_ID=${SHARED_PROJECT_ID} ❌ (shell substitution, not Dokploy) +- SHARED_PROJECT_ID={{project.SHARED_PROJECT_ID}} ❌ (missing $) +``` + +### How to Verify Configuration + +Check portal container environment: +```bash +# SSH into Dokploy host +ssh user@10.100.0.20 + +# Inspect portal container +docker exec ai-stack-deployer env | grep SHARED + +# Should show: +SHARED_PROJECT_ID=2y2Glhz5Wy0dBNf6BOR_- +SHARED_ENVIRONMENT_ID=RqE9OFMdLwkzN7pif1xN8 +``` + +--- + +## Environment-Specific Shared Projects + +You can have **separate shared projects for dev/staging/prod**: + +| Portal Environment | Shared Project | Purpose | +|--------------------|----------------|---------| +| Dev | `ai-stack-portal-dev` | Development user stacks | +| Staging | `ai-stack-portal-staging` | Staging user stacks | +| Prod | `ai-stack-portal` | Production user stacks | + +Each portal deployment references its own shared project: +- `portal-dev.ai.flexinit.nl` → `ai-stack-portal-dev` +- `portal-staging.ai.flexinit.nl` → `ai-stack-portal-staging` +- `portal.ai.flexinit.nl` → `ai-stack-portal` + +--- + +## Migration from Legacy + +If you're currently using the legacy behavior (separate projects per user): + +### Option 1: Gradual Migration +- New deployments use shared project +- Old deployments remain in separate projects +- Migrate old stacks manually over time + +### Option 2: Full Migration +1. Create shared project +2. Set project-level variables +3. Redeploy all user stacks to shared project +4. Delete old separate projects + +**Note**: Migration requires downtime for each stack being moved. + +--- + +## Reference + +- **Environment Variable Syntax**: See Dokploy docs on project-level variables +- **Code Location**: `src/orchestrator/production-deployer.ts` (lines 178-200) +- **Example IDs**: `.env.example` (lines 25-27) + +--- + +**Questions?** Check the main deployment guide: `DOKPLOY_DEPLOYMENT.md` diff --git a/docs/TESTING.md b/docs/TESTING.md index 7d5f455..e4363a4 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -261,3 +261,113 @@ Authorization: token |-----|---------| | `GITEA_API_TOKEN` | Gitea API access for workflow status | | `DOKPLOY_API_TOKEN` | Dokploy deployment API (BWS ID: `6b3618fc-ba02-49bc-bdc8-b3c9004087bc`) | + +--- + +## Testing Session: 2026-01-13 + +### Session Summary + +**Goal:** Verify multi-environment deployment setup and shared project configuration. + +### Completed Tasks + +| Task | Status | Evidence | +|------|--------|----------| +| Workflow separation (dev/staging/main) | ✅ | Committed as `eb2745d` | +| Dollar sign escaping (`$${{project.VAR}}`) | ✅ | Verified in all docker-compose.*.yml | +| Shared project exists | ✅ | `ai-stack-portal` (ID: `2y2Glhz5Wy0dBNf6BOR_-`) | +| Environment IDs retrieved | ✅ | See below | +| Local dev server health | ✅ | `/health` returns healthy | + +### Environment IDs + +``` +Project: ai-stack-portal +ID: 2y2Glhz5Wy0dBNf6BOR_- + +Environments: +- production: _dKAmxVcadqi-z73wKpEB (default) +- deployments: RqE9OFMdLwkzN7pif1xN8 (for user stacks) +- test: KVKn5fXGz10g7KVxPWOQj +``` + +### Blockers Identified + +#### BLOCKER: Dokploy API Token Permissions + +**Symptom:** All Dokploy API calls return `Forbidden` + +```bash +# Previously working +curl -s "https://app.flexinit.nl/api/project.all" -H "x-api-key: $DOKPLOY_API_TOKEN" +# Now returns: Forbidden + +# Environment endpoint +curl -s "https://app.flexinit.nl/api/environment.one?environmentId=RqE9OFMdLwkzN7pif1xN8" -H "x-api-key: $DOKPLOY_API_TOKEN" +# Returns: Forbidden +``` + +**Root Cause:** The API token `app_deployment...` has been revoked or has limited scope. + +**Impact:** +- Cannot verify Docker image exists in registry +- Cannot test name availability (requires `environment.one`) +- Cannot create applications or compose stacks +- Cannot deploy portal to Dokploy + +**Resolution Required:** +1. Log into Dokploy UI at https://app.flexinit.nl +2. Navigate to Settings → API Keys +3. Generate new API key with full permissions: + - Read/Write access to projects + - Read/Write access to applications + - Read/Write access to compose stacks + - Read/Write access to domains +4. Update `.env` with new token +5. Update BWS secret (ID: `6b3618fc-ba02-49bc-bdc8-b3c9004087bc`) + +### Local Testing Results + +```bash +# Health check - WORKS +curl -s "http://localhost:3000/health" +# {"status":"healthy","timestamp":"2026-01-13T13:01:46.100Z","version":"0.2.0",...} + +# Name check - FAILS (API token issue) +curl -s "http://localhost:3000/api/check/test-stack" +# {"available":false,"valid":false,"error":"Failed to check availability"} +``` + +### Required .env Configuration + +```bash +# Added for shared project deployment +SHARED_PROJECT_ID=2y2Glhz5Wy0dBNf6BOR_- +SHARED_ENVIRONMENT_ID=RqE9OFMdLwkzN7pif1xN8 +``` + +### Next Steps After Token Fix + +1. Verify `project.all` API works with new token +2. Deploy portal to Dokploy (docker-compose.dev.yml) +3. Test end-to-end stack deployment +4. Verify stacks deploy to shared project +5. Clean up test deployments + +### Commands Reference + +```bash +# Test API token +source .env && curl -s "https://app.flexinit.nl/api/project.all" \ + -H "x-api-key: $DOKPLOY_API_TOKEN" | jq '.[].name' + +# Get environment applications +source .env && curl -s "https://app.flexinit.nl/api/environment.one?environmentId=RqE9OFMdLwkzN7pif1xN8" \ + -H "x-api-key: $DOKPLOY_API_TOKEN" | jq '.applications' + +# Deploy test stack +curl -X POST http://localhost:3000/api/deploy \ + -H "Content-Type: application/json" \ + -d '{"name":"test-'$(date +%s | tail -c 4)'"}' +```