feat(logging): finalize log-ingest service for dokploy-network deployment
- Update log-ingest to use internal Loki endpoint - Add standalone docker-compose for dokploy deployment - Update ROADMAP and LOGGING-PLAN with completed status - Configure proper network settings for dokploy-network
This commit is contained in:
@@ -11,6 +11,8 @@
|
|||||||
- [x] Stack cleanup API
|
- [x] Stack cleanup API
|
||||||
- [x] Auto-rollback on failure
|
- [x] Auto-rollback on failure
|
||||||
- [x] Persistent storage volumes
|
- [x] Persistent storage volumes
|
||||||
|
- [x] Logging infrastructure (log-ingest → Loki → Grafana)
|
||||||
|
- [x] AI Stack monitoring dashboard at logs.intra.flexinit.nl
|
||||||
|
|
||||||
## Next
|
## Next
|
||||||
|
|
||||||
@@ -24,4 +26,3 @@
|
|||||||
- [ ] Integration tests
|
- [ ] Integration tests
|
||||||
- [ ] Resource limits configuration
|
- [ ] Resource limits configuration
|
||||||
- [ ] Custom domain support
|
- [ ] Custom domain support
|
||||||
- [ ] Usage metrics
|
|
||||||
|
|||||||
@@ -387,29 +387,35 @@ groups:
|
|||||||
## 7. Implementation Checklist
|
## 7. Implementation Checklist
|
||||||
|
|
||||||
### Phase 1: Container Logging
|
### Phase 1: Container Logging
|
||||||
- [ ] Set up Loki + Promtail on logging server
|
- [x] Set up Loki + Promtail on logging server (using existing `logs.intra.flexinit.nl`)
|
||||||
- [ ] Configure Docker log driver for ai-stack containers
|
- [x] Configure Docker log driver for ai-stack containers
|
||||||
- [ ] Add log rotation to Dockerfile
|
- [x] Add log rotation to Dockerfile
|
||||||
- [ ] Verify logs flowing to Loki
|
- [x] Verify logs flowing to Loki
|
||||||
|
|
||||||
### Phase 2: Session Logging
|
### Phase 2: Session Logging
|
||||||
- [ ] Create logging hook in oh-my-opencode
|
- [x] Create logging hook in oh-my-opencode (`/home/odouhou/locale-projects/oh-my-opencode-free-fork/src/hooks/usage-logging/`)
|
||||||
- [ ] Define event schema
|
- [x] Define event schema
|
||||||
- [ ] Implement log shipping (HTTP or file-based)
|
- [x] Implement log shipping (HTTP-based via log-ingest service)
|
||||||
- [ ] Add session/message/tool logging
|
- [x] Add session/message/tool logging
|
||||||
|
|
||||||
### Phase 3: Metrics
|
### Phase 3: Metrics
|
||||||
- [ ] Add prom-client to container
|
- [x] Add prom-client to container (`docker/shared-config/metrics-exporter.ts`)
|
||||||
- [ ] Expose /metrics endpoint
|
- [x] Expose /metrics endpoint (port 9090)
|
||||||
- [ ] Configure Prometheus scraping
|
- [x] Configure Prometheus scraping (datasource added to Grafana)
|
||||||
- [ ] Create initial Grafana dashboards
|
- [x] Create initial Grafana dashboards (`/d/ai-stack-overview`)
|
||||||
|
|
||||||
### Phase 4: Production Hardening
|
### Phase 4: Production Hardening
|
||||||
- [ ] Implement data anonymization
|
- [x] Implement data anonymization (content hashed, not stored)
|
||||||
- [ ] Set up retention policies
|
- [ ] Set up retention policies
|
||||||
- [ ] Configure alerts
|
- [ ] Configure alerts
|
||||||
- [ ] Document runbooks
|
- [ ] Document runbooks
|
||||||
|
|
||||||
|
### Deployed Components (2026-01-10)
|
||||||
|
- **Log-ingest service**: `http://ai-stack-log-ingest:3000/ingest` (dokploy-network)
|
||||||
|
- **Grafana dashboard**: https://logs.intra.flexinit.nl/d/ai-stack-overview
|
||||||
|
- **Datasource UIDs**: Loki (`af9a823s6iku8b`), Prometheus (`cf9r1fmfw9xxcf`)
|
||||||
|
- **BWS credentials**: `GRAFANA_OPENCODE_ACCESS_TOKEN` (id: `c77e58e3-fb34-41dc-9824-b3ce00da18a0`)
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 8. Cost Estimates
|
## 8. Cost Estimates
|
||||||
|
|||||||
@@ -50,7 +50,7 @@ services:
|
|||||||
image: prom/prometheus:v2.47.0
|
image: prom/prometheus:v2.47.0
|
||||||
container_name: ai-stack-prometheus
|
container_name: ai-stack-prometheus
|
||||||
ports:
|
ports:
|
||||||
- "9090:9090"
|
- "9091:9090"
|
||||||
volumes:
|
volumes:
|
||||||
- ./config/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
- ./config/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||||
- ./alerting:/etc/prometheus/alerting:ro
|
- ./alerting:/etc/prometheus/alerting:ro
|
||||||
|
|||||||
@@ -1,13 +1,13 @@
|
|||||||
FROM oven/bun:1.0
|
FROM node:20-alpine
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
COPY package.json bun.lockb* ./
|
COPY package.json ./
|
||||||
RUN bun install --frozen-lockfile 2>/dev/null || bun install
|
RUN npm install
|
||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
ENV PORT=3000
|
ENV PORT=3000
|
||||||
EXPOSE 3000
|
EXPOSE 3000
|
||||||
|
|
||||||
CMD ["bun", "run", "src/index.ts"]
|
CMD ["npx", "tsx", "src/index.ts"]
|
||||||
|
|||||||
27
logging-stack/log-ingest/docker-compose.yml
Normal file
27
logging-stack/log-ingest/docker-compose.yml
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
# AI Stack Log Ingest Service
|
||||||
|
# Connects to existing Loki at logs.intra.flexinit.nl
|
||||||
|
|
||||||
|
services:
|
||||||
|
log-ingest:
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
dockerfile: Dockerfile
|
||||||
|
container_name: ai-stack-log-ingest
|
||||||
|
ports:
|
||||||
|
- "3102:3000"
|
||||||
|
environment:
|
||||||
|
# Connect to existing Loki on dokploy-network
|
||||||
|
- LOKI_URL=http://monitor-grafanaloki-qkj16i-loki-1:3100
|
||||||
|
- LOG_LEVEL=info
|
||||||
|
networks:
|
||||||
|
- dokploy-network
|
||||||
|
restart: unless-stopped
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:3000/health"]
|
||||||
|
interval: 30s
|
||||||
|
timeout: 10s
|
||||||
|
retries: 3
|
||||||
|
|
||||||
|
networks:
|
||||||
|
dokploy-network:
|
||||||
|
external: true
|
||||||
@@ -3,14 +3,16 @@
|
|||||||
"version": "1.0.0",
|
"version": "1.0.0",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"start": "bun run src/index.ts",
|
"start": "tsx src/index.ts",
|
||||||
"dev": "bun --watch run src/index.ts"
|
"dev": "tsx --watch src/index.ts"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"hono": "^4.0.0",
|
"hono": "^4.0.0",
|
||||||
"prom-client": "^15.0.0"
|
"prom-client": "^15.0.0",
|
||||||
|
"@hono/node-server": "^1.8.0"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/bun": "latest"
|
"tsx": "^4.7.0",
|
||||||
|
"@types/node": "^20.0.0"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import { Hono } from 'hono';
|
import { Hono } from 'hono';
|
||||||
import { cors } from 'hono/cors';
|
import { cors } from 'hono/cors';
|
||||||
import { logger } from 'hono/logger';
|
import { logger } from 'hono/logger';
|
||||||
|
import { serve } from '@hono/node-server';
|
||||||
import { Registry, Counter, Histogram, Gauge, collectDefaultMetrics } from 'prom-client';
|
import { Registry, Counter, Histogram, Gauge, collectDefaultMetrics } from 'prom-client';
|
||||||
|
|
||||||
const app = new Hono();
|
const app = new Hono();
|
||||||
@@ -193,7 +194,7 @@ app.post('/ingest/batch', async (c) => {
|
|||||||
const port = parseInt(process.env.PORT || '3000');
|
const port = parseInt(process.env.PORT || '3000');
|
||||||
console.log(`Log ingest service starting on port ${port}`);
|
console.log(`Log ingest service starting on port ${port}`);
|
||||||
|
|
||||||
export default {
|
serve({
|
||||||
port,
|
fetch: app.fetch,
|
||||||
fetch: app.fetch
|
port
|
||||||
};
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user