ai-stack-deployer/src/orchestrator/production-deployer.ts

/**
 * Production-Grade Deployment Orchestrator
 *
 * Features:
 * - Complete deployment lifecycle management
 * - Idempotency checks at every stage
 * - Automatic rollback on failure
 * - Health verification
 * - Comprehensive logging
 * - State tracking
 */

import { DokployProductionClient } from '../api/dokploy-production.js';

export interface DeploymentConfig {
  stackName: string;
  dockerImage: string;
  domainSuffix: string;
  port?: number;
  healthCheckTimeout?: number;
  healthCheckInterval?: number;
  registryId?: string;
}

export interface DeploymentState {
  id: string;
  stackName: string;
  phase:
    | 'initializing'
    | 'creating_project'
    | 'getting_environment'
    | 'creating_application'
    | 'configuring_application'
    | 'creating_domain'
    | 'deploying'
    | 'verifying_health'
    | 'completed'
    | 'failed'
    | 'rolling_back';
  status: 'success' | 'failure' | 'in_progress';
  progress: number;
  message: string;
  url?: string;
  error?: {
    phase: string;
    message: string;
    code?: string;
  };
  resources: {
    projectId?: string;
    environmentId?: string;
    applicationId?: string;
    domainId?: string;
  };
  timestamps: {
    started: string;
    completed?: string;
  };
}

export interface DeploymentResult {
  success: boolean;
  state: DeploymentState;
  logs: string[];
}

export type ProgressCallback = (state: DeploymentState) => void;

export class ProductionDeployer {
  private client: DokployProductionClient;
  private progressCallback?: ProgressCallback;

  constructor(client: DokployProductionClient, progressCallback?: ProgressCallback) {
    this.client = client;
    this.progressCallback = progressCallback;
  }

  private notifyProgress(state: DeploymentState): void {
    if (this.progressCallback) {
      this.progressCallback({ ...state });
    }
  }

  /**
   * Deploy a complete AI stack with full production safeguards
   */
  async deploy(config: DeploymentConfig): Promise<DeploymentResult> {
    const state: DeploymentState = {
      id: `dep_${Date.now()}_${Math.random().toString(36).substring(2, 9)}`,
      stackName: config.stackName,
      phase: 'initializing',
      status: 'in_progress',
      progress: 0,
      message: 'Initializing deployment',
      resources: {},
      timestamps: {
        started: new Date().toISOString(),
      },
    };

    this.notifyProgress(state);

    try {
      // Phase 1: Project Creation (Idempotent)
      await this.createOrFindProject(state, config);
      this.notifyProgress(state);

      // Phase 2: Get Environment ID
      await this.getEnvironment(state);
      this.notifyProgress(state);

      // Phase 3: Application Creation (Idempotent)
      await this.createOrFindApplication(state, config);
      this.notifyProgress(state);

      // Phase 4: Configure Application
      await this.configureApplication(state, config);
      this.notifyProgress(state);

      // Phase 5: Domain Creation (Idempotent)
      await this.createOrFindDomain(state, config);
      this.notifyProgress(state);

      // Phase 6: Deploy Application
      await this.deployApplication(state);
      this.notifyProgress(state);

      // Phase 7: Health Verification
      await this.verifyHealth(state, config);
      this.notifyProgress(state);

      // Success
      state.phase = 'completed';
      state.status = 'success';
      state.progress = 100;
      state.message = 'Deployment completed successfully';
      state.timestamps.completed = new Date().toISOString();

      this.notifyProgress(state);

      return {
        success: true,
        state,
        logs: this.client.getLogs().map(l => JSON.stringify(l)),
      };

    } catch (error) {
      // Failure - Initiate Rollback
      state.status = 'failure';
      state.error = {
        phase: state.phase,
        message: error instanceof Error ? error.message : String(error),
        code: 'DEPLOYMENT_FAILED',
      };

      this.notifyProgress(state);

      console.error(`Deployment failed at phase ${state.phase}:`, error);

      // Attempt rollback
      await this.rollback(state);
      this.notifyProgress(state);

      state.timestamps.completed = new Date().toISOString();

      this.notifyProgress(state);

      return {
        success: false,
        state,
        logs: this.client.getLogs().map(l => JSON.stringify(l)),
      };
    }
  }

  private async createOrFindProject(
    state: DeploymentState,
    config: DeploymentConfig
  ): Promise<void> {
    state.phase = 'creating_project';
    state.progress = 10;
    state.message = 'Creating or finding project';

    const projectName = `ai-stack-${config.stackName}`;

    // Idempotency: Check if project already exists
    const existingProject = await this.client.findProjectByName(projectName);

    if (existingProject) {
      console.log(`Project ${projectName} already exists, reusing...`);
      state.resources.projectId = existingProject.project.projectId;
      // Also capture environment ID if available
      if (existingProject.environmentId) {
        state.resources.environmentId = existingProject.environmentId;
      }
      state.message = 'Found existing project';
      return;
    }

    // Create new project (returns both project and environment)
    const response = await this.client.createProject(
      projectName,
      `AI Stack for ${config.stackName}`
    );

    console.log('Project and environment created:', JSON.stringify(response, null, 2));

    if (!response.project.projectId) {
      throw new Error(`Project creation succeeded but projectId is missing. Response: ${JSON.stringify(response)}`);
    }

    state.resources.projectId = response.project.projectId;
    state.resources.environmentId = response.environment.environmentId;
    state.message = 'Project and environment created';
  }

  private async getEnvironment(state: DeploymentState): Promise<void> {
    state.phase = 'getting_environment';
    state.progress = 25;
    state.message = 'Getting environment ID';

    // Skip if we already have environment ID from project creation
    if (state.resources.environmentId) {
      console.log('Environment ID already available from project creation');
      state.message = 'Environment ID already available';
      return;
    }

    if (!state.resources.projectId) {
      throw new Error('Project ID not available');
    }

    const environment = await this.client.getDefaultEnvironment(state.resources.projectId);
    state.resources.environmentId = environment.environmentId;
    state.message = 'Environment ID retrieved';
  }

  private async createOrFindApplication(
    state: DeploymentState,
    config: DeploymentConfig
  ): Promise<void> {
    state.phase = 'creating_application';
    state.progress = 40;
    state.message = 'Creating application';

    if (!state.resources.environmentId) {
      throw new Error('Environment ID not available');
    }

    const appName = `opencode-${config.stackName}`;

    // Note: Idempotency for applications requires listing all applications
    // in the environment and checking by name. For now, we rely on API
    // to handle duplicates or we can add this later.

    const application = await this.client.createApplication(
      appName,
      state.resources.environmentId
    );

    state.resources.applicationId = application.applicationId;
    state.message = 'Application created';
  }

  private async configureApplication(
    state: DeploymentState,
    config: DeploymentConfig
  ): Promise<void> {
    state.phase = 'configuring_application';
    state.progress = 50;
    state.message = 'Configuring application with Docker image';

    if (!state.resources.applicationId) {
      throw new Error('Application ID not available');
    }

    await this.client.updateApplication(state.resources.applicationId, {
      dockerImage: config.dockerImage,
      sourceType: 'docker',
      registryId: config.registryId,
    });

    state.progress = 52;
    state.message = 'Setting environment variables for logging';

    const envVars = [
      `STACK_NAME=${config.stackName}`,
      `USAGE_LOGGING_ENABLED=true`,
      `LOG_INGEST_URL=${process.env.LOG_INGEST_URL || 'http://10.100.0.20:3102/ingest'}`,
      `METRICS_PORT=9090`,
      // TUI Support: Terminal environment for proper TUI rendering in web browser
      `TERM=xterm-256color`,
      `COLORTERM=truecolor`,
      `LANG=en_US.UTF-8`,
      `LC_ALL=en_US.UTF-8`,
    ].join('\n');

    await this.client.setApplicationEnv(state.resources.applicationId, envVars);

    state.progress = 55;
    state.message = 'Creating persistent storage';

    const volumeName = `portal-ai-workspace-${config.stackName}`;
    try {
      await this.client.createMount(
        state.resources.applicationId,
        volumeName,
        '/workspace'
      );
      console.log(`Created persistent volume: ${volumeName}`);
    } catch (error) {
      console.warn(`Volume creation failed (may already exist): ${error}`);
    }

    state.message = 'Application configured with storage';
  }

  private async createOrFindDomain(
    state: DeploymentState,
    config: DeploymentConfig
  ): Promise<void> {
    state.phase = 'creating_domain';
    state.progress = 70;
    state.message = 'Creating domain';

    if (!state.resources.applicationId) {
      throw new Error('Application ID not available');
    }

    const host = `${config.stackName}.${config.domainSuffix}`;
    const port = config.port || 8080;

    // Note: Idempotency for domains would require listing existing domains
    // for the application. For now, API should handle duplicates.

    const domain = await this.client.createDomain(
      host,
      state.resources.applicationId,
      true,
      port
    );

    state.resources.domainId = domain.domainId;
    state.url = `https://${host}`;
    state.message = 'Domain created';
  }

  private async deployApplication(state: DeploymentState): Promise<void> {
    state.phase = 'deploying';
    state.progress = 85;
    state.message = 'Triggering deployment';

    if (!state.resources.applicationId) {
      throw new Error('Application ID not available');
    }

    await this.client.deployApplication(state.resources.applicationId);
    state.message = 'Deployment triggered';
  }

  private async verifyHealth(
    state: DeploymentState,
    config: DeploymentConfig
  ): Promise<void> {
    state.phase = 'verifying_health';
    state.progress = 95;
    state.message = 'Verifying application status via Dokploy';

    if (!state.resources.applicationId) {
      throw new Error('Application ID not available');
    }

    const timeout = config.healthCheckTimeout || 60000;
    const interval = config.healthCheckInterval || 3000;
    const startTime = Date.now();

    while (Date.now() - startTime < timeout) {
      try {
        const app = await this.client.getApplication(state.resources.applicationId);
        const appStatus = app.applicationStatus;
        console.log(`Application status: ${appStatus}`);
        
        if (appStatus === 'done') {
          state.message = 'Waiting for SSL certificate provisioning...';
          state.progress = 98;
          this.notifyProgress(state);
          
          await this.sleep(15000);
          
          state.message = 'Application deployed successfully';
          return;
        }
        
        if (appStatus === 'error') {
          throw new Error('Application deployment failed in Dokploy');
        }
      } catch (error) {
        console.log(`Status check failed: ${error}`);
      }

      const elapsed = Math.round((Date.now() - startTime) / 1000);
      state.message = `Waiting for application to start (${elapsed}s)...`;
      this.notifyProgress(state);

      await this.sleep(interval);
    }

    throw new Error('Health check timeout - application did not become ready');
  }

  private async rollback(state: DeploymentState): Promise<void> {
    console.log('Initiating rollback...');
    state.phase = 'rolling_back';
    state.message = 'Rolling back deployment';

    try {
      if (state.resources.domainId) {
        console.log(`Rolling back: deleting domain ${state.resources.domainId}`);
        try {
          await this.client.deleteDomain(state.resources.domainId);
        } catch (error) {
          console.error('Failed to delete domain during rollback:', error);
        }
      }

      if (state.resources.applicationId) {
        console.log(`Rolling back: deleting application ${state.resources.applicationId}`);
        try {
          await this.client.deleteApplication(state.resources.applicationId);
        } catch (error) {
          console.error('Failed to delete application during rollback:', error);
        }
      }

      if (state.resources.projectId) {
        console.log(`Rolling back: deleting project ${state.resources.projectId}`);
        try {
          await this.client.deleteProject(state.resources.projectId);
        } catch (error) {
          console.error('Failed to delete project during rollback:', error);
        }
      }

      state.message = 'Rollback completed';
    } catch (error) {
      console.error('Rollback failed:', error);
      state.message = 'Rollback failed - manual cleanup required';
    }
  }

  private sleep(ms: number): Promise<void> {
    return new Promise(resolve => setTimeout(resolve, ms));
  }
}