cmdb-insight/backend/src/services/database/migrate-to-normalized-schema.ts

/**
 * Migration script to migrate from configured_object_types to normalized schema structure
 * 
 * This script:
 * 1. Creates schemas table if it doesn't exist
 * 2. Migrates unique schemas from configured_object_types to schemas
 * 3. Adds schema_id and enabled columns to object_types if they don't exist
 * 4. Migrates object types from configured_object_types to object_types with schema_id FK
 * 5. Drops configured_object_types table after successful migration
 */

import { logger } from '../logger.js';
import { normalizedCacheStore } from '../normalizedCacheStore.js';
import type { DatabaseAdapter } from './interface.js';

export async function migrateToNormalizedSchema(): Promise<void> {
  const db = (normalizedCacheStore as any).db;
  if (!db) {
    throw new Error('Database not available');
  }

  await db.ensureInitialized?.();

  logger.info('Migration: Starting migration to normalized schema structure...');

  try {
    await db.transaction(async (txDb: DatabaseAdapter) => {
      // Step 1: Check if configured_object_types table exists
      let configuredTableExists = false;
      try {
        if (txDb.isPostgres) {
          const result = await txDb.queryOne<{ count: number }>(`
            SELECT COUNT(*) as count
            FROM information_schema.tables
            WHERE table_schema = 'public' AND table_name = 'configured_object_types'
          `);
          configuredTableExists = (result?.count || 0) > 0;
        } else {
          const result = await txDb.queryOne<{ count: number }>(`
            SELECT COUNT(*) as count
            FROM sqlite_master
            WHERE type='table' AND name='configured_object_types'
          `);
          configuredTableExists = (result?.count || 0) > 0;
        }
      } catch (error) {
        logger.debug('Migration: configured_object_types table check failed (may not exist)', error);
      }

      if (!configuredTableExists) {
        logger.info('Migration: configured_object_types table does not exist, skipping migration');
        return;
      }

      // Step 2: Check if schemas table exists, create if not
      let schemasTableExists = false;
      try {
        if (txDb.isPostgres) {
          const result = await txDb.queryOne<{ count: number }>(`
            SELECT COUNT(*) as count
            FROM information_schema.tables
            WHERE table_schema = 'public' AND table_name = 'schemas'
          `);
          schemasTableExists = (result?.count || 0) > 0;
        } else {
          const result = await txDb.queryOne<{ count: number }>(`
            SELECT COUNT(*) as count
            FROM sqlite_master
            WHERE type='table' AND name='schemas'
          `);
          schemasTableExists = (result?.count || 0) > 0;
        }
      } catch (error) {
        logger.debug('Migration: schemas table check failed', error);
      }

      if (!schemasTableExists) {
        logger.info('Migration: Creating schemas table...');
        if (txDb.isPostgres) {
          await txDb.execute(`
            CREATE TABLE IF NOT EXISTS schemas (
              id SERIAL PRIMARY KEY,
              jira_schema_id TEXT NOT NULL UNIQUE,
              name TEXT NOT NULL,
              description TEXT,
              discovered_at TIMESTAMP NOT NULL DEFAULT NOW(),
              updated_at TIMESTAMP NOT NULL DEFAULT NOW()
            )
          `);
          await txDb.execute(`
            CREATE INDEX IF NOT EXISTS idx_schemas_jira_schema_id ON schemas(jira_schema_id)
          `);
          await txDb.execute(`
            CREATE INDEX IF NOT EXISTS idx_schemas_name ON schemas(name)
          `);
        } else {
          await txDb.execute(`
            CREATE TABLE IF NOT EXISTS schemas (
              id INTEGER PRIMARY KEY AUTOINCREMENT,
              jira_schema_id TEXT NOT NULL UNIQUE,
              name TEXT NOT NULL,
              description TEXT,
              discovered_at TEXT NOT NULL DEFAULT (datetime('now')),
              updated_at TEXT NOT NULL DEFAULT (datetime('now'))
            )
          `);
          await txDb.execute(`
            CREATE INDEX IF NOT EXISTS idx_schemas_jira_schema_id ON schemas(jira_schema_id)
          `);
          await txDb.execute(`
            CREATE INDEX IF NOT EXISTS idx_schemas_name ON schemas(name)
          `);
        }
      }

      // Step 3: Migrate unique schemas from configured_object_types to schemas
      logger.info('Migration: Migrating schemas from configured_object_types...');
      const schemaRows = await txDb.query<{
        schema_id: string;
        schema_name: string;
        min_discovered_at: string;
        max_updated_at: string;
      }>(`
        SELECT 
          schema_id,
          schema_name,
          MIN(discovered_at) as min_discovered_at,
          MAX(updated_at) as max_updated_at
        FROM configured_object_types
        GROUP BY schema_id, schema_name
      `);

      for (const schemaRow of schemaRows) {
        if (txDb.isPostgres) {
          await txDb.execute(`
            INSERT INTO schemas (jira_schema_id, name, description, discovered_at, updated_at)
            VALUES (?, ?, ?, ?, ?)
            ON CONFLICT(jira_schema_id) DO UPDATE SET
              name = excluded.name,
              updated_at = excluded.updated_at
          `, [
            schemaRow.schema_id,
            schemaRow.schema_name,
            null,
            schemaRow.min_discovered_at,
            schemaRow.max_updated_at,
          ]);
        } else {
          await txDb.execute(`
            INSERT INTO schemas (jira_schema_id, name, description, discovered_at, updated_at)
            VALUES (?, ?, ?, ?, ?)
            ON CONFLICT(jira_schema_id) DO UPDATE SET
              name = excluded.name,
              updated_at = excluded.updated_at
          `, [
            schemaRow.schema_id,
            schemaRow.schema_name,
            null,
            schemaRow.min_discovered_at,
            schemaRow.max_updated_at,
          ]);
        }
      }
      logger.info(`Migration: Migrated ${schemaRows.length} schemas`);

      // Step 4: Check if object_types has schema_id and enabled columns
      let hasSchemaId = false;
      let hasEnabled = false;
      try {
        if (txDb.isPostgres) {
          const columns = await txDb.query<{ column_name: string }>(`
            SELECT column_name
            FROM information_schema.columns
            WHERE table_schema = 'public' AND table_name = 'object_types'
          `);
          hasSchemaId = columns.some((c: { column_name: string }) => c.column_name === 'schema_id');
          hasEnabled = columns.some((c: { column_name: string }) => c.column_name === 'enabled');
        } else {
          const tableInfo = await txDb.query<{ name: string }>(`
            PRAGMA table_info(object_types)
          `);
          hasSchemaId = tableInfo.some((c: { name: string }) => c.name === 'schema_id');
          hasEnabled = tableInfo.some((c: { name: string }) => c.name === 'enabled');
        }
      } catch (error) {
        logger.warn('Migration: Could not check object_types columns', error);
      }

      // Step 5: Add schema_id and enabled columns if they don't exist
      if (!hasSchemaId) {
        logger.info('Migration: Adding schema_id column to object_types...');
        if (txDb.isPostgres) {
          await txDb.execute(`
            ALTER TABLE object_types
            ADD COLUMN schema_id INTEGER REFERENCES schemas(id) ON DELETE CASCADE
          `);
        } else {
          // SQLite doesn't support ALTER TABLE ADD COLUMN with FK, so we'll handle it differently
          // For now, just add the column without FK constraint
          await txDb.execute(`
            ALTER TABLE object_types
            ADD COLUMN schema_id INTEGER
          `);
        }
      }

      if (!hasEnabled) {
        logger.info('Migration: Adding enabled column to object_types...');
        if (txDb.isPostgres) {
          await txDb.execute(`
            ALTER TABLE object_types
            ADD COLUMN enabled BOOLEAN NOT NULL DEFAULT FALSE
          `);
        } else {
          await txDb.execute(`
            ALTER TABLE object_types
            ADD COLUMN enabled INTEGER NOT NULL DEFAULT 0
          `);
        }
      }

      // Step 6: Migrate object types from configured_object_types to object_types
      logger.info('Migration: Migrating object types from configured_object_types...');
      const configuredTypes = await txDb.query<{
        schema_id: string;
        object_type_id: number;
        object_type_name: string;
        display_name: string;
        description: string | null;
        object_count: number;
        enabled: boolean | number;
        discovered_at: string;
        updated_at: string;
      }>(`
        SELECT 
          schema_id,
          object_type_id,
          object_type_name,
          display_name,
          description,
          object_count,
          enabled,
          discovered_at,
          updated_at
        FROM configured_object_types
      `);

      let migratedCount = 0;
      for (const configuredType of configuredTypes) {
        // Get schema_id (FK) from schemas table
        const schemaRow = await txDb.queryOne<{ id: number }>(
          `SELECT id FROM schemas WHERE jira_schema_id = ?`,
          [configuredType.schema_id]
        );

        if (!schemaRow) {
          logger.warn(`Migration: Schema ${configuredType.schema_id} not found, skipping object type ${configuredType.object_type_name}`);
          continue;
        }

        // Check if object type already exists in object_types
        const existingType = await txDb.queryOne<{ jira_type_id: number }>(
          `SELECT jira_type_id FROM object_types WHERE jira_type_id = ?`,
          [configuredType.object_type_id]
        );

        if (existingType) {
          // Update existing object type with schema_id and enabled
          if (txDb.isPostgres) {
            await txDb.execute(`
              UPDATE object_types
              SET 
                schema_id = ?,
                enabled = ?,
                display_name = COALESCE(display_name, ?),
                description = COALESCE(description, ?),
                object_count = COALESCE(object_count, ?),
                updated_at = ?
              WHERE jira_type_id = ?
            `, [
              schemaRow.id,
              typeof configuredType.enabled === 'boolean' ? configuredType.enabled : configuredType.enabled === 1,
              configuredType.display_name,
              configuredType.description,
              configuredType.object_count,
              configuredType.updated_at,
              configuredType.object_type_id,
            ]);
          } else {
            await txDb.execute(`
              UPDATE object_types
              SET 
                schema_id = ?,
                enabled = ?,
                display_name = COALESCE(display_name, ?),
                description = COALESCE(description, ?),
                object_count = COALESCE(object_count, ?),
                updated_at = ?
              WHERE jira_type_id = ?
            `, [
              schemaRow.id,
              typeof configuredType.enabled === 'boolean' ? (configuredType.enabled ? 1 : 0) : configuredType.enabled,
              configuredType.display_name,
              configuredType.description,
              configuredType.object_count,
              configuredType.updated_at,
              configuredType.object_type_id,
            ]);
          }
        } else {
          // Insert new object type
          // Note: We need sync_priority - use default 0
          if (txDb.isPostgres) {
            await txDb.execute(`
              INSERT INTO object_types (
                schema_id, jira_type_id, type_name, display_name, description,
                sync_priority, object_count, enabled, discovered_at, updated_at
              )
              VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
            `, [
              schemaRow.id,
              configuredType.object_type_id,
              configuredType.object_type_name,
              configuredType.display_name,
              configuredType.description,
              0, // sync_priority
              configuredType.object_count,
              typeof configuredType.enabled === 'boolean' ? configuredType.enabled : configuredType.enabled === 1,
              configuredType.discovered_at,
              configuredType.updated_at,
            ]);
          } else {
            await txDb.execute(`
              INSERT INTO object_types (
                schema_id, jira_type_id, type_name, display_name, description,
                sync_priority, object_count, enabled, discovered_at, updated_at
              )
              VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
            `, [
              schemaRow.id,
              configuredType.object_type_id,
              configuredType.object_type_name,
              configuredType.display_name,
              configuredType.description,
              0, // sync_priority
              configuredType.object_count,
              typeof configuredType.enabled === 'boolean' ? (configuredType.enabled ? 1 : 0) : configuredType.enabled,
              configuredType.discovered_at,
              configuredType.updated_at,
            ]);
          }
        }
        migratedCount++;
      }
      logger.info(`Migration: Migrated ${migratedCount} object types`);

      // Step 7: Fix UNIQUE constraints on object_types
      logger.info('Migration: Fixing UNIQUE constraints on object_types...');
      try {
        // Remove old UNIQUE constraint on type_name if it exists
        if (txDb.isPostgres) {
          // Check if constraint exists
          const constraintExists = await txDb.queryOne<{ count: number }>(`
            SELECT COUNT(*) as count
            FROM pg_constraint
            WHERE conname = 'object_types_type_name_key'
          `);
          
          if (constraintExists && constraintExists.count > 0) {
            logger.info('Migration: Dropping old UNIQUE constraint on type_name...');
            await txDb.execute(`ALTER TABLE object_types DROP CONSTRAINT IF EXISTS object_types_type_name_key`);
          }
          
          // Add new UNIQUE constraint on (schema_id, type_name)
          const newConstraintExists = await txDb.queryOne<{ count: number }>(`
            SELECT COUNT(*) as count
            FROM pg_constraint
            WHERE conname = 'object_types_schema_id_type_name_key'
          `);
          
          if (!newConstraintExists || newConstraintExists.count === 0) {
            logger.info('Migration: Adding UNIQUE constraint on (schema_id, type_name)...');
            await txDb.execute(`
              ALTER TABLE object_types
              ADD CONSTRAINT object_types_schema_id_type_name_key UNIQUE (schema_id, type_name)
            `);
          }
        } else {
          // SQLite: UNIQUE constraints are part of table definition, so we need to recreate
          // For now, just log a warning - SQLite doesn't support DROP CONSTRAINT easily
          logger.info('Migration: SQLite UNIQUE constraints are handled in table definition');
        }
      } catch (error) {
        logger.warn('Migration: Could not fix UNIQUE constraints (may already be correct)', error);
      }

      // Step 8: Add indexes if they don't exist
      logger.info('Migration: Adding indexes...');
      try {
        await txDb.execute(`CREATE INDEX IF NOT EXISTS idx_object_types_schema_id ON object_types(schema_id)`);
        await txDb.execute(`CREATE INDEX IF NOT EXISTS idx_object_types_enabled ON object_types(enabled)`);
        await txDb.execute(`CREATE INDEX IF NOT EXISTS idx_object_types_schema_enabled ON object_types(schema_id, enabled)`);
      } catch (error) {
        logger.warn('Migration: Some indexes may already exist', error);
      }

      // Step 9: Drop configured_object_types table
      logger.info('Migration: Dropping configured_object_types table...');
      await txDb.execute(`DROP TABLE IF EXISTS configured_object_types`);
      logger.info('Migration: Dropped configured_object_types table');
    });

    logger.info('Migration: Migration to normalized schema structure completed successfully');
  } catch (error) {
    logger.error('Migration: Failed to migrate to normalized schema structure', error);
    throw error;
  }
}