/** * Schema Sync Service * * Unified service for synchronizing Jira Assets schema configuration to local database. * Implements the complete sync flow as specified in the refactor plan. */ import { logger } from './logger.js'; import { getDatabaseAdapter } from './database/singleton.js'; import type { DatabaseAdapter } from './database/interface.js'; import { config } from '../config/env.js'; import { toCamelCase, toPascalCase, mapJiraType, determineSyncPriority } from './schemaUtils.js'; // ============================================================================= // Types // ============================================================================= interface JiraSchema { id: number; name: string; objectSchemaKey?: string; status?: string; description?: string; created?: string; updated?: string; objectCount?: number; objectTypeCount?: number; } interface JiraObjectType { id: number; name: string; type?: number; description?: string; icon?: { id: number; name: string; url16?: string; url48?: string; }; position?: number; created?: string; updated?: string; objectCount?: number; parentObjectTypeId?: number | null; objectSchemaId: number; inherited?: boolean; abstractObjectType?: boolean; } interface JiraAttribute { id: number; objectType?: { id: number; name: string; }; name: string; label?: boolean; type: number; description?: string; defaultType?: { id: number; name: string; } | null; typeValue?: string | null; typeValueMulti?: string[]; additionalValue?: string | null; referenceType?: { id: number; name: string; description?: string; color?: string; url16?: string | null; removable?: boolean; objectSchemaId?: number; } | null; referenceObjectTypeId?: number | null; referenceObjectType?: { id: number; name: string; objectSchemaId?: number; } | null; editable?: boolean; system?: boolean; sortable?: boolean; summable?: boolean; indexed?: boolean; minimumCardinality?: number; maximumCardinality?: number; suffix?: string; removable?: boolean; hidden?: boolean; includeChildObjectTypes?: boolean; uniqueAttribute?: boolean; regexValidation?: string | null; iql?: string | null; options?: string; position?: number; } export interface SyncResult { success: boolean; schemasProcessed: number; objectTypesProcessed: number; attributesProcessed: number; schemasDeleted: number; objectTypesDeleted: number; attributesDeleted: number; errors: SyncError[]; duration: number; // milliseconds } export interface SyncError { type: 'schema' | 'objectType' | 'attribute'; id: string | number; message: string; } export interface SyncProgress { status: 'idle' | 'running' | 'completed' | 'failed'; currentSchema?: string; currentObjectType?: string; schemasTotal: number; schemasCompleted: number; objectTypesTotal: number; objectTypesCompleted: number; startedAt?: Date; estimatedCompletion?: Date; } // ============================================================================= // SchemaSyncService Implementation // ============================================================================= class SchemaSyncService { private db: DatabaseAdapter; private isPostgres: boolean; private baseUrl: string; private progress: SyncProgress = { status: 'idle', schemasTotal: 0, schemasCompleted: 0, objectTypesTotal: 0, objectTypesCompleted: 0, }; // Rate limiting configuration private readonly RATE_LIMIT_DELAY_MS = 150; // 150ms between requests private readonly MAX_RETRIES = 3; private readonly RETRY_DELAY_MS = 1000; constructor() { this.db = getDatabaseAdapter(); this.isPostgres = (this.db.isPostgres === true); this.baseUrl = `${config.jiraHost}/rest/assets/1.0`; } /** * Get authentication headers for API requests */ private getHeaders(): Record { const token = config.jiraServiceAccountToken; if (!token) { throw new Error('JIRA_SERVICE_ACCOUNT_TOKEN not configured. Schema sync requires a service account token.'); } return { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/json', 'Accept': 'application/json', }; } /** * Rate limiting delay */ private delay(ms: number): Promise { return new Promise(resolve => setTimeout(resolve, ms)); } /** * Fetch with rate limiting and retry logic */ private async fetchWithRateLimit( url: string, retries: number = this.MAX_RETRIES ): Promise { await this.delay(this.RATE_LIMIT_DELAY_MS); try { const response = await fetch(url, { headers: this.getHeaders(), }); // Handle rate limiting (429) if (response.status === 429) { const retryAfter = parseInt(response.headers.get('Retry-After') || '5', 10); logger.warn(`SchemaSync: Rate limited, waiting ${retryAfter}s before retry`); await this.delay(retryAfter * 1000); return this.fetchWithRateLimit(url, retries); } // Handle server errors with retry if (response.status >= 500 && retries > 0) { logger.warn(`SchemaSync: Server error ${response.status}, retrying (${retries} attempts left)`); await this.delay(this.RETRY_DELAY_MS); return this.fetchWithRateLimit(url, retries - 1); } if (!response.ok) { const text = await response.text(); throw new Error(`HTTP ${response.status}: ${text}`); } return await response.json() as T; } catch (error) { if (retries > 0 && error instanceof Error && !error.message.includes('HTTP')) { logger.warn(`SchemaSync: Network error, retrying (${retries} attempts left)`, error); await this.delay(this.RETRY_DELAY_MS); return this.fetchWithRateLimit(url, retries - 1); } throw error; } } /** * Fetch all schemas from Jira */ private async fetchSchemas(): Promise { const url = `${this.baseUrl}/objectschema/list`; logger.debug(`SchemaSync: Fetching schemas from ${url}`); const result = await this.fetchWithRateLimit<{ objectschemas?: JiraSchema[] } | JiraSchema[]>(url); // Handle different response formats if (Array.isArray(result)) { return result; } else if (result && typeof result === 'object' && 'objectschemas' in result) { return result.objectschemas || []; } logger.warn('SchemaSync: Unexpected schema list response format', result); return []; } /** * Fetch schema details */ private async fetchSchemaDetails(schemaId: number): Promise { const url = `${this.baseUrl}/objectschema/${schemaId}`; logger.debug(`SchemaSync: Fetching schema details for ${schemaId}`); return await this.fetchWithRateLimit(url); } /** * Fetch all object types for a schema (flat list) */ private async fetchObjectTypes(schemaId: number): Promise { const url = `${this.baseUrl}/objectschema/${schemaId}/objecttypes/flat`; logger.debug(`SchemaSync: Fetching object types for schema ${schemaId}`); try { const result = await this.fetchWithRateLimit(url); return Array.isArray(result) ? result : []; } catch (error) { // Fallback to regular endpoint if flat endpoint fails logger.warn(`SchemaSync: Flat endpoint failed, trying regular endpoint`, error); const fallbackUrl = `${this.baseUrl}/objectschema/${schemaId}/objecttypes`; const fallbackResult = await this.fetchWithRateLimit<{ objectTypes?: JiraObjectType[] } | JiraObjectType[]>(fallbackUrl); if (Array.isArray(fallbackResult)) { return fallbackResult; } else if (fallbackResult && typeof fallbackResult === 'object' && 'objectTypes' in fallbackResult) { return fallbackResult.objectTypes || []; } return []; } } /** * Fetch object type details */ private async fetchObjectTypeDetails(typeId: number): Promise { const url = `${this.baseUrl}/objecttype/${typeId}`; logger.debug(`SchemaSync: Fetching object type details for ${typeId}`); return await this.fetchWithRateLimit(url); } /** * Fetch attributes for an object type */ private async fetchAttributes(typeId: number): Promise { const url = `${this.baseUrl}/objecttype/${typeId}/attributes`; logger.debug(`SchemaSync: Fetching attributes for object type ${typeId}`); try { const result = await this.fetchWithRateLimit(url); return Array.isArray(result) ? result : []; } catch (error) { logger.warn(`SchemaSync: Failed to fetch attributes for type ${typeId}`, error); return []; } } /** * Parse Jira attribute to database format */ private parseAttribute( attr: JiraAttribute, allTypeConfigs: Map ): { jiraId: number; name: string; fieldName: string; type: string; isMultiple: boolean; isEditable: boolean; isRequired: boolean; isSystem: boolean; referenceTypeName?: string; description?: string; // Additional fields from plan label?: boolean; sortable?: boolean; summable?: boolean; indexed?: boolean; suffix?: string; removable?: boolean; hidden?: boolean; includeChildObjectTypes?: boolean; uniqueAttribute?: boolean; regexValidation?: string | null; iql?: string | null; options?: string; position?: number; } { const typeId = attr.type || attr.defaultType?.id || 0; let type = mapJiraType(typeId); const isMultiple = (attr.maximumCardinality ?? 1) > 1 || attr.maximumCardinality === -1; const isEditable = attr.editable !== false && !attr.hidden; const isRequired = (attr.minimumCardinality ?? 0) > 0; const isSystem = attr.system === true; // CRITICAL: Jira sometimes returns type=1 (integer) for reference attributes! // The presence of referenceObjectTypeId is the true indicator of a reference type. const refTypeId = attr.referenceObjectTypeId || attr.referenceObject?.id || attr.referenceType?.id; if (refTypeId) { type = 'reference'; } const result: ReturnType = { jiraId: attr.id, name: attr.name, fieldName: toCamelCase(attr.name), type, isMultiple, isEditable, isRequired, isSystem, description: attr.description, label: attr.label, sortable: attr.sortable, summable: attr.summable, indexed: attr.indexed, suffix: attr.suffix, removable: attr.removable, hidden: attr.hidden, includeChildObjectTypes: attr.includeChildObjectTypes, uniqueAttribute: attr.uniqueAttribute, regexValidation: attr.regexValidation, iql: attr.iql, options: attr.options, position: attr.position, }; // Handle reference types - add reference metadata if (type === 'reference' && refTypeId) { const refConfig = allTypeConfigs.get(refTypeId); result.referenceTypeName = refConfig?.typeName || attr.referenceObjectType?.name || attr.referenceType?.name || `Type${refTypeId}`; } return result; } /** * Sync all schemas and their complete structure */ async syncAll(): Promise { const startTime = Date.now(); const errors: SyncError[] = []; this.progress = { status: 'running', schemasTotal: 0, schemasCompleted: 0, objectTypesTotal: 0, objectTypesCompleted: 0, startedAt: new Date(), }; try { logger.info('SchemaSync: Starting full schema synchronization...'); // Step 1: Fetch all schemas const schemas = await this.fetchSchemas(); this.progress.schemasTotal = schemas.length; logger.info(`SchemaSync: Found ${schemas.length} schemas to sync`); if (schemas.length === 0) { throw new Error('No schemas found in Jira Assets'); } // Track Jira IDs for cleanup const jiraSchemaIds = new Set(); const jiraObjectTypeIds = new Map>(); // schemaId -> Set const jiraAttributeIds = new Map>(); // typeName -> Set let schemasProcessed = 0; let objectTypesProcessed = 0; let attributesProcessed = 0; let schemasDeleted = 0; let objectTypesDeleted = 0; let attributesDeleted = 0; await this.db.transaction(async (txDb) => { // Step 2: Process each schema for (const schema of schemas) { try { this.progress.currentSchema = schema.name; const schemaIdStr = schema.id.toString(); jiraSchemaIds.add(schemaIdStr); // Fetch schema details let schemaDetails: JiraSchema; try { schemaDetails = await this.fetchSchemaDetails(schema.id); } catch (error) { logger.warn(`SchemaSync: Failed to fetch details for schema ${schema.id}, using list data`, error); schemaDetails = schema; } const now = new Date().toISOString(); const objectSchemaKey = schemaDetails.objectSchemaKey || schemaDetails.name || schemaIdStr; // Upsert schema if (txDb.isPostgres) { await txDb.execute(` INSERT INTO schemas (jira_schema_id, name, object_schema_key, status, description, discovered_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(jira_schema_id) DO UPDATE SET name = excluded.name, object_schema_key = excluded.object_schema_key, status = excluded.status, description = excluded.description, updated_at = excluded.updated_at `, [ schemaIdStr, schemaDetails.name, objectSchemaKey, schemaDetails.status || null, schemaDetails.description || null, now, now, ]); } else { await txDb.execute(` INSERT INTO schemas (jira_schema_id, name, object_schema_key, status, description, discovered_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(jira_schema_id) DO UPDATE SET name = excluded.name, object_schema_key = excluded.object_schema_key, status = excluded.status, description = excluded.description, updated_at = excluded.updated_at `, [ schemaIdStr, schemaDetails.name, objectSchemaKey, schemaDetails.status || null, schemaDetails.description || null, now, now, ]); } // Get schema FK const schemaRow = await txDb.queryOne<{ id: number }>( `SELECT id FROM schemas WHERE jira_schema_id = ?`, [schemaIdStr] ); if (!schemaRow) { throw new Error(`Failed to get schema FK for ${schemaIdStr}`); } const schemaIdFk = schemaRow.id; // Step 3: Fetch all object types for this schema const objectTypes = await this.fetchObjectTypes(schema.id); logger.info(`SchemaSync: Found ${objectTypes.length} object types in schema ${schema.name}`); const typeConfigs = new Map(); jiraObjectTypeIds.set(schemaIdStr, new Set()); // Build type name mapping for (const objType of objectTypes) { const typeName = toPascalCase(objType.name); typeConfigs.set(objType.id, { name: objType.name, typeName, }); jiraObjectTypeIds.get(schemaIdStr)!.add(objType.id); } // Step 4: Store object types for (const objType of objectTypes) { try { this.progress.currentObjectType = objType.name; const typeName = toPascalCase(objType.name); const objectCount = objType.objectCount || 0; const syncPriority = determineSyncPriority(objType.name, objectCount); // Upsert object type if (txDb.isPostgres) { await txDb.execute(` INSERT INTO object_types ( schema_id, jira_type_id, type_name, display_name, description, sync_priority, object_count, enabled, discovered_at, updated_at ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(schema_id, jira_type_id) DO UPDATE SET display_name = excluded.display_name, description = excluded.description, sync_priority = excluded.sync_priority, object_count = excluded.object_count, updated_at = excluded.updated_at `, [ schemaIdFk, objType.id, typeName, objType.name, objType.description || null, syncPriority, objectCount, false, // Default: disabled now, now, ]); } else { await txDb.execute(` INSERT INTO object_types ( schema_id, jira_type_id, type_name, display_name, description, sync_priority, object_count, enabled, discovered_at, updated_at ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(schema_id, jira_type_id) DO UPDATE SET display_name = excluded.display_name, description = excluded.description, sync_priority = excluded.sync_priority, object_count = excluded.object_count, updated_at = excluded.updated_at `, [ schemaIdFk, objType.id, typeName, objType.name, objType.description || null, syncPriority, objectCount, 0, // Default: disabled (0 = false in SQLite) now, now, ]); } objectTypesProcessed++; // Step 5: Fetch and store attributes const attributes = await this.fetchAttributes(objType.id); logger.info(`SchemaSync: Fetched ${attributes.length} attributes for ${objType.name} (type ${objType.id})`); if (!jiraAttributeIds.has(typeName)) { jiraAttributeIds.set(typeName, new Set()); } if (attributes.length === 0) { logger.warn(`SchemaSync: No attributes found for ${objType.name} (type ${objType.id})`); } for (const jiraAttr of attributes) { try { const attrDef = this.parseAttribute(jiraAttr, typeConfigs); jiraAttributeIds.get(typeName)!.add(attrDef.jiraId); // Upsert attribute if (txDb.isPostgres) { await txDb.execute(` INSERT INTO attributes ( jira_attr_id, object_type_name, attr_name, field_name, attr_type, is_multiple, is_editable, is_required, is_system, reference_type_name, description, position, discovered_at ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(jira_attr_id, object_type_name) DO UPDATE SET attr_name = excluded.attr_name, field_name = excluded.field_name, attr_type = excluded.attr_type, is_multiple = excluded.is_multiple, is_editable = excluded.is_editable, is_required = excluded.is_required, is_system = excluded.is_system, reference_type_name = excluded.reference_type_name, description = excluded.description, position = excluded.position `, [ attrDef.jiraId, typeName, attrDef.name, attrDef.fieldName, attrDef.type, attrDef.isMultiple, attrDef.isEditable, attrDef.isRequired, attrDef.isSystem, attrDef.referenceTypeName || null, attrDef.description || null, attrDef.position ?? 0, now, ]); } else { await txDb.execute(` INSERT INTO attributes ( jira_attr_id, object_type_name, attr_name, field_name, attr_type, is_multiple, is_editable, is_required, is_system, reference_type_name, description, position, discovered_at ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(jira_attr_id, object_type_name) DO UPDATE SET attr_name = excluded.attr_name, field_name = excluded.field_name, attr_type = excluded.attr_type, is_multiple = excluded.is_multiple, is_editable = excluded.is_editable, is_required = excluded.is_required, is_system = excluded.is_system, reference_type_name = excluded.reference_type_name, description = excluded.description, position = excluded.position `, [ attrDef.jiraId, typeName, attrDef.name, attrDef.fieldName, attrDef.type, attrDef.isMultiple ? 1 : 0, attrDef.isEditable ? 1 : 0, attrDef.isRequired ? 1 : 0, attrDef.isSystem ? 1 : 0, attrDef.referenceTypeName || null, attrDef.description || null, attrDef.position ?? 0, now, ]); } attributesProcessed++; } catch (error) { logger.error(`SchemaSync: Failed to process attribute ${jiraAttr.id} (${jiraAttr.name}) for ${objType.name}`, error); if (error instanceof Error) { logger.error(`SchemaSync: Attribute error details: ${error.message}`, error.stack); } errors.push({ type: 'attribute', id: jiraAttr.id, message: error instanceof Error ? error.message : String(error), }); } } logger.info(`SchemaSync: Processed ${attributesProcessed} attributes for ${objType.name} (type ${objType.id})`); this.progress.objectTypesCompleted++; } catch (error) { logger.warn(`SchemaSync: Failed to process object type ${objType.id}`, error); errors.push({ type: 'objectType', id: objType.id, message: error instanceof Error ? error.message : String(error), }); } } this.progress.schemasCompleted++; schemasProcessed++; } catch (error) { logger.error(`SchemaSync: Failed to process schema ${schema.id}`, error); errors.push({ type: 'schema', id: schema.id.toString(), message: error instanceof Error ? error.message : String(error), }); } } // Step 6: Clean up orphaned records (hard delete) logger.info('SchemaSync: Cleaning up orphaned records...'); // Delete orphaned schemas const allLocalSchemas = await txDb.query<{ jira_schema_id: string }>( `SELECT jira_schema_id FROM schemas` ); for (const localSchema of allLocalSchemas) { if (!jiraSchemaIds.has(localSchema.jira_schema_id)) { logger.info(`SchemaSync: Deleting orphaned schema ${localSchema.jira_schema_id}`); await txDb.execute(`DELETE FROM schemas WHERE jira_schema_id = ?`, [localSchema.jira_schema_id]); schemasDeleted++; } } // Delete orphaned object types // First, get all object types from all remaining schemas const allLocalObjectTypes = await txDb.query<{ schema_id: number; jira_type_id: number; jira_schema_id: string }>( `SELECT ot.schema_id, ot.jira_type_id, s.jira_schema_id FROM object_types ot JOIN schemas s ON ot.schema_id = s.id` ); for (const localType of allLocalObjectTypes) { const schemaIdStr = localType.jira_schema_id; const typeIds = jiraObjectTypeIds.get(schemaIdStr); // If schema doesn't exist in Jira anymore, or type doesn't exist in schema if (!jiraSchemaIds.has(schemaIdStr) || (typeIds && !typeIds.has(localType.jira_type_id))) { logger.info(`SchemaSync: Deleting orphaned object type ${localType.jira_type_id} from schema ${schemaIdStr}`); await txDb.execute( `DELETE FROM object_types WHERE schema_id = ? AND jira_type_id = ?`, [localType.schema_id, localType.jira_type_id] ); objectTypesDeleted++; } } // Delete orphaned attributes // Get all attributes and check against synced types const allLocalAttributes = await txDb.query<{ object_type_name: string; jira_attr_id: number }>( `SELECT object_type_name, jira_attr_id FROM attributes` ); for (const localAttr of allLocalAttributes) { const attrIds = jiraAttributeIds.get(localAttr.object_type_name); // If type wasn't synced or attribute doesn't exist in type if (!attrIds || !attrIds.has(localAttr.jira_attr_id)) { logger.info(`SchemaSync: Deleting orphaned attribute ${localAttr.jira_attr_id} from type ${localAttr.object_type_name}`); await txDb.execute( `DELETE FROM attributes WHERE object_type_name = ? AND jira_attr_id = ?`, [localAttr.object_type_name, localAttr.jira_attr_id] ); attributesDeleted++; } } logger.info(`SchemaSync: Cleanup complete - ${schemasDeleted} schemas, ${objectTypesDeleted} object types, ${attributesDeleted} attributes deleted`); }); const duration = Date.now() - startTime; this.progress.status = 'completed'; logger.info(`SchemaSync: Synchronization complete in ${duration}ms - ${schemasProcessed} schemas, ${objectTypesProcessed} object types, ${attributesProcessed} attributes, ${schemasDeleted} deleted schemas, ${objectTypesDeleted} deleted types, ${attributesDeleted} deleted attributes`); if (attributesProcessed === 0) { logger.warn(`SchemaSync: WARNING - No attributes were saved! Check logs for errors.`); } if (errors.length > 0) { logger.warn(`SchemaSync: Sync completed with ${errors.length} errors:`, errors); } return { success: errors.length === 0, schemasProcessed, objectTypesProcessed, attributesProcessed, schemasDeleted, objectTypesDeleted, attributesDeleted, errors, duration, }; } catch (error) { this.progress.status = 'failed'; logger.error('SchemaSync: Synchronization failed', error); throw error; } } /** * Sync a single schema by ID */ async syncSchema(schemaId: number): Promise { // For single schema sync, we can reuse syncAll logic but filter // For now, just call syncAll (it's idempotent) logger.info(`SchemaSync: Syncing single schema ${schemaId}`); return this.syncAll(); } /** * Get sync status/progress */ getProgress(): SyncProgress { return { ...this.progress }; } } // Export singleton instance export const schemaSyncService = new SchemaSyncService();