/** * Data Integrity Service * * Handles validation and repair of broken references and other data integrity issues. */ import { logger } from './logger.js'; import { normalizedCacheStore as cacheStore } from './normalizedCacheStore.js'; import { jiraAssetsClient, JiraObjectNotFoundError } from './jiraAssetsClient.js'; import type { CMDBObject } from '../generated/jira-types.js'; import type { DatabaseAdapter } from './database/interface.js'; export interface BrokenReference { object_id: string; attribute_id: number; reference_object_id: string; field_name: string; object_type_name: string; object_key: string; label: string; } export interface RepairResult { total: number; repaired: number; deleted: number; failed: number; errors: Array<{ reference: BrokenReference; error: string }>; } export interface ValidationResult { brokenReferences: number; objectsWithBrokenRefs: number; lastValidated: string; } class DataIntegrityService { /** * Validate all references in the cache */ async validateReferences(): Promise { const brokenCount = await cacheStore.getBrokenReferencesCount(); // Count unique objects with broken references const brokenRefs = await cacheStore.getBrokenReferences(10000, 0); const uniqueObjectIds = new Set(brokenRefs.map(ref => ref.object_id)); return { brokenReferences: brokenCount, objectsWithBrokenRefs: uniqueObjectIds.size, lastValidated: new Date().toISOString(), }; } /** * Repair broken references * * @param mode - 'delete': Remove broken references, 'fetch': Try to fetch missing objects from Jira, 'dry-run': Just report * @param batchSize - Number of references to process at a time * @param maxRepairs - Maximum number of repairs to attempt (0 = unlimited) */ async repairBrokenReferences( mode: 'delete' | 'fetch' | 'dry-run' = 'fetch', batchSize: number = 100, maxRepairs: number = 0 ): Promise { const result: RepairResult = { total: 0, repaired: 0, deleted: 0, failed: 0, errors: [], }; let offset = 0; let processed = 0; while (true) { // Fetch batch of broken references const brokenRefs = await cacheStore.getBrokenReferences(batchSize, offset); if (brokenRefs.length === 0) break; result.total += brokenRefs.length; for (const ref of brokenRefs) { // Check max repairs limit if (maxRepairs > 0 && processed >= maxRepairs) { logger.info(`DataIntegrityService: Reached max repairs limit (${maxRepairs})`); break; } try { if (mode === 'dry-run') { // Just count, don't repair processed++; continue; } if (mode === 'fetch') { // Try to fetch the referenced object from Jira const fetchResult = await this.validateAndFetchReference(ref.reference_object_id); if (fetchResult.exists && fetchResult.object) { // Object was successfully fetched and cached logger.debug(`DataIntegrityService: Repaired reference from ${ref.object_key}.${ref.field_name} to ${ref.reference_object_id}`); result.repaired++; } else { // Object doesn't exist in Jira, delete the reference await this.deleteBrokenReference(ref); logger.debug(`DataIntegrityService: Deleted broken reference from ${ref.object_key}.${ref.field_name} to ${ref.reference_object_id} (object not found in Jira)`); result.deleted++; } } else if (mode === 'delete') { // Directly delete the broken reference await this.deleteBrokenReference(ref); result.deleted++; } processed++; } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); logger.error(`DataIntegrityService: Failed to repair reference from ${ref.object_key}.${ref.field_name} to ${ref.reference_object_id}`, error); result.failed++; result.errors.push({ reference: ref, error: errorMessage, }); } } // Check if we should continue if (brokenRefs.length < batchSize || (maxRepairs > 0 && processed >= maxRepairs)) { break; } offset += batchSize; } logger.info(`DataIntegrityService: Repair completed - Total: ${result.total}, Repaired: ${result.repaired}, Deleted: ${result.deleted}, Failed: ${result.failed}`); return result; } /** * Validate and fetch a referenced object */ private async validateAndFetchReference( referenceObjectId: string ): Promise<{ exists: boolean; object?: CMDBObject }> { // 1. Check cache first const db = (cacheStore as any).db; if (db) { const typedDb = db as DatabaseAdapter; const objRow = await typedDb.queryOne<{ id: string; object_type_name: string; }>(` SELECT id, object_type_name FROM objects WHERE id = ? `, [referenceObjectId]); if (objRow) { const cached = await cacheStore.getObject(objRow.object_type_name as any, referenceObjectId); if (cached) { return { exists: true, object: cached }; } } } // 2. Try to fetch from Jira try { const jiraObj = await jiraAssetsClient.getObject(referenceObjectId); if (jiraObj) { // Parse and cache const parsed = await jiraAssetsClient.parseObject(jiraObj); if (parsed) { await cacheStore.upsertObject(parsed._objectType, parsed); await cacheStore.extractAndStoreRelations(parsed._objectType, parsed); return { exists: true, object: parsed }; } } } catch (error) { if (error instanceof JiraObjectNotFoundError) { return { exists: false }; } // Re-throw other errors throw error; } return { exists: false }; } /** * Delete a broken reference */ private async deleteBrokenReference(ref: BrokenReference): Promise { const db = (cacheStore as any).db; if (!db) { throw new Error('Database not available'); } await db.execute(` DELETE FROM attribute_values WHERE object_id = ? AND attribute_id = ? AND reference_object_id = ? `, [ref.object_id, ref.attribute_id, ref.reference_object_id]); } /** * Cleanup orphaned attribute values (values without parent object) */ async cleanupOrphanedAttributeValues(): Promise { const db = (cacheStore as any).db; if (!db) { throw new Error('Database not available'); } const result = await db.execute(` DELETE FROM attribute_values WHERE object_id NOT IN (SELECT id FROM objects) `); logger.info(`DataIntegrityService: Cleaned up ${result} orphaned attribute values`); return result; } /** * Cleanup orphaned relations (relations where source or target doesn't exist) */ async cleanupOrphanedRelations(): Promise { const db = (cacheStore as any).db; if (!db) { throw new Error('Database not available'); } const result = await db.execute(` DELETE FROM object_relations WHERE source_id NOT IN (SELECT id FROM objects) OR target_id NOT IN (SELECT id FROM objects) `); logger.info(`DataIntegrityService: Cleaned up ${result} orphaned relations`); return result; } /** * Full integrity check and repair */ async fullIntegrityCheck(repair: boolean = false): Promise<{ validation: ValidationResult; repair?: RepairResult; orphanedValues: number; orphanedRelations: number; }> { logger.info('DataIntegrityService: Starting full integrity check...'); const validation = await this.validateReferences(); const orphanedValues = await this.cleanupOrphanedAttributeValues(); const orphanedRelations = await this.cleanupOrphanedRelations(); let repairResult: RepairResult | undefined; if (repair) { repairResult = await this.repairBrokenReferences('fetch', 100, 0); } logger.info('DataIntegrityService: Integrity check completed', { brokenReferences: validation.brokenReferences, orphanedValues, orphanedRelations, repaired: repairResult?.repaired || 0, deleted: repairResult?.deleted || 0, }); return { validation, repair: repairResult, orphanedValues, orphanedRelations, }; } } export const dataIntegrityService = new DataIntegrityService();