Add database adapter system, production deployment configs, and new dashboard components
- Add PostgreSQL and SQLite database adapters with factory pattern - Add migration script for SQLite to PostgreSQL - Add production Dockerfiles and docker-compose configs - Add deployment documentation and scripts - Add BIA sync dashboard and matching service - Add data completeness configuration and components - Add new dashboard components (BusinessImportanceComparison, ComplexityDynamics, etc.) - Update various services and routes - Remove deprecated management-parameters.json and taxonomy files
This commit is contained in:
@@ -91,7 +91,7 @@ class SyncEngine {
|
||||
this.isRunning = true;
|
||||
|
||||
// Check if we need a full sync
|
||||
const stats = cacheStore.getStats();
|
||||
const stats = await cacheStore.getStats();
|
||||
const lastFullSync = stats.lastFullSync;
|
||||
const needsFullSync = !stats.isWarm || !lastFullSync || this.isStale(lastFullSync, 24 * 60 * 60 * 1000);
|
||||
|
||||
@@ -175,8 +175,8 @@ class SyncEngine {
|
||||
|
||||
// Update sync metadata
|
||||
const now = new Date().toISOString();
|
||||
cacheStore.setSyncMetadata('lastFullSync', now);
|
||||
cacheStore.setSyncMetadata('lastIncrementalSync', now);
|
||||
await cacheStore.setSyncMetadata('lastFullSync', now);
|
||||
await cacheStore.setSyncMetadata('lastIncrementalSync', now);
|
||||
this.lastIncrementalSync = new Date();
|
||||
|
||||
const duration = Date.now() - startTime;
|
||||
@@ -223,31 +223,52 @@ class SyncEngine {
|
||||
|
||||
// Fetch all objects from Jira
|
||||
const jiraObjects = await jiraAssetsClient.getAllObjectsOfType(typeName, this.batchSize);
|
||||
logger.info(`SyncEngine: Fetched ${jiraObjects.length} ${typeName} objects from Jira`);
|
||||
|
||||
// Parse and cache objects
|
||||
const parsedObjects: CMDBObject[] = [];
|
||||
const failedObjects: Array<{ id: string; key: string; label: string; reason: string }> = [];
|
||||
|
||||
for (const jiraObj of jiraObjects) {
|
||||
const parsed = jiraAssetsClient.parseObject(jiraObj);
|
||||
if (parsed) {
|
||||
parsedObjects.push(parsed);
|
||||
} else {
|
||||
// Track objects that failed to parse
|
||||
failedObjects.push({
|
||||
id: jiraObj.id?.toString() || 'unknown',
|
||||
key: jiraObj.objectKey || 'unknown',
|
||||
label: jiraObj.label || 'unknown',
|
||||
reason: 'parseObject returned null',
|
||||
});
|
||||
logger.warn(`SyncEngine: Failed to parse ${typeName} object: ${jiraObj.objectKey || jiraObj.id} (${jiraObj.label || 'unknown label'})`);
|
||||
}
|
||||
}
|
||||
|
||||
// Log parsing statistics
|
||||
if (failedObjects.length > 0) {
|
||||
logger.warn(`SyncEngine: ${failedObjects.length} ${typeName} objects failed to parse:`, failedObjects.map(o => `${o.key} (${o.label})`).join(', '));
|
||||
}
|
||||
|
||||
// Batch upsert to cache
|
||||
if (parsedObjects.length > 0) {
|
||||
cacheStore.batchUpsertObjects(typeName, parsedObjects);
|
||||
await cacheStore.batchUpsertObjects(typeName, parsedObjects);
|
||||
objectsProcessed = parsedObjects.length;
|
||||
|
||||
// Extract relations
|
||||
for (const obj of parsedObjects) {
|
||||
cacheStore.extractAndStoreRelations(typeName, obj);
|
||||
await cacheStore.extractAndStoreRelations(typeName, obj);
|
||||
relationsExtracted++;
|
||||
}
|
||||
}
|
||||
|
||||
const duration = Date.now() - startTime;
|
||||
logger.debug(`SyncEngine: Synced ${objectsProcessed} ${typeName} objects in ${duration}ms`);
|
||||
const skippedCount = jiraObjects.length - objectsProcessed;
|
||||
if (skippedCount > 0) {
|
||||
logger.warn(`SyncEngine: Synced ${objectsProcessed}/${jiraObjects.length} ${typeName} objects in ${duration}ms (${skippedCount} skipped)`);
|
||||
} else {
|
||||
logger.debug(`SyncEngine: Synced ${objectsProcessed} ${typeName} objects in ${duration}ms`);
|
||||
}
|
||||
|
||||
return {
|
||||
objectType: typeName,
|
||||
@@ -304,7 +325,7 @@ class SyncEngine {
|
||||
|
||||
try {
|
||||
// Get the last sync time
|
||||
const lastSyncStr = cacheStore.getSyncMetadata('lastIncrementalSync');
|
||||
const lastSyncStr = await cacheStore.getSyncMetadata('lastIncrementalSync');
|
||||
const since = lastSyncStr
|
||||
? new Date(lastSyncStr)
|
||||
: new Date(Date.now() - 60000); // Default: last minute
|
||||
@@ -317,7 +338,7 @@ class SyncEngine {
|
||||
// If no objects returned (e.g., Data Center doesn't support IQL incremental sync),
|
||||
// check if we should trigger a full sync instead
|
||||
if (updatedObjects.length === 0) {
|
||||
const lastFullSyncStr = cacheStore.getSyncMetadata('lastFullSync');
|
||||
const lastFullSyncStr = await cacheStore.getSyncMetadata('lastFullSync');
|
||||
if (lastFullSyncStr) {
|
||||
const lastFullSync = new Date(lastFullSyncStr);
|
||||
const fullSyncAge = Date.now() - lastFullSync.getTime();
|
||||
@@ -334,7 +355,7 @@ class SyncEngine {
|
||||
|
||||
// Update timestamp even if no objects were synced
|
||||
const now = new Date();
|
||||
cacheStore.setSyncMetadata('lastIncrementalSync', now.toISOString());
|
||||
await cacheStore.setSyncMetadata('lastIncrementalSync', now.toISOString());
|
||||
this.lastIncrementalSync = now;
|
||||
|
||||
return { success: true, updatedCount: 0 };
|
||||
@@ -346,15 +367,15 @@ class SyncEngine {
|
||||
const parsed = jiraAssetsClient.parseObject(jiraObj);
|
||||
if (parsed) {
|
||||
const typeName = parsed._objectType as CMDBObjectTypeName;
|
||||
cacheStore.upsertObject(typeName, parsed);
|
||||
cacheStore.extractAndStoreRelations(typeName, parsed);
|
||||
await cacheStore.upsertObject(typeName, parsed);
|
||||
await cacheStore.extractAndStoreRelations(typeName, parsed);
|
||||
updatedCount++;
|
||||
}
|
||||
}
|
||||
|
||||
// Update sync metadata
|
||||
const now = new Date();
|
||||
cacheStore.setSyncMetadata('lastIncrementalSync', now.toISOString());
|
||||
await cacheStore.setSyncMetadata('lastIncrementalSync', now.toISOString());
|
||||
this.lastIncrementalSync = now;
|
||||
|
||||
if (updatedCount > 0) {
|
||||
@@ -413,14 +434,14 @@ class SyncEngine {
|
||||
const parsed = jiraAssetsClient.parseObject(jiraObj);
|
||||
if (!parsed) return false;
|
||||
|
||||
cacheStore.upsertObject(typeName, parsed);
|
||||
cacheStore.extractAndStoreRelations(typeName, parsed);
|
||||
await cacheStore.upsertObject(typeName, parsed);
|
||||
await cacheStore.extractAndStoreRelations(typeName, parsed);
|
||||
|
||||
return true;
|
||||
} catch (error) {
|
||||
// If object was deleted from Jira, remove it from our cache
|
||||
if (error instanceof JiraObjectNotFoundError) {
|
||||
const deleted = cacheStore.deleteObject(typeName, objectId);
|
||||
const deleted = await cacheStore.deleteObject(typeName, objectId);
|
||||
if (deleted) {
|
||||
logger.info(`SyncEngine: Removed deleted object ${typeName}/${objectId} from cache`);
|
||||
}
|
||||
@@ -438,8 +459,8 @@ class SyncEngine {
|
||||
/**
|
||||
* Get current sync engine status
|
||||
*/
|
||||
getStatus(): SyncEngineStatus {
|
||||
const stats = cacheStore.getStats();
|
||||
async getStatus(): Promise<SyncEngineStatus> {
|
||||
const stats = await cacheStore.getStats();
|
||||
|
||||
let nextIncrementalSync: string | null = null;
|
||||
if (this.isRunning && this.lastIncrementalSync) {
|
||||
|
||||
Reference in New Issue
Block a user