Bert Hausmans
cdee0e8819
- Restore blue PageHeader on Dashboard (/app-components) - Update homepage (/) with subtle header design without blue bar - Add uniform PageHeader styling to application edit page - Fix Rapporten link on homepage to point to /reports overview - Improve header descriptions spacing for better readability
786 lines
31 KiB
TypeScript
786 lines
31 KiB
TypeScript
/**
|
|
* SyncEngine - Background synchronization service
|
|
*
|
|
* Handles:
|
|
* - Full sync at startup and periodically
|
|
* - Incremental sync every 30 seconds
|
|
* - Schema-driven sync for all object types
|
|
*/
|
|
|
|
import { logger } from './logger.js';
|
|
import { normalizedCacheStore as cacheStore } from './normalizedCacheStore.js';
|
|
import { jiraAssetsClient, JiraObjectNotFoundError } from './jiraAssetsClient.js';
|
|
import { OBJECT_TYPES, getObjectTypesBySyncPriority } from '../generated/jira-schema.js';
|
|
import type { CMDBObject, CMDBObjectTypeName } from '../generated/jira-types.js';
|
|
import { schemaDiscoveryService } from './schemaDiscoveryService.js';
|
|
import type { ObjectEntry } from '../domain/jiraAssetsPayload.js';
|
|
|
|
// =============================================================================
|
|
// Types
|
|
// =============================================================================
|
|
|
|
export interface SyncStats {
|
|
objectType: string;
|
|
objectsProcessed: number;
|
|
relationsExtracted: number;
|
|
duration: number;
|
|
}
|
|
|
|
export interface SyncResult {
|
|
success: boolean;
|
|
stats: SyncStats[];
|
|
totalObjects: number;
|
|
totalRelations: number;
|
|
duration: number;
|
|
error?: string;
|
|
}
|
|
|
|
export interface SyncEngineStatus {
|
|
isRunning: boolean;
|
|
isSyncing: boolean;
|
|
lastFullSync: string | null;
|
|
lastIncrementalSync: string | null;
|
|
nextIncrementalSync: string | null;
|
|
incrementalInterval: number;
|
|
}
|
|
|
|
// =============================================================================
|
|
// Configuration
|
|
// =============================================================================
|
|
|
|
const DEFAULT_INCREMENTAL_INTERVAL = 30_000; // 30 seconds
|
|
const DEFAULT_BATCH_SIZE = 50;
|
|
|
|
// =============================================================================
|
|
// Sync Engine Implementation
|
|
// =============================================================================
|
|
|
|
class SyncEngine {
|
|
private isRunning: boolean = false;
|
|
private isSyncing: boolean = false; // For full/incremental syncs
|
|
private syncingTypes: Set<CMDBObjectTypeName> = new Set(); // Track which types are being synced
|
|
private incrementalTimer: NodeJS.Timeout | null = null;
|
|
private incrementalInterval: number;
|
|
private batchSize: number;
|
|
private lastIncrementalSync: Date | null = null;
|
|
private lastConfigCheck: number = 0; // Track last config check time to avoid log spam
|
|
|
|
constructor() {
|
|
this.incrementalInterval = parseInt(
|
|
process.env.SYNC_INCREMENTAL_INTERVAL_MS || String(DEFAULT_INCREMENTAL_INTERVAL),
|
|
10
|
|
);
|
|
this.batchSize = parseInt(
|
|
process.env.JIRA_API_BATCH_SIZE || String(DEFAULT_BATCH_SIZE),
|
|
10
|
|
);
|
|
}
|
|
|
|
// ==========================================================================
|
|
// Lifecycle
|
|
// ==========================================================================
|
|
|
|
/**
|
|
* Initialize the sync engine
|
|
* Performs initial sync if cache is cold, then starts incremental sync
|
|
* Note: Sync engine uses service account token from .env (JIRA_SERVICE_ACCOUNT_TOKEN)
|
|
* for all read operations. Write operations require user PAT from profile settings.
|
|
*/
|
|
async initialize(): Promise<void> {
|
|
if (this.isRunning) {
|
|
logger.warn('SyncEngine: Already running');
|
|
return;
|
|
}
|
|
|
|
logger.info('SyncEngine: Initializing...');
|
|
logger.info('SyncEngine: Sync uses service account token (JIRA_SERVICE_ACCOUNT_TOKEN) from .env');
|
|
this.isRunning = true;
|
|
|
|
// Check if configuration is complete before starting scheduler
|
|
const { schemaConfigurationService } = await import('./schemaConfigurationService.js');
|
|
const isConfigured = await schemaConfigurationService.isConfigurationComplete();
|
|
|
|
// Start incremental sync scheduler if token is available AND configuration is complete
|
|
if (jiraAssetsClient.hasToken()) {
|
|
if (isConfigured) {
|
|
this.startIncrementalSyncScheduler();
|
|
logger.info('SyncEngine: Incremental sync scheduler started (configuration complete)');
|
|
} else {
|
|
logger.info('SyncEngine: Incremental sync scheduler NOT started - schema configuration not complete. Please configure object types in settings first.');
|
|
// Start scheduler but it will check configuration on each run
|
|
// This allows scheduler to start automatically when configuration is completed later
|
|
this.startIncrementalSyncScheduler();
|
|
logger.info('SyncEngine: Incremental sync scheduler started (will check configuration on each run)');
|
|
}
|
|
} else {
|
|
logger.info('SyncEngine: Service account token not configured, incremental sync disabled');
|
|
}
|
|
|
|
logger.info('SyncEngine: Initialized (using service account token for sync operations)');
|
|
}
|
|
|
|
/**
|
|
* Stop the sync engine
|
|
*/
|
|
stop(): void {
|
|
logger.info('SyncEngine: Stopping...');
|
|
this.isRunning = false;
|
|
|
|
if (this.incrementalTimer) {
|
|
clearInterval(this.incrementalTimer);
|
|
this.incrementalTimer = null;
|
|
}
|
|
|
|
logger.info('SyncEngine: Stopped');
|
|
}
|
|
|
|
/**
|
|
* Check if a timestamp is stale
|
|
*/
|
|
private isStale(timestamp: string, maxAgeMs: number): boolean {
|
|
const age = Date.now() - new Date(timestamp).getTime();
|
|
return age > maxAgeMs;
|
|
}
|
|
|
|
// ==========================================================================
|
|
// Full Sync
|
|
// ==========================================================================
|
|
|
|
/**
|
|
* Perform a full sync of all object types
|
|
* Uses service account token from .env (JIRA_SERVICE_ACCOUNT_TOKEN)
|
|
*/
|
|
async fullSync(): Promise<SyncResult> {
|
|
// Check if service account token is configured (sync uses service account token)
|
|
if (!jiraAssetsClient.hasToken()) {
|
|
logger.warn('SyncEngine: Jira service account token not configured, cannot perform sync');
|
|
return {
|
|
success: false,
|
|
stats: [],
|
|
totalObjects: 0,
|
|
totalRelations: 0,
|
|
duration: 0,
|
|
error: 'Jira service account token (JIRA_SERVICE_ACCOUNT_TOKEN) not configured in .env. Please configure it to enable sync operations.',
|
|
};
|
|
}
|
|
|
|
if (this.isSyncing) {
|
|
logger.warn('SyncEngine: Sync already in progress');
|
|
return {
|
|
success: false,
|
|
stats: [],
|
|
totalObjects: 0,
|
|
totalRelations: 0,
|
|
duration: 0,
|
|
error: 'Sync already in progress',
|
|
};
|
|
}
|
|
|
|
this.isSyncing = true;
|
|
const startTime = Date.now();
|
|
const stats: SyncStats[] = [];
|
|
let totalObjects = 0;
|
|
let totalRelations = 0;
|
|
|
|
logger.info('SyncEngine: Starting full sync...');
|
|
|
|
try {
|
|
// Check if configuration is complete
|
|
const { schemaConfigurationService } = await import('./schemaConfigurationService.js');
|
|
const isConfigured = await schemaConfigurationService.isConfigurationComplete();
|
|
if (!isConfigured) {
|
|
throw new Error('Schema configuration not complete. Please configure at least one object type to be synced in the settings page.');
|
|
}
|
|
|
|
// Get enabled object types from configuration
|
|
logger.info('SyncEngine: Fetching enabled object types from configuration...');
|
|
const enabledTypes = await schemaConfigurationService.getEnabledObjectTypes();
|
|
logger.info(`SyncEngine: Found ${enabledTypes.length} enabled object types to sync`);
|
|
|
|
if (enabledTypes.length === 0) {
|
|
throw new Error('No object types enabled for syncing. Please enable at least one object type in the settings page.');
|
|
}
|
|
|
|
// Schema discovery will happen automatically when needed (e.g., for relation extraction)
|
|
// It's no longer required upfront - the user has already configured which object types to sync
|
|
logger.info('SyncEngine: Starting object sync for configured object types...');
|
|
|
|
// Sync each enabled object type
|
|
for (const enabledType of enabledTypes) {
|
|
try {
|
|
const typeStat = await this.syncConfiguredObjectType(enabledType);
|
|
stats.push(typeStat);
|
|
totalObjects += typeStat.objectsProcessed;
|
|
totalRelations += typeStat.relationsExtracted;
|
|
} catch (error) {
|
|
logger.error(`SyncEngine: Failed to sync ${enabledType.displayName}`, error);
|
|
stats.push({
|
|
objectType: enabledType.displayName,
|
|
objectsProcessed: 0,
|
|
relationsExtracted: 0,
|
|
duration: 0,
|
|
});
|
|
}
|
|
}
|
|
|
|
// Update sync metadata
|
|
const now = new Date().toISOString();
|
|
await cacheStore.setSyncMetadata('lastFullSync', now);
|
|
await cacheStore.setSyncMetadata('lastIncrementalSync', now);
|
|
this.lastIncrementalSync = new Date();
|
|
|
|
const duration = Date.now() - startTime;
|
|
logger.info(`SyncEngine: Full sync complete. ${totalObjects} objects, ${totalRelations} relations in ${duration}ms`);
|
|
|
|
return {
|
|
success: true,
|
|
stats,
|
|
totalObjects,
|
|
totalRelations,
|
|
duration,
|
|
};
|
|
} catch (error) {
|
|
logger.error('SyncEngine: Full sync failed', error);
|
|
return {
|
|
success: false,
|
|
stats,
|
|
totalObjects,
|
|
totalRelations,
|
|
duration: Date.now() - startTime,
|
|
error: error instanceof Error ? error.message : 'Unknown error',
|
|
};
|
|
} finally {
|
|
this.isSyncing = false;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Store an object and all its nested referenced objects recursively
|
|
* This method processes the entire object tree, storing all nested objects
|
|
* and extracting all relations, while preventing infinite loops with circular references.
|
|
*
|
|
* @param entry - The object entry to store (in ObjectEntry format from API)
|
|
* @param typeName - The type name of the object
|
|
* @param processedIds - Set of already processed object IDs (to prevent duplicates and circular refs)
|
|
* @returns Statistics about objects stored and relations extracted
|
|
*/
|
|
private async storeObjectTree(
|
|
entry: ObjectEntry,
|
|
typeName: CMDBObjectTypeName,
|
|
processedIds: Set<string>
|
|
): Promise<{ objectsStored: number; relationsExtracted: number }> {
|
|
const entryId = String(entry.id);
|
|
|
|
// Skip if already processed (handles circular references)
|
|
if (processedIds.has(entryId)) {
|
|
logger.debug(`SyncEngine: Skipping already processed object ${entry.objectKey || entryId} of type ${typeName}`);
|
|
return { objectsStored: 0, relationsExtracted: 0 };
|
|
}
|
|
processedIds.add(entryId);
|
|
|
|
let objectsStored = 0;
|
|
let relationsExtracted = 0;
|
|
|
|
try {
|
|
logger.debug(`SyncEngine: [Recursive] Storing object tree for ${entry.objectKey || entryId} of type ${typeName} (depth: ${processedIds.size - 1})`);
|
|
|
|
// 1. Adapt and parse the object
|
|
const adapted = jiraAssetsClient.adaptObjectEntryToJiraAssetsObject(entry);
|
|
if (!adapted) {
|
|
logger.warn(`SyncEngine: Failed to adapt object ${entry.objectKey || entryId}`);
|
|
return { objectsStored: 0, relationsExtracted: 0 };
|
|
}
|
|
|
|
const parsed = await jiraAssetsClient.parseObject(adapted);
|
|
if (!parsed) {
|
|
logger.warn(`SyncEngine: Failed to parse object ${entry.objectKey || entryId}`);
|
|
return { objectsStored: 0, relationsExtracted: 0 };
|
|
}
|
|
|
|
// 2. Store the object
|
|
await cacheStore.upsertObject(typeName, parsed);
|
|
objectsStored++;
|
|
logger.debug(`SyncEngine: Stored object ${parsed.objectKey || parsed.id} of type ${typeName}`);
|
|
|
|
// 3. Schema discovery must be manually triggered via API endpoints
|
|
// No automatic discovery
|
|
|
|
// 4. Extract and store relations for this object
|
|
await cacheStore.extractAndStoreRelations(typeName, parsed);
|
|
relationsExtracted++;
|
|
logger.debug(`SyncEngine: Extracted relations for object ${parsed.objectKey || parsed.id}`);
|
|
|
|
// 5. Recursively process nested referenced objects
|
|
// Note: Lookup maps should already be initialized by getAllObjectsOfType
|
|
// Use a separate Set for extraction to avoid conflicts with storage tracking
|
|
const extractionProcessedIds = new Set<string>();
|
|
const nestedRefs = jiraAssetsClient.extractNestedReferencedObjects(
|
|
entry,
|
|
extractionProcessedIds, // Separate Set for extraction (prevents infinite loops in traversal)
|
|
5, // max depth
|
|
0 // current depth
|
|
);
|
|
|
|
if (nestedRefs.length > 0) {
|
|
logger.debug(`SyncEngine: [Recursive] Found ${nestedRefs.length} nested referenced objects for ${entry.objectKey || entryId}`);
|
|
|
|
// Group by type for better logging
|
|
const refsByType = new Map<string, number>();
|
|
for (const ref of nestedRefs) {
|
|
refsByType.set(ref.typeName, (refsByType.get(ref.typeName) || 0) + 1);
|
|
}
|
|
const typeSummary = Array.from(refsByType.entries())
|
|
.map(([type, count]) => `${count} ${type}`)
|
|
.join(', ');
|
|
logger.debug(`SyncEngine: [Recursive] Nested objects by type: ${typeSummary}`);
|
|
}
|
|
|
|
// 6. Recursively store each nested object
|
|
for (const { entry: nestedEntry, typeName: nestedTypeName } of nestedRefs) {
|
|
logger.debug(`SyncEngine: [Recursive] Processing nested object ${nestedEntry.objectKey || nestedEntry.id} of type ${nestedTypeName}`);
|
|
const nestedResult = await this.storeObjectTree(
|
|
nestedEntry,
|
|
nestedTypeName as CMDBObjectTypeName,
|
|
processedIds
|
|
);
|
|
objectsStored += nestedResult.objectsStored;
|
|
relationsExtracted += nestedResult.relationsExtracted;
|
|
}
|
|
|
|
logger.debug(`SyncEngine: [Recursive] Completed storing object tree for ${entry.objectKey || entryId}: ${objectsStored} objects, ${relationsExtracted} relations`);
|
|
return { objectsStored, relationsExtracted };
|
|
} catch (error) {
|
|
logger.error(`SyncEngine: Failed to store object tree for ${entry.objectKey || entryId}`, error);
|
|
return { objectsStored, relationsExtracted };
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Sync a configured object type (from schema configuration)
|
|
*/
|
|
private async syncConfiguredObjectType(enabledType: {
|
|
schemaId: string;
|
|
objectTypeId: number;
|
|
objectTypeName: string;
|
|
displayName: string;
|
|
}): Promise<SyncStats> {
|
|
const startTime = Date.now();
|
|
let objectsProcessed = 0;
|
|
let relationsExtracted = 0;
|
|
|
|
try {
|
|
logger.info(`SyncEngine: Syncing ${enabledType.displayName} (${enabledType.objectTypeName}) from schema ${enabledType.schemaId}...`);
|
|
|
|
// Fetch all objects from Jira using the configured schema and object type
|
|
// This returns raw entries for recursive processing (includeAttributesDeep=2 provides nested data)
|
|
const { objects: jiraObjects, rawEntries } = await jiraAssetsClient.getAllObjectsOfType(
|
|
enabledType.displayName, // Use display name for Jira API
|
|
this.batchSize,
|
|
enabledType.schemaId
|
|
);
|
|
logger.info(`SyncEngine: Fetched ${jiraObjects.length} ${enabledType.displayName} objects from Jira (schema: ${enabledType.schemaId})`);
|
|
|
|
// Schema discovery must be manually triggered via API endpoints
|
|
// No automatic discovery
|
|
|
|
// Use objectTypeName for cache storage (PascalCase)
|
|
const typeName = enabledType.objectTypeName as CMDBObjectTypeName;
|
|
|
|
// Process each main object recursively using storeObjectTree
|
|
// This will store the object and all its nested referenced objects
|
|
const processedIds = new Set<string>(); // Track processed objects to prevent duplicates and circular refs
|
|
const failedObjects: Array<{ id: string; key: string; label: string; reason: string }> = [];
|
|
|
|
if (rawEntries && rawEntries.length > 0) {
|
|
logger.info(`SyncEngine: Processing ${rawEntries.length} ${enabledType.displayName} objects recursively...`);
|
|
|
|
for (const rawEntry of rawEntries) {
|
|
try {
|
|
const result = await this.storeObjectTree(rawEntry, typeName, processedIds);
|
|
objectsProcessed += result.objectsStored;
|
|
relationsExtracted += result.relationsExtracted;
|
|
} catch (error) {
|
|
const entryId = String(rawEntry.id);
|
|
failedObjects.push({
|
|
id: entryId,
|
|
key: rawEntry.objectKey || 'unknown',
|
|
label: rawEntry.label || 'unknown',
|
|
reason: error instanceof Error ? error.message : 'Unknown error',
|
|
});
|
|
logger.warn(`SyncEngine: Failed to store object tree for ${enabledType.displayName} object: ${rawEntry.objectKey || entryId} (${rawEntry.label || 'unknown label'})`, error);
|
|
}
|
|
}
|
|
} else {
|
|
// Fallback: if rawEntries not available, use adapted objects (less efficient, no recursion)
|
|
logger.warn(`SyncEngine: Raw entries not available, using fallback linear processing (no recursive nesting)`);
|
|
|
|
const parsedObjects: CMDBObject[] = [];
|
|
for (const jiraObj of jiraObjects) {
|
|
const parsed = await jiraAssetsClient.parseObject(jiraObj);
|
|
if (parsed) {
|
|
parsedObjects.push(parsed);
|
|
} else {
|
|
failedObjects.push({
|
|
id: jiraObj.id?.toString() || 'unknown',
|
|
key: jiraObj.objectKey || 'unknown',
|
|
label: jiraObj.label || 'unknown',
|
|
reason: 'parseObject returned null',
|
|
});
|
|
logger.warn(`SyncEngine: Failed to parse ${enabledType.displayName} object: ${jiraObj.objectKey || jiraObj.id} (${jiraObj.label || 'unknown label'})`);
|
|
}
|
|
}
|
|
|
|
if (parsedObjects.length > 0) {
|
|
await cacheStore.batchUpsertObjects(typeName, parsedObjects);
|
|
objectsProcessed = parsedObjects.length;
|
|
|
|
// Extract relations
|
|
for (const obj of parsedObjects) {
|
|
await cacheStore.extractAndStoreRelations(typeName, obj);
|
|
relationsExtracted++;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Log parsing statistics
|
|
if (failedObjects.length > 0) {
|
|
logger.warn(`SyncEngine: ${failedObjects.length} ${enabledType.displayName} objects failed to process:`, failedObjects.map(o => `${o.key} (${o.label}): ${o.reason}`).join(', '));
|
|
}
|
|
|
|
const duration = Date.now() - startTime;
|
|
const skippedCount = jiraObjects.length - objectsProcessed;
|
|
if (skippedCount > 0) {
|
|
logger.warn(`SyncEngine: Synced ${objectsProcessed}/${jiraObjects.length} ${enabledType.displayName} objects in ${duration}ms (${skippedCount} skipped)`);
|
|
} else {
|
|
logger.debug(`SyncEngine: Synced ${objectsProcessed} ${enabledType.displayName} objects in ${duration}ms`);
|
|
}
|
|
|
|
return {
|
|
objectType: enabledType.displayName,
|
|
objectsProcessed,
|
|
relationsExtracted,
|
|
duration,
|
|
};
|
|
} catch (error) {
|
|
logger.error(`SyncEngine: Failed to sync ${enabledType.displayName}`, error);
|
|
return {
|
|
objectType: enabledType.displayName,
|
|
objectsProcessed,
|
|
relationsExtracted,
|
|
duration: Date.now() - startTime,
|
|
};
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Sync a single object type (legacy method, kept for backward compatibility)
|
|
*/
|
|
private async syncObjectType(typeName: CMDBObjectTypeName): Promise<SyncStats> {
|
|
// This method is deprecated - use syncConfiguredObjectType instead
|
|
logger.warn(`SyncEngine: syncObjectType(${typeName}) is deprecated, use configured object types instead`);
|
|
return {
|
|
objectType: typeName,
|
|
objectsProcessed: 0,
|
|
relationsExtracted: 0,
|
|
duration: 0,
|
|
};
|
|
}
|
|
|
|
// ==========================================================================
|
|
// Incremental Sync
|
|
// ==========================================================================
|
|
|
|
/**
|
|
* Start the incremental sync scheduler
|
|
* The scheduler will check configuration on each run and only sync if configuration is complete
|
|
*/
|
|
private startIncrementalSyncScheduler(): void {
|
|
if (this.incrementalTimer) {
|
|
clearInterval(this.incrementalTimer);
|
|
}
|
|
|
|
logger.info(`SyncEngine: Starting incremental sync scheduler (every ${this.incrementalInterval}ms)`);
|
|
logger.info('SyncEngine: Scheduler will only perform syncs when schema configuration is complete');
|
|
|
|
this.incrementalTimer = setInterval(() => {
|
|
if (!this.isSyncing && this.isRunning) {
|
|
// incrementalSync() will check if configuration is complete before syncing
|
|
this.incrementalSync().catch(err => {
|
|
logger.error('SyncEngine: Incremental sync failed', err);
|
|
});
|
|
}
|
|
}, this.incrementalInterval);
|
|
}
|
|
|
|
/**
|
|
* Perform an incremental sync (only updated objects)
|
|
* Uses service account token from .env (JIRA_SERVICE_ACCOUNT_TOKEN)
|
|
*
|
|
* Note: On Jira Data Center, IQL-based incremental sync is not supported.
|
|
* We instead check if a periodic full sync is needed.
|
|
*/
|
|
async incrementalSync(): Promise<{ success: boolean; updatedCount: number }> {
|
|
// Check if service account token is configured (sync uses service account token)
|
|
if (!jiraAssetsClient.hasToken()) {
|
|
logger.debug('SyncEngine: Jira service account token not configured, skipping incremental sync');
|
|
return { success: false, updatedCount: 0 };
|
|
}
|
|
|
|
// Check if configuration is complete before attempting sync
|
|
const { schemaConfigurationService } = await import('./schemaConfigurationService.js');
|
|
const isConfigured = await schemaConfigurationService.isConfigurationComplete();
|
|
if (!isConfigured) {
|
|
// Don't log on every interval - only log once per minute to avoid spam
|
|
const now = Date.now();
|
|
if (!this.lastConfigCheck || now - this.lastConfigCheck > 60000) {
|
|
logger.debug('SyncEngine: Schema configuration not complete, skipping incremental sync. Please configure object types in settings.');
|
|
this.lastConfigCheck = now;
|
|
}
|
|
return { success: false, updatedCount: 0 };
|
|
}
|
|
|
|
// Get enabled object types - will be used later to filter updated objects
|
|
const enabledTypes = await schemaConfigurationService.getEnabledObjectTypes();
|
|
if (enabledTypes.length === 0) {
|
|
logger.debug('SyncEngine: No enabled object types, skipping incremental sync');
|
|
return { success: false, updatedCount: 0 };
|
|
}
|
|
|
|
if (this.isSyncing) {
|
|
return { success: false, updatedCount: 0 };
|
|
}
|
|
|
|
this.isSyncing = true;
|
|
|
|
try {
|
|
// Get the last sync time
|
|
const lastSyncStr = await cacheStore.getSyncMetadata('lastIncrementalSync');
|
|
const since = lastSyncStr
|
|
? new Date(lastSyncStr)
|
|
: new Date(Date.now() - 60000); // Default: last minute
|
|
|
|
logger.debug(`SyncEngine: Incremental sync since ${since.toISOString()}`);
|
|
|
|
// Get enabled object types to filter incremental sync
|
|
const enabledTypes = await schemaConfigurationService.getEnabledObjectTypes();
|
|
const enabledTypeNames = new Set(enabledTypes.map(et => et.objectTypeName));
|
|
|
|
if (enabledTypeNames.size === 0) {
|
|
logger.debug('SyncEngine: No enabled object types, skipping incremental sync');
|
|
return { success: false, updatedCount: 0 };
|
|
}
|
|
|
|
// Fetch updated objects from Jira
|
|
const updatedObjects = await jiraAssetsClient.getUpdatedObjectsSince(since, this.batchSize);
|
|
|
|
// If no objects returned (e.g., Data Center doesn't support IQL incremental sync),
|
|
// check if we should trigger a full sync instead
|
|
if (updatedObjects.length === 0) {
|
|
const lastFullSyncStr = await cacheStore.getSyncMetadata('lastFullSync');
|
|
if (lastFullSyncStr) {
|
|
const lastFullSync = new Date(lastFullSyncStr);
|
|
const fullSyncAge = Date.now() - lastFullSync.getTime();
|
|
const FULL_SYNC_INTERVAL = 15 * 60 * 1000; // 15 minutes
|
|
|
|
if (fullSyncAge > FULL_SYNC_INTERVAL) {
|
|
logger.info('SyncEngine: Triggering periodic full sync (incremental not available)');
|
|
// Release the lock before calling fullSync
|
|
this.isSyncing = false;
|
|
await this.fullSync();
|
|
return { success: true, updatedCount: 0 };
|
|
}
|
|
}
|
|
|
|
// Update timestamp even if no objects were synced
|
|
const now = new Date();
|
|
await cacheStore.setSyncMetadata('lastIncrementalSync', now.toISOString());
|
|
this.lastIncrementalSync = now;
|
|
|
|
return { success: true, updatedCount: 0 };
|
|
}
|
|
|
|
// Schema discovery must be manually triggered via API endpoints
|
|
// No automatic discovery
|
|
|
|
let updatedCount = 0;
|
|
const processedIds = new Set<string>(); // Track processed objects for recursive sync
|
|
|
|
// Filter updated objects to only process enabled object types
|
|
// Use recursive processing to handle nested references
|
|
for (const jiraObj of updatedObjects) {
|
|
const parsed = await jiraAssetsClient.parseObject(jiraObj);
|
|
if (parsed) {
|
|
const typeName = parsed._objectType as CMDBObjectTypeName;
|
|
|
|
// Only sync if this object type is enabled
|
|
if (!enabledTypeNames.has(typeName)) {
|
|
logger.debug(`SyncEngine: Skipping ${typeName} in incremental sync - not enabled`);
|
|
continue;
|
|
}
|
|
|
|
// Get raw entry for recursive processing
|
|
const objectId = parsed.id;
|
|
try {
|
|
const entry = await jiraAssetsClient.getObjectEntry(objectId);
|
|
if (entry) {
|
|
// Use recursive storeObjectTree to process object and all nested references
|
|
const result = await this.storeObjectTree(entry, typeName, processedIds);
|
|
if (result.objectsStored > 0) {
|
|
updatedCount++;
|
|
logger.debug(`SyncEngine: Incremental sync processed ${objectId}: ${result.objectsStored} objects, ${result.relationsExtracted} relations`);
|
|
}
|
|
} else {
|
|
// Fallback to linear processing if raw entry not available
|
|
await cacheStore.upsertObject(typeName, parsed);
|
|
await cacheStore.extractAndStoreRelations(typeName, parsed);
|
|
updatedCount++;
|
|
}
|
|
} catch (error) {
|
|
logger.warn(`SyncEngine: Failed to get raw entry for ${objectId}, using fallback`, error);
|
|
// Fallback to linear processing
|
|
await cacheStore.upsertObject(typeName, parsed);
|
|
await cacheStore.extractAndStoreRelations(typeName, parsed);
|
|
updatedCount++;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Update sync metadata
|
|
const now = new Date();
|
|
await cacheStore.setSyncMetadata('lastIncrementalSync', now.toISOString());
|
|
this.lastIncrementalSync = now;
|
|
|
|
if (updatedCount > 0) {
|
|
logger.info(`SyncEngine: Incremental sync updated ${updatedCount} objects`);
|
|
}
|
|
|
|
return { success: true, updatedCount };
|
|
} catch (error) {
|
|
logger.error('SyncEngine: Incremental sync failed', error);
|
|
return { success: false, updatedCount: 0 };
|
|
} finally {
|
|
this.isSyncing = false;
|
|
}
|
|
}
|
|
|
|
// ==========================================================================
|
|
// Manual Sync Triggers
|
|
// ==========================================================================
|
|
|
|
/**
|
|
* Trigger a sync for a specific object type
|
|
* Only syncs if the object type is enabled in configuration
|
|
* Allows concurrent syncs for different types, but blocks if:
|
|
* - A full sync is in progress
|
|
* - An incremental sync is in progress
|
|
* - This specific type is already being synced
|
|
*/
|
|
async syncType(typeName: CMDBObjectTypeName): Promise<SyncStats> {
|
|
// Block if a full or incremental sync is running
|
|
if (this.isSyncing) {
|
|
throw new Error('Full or incremental sync already in progress');
|
|
}
|
|
|
|
// Block if this specific type is already being synced
|
|
if (this.syncingTypes.has(typeName)) {
|
|
throw new Error(`Sync already in progress for ${typeName}`);
|
|
}
|
|
|
|
// Check if this type is enabled in configuration
|
|
const { schemaConfigurationService } = await import('./schemaConfigurationService.js');
|
|
const enabledTypes = await schemaConfigurationService.getEnabledObjectTypes();
|
|
const enabledType = enabledTypes.find(et => et.objectTypeName === typeName);
|
|
|
|
if (!enabledType) {
|
|
throw new Error(`Object type ${typeName} is not enabled for syncing. Please enable it in the Schema Configuration settings page.`);
|
|
}
|
|
|
|
this.syncingTypes.add(typeName);
|
|
|
|
try {
|
|
return await this.syncConfiguredObjectType(enabledType);
|
|
} finally {
|
|
this.syncingTypes.delete(typeName);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Force sync a single object
|
|
* Only syncs if the object type is enabled in configuration
|
|
* If the object was deleted from Jira, it will be removed from the local cache
|
|
* Uses recursive processing to store nested referenced objects
|
|
*/
|
|
async syncObject(typeName: CMDBObjectTypeName, objectId: string): Promise<boolean> {
|
|
try {
|
|
// Check if this type is enabled in configuration
|
|
const { schemaConfigurationService } = await import('./schemaConfigurationService.js');
|
|
const enabledTypes = await schemaConfigurationService.getEnabledObjectTypes();
|
|
const isEnabled = enabledTypes.some(et => et.objectTypeName === typeName);
|
|
|
|
if (!isEnabled) {
|
|
logger.warn(`SyncEngine: Cannot sync object ${objectId} - type ${typeName} is not enabled for syncing`);
|
|
return false;
|
|
}
|
|
|
|
// Schema discovery must be manually triggered via API endpoints
|
|
// No automatic discovery
|
|
|
|
// Get raw ObjectEntry for recursive processing
|
|
const entry = await jiraAssetsClient.getObjectEntry(objectId);
|
|
if (!entry) return false;
|
|
|
|
// Use recursive storeObjectTree to process object and all nested references
|
|
const processedIds = new Set<string>();
|
|
const result = await this.storeObjectTree(entry, typeName, processedIds);
|
|
|
|
if (result.objectsStored > 0) {
|
|
logger.info(`SyncEngine: Synced object ${objectId} recursively: ${result.objectsStored} objects, ${result.relationsExtracted} relations`);
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
} catch (error) {
|
|
// If object was deleted from Jira, remove it from our cache
|
|
if (error instanceof JiraObjectNotFoundError) {
|
|
const deleted = await cacheStore.deleteObject(typeName, objectId);
|
|
if (deleted) {
|
|
logger.info(`SyncEngine: Removed deleted object ${typeName}/${objectId} from cache`);
|
|
}
|
|
return false;
|
|
}
|
|
logger.error(`SyncEngine: Failed to sync object ${objectId}`, error);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
// ==========================================================================
|
|
// Status
|
|
// ==========================================================================
|
|
|
|
/**
|
|
* Get current sync engine status
|
|
*/
|
|
async getStatus(): Promise<SyncEngineStatus> {
|
|
const stats = await cacheStore.getStats();
|
|
|
|
let nextIncrementalSync: string | null = null;
|
|
if (this.isRunning && this.lastIncrementalSync) {
|
|
const nextTime = new Date(this.lastIncrementalSync.getTime() + this.incrementalInterval);
|
|
nextIncrementalSync = nextTime.toISOString();
|
|
}
|
|
|
|
return {
|
|
isRunning: this.isRunning,
|
|
isSyncing: this.isSyncing,
|
|
lastFullSync: stats.lastFullSync,
|
|
lastIncrementalSync: stats.lastIncrementalSync,
|
|
nextIncrementalSync,
|
|
incrementalInterval: this.incrementalInterval,
|
|
};
|
|
}
|
|
}
|
|
|
|
// Export singleton instance
|
|
export const syncEngine = new SyncEngine();
|
|
|