Improve Team-indeling dashboard UI and cache invalidation
- Replace 'TEAM' label with Type attribute (Business/Enabling/Staf) in team blocks - Make Type labels larger (text-sm) and brighter colors - Make SUBTEAM label less bright (indigo-300) and smaller (text-[10px]) - Add 'FTE' suffix to bandbreedte values in header and application blocks - Add Platform and Connected Device labels to application blocks - Show Platform FTE and Workloads FTE separately in Platform blocks - Add spacing between Regiemodel letter and count value - Add cache invalidation for Team Dashboard when applications are updated - Enrich team references with Type attribute in getSubteamToTeamMapping
This commit is contained in:
463
backend/src/services/syncEngine.ts
Normal file
463
backend/src/services/syncEngine.ts
Normal file
@@ -0,0 +1,463 @@
|
||||
/**
|
||||
* SyncEngine - Background synchronization service
|
||||
*
|
||||
* Handles:
|
||||
* - Full sync at startup and periodically
|
||||
* - Incremental sync every 30 seconds
|
||||
* - Schema-driven sync for all object types
|
||||
*/
|
||||
|
||||
import { logger } from './logger.js';
|
||||
import { cacheStore } from './cacheStore.js';
|
||||
import { jiraAssetsClient, JiraObjectNotFoundError } from './jiraAssetsClient.js';
|
||||
import { OBJECT_TYPES, getObjectTypesBySyncPriority } from '../generated/jira-schema.js';
|
||||
import type { CMDBObject, CMDBObjectTypeName } from '../generated/jira-types.js';
|
||||
|
||||
// =============================================================================
|
||||
// Types
|
||||
// =============================================================================
|
||||
|
||||
export interface SyncStats {
|
||||
objectType: string;
|
||||
objectsProcessed: number;
|
||||
relationsExtracted: number;
|
||||
duration: number;
|
||||
}
|
||||
|
||||
export interface SyncResult {
|
||||
success: boolean;
|
||||
stats: SyncStats[];
|
||||
totalObjects: number;
|
||||
totalRelations: number;
|
||||
duration: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface SyncEngineStatus {
|
||||
isRunning: boolean;
|
||||
isSyncing: boolean;
|
||||
lastFullSync: string | null;
|
||||
lastIncrementalSync: string | null;
|
||||
nextIncrementalSync: string | null;
|
||||
incrementalInterval: number;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Configuration
|
||||
// =============================================================================
|
||||
|
||||
const DEFAULT_INCREMENTAL_INTERVAL = 30_000; // 30 seconds
|
||||
const DEFAULT_BATCH_SIZE = 50;
|
||||
|
||||
// =============================================================================
|
||||
// Sync Engine Implementation
|
||||
// =============================================================================
|
||||
|
||||
class SyncEngine {
|
||||
private isRunning: boolean = false;
|
||||
private isSyncing: boolean = false; // For full/incremental syncs
|
||||
private syncingTypes: Set<CMDBObjectTypeName> = new Set(); // Track which types are being synced
|
||||
private incrementalTimer: NodeJS.Timeout | null = null;
|
||||
private incrementalInterval: number;
|
||||
private batchSize: number;
|
||||
private lastIncrementalSync: Date | null = null;
|
||||
|
||||
constructor() {
|
||||
this.incrementalInterval = parseInt(
|
||||
process.env.SYNC_INCREMENTAL_INTERVAL_MS || String(DEFAULT_INCREMENTAL_INTERVAL),
|
||||
10
|
||||
);
|
||||
this.batchSize = parseInt(
|
||||
process.env.JIRA_API_BATCH_SIZE || String(DEFAULT_BATCH_SIZE),
|
||||
10
|
||||
);
|
||||
}
|
||||
|
||||
// ==========================================================================
|
||||
// Lifecycle
|
||||
// ==========================================================================
|
||||
|
||||
/**
|
||||
* Initialize the sync engine
|
||||
* Performs initial sync if cache is cold, then starts incremental sync
|
||||
*/
|
||||
async initialize(): Promise<void> {
|
||||
if (this.isRunning) {
|
||||
logger.warn('SyncEngine: Already running');
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info('SyncEngine: Initializing...');
|
||||
this.isRunning = true;
|
||||
|
||||
// Check if we need a full sync
|
||||
const stats = cacheStore.getStats();
|
||||
const lastFullSync = stats.lastFullSync;
|
||||
const needsFullSync = !stats.isWarm || !lastFullSync || this.isStale(lastFullSync, 24 * 60 * 60 * 1000);
|
||||
|
||||
if (needsFullSync) {
|
||||
logger.info('SyncEngine: Cache is cold or stale, starting full sync in background...');
|
||||
// Run full sync in background (non-blocking)
|
||||
this.fullSync().catch(err => {
|
||||
logger.error('SyncEngine: Background full sync failed', err);
|
||||
});
|
||||
} else {
|
||||
logger.info('SyncEngine: Cache is warm, skipping initial full sync');
|
||||
}
|
||||
|
||||
// Start incremental sync scheduler
|
||||
this.startIncrementalSyncScheduler();
|
||||
|
||||
logger.info('SyncEngine: Initialized');
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the sync engine
|
||||
*/
|
||||
stop(): void {
|
||||
logger.info('SyncEngine: Stopping...');
|
||||
this.isRunning = false;
|
||||
|
||||
if (this.incrementalTimer) {
|
||||
clearInterval(this.incrementalTimer);
|
||||
this.incrementalTimer = null;
|
||||
}
|
||||
|
||||
logger.info('SyncEngine: Stopped');
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a timestamp is stale
|
||||
*/
|
||||
private isStale(timestamp: string, maxAgeMs: number): boolean {
|
||||
const age = Date.now() - new Date(timestamp).getTime();
|
||||
return age > maxAgeMs;
|
||||
}
|
||||
|
||||
// ==========================================================================
|
||||
// Full Sync
|
||||
// ==========================================================================
|
||||
|
||||
/**
|
||||
* Perform a full sync of all object types
|
||||
*/
|
||||
async fullSync(): Promise<SyncResult> {
|
||||
if (this.isSyncing) {
|
||||
logger.warn('SyncEngine: Sync already in progress');
|
||||
return {
|
||||
success: false,
|
||||
stats: [],
|
||||
totalObjects: 0,
|
||||
totalRelations: 0,
|
||||
duration: 0,
|
||||
error: 'Sync already in progress',
|
||||
};
|
||||
}
|
||||
|
||||
this.isSyncing = true;
|
||||
const startTime = Date.now();
|
||||
const stats: SyncStats[] = [];
|
||||
let totalObjects = 0;
|
||||
let totalRelations = 0;
|
||||
|
||||
logger.info('SyncEngine: Starting full sync...');
|
||||
|
||||
try {
|
||||
// Get object types sorted by sync priority
|
||||
const objectTypes = getObjectTypesBySyncPriority();
|
||||
|
||||
for (const typeDef of objectTypes) {
|
||||
const typeStat = await this.syncObjectType(typeDef.typeName as CMDBObjectTypeName);
|
||||
stats.push(typeStat);
|
||||
totalObjects += typeStat.objectsProcessed;
|
||||
totalRelations += typeStat.relationsExtracted;
|
||||
}
|
||||
|
||||
// Update sync metadata
|
||||
const now = new Date().toISOString();
|
||||
cacheStore.setSyncMetadata('lastFullSync', now);
|
||||
cacheStore.setSyncMetadata('lastIncrementalSync', now);
|
||||
this.lastIncrementalSync = new Date();
|
||||
|
||||
const duration = Date.now() - startTime;
|
||||
logger.info(`SyncEngine: Full sync complete. ${totalObjects} objects, ${totalRelations} relations in ${duration}ms`);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
stats,
|
||||
totalObjects,
|
||||
totalRelations,
|
||||
duration,
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('SyncEngine: Full sync failed', error);
|
||||
return {
|
||||
success: false,
|
||||
stats,
|
||||
totalObjects,
|
||||
totalRelations,
|
||||
duration: Date.now() - startTime,
|
||||
error: error instanceof Error ? error.message : 'Unknown error',
|
||||
};
|
||||
} finally {
|
||||
this.isSyncing = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync a single object type
|
||||
*/
|
||||
private async syncObjectType(typeName: CMDBObjectTypeName): Promise<SyncStats> {
|
||||
const startTime = Date.now();
|
||||
let objectsProcessed = 0;
|
||||
let relationsExtracted = 0;
|
||||
|
||||
try {
|
||||
const typeDef = OBJECT_TYPES[typeName];
|
||||
if (!typeDef) {
|
||||
logger.warn(`SyncEngine: Unknown type ${typeName}`);
|
||||
return { objectType: typeName, objectsProcessed: 0, relationsExtracted: 0, duration: 0 };
|
||||
}
|
||||
|
||||
logger.debug(`SyncEngine: Syncing ${typeName}...`);
|
||||
|
||||
// Fetch all objects from Jira
|
||||
const jiraObjects = await jiraAssetsClient.getAllObjectsOfType(typeName, this.batchSize);
|
||||
|
||||
// Parse and cache objects
|
||||
const parsedObjects: CMDBObject[] = [];
|
||||
|
||||
for (const jiraObj of jiraObjects) {
|
||||
const parsed = jiraAssetsClient.parseObject(jiraObj);
|
||||
if (parsed) {
|
||||
parsedObjects.push(parsed);
|
||||
}
|
||||
}
|
||||
|
||||
// Batch upsert to cache
|
||||
if (parsedObjects.length > 0) {
|
||||
cacheStore.batchUpsertObjects(typeName, parsedObjects);
|
||||
objectsProcessed = parsedObjects.length;
|
||||
|
||||
// Extract relations
|
||||
for (const obj of parsedObjects) {
|
||||
cacheStore.extractAndStoreRelations(typeName, obj);
|
||||
relationsExtracted++;
|
||||
}
|
||||
}
|
||||
|
||||
const duration = Date.now() - startTime;
|
||||
logger.debug(`SyncEngine: Synced ${objectsProcessed} ${typeName} objects in ${duration}ms`);
|
||||
|
||||
return {
|
||||
objectType: typeName,
|
||||
objectsProcessed,
|
||||
relationsExtracted,
|
||||
duration,
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error(`SyncEngine: Failed to sync ${typeName}`, error);
|
||||
return {
|
||||
objectType: typeName,
|
||||
objectsProcessed,
|
||||
relationsExtracted,
|
||||
duration: Date.now() - startTime,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ==========================================================================
|
||||
// Incremental Sync
|
||||
// ==========================================================================
|
||||
|
||||
/**
|
||||
* Start the incremental sync scheduler
|
||||
*/
|
||||
private startIncrementalSyncScheduler(): void {
|
||||
if (this.incrementalTimer) {
|
||||
clearInterval(this.incrementalTimer);
|
||||
}
|
||||
|
||||
logger.info(`SyncEngine: Starting incremental sync scheduler (every ${this.incrementalInterval}ms)`);
|
||||
|
||||
this.incrementalTimer = setInterval(() => {
|
||||
if (!this.isSyncing && this.isRunning) {
|
||||
this.incrementalSync().catch(err => {
|
||||
logger.error('SyncEngine: Incremental sync failed', err);
|
||||
});
|
||||
}
|
||||
}, this.incrementalInterval);
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform an incremental sync (only updated objects)
|
||||
*
|
||||
* Note: On Jira Data Center, IQL-based incremental sync is not supported.
|
||||
* We instead check if a periodic full sync is needed.
|
||||
*/
|
||||
async incrementalSync(): Promise<{ success: boolean; updatedCount: number }> {
|
||||
if (this.isSyncing) {
|
||||
return { success: false, updatedCount: 0 };
|
||||
}
|
||||
|
||||
this.isSyncing = true;
|
||||
|
||||
try {
|
||||
// Get the last sync time
|
||||
const lastSyncStr = cacheStore.getSyncMetadata('lastIncrementalSync');
|
||||
const since = lastSyncStr
|
||||
? new Date(lastSyncStr)
|
||||
: new Date(Date.now() - 60000); // Default: last minute
|
||||
|
||||
logger.debug(`SyncEngine: Incremental sync since ${since.toISOString()}`);
|
||||
|
||||
// Fetch updated objects from Jira
|
||||
const updatedObjects = await jiraAssetsClient.getUpdatedObjectsSince(since, this.batchSize);
|
||||
|
||||
// If no objects returned (e.g., Data Center doesn't support IQL incremental sync),
|
||||
// check if we should trigger a full sync instead
|
||||
if (updatedObjects.length === 0) {
|
||||
const lastFullSyncStr = cacheStore.getSyncMetadata('lastFullSync');
|
||||
if (lastFullSyncStr) {
|
||||
const lastFullSync = new Date(lastFullSyncStr);
|
||||
const fullSyncAge = Date.now() - lastFullSync.getTime();
|
||||
const FULL_SYNC_INTERVAL = 15 * 60 * 1000; // 15 minutes
|
||||
|
||||
if (fullSyncAge > FULL_SYNC_INTERVAL) {
|
||||
logger.info('SyncEngine: Triggering periodic full sync (incremental not available)');
|
||||
// Release the lock before calling fullSync
|
||||
this.isSyncing = false;
|
||||
await this.fullSync();
|
||||
return { success: true, updatedCount: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
// Update timestamp even if no objects were synced
|
||||
const now = new Date();
|
||||
cacheStore.setSyncMetadata('lastIncrementalSync', now.toISOString());
|
||||
this.lastIncrementalSync = now;
|
||||
|
||||
return { success: true, updatedCount: 0 };
|
||||
}
|
||||
|
||||
let updatedCount = 0;
|
||||
|
||||
for (const jiraObj of updatedObjects) {
|
||||
const parsed = jiraAssetsClient.parseObject(jiraObj);
|
||||
if (parsed) {
|
||||
const typeName = parsed._objectType as CMDBObjectTypeName;
|
||||
cacheStore.upsertObject(typeName, parsed);
|
||||
cacheStore.extractAndStoreRelations(typeName, parsed);
|
||||
updatedCount++;
|
||||
}
|
||||
}
|
||||
|
||||
// Update sync metadata
|
||||
const now = new Date();
|
||||
cacheStore.setSyncMetadata('lastIncrementalSync', now.toISOString());
|
||||
this.lastIncrementalSync = now;
|
||||
|
||||
if (updatedCount > 0) {
|
||||
logger.info(`SyncEngine: Incremental sync updated ${updatedCount} objects`);
|
||||
}
|
||||
|
||||
return { success: true, updatedCount };
|
||||
} catch (error) {
|
||||
logger.error('SyncEngine: Incremental sync failed', error);
|
||||
return { success: false, updatedCount: 0 };
|
||||
} finally {
|
||||
this.isSyncing = false;
|
||||
}
|
||||
}
|
||||
|
||||
// ==========================================================================
|
||||
// Manual Sync Triggers
|
||||
// ==========================================================================
|
||||
|
||||
/**
|
||||
* Trigger a sync for a specific object type
|
||||
* Allows concurrent syncs for different types, but blocks if:
|
||||
* - A full sync is in progress
|
||||
* - An incremental sync is in progress
|
||||
* - This specific type is already being synced
|
||||
*/
|
||||
async syncType(typeName: CMDBObjectTypeName): Promise<SyncStats> {
|
||||
// Block if a full or incremental sync is running
|
||||
if (this.isSyncing) {
|
||||
throw new Error('Full or incremental sync already in progress');
|
||||
}
|
||||
|
||||
// Block if this specific type is already being synced
|
||||
if (this.syncingTypes.has(typeName)) {
|
||||
throw new Error(`Sync already in progress for ${typeName}`);
|
||||
}
|
||||
|
||||
this.syncingTypes.add(typeName);
|
||||
|
||||
try {
|
||||
return await this.syncObjectType(typeName);
|
||||
} finally {
|
||||
this.syncingTypes.delete(typeName);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Force sync a single object
|
||||
* If the object was deleted from Jira, it will be removed from the local cache
|
||||
*/
|
||||
async syncObject(typeName: CMDBObjectTypeName, objectId: string): Promise<boolean> {
|
||||
try {
|
||||
const jiraObj = await jiraAssetsClient.getObject(objectId);
|
||||
if (!jiraObj) return false;
|
||||
|
||||
const parsed = jiraAssetsClient.parseObject(jiraObj);
|
||||
if (!parsed) return false;
|
||||
|
||||
cacheStore.upsertObject(typeName, parsed);
|
||||
cacheStore.extractAndStoreRelations(typeName, parsed);
|
||||
|
||||
return true;
|
||||
} catch (error) {
|
||||
// If object was deleted from Jira, remove it from our cache
|
||||
if (error instanceof JiraObjectNotFoundError) {
|
||||
const deleted = cacheStore.deleteObject(typeName, objectId);
|
||||
if (deleted) {
|
||||
logger.info(`SyncEngine: Removed deleted object ${typeName}/${objectId} from cache`);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
logger.error(`SyncEngine: Failed to sync object ${objectId}`, error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// ==========================================================================
|
||||
// Status
|
||||
// ==========================================================================
|
||||
|
||||
/**
|
||||
* Get current sync engine status
|
||||
*/
|
||||
getStatus(): SyncEngineStatus {
|
||||
const stats = cacheStore.getStats();
|
||||
|
||||
let nextIncrementalSync: string | null = null;
|
||||
if (this.isRunning && this.lastIncrementalSync) {
|
||||
const nextTime = new Date(this.lastIncrementalSync.getTime() + this.incrementalInterval);
|
||||
nextIncrementalSync = nextTime.toISOString();
|
||||
}
|
||||
|
||||
return {
|
||||
isRunning: this.isRunning,
|
||||
isSyncing: this.isSyncing,
|
||||
lastFullSync: stats.lastFullSync,
|
||||
lastIncrementalSync: stats.lastIncrementalSync,
|
||||
nextIncrementalSync,
|
||||
incrementalInterval: this.incrementalInterval,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
export const syncEngine = new SyncEngine();
|
||||
|
||||
Reference in New Issue
Block a user