Add authentication, user management, and database migration features

- Implement OAuth 2.0 and PAT authentication methods
- Add user management, roles, and profile functionality
- Add database migrations and admin user scripts
- Update services for authentication and user settings
- Add protected routes and permission hooks
- Update documentation for authentication and database access

backend/src/services/dataService.ts

@@ -48,7 +48,9 @@ import { calculateRequiredEffortWithMinMax } from './effortCalculation.js';
 import { calculateApplicationCompleteness } from './dataCompletenessConfig.js';
 
 // Determine if we should use real Jira Assets or mock data
-const useJiraAssets = !!(config.jiraPat && config.jiraSchemaId);
+// Jira PAT is now configured per-user, so we check if schema is configured
+// The actual PAT is provided per-request via middleware
+const useJiraAssets = !!config.jiraSchemaId;
 
 if (useJiraAssets) {
   logger.info('DataService: Using CMDB cache layer with Jira Assets API');
@@ -121,9 +123,40 @@ async function lookupReferences<T extends CMDBObject>(
 
 /**
  * Convert ObjectReference to ReferenceValue format used by frontend
+ * Try to enrich with description from jiraAssetsService cache if available
+ * If not in cache or cache entry has no description, fetch it async
  */
-function toReferenceValue(ref: ObjectReference | null | undefined): ReferenceValue | null {
+async function toReferenceValue(ref: ObjectReference | null | undefined): Promise<ReferenceValue | null> {
   if (!ref) return null;
+  
+  // Try to get enriched ReferenceValue from jiraAssetsService cache (includes description if available)
+  const enriched = useJiraAssets ? jiraAssetsService.getEnrichedReferenceValue(ref.objectKey, ref.objectId) : null;
+  
+  if (enriched && enriched.description) {
+    // Use enriched value with description
+    return enriched;
+  }
+  
+  // Cache miss or no description - fetch it async if using Jira Assets
+  if (useJiraAssets && enriched && !enriched.description) {
+    // We have a cached value but it lacks description - fetch it
+    const fetched = await jiraAssetsService.fetchEnrichedReferenceValue(ref.objectKey, ref.objectId);
+    if (fetched) {
+      return fetched;
+    }
+    // If fetch failed, return the cached value anyway
+    return enriched;
+  }
+  
+  if (useJiraAssets) {
+    // Cache miss - fetch it
+    const fetched = await jiraAssetsService.fetchEnrichedReferenceValue(ref.objectKey, ref.objectId);
+    if (fetched) {
+      return fetched;
+    }
+  }
+  
+  // Fallback to basic conversion without description (if fetch failed or not using Jira Assets)
   return {
     objectId: ref.objectId,
     key: ref.objectKey,
@@ -188,25 +221,49 @@ function extractDisplayValue(value: unknown): string | null {
  * References are now stored as ObjectReference objects directly (not IDs)
  */
 async function toApplicationDetails(app: ApplicationComponent): Promise<ApplicationDetails> {
+  // Debug logging for confluenceSpace from cache
+  logger.info(`[toApplicationDetails] Converting cached object ${app.objectKey || app.id} to ApplicationDetails`);
+  logger.info(`[toApplicationDetails] confluenceSpace from cache: ${app.confluenceSpace} (type: ${typeof app.confluenceSpace})`);
+  
+  // Handle confluenceSpace - it can be a string (URL) or number (legacy), convert to string
+  const confluenceSpaceValue = app.confluenceSpace !== null && app.confluenceSpace !== undefined 
+    ? (typeof app.confluenceSpace === 'string' ? app.confluenceSpace : String(app.confluenceSpace))
+    : null;
+  
   // Ensure factor caches are loaded for factor value lookup
   await ensureFactorCaches();
   
   // Convert ObjectReference to ReferenceValue format
-  const governanceModel = toReferenceValue(app.ictGovernanceModel);
-  // Note: applicationManagementSubteam and applicationManagementTeam are not in the generated schema
-  // They are only available when fetching directly from Jira API (via jiraAssetsClient)
-  const applicationSubteam = toReferenceValue((app as any).applicationManagementSubteam);
-  const applicationTeam = toReferenceValue((app as any).applicationManagementTeam);
-  const applicationType = toReferenceValue(app.applicationManagementApplicationType);
-  const applicationManagementHosting = toReferenceValue(app.applicationManagementHosting);
-  const applicationManagementTAM = toReferenceValue(app.applicationManagementTAM);
-  const hostingType = toReferenceValue(app.applicationComponentHostingType);
-  const businessImpactAnalyse = toReferenceValue(app.businessImpactAnalyse);
-  const platform = toReferenceValue(app.platform);
-  const organisation = toReferenceValue(app.organisation);
-  const businessImportance = toReferenceValue(app.businessImportance);
+  // Fetch descriptions async if not in cache
+  // Use Promise.all to fetch all reference values in parallel for better performance
+  const [
+    governanceModel,
+    applicationSubteam,
+    applicationTeam,
+    applicationType,
+    applicationManagementHosting,
+    applicationManagementTAM,
+    hostingType,
+    businessImpactAnalyse,
+    platform,
+    organisation,
+    businessImportance,
+  ] = await Promise.all([
+    toReferenceValue(app.ictGovernanceModel),
+    toReferenceValue((app as any).applicationManagementSubteam),
+    toReferenceValue((app as any).applicationManagementTeam),
+    toReferenceValue(app.applicationManagementApplicationType),
+    toReferenceValue(app.applicationManagementHosting),
+    toReferenceValue(app.applicationManagementTAM),
+    toReferenceValue(app.applicationComponentHostingType),
+    toReferenceValue(app.businessImpactAnalyse),
+    toReferenceValue(app.platform),
+    toReferenceValue(app.organisation),
+    toReferenceValue(app.businessImportance),
+  ]);
   
   // Look up factor values from cached factor objects (same as toMinimalDetailsForEffort)
+  // Also include descriptions from cache if available
   let dynamicsFactor: ReferenceValue | null = null;
   if (app.applicationManagementDynamicsFactor && typeof app.applicationManagementDynamicsFactor === 'object') {
     const factorObj = dynamicsFactorCache?.get(app.applicationManagementDynamicsFactor.objectId);
@@ -215,6 +272,7 @@ async function toApplicationDetails(app: ApplicationComponent): Promise<Applicat
       key: app.applicationManagementDynamicsFactor.objectKey,
       name: app.applicationManagementDynamicsFactor.label,
       factor: factorObj?.factor ?? undefined,
+      description: factorObj?.description ?? undefined, // Include description from cache
     };
   }
   
@@ -226,6 +284,7 @@ async function toApplicationDetails(app: ApplicationComponent): Promise<Applicat
       key: app.applicationManagementComplexityFactor.objectKey,
       name: app.applicationManagementComplexityFactor.label,
       factor: factorObj?.factor ?? undefined,
+      description: factorObj?.description ?? undefined, // Include description from cache
     };
   }
   
@@ -237,6 +296,7 @@ async function toApplicationDetails(app: ApplicationComponent): Promise<Applicat
       key: app.applicationManagementNumberOfUsers.objectKey,
       name: app.applicationManagementNumberOfUsers.label,
       factor: factorObj?.factor ?? undefined,
+      description: (factorObj as any)?.description ?? undefined, // Include description from cache (may not be in generated types)
     };
   }
   
@@ -286,6 +346,12 @@ async function toApplicationDetails(app: ApplicationComponent): Promise<Applicat
     // Override
     overrideFTE: app.applicationManagementOverrideFTE ?? null,
     requiredEffortApplicationManagement: null,
+    
+    // Enterprise Architect reference
+    reference: app.reference || null,
+    
+    // Confluence Space (URL string)
+    confluenceSpace: confluenceSpaceValue,
   };
   
   // Calculate data completeness percentage
@@ -356,6 +422,12 @@ async function toApplicationDetails(app: ApplicationComponent): Promise<Applicat
     overrideFTE: app.applicationManagementOverrideFTE ?? null,
     requiredEffortApplicationManagement: null,
     dataCompletenessPercentage: Math.round(completenessPercentage * 10) / 10, // Round to 1 decimal
+    
+    // Enterprise Architect reference
+    reference: app.reference || null,
+    
+    // Confluence Space (URL string)
+    confluenceSpace: confluenceSpaceValue,
   };
 }
 
@@ -397,11 +469,18 @@ function clearFactorCaches(): void {
  * This avoids the overhead of toApplicationDetails while providing enough data for effort calculation
  * Note: ensureFactorCaches() must be called before using this function
  */
-function toMinimalDetailsForEffort(app: ApplicationComponent): ApplicationDetails {
-  const governanceModel = toReferenceValue(app.ictGovernanceModel);
-  const applicationType = toReferenceValue(app.applicationManagementApplicationType);
-  const businessImpactAnalyse = toReferenceValue(app.businessImpactAnalyse);
-  const applicationManagementHosting = toReferenceValue(app.applicationManagementHosting);
+async function toMinimalDetailsForEffort(app: ApplicationComponent): Promise<ApplicationDetails> {
+  const [
+    governanceModel,
+    applicationType,
+    businessImpactAnalyse,
+    applicationManagementHosting,
+  ] = await Promise.all([
+    toReferenceValue(app.ictGovernanceModel),
+    toReferenceValue(app.applicationManagementApplicationType),
+    toReferenceValue(app.businessImpactAnalyse),
+    toReferenceValue(app.applicationManagementHosting),
+  ]);
   
   // Look up factor values from cached factor objects
   let dynamicsFactor: ReferenceValue | null = null;
@@ -434,6 +513,7 @@ function toMinimalDetailsForEffort(app: ApplicationComponent): ApplicationDetail
       key: app.applicationManagementNumberOfUsers.objectKey,
       name: app.applicationManagementNumberOfUsers.label,
       factor: factorObj?.factor ?? undefined,
+      description: (factorObj as any)?.description ?? undefined, // Include description from cache (may not be in generated types)
     };
   }
   
@@ -474,23 +554,38 @@ function toMinimalDetailsForEffort(app: ApplicationComponent): ApplicationDetail
 /**
  * Convert ApplicationComponent to ApplicationListItem (lighter weight, for lists)
  */
-function toApplicationListItem(app: ApplicationComponent): ApplicationListItem {
+async function toApplicationListItem(app: ApplicationComponent): Promise<ApplicationListItem> {
   // Use direct ObjectReference conversion instead of lookups
-  const governanceModel = toReferenceValue(app.ictGovernanceModel);
-  const dynamicsFactor = toReferenceValue(app.applicationManagementDynamicsFactor);
-  const complexityFactor = toReferenceValue(app.applicationManagementComplexityFactor);
-  // Note: Team/Subteam fields are not in generated schema, use type assertion
-  const applicationSubteam = toReferenceValue((app as any).applicationManagementSubteam);
-  const applicationTeam = toReferenceValue((app as any).applicationManagementTeam);
-  const applicationType = toReferenceValue(app.applicationManagementApplicationType);
-  const platform = toReferenceValue(app.platform);
+  // Fetch all reference values in parallel
+  const [
+    governanceModel,
+    dynamicsFactor,
+    complexityFactor,
+    applicationSubteam,
+    applicationTeam,
+    applicationType,
+    platform,
+    applicationManagementHosting,
+    applicationManagementTAM,
+    businessImpactAnalyse,
+    minimalDetails,
+  ] = await Promise.all([
+    toReferenceValue(app.ictGovernanceModel),
+    toReferenceValue(app.applicationManagementDynamicsFactor),
+    toReferenceValue(app.applicationManagementComplexityFactor),
+    toReferenceValue((app as any).applicationManagementSubteam),
+    toReferenceValue((app as any).applicationManagementTeam),
+    toReferenceValue(app.applicationManagementApplicationType),
+    toReferenceValue(app.platform),
+    toReferenceValue(app.applicationManagementHosting),
+    toReferenceValue(app.applicationManagementTAM),
+    toReferenceValue(app.businessImpactAnalyse),
+    toMinimalDetailsForEffort(app),
+  ]);
+  
   const applicationFunctions = toReferenceValues(app.applicationFunction);
-  const applicationManagementHosting = toReferenceValue(app.applicationManagementHosting);
-  const applicationManagementTAM = toReferenceValue(app.applicationManagementTAM);
-  const businessImpactAnalyse = toReferenceValue(app.businessImpactAnalyse);
   
   // Calculate effort using minimal details
-  const minimalDetails = toMinimalDetailsForEffort(app);
   const effortResult = calculateRequiredEffortWithMinMax(minimalDetails);
   
   const result: ApplicationListItem = {
@@ -518,12 +613,17 @@ function toApplicationListItem(app: ApplicationComponent): ApplicationListItem {
   
   // Calculate data completeness percentage
   // Convert ApplicationListItem to format expected by completeness calculator
+  const [organisationRef, hostingTypeRef] = await Promise.all([
+    toReferenceValue(app.organisation),
+    toReferenceValue(app.applicationComponentHostingType),
+  ]);
+  
   const appForCompleteness = {
-    organisation: toReferenceValue(app.organisation)?.name || null,
+    organisation: organisationRef?.name || null,
     applicationFunctions: result.applicationFunctions,
     status: result.status,
     businessImpactAnalyse: result.businessImpactAnalyse,
-    hostingType: toReferenceValue(app.applicationComponentHostingType),
+    hostingType: hostingTypeRef,
     supplierProduct: app.supplierProduct?.label || null,
     businessOwner: app.businessOwner?.label || null,
     systemOwner: app.systemOwner?.label || null,
@@ -535,7 +635,7 @@ function toApplicationListItem(app: ApplicationComponent): ApplicationListItem {
     applicationManagementTAM: result.applicationManagementTAM,
     dynamicsFactor: result.dynamicsFactor,
     complexityFactor: result.complexityFactor,
-    numberOfUsers: toReferenceValue(app.applicationManagementNumberOfUsers),
+    numberOfUsers: await toReferenceValue(app.applicationManagementNumberOfUsers),
   };
   
   const completenessPercentage = calculateApplicationCompleteness(appForCompleteness);
@@ -718,8 +818,8 @@ export const dataService = {
     // Ensure factor caches are loaded for effort calculation
     await ensureFactorCaches();
     
-    // Convert to list items (synchronous now)
-    const applications = paginatedApps.map(toApplicationListItem);
+    // Convert to list items (async now to fetch descriptions)
+    const applications = await Promise.all(paginatedApps.map(toApplicationListItem));
     
     return {
       applications,
@@ -1221,8 +1321,8 @@ export const dataService = {
     for (const app of apps) {
       // Get team from application (via subteam lookup if needed)
       let team: ReferenceValue | null = null;
-      const applicationSubteam = toReferenceValue((app as any).applicationManagementSubteam);
-      const applicationTeam = toReferenceValue((app as any).applicationManagementTeam);
+      const applicationSubteam = await toReferenceValue((app as any).applicationManagementSubteam);
+      const applicationTeam = await toReferenceValue((app as any).applicationManagementTeam);
       
       // Prefer direct team assignment, otherwise try to get from subteam
       if (applicationTeam) {
@@ -1265,7 +1365,7 @@ export const dataService = {
       
       // Get BIA value
       if (app.businessImpactAnalyse) {
-        const biaRef = toReferenceValue(app.businessImpactAnalyse);
+        const biaRef = await toReferenceValue(app.businessImpactAnalyse);
         if (biaRef) {
           const biaNum = biaToNumeric(biaRef.name);
           if (biaNum !== null) metrics.biaValues.push(biaNum);
@@ -1274,7 +1374,7 @@ export const dataService = {
       
       // Get governance maturity
       if (app.ictGovernanceModel) {
-        const govRef = toReferenceValue(app.ictGovernanceModel);
+        const govRef = await toReferenceValue(app.ictGovernanceModel);
         if (govRef) {
           const maturity = governanceToMaturity(govRef.name);
           if (maturity !== null) metrics.governanceValues.push(maturity);
@@ -1327,6 +1427,10 @@ export const dataService = {
 
   async testConnection(): Promise<boolean> {
     if (!useJiraAssets) return true;
+    // Only test connection if token is configured
+    if (!jiraAssetsClient.hasToken()) {
+      return false;
+    }
     return jiraAssetsClient.testConnection();
   },
 
@@ -1413,7 +1517,7 @@ export const dataService = {
       if (!app.id || !app.label) continue;
 
       // Extract Business Importance from app object
-      const businessImportanceRef = toReferenceValue(app.businessImportance);
+      const businessImportanceRef = await toReferenceValue(app.businessImportance);
       const businessImportanceName = businessImportanceRef?.name || null;
       
       // Normalize Business Importance
@@ -1436,7 +1540,7 @@ export const dataService = {
       }
 
       // Extract BIA from app object
-      const businessImpactAnalyseRef = toReferenceValue(app.businessImpactAnalyse);
+      const businessImpactAnalyseRef = await toReferenceValue(app.businessImpactAnalyse);
       
       // Normalize BIA Class
       let biaClass: string | null = null;