Skip to content

Commit 1c5d011

Browse files
committed
Improve cdsExtractorLog for debugging performance
1 parent 3573995 commit 1c5d011

File tree

6 files changed

+606
-33
lines changed

6 files changed

+606
-33
lines changed

extractors/cds/tools/cds-extractor.ts

Lines changed: 44 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,16 @@ import { buildEnhancedCdsProjectDependencyGraph } from './src/cds/parser/graph';
1515
import { runJavaScriptExtractor } from './src/codeql';
1616
import { addCompilationDiagnostic } from './src/diagnostics';
1717
import { configureLgtmIndexFilters, setupAndValidateEnvironment } from './src/environment';
18-
import { cdsExtractorLog, setSourceRootDirectory } from './src/logging';
18+
import {
19+
cdsExtractorLog,
20+
setSourceRootDirectory,
21+
logExtractorStart,
22+
logExtractorEnd,
23+
logPerformanceMilestone,
24+
startPerformanceTracking,
25+
endPerformanceTracking,
26+
logMemoryUsage,
27+
} from './src/logging';
1928
import { installDependencies } from './src/packageManager';
2029
import { RunMode } from './src/runMode';
2130
import { validateArguments } from './src/utils';
@@ -35,21 +44,28 @@ const { runMode, sourceRoot } = validationResult.args!;
3544
// Initialize the unified logging system with the source root directory
3645
setSourceRootDirectory(sourceRoot);
3746

47+
// Log the start of the CDS extractor session
48+
logExtractorStart(runMode, sourceRoot);
49+
logMemoryUsage('Extractor Start');
50+
3851
// Check for autobuild mode
3952
if (runMode === (RunMode.AUTOBUILD as string)) {
4053
cdsExtractorLog('info', 'Autobuild mode is not implemented yet.');
54+
logExtractorEnd(false, 'Terminated: Autobuild mode not implemented');
4155
process.exit(1);
4256
}
4357

4458
// Setup the environment and validate all requirements first, before changing directory
4559
// This ensures we can properly locate the CodeQL tools
60+
startPerformanceTracking('Environment Setup');
4661
const {
4762
success: envSetupSuccess,
4863
errorMessages,
4964
codeqlExePath,
5065
autobuildScriptPath,
5166
platformInfo,
5267
} = setupAndValidateEnvironment(sourceRoot);
68+
endPerformanceTracking('Environment Setup');
5369

5470
if (!envSetupSuccess) {
5571
const codeqlExe = platformInfo.isWindows ? 'codeql.exe' : 'codeql';
@@ -60,6 +76,7 @@ if (!envSetupSuccess) {
6076
)}.`,
6177
);
6278
// Exit with an error code when environment setup fails.
79+
logExtractorEnd(false, 'Terminated: Environment setup failed');
6380
process.exit(1);
6481
}
6582

@@ -78,17 +95,21 @@ cdsExtractorLog('info', 'Building enhanced CDS project dependency graph...');
7895
// Build the enhanced dependency graph using the new enhanced parser
7996
let dependencyGraph;
8097

98+
startPerformanceTracking('Dependency Graph Build');
8199
try {
82100
dependencyGraph = buildEnhancedCdsProjectDependencyGraph(sourceRoot, runMode, __dirname);
101+
endPerformanceTracking('Dependency Graph Build');
83102

84-
cdsExtractorLog(
85-
'info',
86-
`Enhanced dependency graph created with ${dependencyGraph.projects.size} projects and ${dependencyGraph.statusSummary.totalCdsFiles} CDS files`,
103+
logPerformanceMilestone(
104+
'Dependency graph created',
105+
`${dependencyGraph.projects.size} projects, ${dependencyGraph.statusSummary.totalCdsFiles} CDS files`,
87106
);
107+
logMemoryUsage('After Dependency Graph');
88108

89109
// Handle debug modes early - these modes should exit after completing their specific tasks
90110
if (isDebugParserMode(runMode)) {
91111
const debugSuccess = handleDebugParserMode(dependencyGraph, sourceRoot, __dirname);
112+
logExtractorEnd(debugSuccess, `Debug parser mode completed: ${runMode}`);
92113
process.exit(debugSuccess ? 0 : 1);
93114
}
94115

@@ -131,13 +152,16 @@ try {
131152
} catch (error) {
132153
cdsExtractorLog('error', `Failed to build enhanced dependency graph: ${String(error)}`);
133154
// Exit with error since we can't continue without a proper dependency graph
155+
logExtractorEnd(false, 'Terminated: Dependency graph build failed');
134156
process.exit(1);
135157
}
136158

137159
// Install dependencies of discovered CAP/CDS projects
138160
cdsExtractorLog('info', 'Installing dependencies for discovered CDS projects...');
139161

162+
startPerformanceTracking('Dependency Installation');
140163
const projectCacheDirMap = installDependencies(dependencyGraph, sourceRoot, codeqlExePath);
164+
endPerformanceTracking('Dependency Installation');
141165

142166
const cdsFilePathsToProcess: string[] = [];
143167

@@ -162,10 +186,13 @@ if (isDebugMode(runMode)) {
162186

163187
// Initialize CDS command cache early to avoid repeated testing during compilation
164188
// This is a critical optimization that avoids testing commands for every single file
189+
startPerformanceTracking('CDS Command Cache Initialization');
165190
try {
166191
determineCdsCommand(undefined, sourceRoot);
192+
endPerformanceTracking('CDS Command Cache Initialization');
167193
cdsExtractorLog('info', 'CDS command cache initialized successfully');
168194
} catch (error) {
195+
endPerformanceTracking('CDS Command Cache Initialization');
169196
cdsExtractorLog('warn', `CDS command cache initialization failed: ${String(error)}`);
170197
// Continue anyway - individual calls will handle fallbacks
171198
}
@@ -175,13 +202,15 @@ cdsExtractorLog(
175202
`Found ${cdsFilePathsToProcess.length} total CDS files, ${dependencyGraph.statusSummary.totalCdsFiles} CDS files in dependency graph`,
176203
);
177204

205+
startPerformanceTracking('CDS Compilation');
178206
try {
179207
// Use the new orchestrated compilation approach with debug awareness
180208
orchestrateCompilation(dependencyGraph, projectCacheDirMap, codeqlExePath, isDebugMode(runMode));
181209

182210
// Check if we should exit for debug modes after successful compilation
183211
if (isDebugCompilerMode(runMode)) {
184212
const debugSuccess = handleDebugCompilerMode(dependencyGraph, runMode);
213+
logExtractorEnd(debugSuccess, `Debug mode completed: ${runMode}`);
185214
process.exit(debugSuccess ? 0 : 1);
186215
}
187216

@@ -199,7 +228,12 @@ try {
199228

200229
// Don't exit with error - let the JavaScript extractor run on whatever was compiled
201230
}
231+
232+
endPerformanceTracking('CDS Compilation');
233+
logPerformanceMilestone('CDS compilation completed');
234+
logMemoryUsage('After CDS Compilation');
202235
} catch (error) {
236+
endPerformanceTracking('CDS Compilation');
203237
cdsExtractorLog('error', `Compilation orchestration failed: ${String(error)}`);
204238

205239
// Add diagnostic for the overall failure
@@ -216,9 +250,15 @@ try {
216250
configureLgtmIndexFilters();
217251

218252
// Run CodeQL's JavaScript extractor to process the compiled JSON files.
253+
startPerformanceTracking('JavaScript Extraction');
219254
const extractorResult = runJavaScriptExtractor(sourceRoot, autobuildScriptPath, codeqlExePath);
255+
endPerformanceTracking('JavaScript Extraction');
256+
220257
if (!extractorResult.success && extractorResult.error) {
221258
cdsExtractorLog('error', `Error running JavaScript extractor: ${extractorResult.error}`);
259+
logExtractorEnd(false, 'JavaScript extractor failed');
260+
} else {
261+
logExtractorEnd(true, 'CDS extraction completed successfully');
222262
}
223263

224264
// Use the `cds-extractor.js` name in the log message as that is the name of the script

extractors/cds/tools/src/logging/cdsExtractorLog.ts

Lines changed: 169 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,22 @@ import type { LogLevel } from './types';
77
*/
88
let sourceRootDirectory: string | undefined;
99

10+
/**
11+
* Unique session ID for this CDS extractor run to help distinguish
12+
* between multiple concurrent or sequential runs in logs.
13+
*/
14+
const sessionId = Math.random().toString(36).substring(2, 8);
15+
16+
/**
17+
* Start time of the CDS extractor session for performance tracking.
18+
*/
19+
const extractorStartTime = Date.now();
20+
21+
/**
22+
* Performance tracking state for timing critical operations.
23+
*/
24+
const performanceTracking = new Map<string, number>();
25+
1026
/**
1127
* Sets the source root directory for path filtering in log messages.
1228
* This should typically be called once at the start of the CDS extractor.
@@ -17,9 +33,158 @@ export function setSourceRootDirectory(sourceRoot: string): void {
1733
sourceRootDirectory = sourceRoot;
1834
}
1935

36+
/**
37+
* Calculates elapsed time from start and formats it with appropriate units.
38+
*
39+
* @param startTime - The start timestamp in milliseconds
40+
* @param endTime - The end timestamp in milliseconds (defaults to current time)
41+
* @returns Formatted duration string
42+
*/
43+
function formatDuration(startTime: number, endTime: number = Date.now()): string {
44+
const durationMs = endTime - startTime;
45+
46+
if (durationMs < 1000) {
47+
return `${durationMs}ms`;
48+
} else if (durationMs < 60000) {
49+
return `${(durationMs / 1000).toFixed(2)}s`;
50+
} else {
51+
const minutes = Math.floor(durationMs / 60000);
52+
const seconds = ((durationMs % 60000) / 1000).toFixed(2);
53+
return `${minutes}m ${seconds}s`;
54+
}
55+
}
56+
57+
/**
58+
* Starts tracking performance for a named operation.
59+
*
60+
* @param operationName - Name of the operation to track
61+
*/
62+
export function startPerformanceTracking(operationName: string): void {
63+
performanceTracking.set(operationName, Date.now());
64+
cdsExtractorLog('debug', `Started: ${operationName}`);
65+
}
66+
67+
/**
68+
* Ends tracking performance for a named operation and logs the duration.
69+
*
70+
* @param operationName - Name of the operation to stop tracking
71+
*/
72+
export function endPerformanceTracking(operationName: string): void {
73+
const startTime = performanceTracking.get(operationName);
74+
if (startTime) {
75+
const duration = formatDuration(startTime);
76+
performanceTracking.delete(operationName);
77+
cdsExtractorLog('info', `Completed: ${operationName} (took ${duration})`);
78+
} else {
79+
cdsExtractorLog('warn', `No start time found for operation: ${operationName}`);
80+
}
81+
}
82+
83+
/**
84+
* Logs a performance milestone with timing information.
85+
*
86+
* @param milestone - Description of the milestone reached
87+
* @param additionalInfo - Optional additional information to include
88+
*/
89+
export function logPerformanceMilestone(milestone: string, additionalInfo?: string): void {
90+
const currentTime = Date.now();
91+
const overallDuration = formatDuration(extractorStartTime, currentTime);
92+
const info = additionalInfo ? ` - ${additionalInfo}` : '';
93+
cdsExtractorLog('info', `MILESTONE: ${milestone} (after ${overallDuration})${info}`);
94+
}
95+
96+
/**
97+
* Logs the start of the CDS extractor session with session information.
98+
*
99+
* @param runMode - The run mode being used
100+
* @param sourceRoot - The source root directory being processed
101+
*/
102+
export function logExtractorStart(runMode: string, sourceRoot: string): void {
103+
cdsExtractorLog('info', `=== CDS EXTRACTOR START [${sessionId}] ===`);
104+
cdsExtractorLog('info', `Run Mode: ${runMode}`);
105+
cdsExtractorLog('info', `Source Root: ${sourceRoot}`);
106+
}
107+
108+
/**
109+
* Logs the end of the CDS extractor session with final performance summary.
110+
*
111+
* @param success - Whether the extraction completed successfully
112+
* @param additionalSummary - Optional additional summary information
113+
*/
114+
export function logExtractorEnd(success: boolean = true, additionalSummary?: string): void {
115+
const endTime = Date.now();
116+
const totalDuration = formatDuration(extractorStartTime, endTime);
117+
const status = success ? 'SUCCESS' : 'FAILURE';
118+
119+
if (additionalSummary) {
120+
cdsExtractorLog('info', additionalSummary);
121+
}
122+
123+
cdsExtractorLog('info', `=== CDS EXTRACTOR END [${sessionId}] - ${status} ===`);
124+
cdsExtractorLog('info', `Total Duration: ${totalDuration}`);
125+
}
126+
127+
/**
128+
* Logs current memory usage for performance debugging.
129+
*
130+
* @param context - Context description for the memory check
131+
*/
132+
export function logMemoryUsage(context: string): void {
133+
if (typeof process !== 'undefined' && process.memoryUsage) {
134+
try {
135+
const memUsage = process.memoryUsage();
136+
const formatBytes = (bytes: number): string => {
137+
if (bytes === 0) return '0 B';
138+
const k = 1024;
139+
const sizes = ['B', 'KB', 'MB', 'GB'];
140+
const i = Math.floor(Math.log(bytes) / Math.log(k));
141+
return `${parseFloat((bytes / Math.pow(k, i)).toFixed(2))} ${sizes[i]}`;
142+
};
143+
144+
cdsExtractorLog(
145+
'debug',
146+
`Memory usage - ${context}: RSS=${formatBytes(memUsage.rss)}, Heap Used=${formatBytes(memUsage.heapUsed)}, Heap Total=${formatBytes(memUsage.heapTotal)}, External=${formatBytes(memUsage.external)}`,
147+
);
148+
} catch {
149+
// Silently ignore errors when memory usage is not available
150+
// This ensures the function never throws and doesn't interrupt the extraction process
151+
}
152+
}
153+
}
154+
155+
/**
156+
* Logs a performance counter with current count and rate information.
157+
*
158+
* @param counterName - Name of the counter
159+
* @param currentCount - Current count value
160+
* @param startTime - Start time for rate calculation (optional)
161+
* @param totalExpected - Total expected count for progress percentage (optional)
162+
*/
163+
export function logPerformanceCounter(
164+
counterName: string,
165+
currentCount: number,
166+
startTime?: number,
167+
totalExpected?: number,
168+
): void {
169+
let message = `${counterName}: ${currentCount}`;
170+
171+
if (totalExpected && totalExpected > 0) {
172+
const percentage = ((currentCount / totalExpected) * 100).toFixed(1);
173+
message += ` / ${totalExpected} (${percentage}%)`;
174+
}
175+
176+
if (startTime) {
177+
const elapsed = Date.now() - startTime;
178+
const rate = elapsed > 0 ? ((currentCount / elapsed) * 1000).toFixed(1) : '0';
179+
message += ` - Rate: ${rate}/sec`;
180+
}
181+
182+
cdsExtractorLog('debug', message);
183+
}
184+
20185
/**
21186
* Unified logging function for the CDS extractor. Provides consistent
22-
* log formatting with level prefixes and path filtering.
187+
* log formatting with level prefixes, elapsed time, session IDs, and path filtering.
23188
*
24189
* @param level - The log level ('debug', 'info', 'warn', 'error')
25190
* @param message - The primary message or data to log
@@ -34,8 +199,9 @@ export function cdsExtractorLog(
34199
throw new Error('Source root directory is not set. Call setSourceRootDirectory() first.');
35200
}
36201

37-
// Convert log level to uppercase for consistent formatting
38-
const levelPrefix = `${level.toUpperCase()}: `;
202+
const currentTime = Date.now();
203+
const elapsedMs = currentTime - extractorStartTime;
204+
const levelPrefix = `[CDS-${sessionId} ${elapsedMs}] ${level.toUpperCase()}: `;
39205

40206
// Process the primary message for path filtering if it's a string
41207
let processedMessage: string;
Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,12 @@
1-
export { cdsExtractorLog, setSourceRootDirectory } from './cdsExtractorLog';
1+
export {
2+
cdsExtractorLog,
3+
setSourceRootDirectory,
4+
logExtractorStart,
5+
logExtractorEnd,
6+
logPerformanceMilestone,
7+
startPerformanceTracking,
8+
endPerformanceTracking,
9+
logMemoryUsage,
10+
logPerformanceCounter,
11+
} from './cdsExtractorLog';
212
export type { LogLevel } from './types';

extractors/cds/tools/test/src/environment.test.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -303,7 +303,9 @@ describe('environment', () => {
303303
expect(jsExtractorRoot).toBe('');
304304
expect(execFileSync).not.toHaveBeenCalled();
305305
expect(consoleSpy.warn).toHaveBeenCalledWith(
306-
'WARN: Cannot resolve JavaScript extractor root because the CodeQL executable path was not provided or found.',
306+
expect.stringMatching(
307+
/^\[CDS-.+ \d+\] WARN: Cannot resolve JavaScript extractor root because the CodeQL executable path was not provided or found\.$/,
308+
),
307309
);
308310
});
309311

extractors/cds/tools/test/src/filesystem.test.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,9 @@ describe('filesystem', () => {
128128

129129
recursivelyRenameJsonFiles('/non-existent/dir');
130130

131-
expect(mockConsoleLog).toHaveBeenCalledWith('INFO: Directory not found: /non-existent/dir');
131+
expect(mockConsoleLog).toHaveBeenCalledWith(
132+
expect.stringMatching(/^\[CDS-.+ \d+\] INFO: Directory not found: \/non-existent\/dir$/),
133+
);
132134
expect(fs.readdirSync).not.toHaveBeenCalled();
133135
});
134136

0 commit comments

Comments
 (0)