[CI] Cache TypeScript build artifacts (#241414)

Implements a GCS-backed cache of TypeScript builds, which ensures only
invalidated projects need to be re-checked, which can speed up CI checks
from 25-30m to 3m in many cases.

When restoring an archive, we use the most recent commit for which there
is an archive. Archives are stored either by commit sha (for the
on-merge pipeline) or PR number (for the pull-request pipeline).

Storing & restoring takes about 20s each. A fully-cached type check is
about 50s.

Based on #240981 from @delanni.

---------

Co-authored-by: Alex Szabo <[email protected]>
Co-authored-by: Tiago Costa <[email protected]>
Co-authored-by: kibanamachine <[email protected]>

Author: 4 people

.buildkite/pipelines/build_api_docs.yml

@@ -1,7 +1,7 @@
 env:
   PUBLISH_API_DOCS_CHANGES: 'true'
 steps:
-  - command: .buildkite/scripts/steps/check_types.sh
+  - command: .buildkite/scripts/steps/typecheck/check_types.sh
     label: 'Check types'
     key: check_types
     agents:

.buildkite/pipelines/on_merge.yml

@@ -106,7 +106,7 @@ steps:
         - exit_status: '-1'
           limit: 3
 
-  - command: .buildkite/scripts/steps/check_types.sh
+  - command: .buildkite/scripts/steps/typecheck/check_types.sh
     label: 'Check Types'
     agents:
       image: family/kibana-ubuntu-2404

.buildkite/pipelines/pull_request/base.yml

@@ -98,7 +98,7 @@ steps:
         - exit_status: '-1'
           limit: 3
 
-  - command: .buildkite/scripts/steps/check_types.sh
+  - command: .buildkite/scripts/steps/typecheck/check_types.sh
     label: 'Check Types'
     agents:
       machineType: c4-standard-4

.buildkite/scripts/common/activate_service_account.sh

@@ -78,6 +78,9 @@ if [[ -z "$EMAIL" ]]; then
     "ci-artifacts.kibana.dev")
       EMAIL="kibana-ci-access-artifacts@$GCLOUD_EMAIL_POSTFIX"
       ;;
+    "ci-typescript-archives")
+      EMAIL="kibana-ci-access-ts-archives@$GCLOUD_EMAIL_POSTFIX"
+      ;;
     "kibana-ci-access-chromium-blds")
       EMAIL="kibana-ci-access-chromium-blds@$GCLOUD_EMAIL_POSTFIX"
       ;;

.buildkite/scripts/steps/check_types.sh …e/scripts/steps/typecheck/check_types.sh.buildkite/scripts/steps/check_types.sh renamed to .buildkite/scripts/steps/typecheck/check_types.sh .buildkite/scripts/steps/check_types.sh renamed to .buildkite/scripts/steps/typecheck/check_types.sh

@@ -7,4 +7,5 @@ source .buildkite/scripts/common/util.sh
 .buildkite/scripts/bootstrap.sh
 
 echo --- Check Types
-node scripts/type_check
+set +e
+node scripts/type_check --with-archive

packages/kbn-ts-type-check-cli/run_type_check_cli.ts

@@ -9,19 +9,22 @@
 
 import Path from 'path';
 import Fsp from 'fs/promises';
-
 import { run } from '@kbn/dev-cli-runner';
 import { createFailError } from '@kbn/dev-cli-errors';
 import { REPO_ROOT } from '@kbn/repo-info';
 import { asyncForEachWithLimit, asyncMapWithLimit } from '@kbn/std';
 import type { SomeDevLog } from '@kbn/some-dev-log';
 import { type TsProject, TS_PROJECTS } from '@kbn/ts-projects';
+import execa from 'execa';
 
 import {
   updateRootRefsConfig,
   cleanupRootRefsConfig,
   ROOT_REFS_CONFIG_PATH,
 } from './root_refs_config';
+import { archiveTSBuildArtifacts } from './src/archive/archive_ts_build_artifacts';
+import { restoreTSBuildArtifacts } from './src/archive/restore_ts_build_artifacts';
+import { LOCAL_CACHE_ROOT } from './src/archive/constants';
 
 const rel = (from: string, to: string) => {
   const path = Path.relative(from, to);
@@ -84,22 +87,46 @@ async function createTypeCheckConfigs(log: SomeDevLog, projects: TsProject[]) {
   );
 }
 
+async function detectLocalChanges(): Promise<boolean> {
+  const { stdout } = await execa('git', ['status', '--porcelain'], {
+    cwd: REPO_ROOT,
+  });
+
+  return stdout.trim().length > 0;
+}
+
 run(
   async ({ log, flagsReader, procRunner }) => {
-    if (flagsReader.boolean('clean-cache')) {
+    const shouldCleanCache = flagsReader.boolean('clean-cache');
+    const shouldUseArchive = flagsReader.boolean('with-archive');
+
+    if (shouldCleanCache) {
       await asyncForEachWithLimit(TS_PROJECTS, 10, async (proj) => {
         await Fsp.rm(Path.resolve(proj.directory, 'target/types'), {
           force: true,
           recursive: true,
         });
       });
-      log.warning('Deleted all typescript caches');
+      await Fsp.rm(LOCAL_CACHE_ROOT, {
+        force: true,
+        recursive: true,
+      });
+      log.warning('Deleted all TypeScript caches');
+      return;
     }
 
     // if the tsconfig.refs.json file is not self-managed then make sure it has
     // a reference to every composite project in the repo
     await updateRootRefsConfig(log);
 
+    if (shouldUseArchive && !shouldCleanCache) {
+      await restoreTSBuildArtifacts(log);
+    } else if (shouldCleanCache && shouldUseArchive) {
+      log.info('Skipping TypeScript cache restore because --clean-cache was provided.');
+    } else {
+      log.verbose('Skipping TypeScript cache restore because --with-archive was not provided.');
+    }
+
     const projectFilter = flagsReader.path('project');
 
     const projects = TS_PROJECTS.filter(
@@ -108,7 +135,7 @@ run(
 
     const created = await createTypeCheckConfigs(log, projects);
 
-    let pluginBuildResult;
+    let didTypeCheckFail = false;
     try {
       log.info(
         `Building TypeScript projects to check types (For visible, though excessive, progress info you can pass --verbose)`
@@ -126,17 +153,28 @@ run(
           relative,
           '--pretty',
           ...(flagsReader.boolean('verbose') ? ['--verbose'] : []),
+          ...(flagsReader.boolean('extended-diagnostics') ? ['--extendedDiagnostics'] : []),
         ],
         env: {
           NODE_OPTIONS: '--max-old-space-size=10240',
         },
         cwd: REPO_ROOT,
         wait: true,
       });
-
-      pluginBuildResult = { failed: false };
     } catch (error) {
-      pluginBuildResult = { failed: true };
+      didTypeCheckFail = true;
+    }
+
+    const hasLocalChanges = shouldUseArchive ? await detectLocalChanges() : false;
+
+    if (shouldUseArchive) {
+      if (hasLocalChanges) {
+        log.info('Skipping TypeScript cache archive because uncommitted changes were detected.');
+      } else {
+        await archiveTSBuildArtifacts(log);
+      }
+    } else {
+      log.verbose('Skipping TypeScript cache archive because --with-archive was not provided.');
     }
 
     // cleanup if requested
@@ -149,7 +187,7 @@ run(
       });
     }
 
-    if (pluginBuildResult.failed) {
+    if (didTypeCheckFail) {
       throw createFailError('Unable to build TS project refs');
     }
   },
@@ -166,7 +204,7 @@ run(
     `,
     flags: {
       string: ['project'],
-      boolean: ['clean-cache', 'cleanup'],
+      boolean: ['clean-cache', 'cleanup', 'extended-diagnostics', 'with-archive'],
       help: `
         --project [path]        Path to a tsconfig.json file determines the project to check
         --help                  Show this message
@@ -175,6 +213,8 @@ run(
                                   files in place makes subsequent executions faster because ts can
                                   identify that none of the imports have changed (it uses creation/update
                                   times) but cleaning them prevents leaving garbage around the repo.
+        --extended-diagnostics  Turn on extended diagnostics in the TypeScript compiler
+        --with-archive          Restore cached artifacts before running and archive results afterwards
       `,
     },
   }

packages/kbn-ts-type-check-cli/src/archive/archive_ts_build_artifacts.test.ts

@@ -0,0 +1,109 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import type { SomeDevLog } from '@kbn/some-dev-log';
+import globby from 'globby';
+import { archiveTSBuildArtifacts } from './archive_ts_build_artifacts';
+import { LocalFileSystem } from './file_system/local_file_system';
+import { getPullRequestNumber, isCiEnvironment, resolveCurrentCommitSha } from './utils';
+
+jest.mock('globby', () => jest.fn());
+
+jest.mock('./utils', () => ({
+  getPullRequestNumber: jest.fn(),
+  isCiEnvironment: jest.fn(),
+  resolveCurrentCommitSha: jest.fn(),
+  withGcsAuth: jest.fn((_, action: () => Promise<unknown>) => action()),
+}));
+
+jest.mock('./file_system/gcs_file_system', () => ({
+  GcsFileSystem: jest.fn().mockImplementation(() => ({
+    updateArchive: jest.fn(),
+  })),
+}));
+
+const mockedGlobby = globby as jest.MockedFunction<typeof globby>;
+const mockedGetPullRequestNumber = getPullRequestNumber as jest.MockedFunction<
+  typeof getPullRequestNumber
+>;
+const mockedIsCiEnvironment = isCiEnvironment as jest.MockedFunction<typeof isCiEnvironment>;
+const mockedResolveCurrentCommitSha = resolveCurrentCommitSha as jest.MockedFunction<
+  typeof resolveCurrentCommitSha
+>;
+
+const createLog = (): SomeDevLog => {
+  return {
+    info: jest.fn(),
+    warning: jest.fn(),
+    error: jest.fn(),
+    debug: jest.fn(),
+  } as unknown as SomeDevLog;
+};
+
+describe('archiveTSBuildArtifacts', () => {
+  let updateSpy: jest.SpyInstance;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    mockedIsCiEnvironment.mockReturnValue(false);
+    mockedGetPullRequestNumber.mockReturnValue(undefined);
+    mockedResolveCurrentCommitSha.mockResolvedValue('');
+    mockedGlobby.mockResolvedValue([]);
+    updateSpy = jest
+      .spyOn(LocalFileSystem.prototype, 'updateArchive')
+      .mockResolvedValue(Promise.resolve() as unknown as void);
+  });
+
+  afterEach(() => {
+    updateSpy.mockRestore();
+  });
+
+  it('logs when no build artifacts are present', async () => {
+    const log = createLog();
+
+    mockedGlobby.mockResolvedValueOnce([]);
+
+    await archiveTSBuildArtifacts(log);
+
+    expect(log.info).toHaveBeenCalledWith('No TypeScript build artifacts found to archive.');
+    expect(updateSpy).not.toHaveBeenCalled();
+  });
+
+  it('warns when the commit SHA cannot be determined', async () => {
+    const log = createLog();
+
+    mockedGlobby.mockResolvedValueOnce(['a']);
+    mockedResolveCurrentCommitSha.mockResolvedValueOnce(undefined);
+
+    await archiveTSBuildArtifacts(log);
+
+    expect(log.warning).toHaveBeenCalledWith(
+      'Unable to determine commit SHA for TypeScript cache archive.'
+    );
+    expect(updateSpy).not.toHaveBeenCalled();
+  });
+
+  it('uses the LocalFileSystem to archive matching artifacts', async () => {
+    const log = createLog();
+    const files = ['target/types/foo.d.ts', 'tsconfig.type_check.json'];
+
+    mockedGlobby.mockResolvedValueOnce(files);
+    mockedResolveCurrentCommitSha.mockResolvedValueOnce('abc123');
+    mockedGetPullRequestNumber.mockReturnValueOnce('789');
+
+    await archiveTSBuildArtifacts(log);
+
+    expect(updateSpy).toHaveBeenCalledTimes(1);
+    expect(updateSpy).toHaveBeenCalledWith({
+      files,
+      prNumber: '789',
+      sha: 'abc123',
+    });
+  });
+});

packages/kbn-ts-type-check-cli/src/archive/archive_ts_build_artifacts.ts

@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+import { REPO_ROOT } from '@kbn/repo-info';
+import type { SomeDevLog } from '@kbn/some-dev-log';
+import globby from 'globby';
+import { CACHE_IGNORE_GLOBS, CACHE_MATCH_GLOBS } from './constants';
+import { GcsFileSystem } from './file_system/gcs_file_system';
+import { LocalFileSystem } from './file_system/local_file_system';
+import {
+  getPullRequestNumber,
+  isCiEnvironment,
+  resolveCurrentCommitSha,
+  withGcsAuth,
+} from './utils';
+
+/**
+ * Archives .tsbuildinfo, type_check.tsconfig.json, and declaration files
+ * in a GCS bucket for cached type checks.
+ */
+export async function archiveTSBuildArtifacts(log: SomeDevLog) {
+  try {
+    const matches = await globby(CACHE_MATCH_GLOBS, {
+      cwd: REPO_ROOT,
+      dot: true,
+      followSymbolicLinks: false,
+      ignore: CACHE_IGNORE_GLOBS,
+    });
+
+    if (matches.length === 0) {
+      log.info('No TypeScript build artifacts found to archive.');
+      return;
+    }
+
+    const commitSha = await resolveCurrentCommitSha();
+
+    if (!commitSha) {
+      log.warning('Unable to determine commit SHA for TypeScript cache archive.');
+      return;
+    }
+
+    const prNumber = getPullRequestNumber();
+
+    const options = { files: matches, sha: commitSha, prNumber };
+
+    if (isCiEnvironment()) {
+      await withGcsAuth(log, () => new GcsFileSystem(log).updateArchive(options));
+    } else {
+      await new LocalFileSystem(log).updateArchive(options);
+    }
+  } catch (error) {
+    const archiveErrorDetails = error instanceof Error ? error.message : String(error);
+    log.warning(`Failed to archive TypeScript build artifacts: ${archiveErrorDetails}`);
+  }
+}