diff --git a/app/core/service/PackageSyncerService.ts b/app/core/service/PackageSyncerService.ts index 0bda5820..41d66f00 100644 --- a/app/core/service/PackageSyncerService.ts +++ b/app/core/service/PackageSyncerService.ts @@ -809,6 +809,7 @@ data sample: ${remoteData.subarray(0, 200).toString()}`; const updateVersions: string[] = []; const differentMetas: [PackageJSONType, Partial][] = []; let syncIndex = 0; + let largeVersionCount = 0; for (const item of versions) { const version: string = item.version; // Skip empty versions, handle abnormal data @@ -910,17 +911,25 @@ data sample: ${remoteData.subarray(0, 200).toString()}`; const allowed = await this.packageVersionFileService.isLargePackageVersionAllowed(scope, name, version); const whiteListVersion = this.packageVersionFileService.unpkgWhiteListVersion; if (!allowed) { - task.error = `Synced version ${version} fail, large package version size: ${size}, allow size: ${this.config.cnpmcore.largePackageVersionSize}, see ${UNPKG_WHITE_LIST_URL}, white list version: ${whiteListVersion}`; - logs.push(`[${isoNow()}] ❌ ${task.error}, log: ${logUrl}`); - logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname} ❌❌❌❌❌`); - await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n')); - this.logger.info( - '[PackageSyncerService.executeTask:fail-large-package-version-size] taskId: %s, targetName: %s, %s', - task.taskId, - task.targetName, - task.error, - ); - return; + largeVersionCount++; + if (largeVersionCount > this.config.cnpmcore.largePackageVersionBlockThreshold) { + task.error = `Synced version ${version} fail, too many large versions (${largeVersionCount}), large package version size: ${size}, allow size: ${this.config.cnpmcore.largePackageVersionSize}, see ${UNPKG_WHITE_LIST_URL}, white list version: ${whiteListVersion}`; + logs.push(`[${isoNow()}] ❌ ${task.error}, log: ${logUrl}`); + logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname} ❌❌❌❌❌`); + await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n')); + this.logger.info( + '[PackageSyncerService.executeTask:fail-large-package-version-size] taskId: %s, targetName: %s, %s', + task.taskId, + task.targetName, + task.error, + ); + return; + } + lastErrorMessage = `large package version size: ${size}, allow size: ${this.config.cnpmcore.largePackageVersionSize}, see ${UNPKG_WHITE_LIST_URL}, white list version: ${whiteListVersion}`; + logs.push(`[${isoNow()}] ⚠️ [${syncIndex}] Synced version ${version} skipped, ${lastErrorMessage}`); + await this.taskService.appendTaskLog(task, logs.join('\n')); + logs = []; + continue; } logs.push( `[${isoNow()}] 🚧 [${syncIndex}] Synced version ${version} size: ${size} too large, it is allowed to sync by unpkg white list, white list version: ${whiteListVersion}`, @@ -1397,6 +1406,7 @@ ${diff.addedVersions.length} added, ${diff.removedVersions.length} removed, calc const updateVersions: string[] = []; let syncIndex = 0; + let largeVersionCount = 0; // #region sync added versions for (const [version, [offsetStart, offsetEnd]] of diff.addedVersions) { // @ts-expect-error JSON.parse accepts Buffer in Node.js, though TypeScript types don't reflect this @@ -1423,17 +1433,25 @@ ${diff.addedVersions.length} added, ${diff.removedVersions.length} removed, calc const allowed = await this.packageVersionFileService.isLargePackageVersionAllowed(scope, name, version); const whiteListVersion = this.packageVersionFileService.unpkgWhiteListVersion; if (!allowed) { - task.error = `Synced version ${version} fail, large package version size: ${size}, allow size: ${this.config.cnpmcore.largePackageVersionSize}, see ${UNPKG_WHITE_LIST_URL}, white list version: ${whiteListVersion}`; - logs.push(`[${isoNow()}] ❌ ${task.error}, log: ${logUrl}`); - logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname} ❌❌❌❌❌`); - await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n')); - this.logger.info( - '[PackageSyncerService.executeTask:fail-large-package-version-size] taskId: %s, targetName: %s, %s', - task.taskId, - task.targetName, - task.error, - ); - return; + largeVersionCount++; + if (largeVersionCount > this.config.cnpmcore.largePackageVersionBlockThreshold) { + task.error = `Synced version ${version} fail, too many large versions (${largeVersionCount}), large package version size: ${size}, allow size: ${this.config.cnpmcore.largePackageVersionSize}, see ${UNPKG_WHITE_LIST_URL}, white list version: ${whiteListVersion}`; + logs.push(`[${isoNow()}] ❌ ${task.error}, log: ${logUrl}`); + logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname} ❌❌❌❌❌`); + await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n')); + this.logger.info( + '[PackageSyncerService.executeTask:fail-large-package-version-size] taskId: %s, targetName: %s, %s', + task.taskId, + task.targetName, + task.error, + ); + return; + } + lastErrorMessage = `large package version size: ${size}, allow size: ${this.config.cnpmcore.largePackageVersionSize}, see ${UNPKG_WHITE_LIST_URL}, white list version: ${whiteListVersion}`; + logs.push(`[${isoNow()}] ⚠️ [${syncIndex}] Synced version ${version} skipped, ${lastErrorMessage}`); + await this.taskService.appendTaskLog(task, logs.join('\n')); + logs = []; + continue; } logs.push( `[${isoNow()}] 🚧 [${syncIndex}] Synced version ${version} size: ${size} too large, it is allowed to sync by unpkg white list, white list version: ${whiteListVersion}`, diff --git a/app/port/config.ts b/app/port/config.ts index 8d82087d..6ccd943b 100644 --- a/app/port/config.ts +++ b/app/port/config.ts @@ -158,6 +158,13 @@ export interface CnpmcoreConfig { * allow large package version size, default is MAX_SAFE_INTEGER */ largePackageVersionSize: number; + /** + * When the number of oversized versions exceeds this threshold, the sync task will fail. + * Oversized versions within the threshold will be skipped and the rest will still be synced. + * e.g. threshold=3 means up to 3 oversized versions are tolerated (skipped), the 4th will fail the task. + * default is 3 + */ + largePackageVersionBlockThreshold: number; /** * enable this would make sync specific version task not append latest version into this task automatically,it would mark the local latest stable version as latest tag. * in most cases, you should set to false to keep the same behavior as source registry. diff --git a/config/config.default.ts b/config/config.default.ts index 1395204f..879d3508 100644 --- a/config/config.default.ts +++ b/config/config.default.ts @@ -57,6 +57,7 @@ export const cnpmcoreConfig: CnpmcoreConfig = { enableSyncUnpkgFiles: true, enableSyncUnpkgFilesWhiteList: false, largePackageVersionSize: Number.MAX_SAFE_INTEGER, + largePackageVersionBlockThreshold: 3, strictSyncSpecivicVersion: false, enableElasticsearch: env('CNPMCORE_CONFIG_ENABLE_ES', 'boolean', false), elasticsearchIndex: 'cnpmcore_packages', diff --git a/test/core/service/PackageSyncerService/executeTask.test.ts b/test/core/service/PackageSyncerService/executeTask.test.ts index 871a6b02..740f1e66 100644 --- a/test/core/service/PackageSyncerService/executeTask.test.ts +++ b/test/core/service/PackageSyncerService/executeTask.test.ts @@ -2324,7 +2324,7 @@ describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => { app.mockAgent().assertNoPendingInterceptors(); }); - it('should mock large package version size block', async () => { + it('should mock large package version size skip when under threshold', async () => { mock.error(NPMRegistry.prototype, 'downloadTarball'); mock.data(NPMRegistry.prototype, 'getFullManifestsBuffer', { data: Buffer.from( @@ -2353,14 +2353,13 @@ describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => { assert.ok(stream); const log = await TestUtil.readStreamToLog(stream); // console.log(log); - assert.match(log, /❌❌❌❌❌ cnpmcore-test-sync-deprecated ❌❌❌❌❌/); assert.match( log, - /Synced version 2.0.0 fail, large package version size: 104857601, allow size: 104857600, see https:\/\/github\.com\/cnpm\/unpkg-white-list/, + /Synced version 2.0.0 skipped, large package version size: 104857601, allow size: 104857600, see https:\/\/github\.com\/cnpm\/unpkg-white-list/, ); }); - it('should mock large package version size block by unpackedSize', async () => { + it('should mock large package version size skip by unpackedSize when under threshold', async () => { mock.error(NPMRegistry.prototype, 'downloadTarball'); mock.data(NPMRegistry.prototype, 'getFullManifestsBuffer', { data: Buffer.from( @@ -2389,13 +2388,58 @@ describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => { assert.ok(stream); const log = await TestUtil.readStreamToLog(stream); // console.log(log); - assert.match(log, /❌❌❌❌❌ cnpmcore-test-sync-deprecated ❌❌❌❌❌/); assert.match( log, - /Synced version 2.0.0 fail, large package version size: 104857601, allow size: 104857600, see https:\/\/github\.com\/cnpm\/unpkg-white-list/, + /Synced version 2.0.0 skipped, large package version size: 104857601, allow size: 104857600, see https:\/\/github\.com\/cnpm\/unpkg-white-list/, ); }); + it('should mock large package version size block when exceeding threshold', async () => { + mock.error(NPMRegistry.prototype, 'downloadTarball'); + mock.data(NPMRegistry.prototype, 'getFullManifestsBuffer', { + data: Buffer.from( + JSON.stringify({ + maintainers: [{ name: 'fengmk2', email: 'fengmk2@gmai.com' }], + versions: { + '1.0.0': { + version: '1.0.0', + dist: { tarball: 'http://foo.com/a.tgz', size: 100 * 1024 * 1024 + 1 }, + }, + '2.0.0': { + version: '2.0.0', + dist: { tarball: 'http://foo.com/b.tgz', size: 100 * 1024 * 1024 + 2 }, + }, + '3.0.0': { + version: '3.0.0', + dist: { tarball: 'http://foo.com/c.tgz', size: 100 * 1024 * 1024 + 3 }, + }, + '4.0.0': { + version: '4.0.0', + dist: { tarball: 'http://foo.com/d.tgz', size: 100 * 1024 * 1024 + 4 }, + }, + }, + }), + ), + res: {}, + headers: {}, + }); + mock(app.config.cnpmcore, 'enableSyncUnpkgFilesWhiteList', true); + mock(app.config.cnpmcore, 'largePackageVersionSize', 100 * 1024 * 1024); + mock(app.config.cnpmcore, 'largePackageVersionBlockThreshold', 3); + const name = 'cnpmcore-test-sync-deprecated'; + await packageSyncerService.createTask(name); + const task = await packageSyncerService.findExecuteTask(); + assert.ok(task); + assert.equal(task.targetName, name); + await packageSyncerService.executeTask(task); + const stream = await packageSyncerService.findTaskLog(task); + assert.ok(stream); + const log = await TestUtil.readStreamToLog(stream); + // console.log(log); + assert.match(log, /❌❌❌❌❌ cnpmcore-test-sync-deprecated ❌❌❌❌❌/); + assert.match(log, /too many large versions/); + }); + it('should mock large package version size allow', async () => { app.mockHttpclient('https://registry.npmjs.org/Buffer/-/Buffer-0.0.0.tgz', 'GET', { data: await TestUtil.readFixturesFile('registry.npmjs.org/foobar/-/foobar-1.0.0.tgz'), diff --git a/test/core/service/PackageSyncerService/executeTaskWithPackument.test.ts b/test/core/service/PackageSyncerService/executeTaskWithPackument.test.ts index 89cd8a30..84d251cd 100644 --- a/test/core/service/PackageSyncerService/executeTaskWithPackument.test.ts +++ b/test/core/service/PackageSyncerService/executeTaskWithPackument.test.ts @@ -2356,7 +2356,7 @@ describe('test/core/service/PackageSyncerService/executeTaskWithPackument.test.t app.mockAgent().assertNoPendingInterceptors(); }); - it('should mock large package version size block', async () => { + it('should mock large package version size skip when under threshold', async () => { mock.error(NPMRegistry.prototype, 'downloadTarball'); mock.data(NPMRegistry.prototype, 'getFullManifestsBuffer', { data: Buffer.from( @@ -2385,14 +2385,13 @@ describe('test/core/service/PackageSyncerService/executeTaskWithPackument.test.t assert.ok(stream); const log = await TestUtil.readStreamToLog(stream); // console.log(log); - assert.match(log, /❌❌❌❌❌ cnpmcore-test-sync-deprecated ❌❌❌❌❌/); assert.match( log, - /Synced version 2.0.0 fail, large package version size: 104857601, allow size: 104857600, see https:\/\/github\.com\/cnpm\/unpkg-white-list/, + /Synced version 2.0.0 skipped, large package version size: 104857601, allow size: 104857600, see https:\/\/github\.com\/cnpm\/unpkg-white-list/, ); }); - it('should mock large package version size block by unpackedSize', async () => { + it('should mock large package version size skip by unpackedSize when under threshold', async () => { mock.error(NPMRegistry.prototype, 'downloadTarball'); mock.data(NPMRegistry.prototype, 'getFullManifestsBuffer', { data: Buffer.from( @@ -2421,13 +2420,58 @@ describe('test/core/service/PackageSyncerService/executeTaskWithPackument.test.t assert.ok(stream); const log = await TestUtil.readStreamToLog(stream); // console.log(log); - assert.match(log, /❌❌❌❌❌ cnpmcore-test-sync-deprecated ❌❌❌❌❌/); assert.match( log, - /Synced version 2.0.0 fail, large package version size: 104857601, allow size: 104857600, see https:\/\/github\.com\/cnpm\/unpkg-white-list/, + /Synced version 2.0.0 skipped, large package version size: 104857601, allow size: 104857600, see https:\/\/github\.com\/cnpm\/unpkg-white-list/, ); }); + it('should mock large package version size block when exceeding threshold', async () => { + mock.error(NPMRegistry.prototype, 'downloadTarball'); + mock.data(NPMRegistry.prototype, 'getFullManifestsBuffer', { + data: Buffer.from( + JSON.stringify({ + maintainers: [{ name: 'fengmk2', email: 'fengmk2@gmai.com' }], + versions: { + '1.0.0': { + version: '1.0.0', + dist: { tarball: 'http://foo.com/a.tgz', size: 100 * 1024 * 1024 + 1 }, + }, + '2.0.0': { + version: '2.0.0', + dist: { tarball: 'http://foo.com/b.tgz', size: 100 * 1024 * 1024 + 2 }, + }, + '3.0.0': { + version: '3.0.0', + dist: { tarball: 'http://foo.com/c.tgz', size: 100 * 1024 * 1024 + 3 }, + }, + '4.0.0': { + version: '4.0.0', + dist: { tarball: 'http://foo.com/d.tgz', size: 100 * 1024 * 1024 + 4 }, + }, + }, + }), + ), + res: {}, + headers: {}, + }); + mock(app.config.cnpmcore, 'enableSyncUnpkgFilesWhiteList', true); + mock(app.config.cnpmcore, 'largePackageVersionSize', 100 * 1024 * 1024); + mock(app.config.cnpmcore, 'largePackageVersionBlockThreshold', 3); + const name = 'cnpmcore-test-sync-deprecated'; + await packageSyncerService.createTask(name); + const task = await packageSyncerService.findExecuteTask(); + assert.ok(task); + assert.equal(task.targetName, name); + await packageSyncerService.executeTask(task); + const stream = await packageSyncerService.findTaskLog(task); + assert.ok(stream); + const log = await TestUtil.readStreamToLog(stream); + // console.log(log); + assert.match(log, /❌❌❌❌❌ cnpmcore-test-sync-deprecated ❌❌❌❌❌/); + assert.match(log, /too many large versions/); + }); + it('should mock large package version size allow', async () => { app.mockHttpclient('https://registry.npmjs.org/Buffer/-/Buffer-0.0.0.tgz', 'GET', { data: await TestUtil.readFixturesFile('registry.npmjs.org/foobar/-/foobar-1.0.0.tgz'), @@ -2466,7 +2510,7 @@ describe('test/core/service/PackageSyncerService/executeTaskWithPackument.test.t app.mockAgent().assertNoPendingInterceptors(); }); - it('issue-943: should block first then allow', async () => { + it('issue-943: should skip large version and sync rest, then allow all', async () => { app.mockHttpclient('https://registry.npmjs.org/Buffer/-/Buffer-0.0.0.tgz', 'GET', { data: await TestUtil.readFixturesFile('registry.npmjs.org/foobar/-/foobar-1.0.0.tgz'), persist: false, @@ -2556,17 +2600,19 @@ describe('test/core/service/PackageSyncerService/executeTaskWithPackument.test.t assert.ok(stream); log = await TestUtil.readStreamToLog(stream); // console.log(log); - assert.match(log, /❌❌❌❌❌ cnpmcore-test-sync-deprecated ❌❌❌❌❌/); - assert.match( - log, - /Synced version 99.0.0-beta.0 fail, large package version size: 104857601, allow size: 104857600, see https:\/\/github\.com\/cnpm\/unpkg-white-list/, - ); - // should at least one version success - // assert.match(log, /Synced version .+? success/); + // large version should be skipped, not fail the task + assert.match(log, /Synced version 99.0.0-beta.0 skipped, large package version size: 104857601/); + // other versions should be synced successfully data = await packageManagerService.listPackageFullManifests('', name); assert(data.data?.versions['0.0.0']); - - // again should allow + assert(data.data?.versions['1.0.0']); + assert(data.data?.versions['1.0.1']); + assert(data.data?.versions['1.1.0']); + assert(data.data?.versions['1.2.0']); + // 99.0.0-beta.0 was skipped + assert(!data.data?.versions['99.0.0-beta.0']); + + // again should allow with whitelist mock.data(NPMRegistry.prototype, 'getFullManifestsBuffer', { data: Buffer.from( JSON.stringify({