Skip to content

feat: skip oversized versions during sync instead of failing the entire task #1022

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
elrrrrrrr merged 4 commits into from
Apr 2, 2026
Merged

feat: skip oversized versions during sync instead of failing the entire task #1022

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e24436b
feat: skip oversized versions during sync instead of failing, with co…
Copilot Apr 2, 2026
a8debb4
docs: clarify largePackageVersionBlockThreshold doc comment
Copilot Apr 2, 2026
7ad090a
Merge branch 'master' into copilot/adjust-version-limits
fengmk2 Apr 2, 2026
e9975ee
fix: format test files to pass oxfmt check
Copilot Apr 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 40 additions & 22 deletions app/core/service/PackageSyncerService.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -809,6 +809,7 @@ data sample: ${remoteData.subarray(0, 200).toString()}`;
const updateVersions: string[] = [];
const differentMetas: [PackageJSONType, Partial<PackageJSONType>][] = [];
let syncIndex = 0;
let largeVersionCount = 0;
for (const item of versions) {
const version: string = item.version;
// Skip empty versions, handle abnormal data
Expand Down Expand Up @@ -910,17 +911,25 @@ data sample: ${remoteData.subarray(0, 200).toString()}`;
const allowed = await this.packageVersionFileService.isLargePackageVersionAllowed(scope, name, version);
const whiteListVersion = this.packageVersionFileService.unpkgWhiteListVersion;
if (!allowed) {
task.error = `Synced version ${version} fail, large package version size: ${size}, allow size: ${this.config.cnpmcore.largePackageVersionSize}, see ${UNPKG_WHITE_LIST_URL}, white list version: ${whiteListVersion}`;
logs.push(`[${isoNow()}] ❌ ${task.error}, log: ${logUrl}`);
logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname} ❌❌❌❌❌`);
await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
this.logger.info(
'[PackageSyncerService.executeTask:fail-large-package-version-size] taskId: %s, targetName: %s, %s',
task.taskId,
task.targetName,
task.error,
);
return;
largeVersionCount++;
if (largeVersionCount > this.config.cnpmcore.largePackageVersionBlockThreshold) {
task.error = `Synced version ${version} fail, too many large versions (${largeVersionCount}), large package version size: ${size}, allow size: ${this.config.cnpmcore.largePackageVersionSize}, see ${UNPKG_WHITE_LIST_URL}, white list version: ${whiteListVersion}`;
logs.push(`[${isoNow()}] ❌ ${task.error}, log: ${logUrl}`);
logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname} ❌❌❌❌❌`);
await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
this.logger.info(
'[PackageSyncerService.executeTask:fail-large-package-version-size] taskId: %s, targetName: %s, %s',
task.taskId,
task.targetName,
task.error,
);
return;
}
lastErrorMessage = `large package version size: ${size}, allow size: ${this.config.cnpmcore.largePackageVersionSize}, see ${UNPKG_WHITE_LIST_URL}, white list version: ${whiteListVersion}`;
logs.push(`[${isoNow()}] ⚠️ [${syncIndex}] Synced version ${version} skipped, ${lastErrorMessage}`);
await this.taskService.appendTaskLog(task, logs.join('\n'));
logs = [];
continue;
}
logs.push(
`[${isoNow()}] 🚧 [${syncIndex}] Synced version ${version} size: ${size} too large, it is allowed to sync by unpkg white list, white list version: ${whiteListVersion}`,
Expand Down Expand Up @@ -1397,6 +1406,7 @@ ${diff.addedVersions.length} added, ${diff.removedVersions.length} removed, calc

const updateVersions: string[] = [];
let syncIndex = 0;
let largeVersionCount = 0;
// #region sync added versions
for (const [version, [offsetStart, offsetEnd]] of diff.addedVersions) {
// @ts-expect-error JSON.parse accepts Buffer in Node.js, though TypeScript types don't reflect this
Expand All @@ -1423,17 +1433,25 @@ ${diff.addedVersions.length} added, ${diff.removedVersions.length} removed, calc
const allowed = await this.packageVersionFileService.isLargePackageVersionAllowed(scope, name, version);
const whiteListVersion = this.packageVersionFileService.unpkgWhiteListVersion;
if (!allowed) {
task.error = `Synced version ${version} fail, large package version size: ${size}, allow size: ${this.config.cnpmcore.largePackageVersionSize}, see ${UNPKG_WHITE_LIST_URL}, white list version: ${whiteListVersion}`;
logs.push(`[${isoNow()}] ❌ ${task.error}, log: ${logUrl}`);
logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname} ❌❌❌❌❌`);
await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
this.logger.info(
'[PackageSyncerService.executeTask:fail-large-package-version-size] taskId: %s, targetName: %s, %s',
task.taskId,
task.targetName,
task.error,
);
return;
largeVersionCount++;
if (largeVersionCount > this.config.cnpmcore.largePackageVersionBlockThreshold) {
task.error = `Synced version ${version} fail, too many large versions (${largeVersionCount}), large package version size: ${size}, allow size: ${this.config.cnpmcore.largePackageVersionSize}, see ${UNPKG_WHITE_LIST_URL}, white list version: ${whiteListVersion}`;
logs.push(`[${isoNow()}] ❌ ${task.error}, log: ${logUrl}`);
logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname} ❌❌❌❌❌`);
await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
this.logger.info(
'[PackageSyncerService.executeTask:fail-large-package-version-size] taskId: %s, targetName: %s, %s',
task.taskId,
task.targetName,
task.error,
);
return;
}
lastErrorMessage = `large package version size: ${size}, allow size: ${this.config.cnpmcore.largePackageVersionSize}, see ${UNPKG_WHITE_LIST_URL}, white list version: ${whiteListVersion}`;
logs.push(`[${isoNow()}] ⚠️ [${syncIndex}] Synced version ${version} skipped, ${lastErrorMessage}`);
await this.taskService.appendTaskLog(task, logs.join('\n'));
logs = [];
continue;
}
logs.push(
`[${isoNow()}] 🚧 [${syncIndex}] Synced version ${version} size: ${size} too large, it is allowed to sync by unpkg white list, white list version: ${whiteListVersion}`,
Expand Down
7 changes: 7 additions & 0 deletions app/port/config.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,13 @@ export interface CnpmcoreConfig {
* allow large package version size, default is MAX_SAFE_INTEGER
*/
largePackageVersionSize: number;
/**
* When the number of oversized versions exceeds this threshold, the sync task will fail.
* Oversized versions within the threshold will be skipped and the rest will still be synced.
* e.g. threshold=3 means up to 3 oversized versions are tolerated (skipped), the 4th will fail the task.
* default is 3
*/
largePackageVersionBlockThreshold: number;
/**
* enable this would make sync specific version task not append latest version into this task automatically,it would mark the local latest stable version as latest tag.
* in most cases, you should set to false to keep the same behavior as source registry.
Expand Down
1 change: 1 addition & 0 deletions config/config.default.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ export const cnpmcoreConfig: CnpmcoreConfig = {
enableSyncUnpkgFiles: true,
enableSyncUnpkgFilesWhiteList: false,
largePackageVersionSize: Number.MAX_SAFE_INTEGER,
largePackageVersionBlockThreshold: 3,
strictSyncSpecivicVersion: false,
enableElasticsearch: env('CNPMCORE_CONFIG_ENABLE_ES', 'boolean', false),
elasticsearchIndex: 'cnpmcore_packages',
Expand Down
56 changes: 50 additions & 6 deletions test/core/service/PackageSyncerService/executeTask.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2324,7 +2324,7 @@ describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => {
app.mockAgent().assertNoPendingInterceptors();
});

it('should mock large package version size block', async () => {
it('should mock large package version size skip when under threshold', async () => {
mock.error(NPMRegistry.prototype, 'downloadTarball');
mock.data(NPMRegistry.prototype, 'getFullManifestsBuffer', {
data: Buffer.from(
Expand Down Expand Up @@ -2353,14 +2353,13 @@ describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => {
assert.ok(stream);
const log = await TestUtil.readStreamToLog(stream);
// console.log(log);
assert.match(log, /❌❌❌❌❌ cnpmcore-test-sync-deprecated ❌❌❌❌❌/);
assert.match(
log,
/Synced version 2.0.0 fail, large package version size: 104857601, allow size: 104857600, see https:\/\/github\.com\/cnpm\/unpkg-white-list/,
/Synced version 2.0.0 skipped, large package version size: 104857601, allow size: 104857600, see https:\/\/github\.com\/cnpm\/unpkg-white-list/,
);
});

it('should mock large package version size block by unpackedSize', async () => {
it('should mock large package version size skip by unpackedSize when under threshold', async () => {
mock.error(NPMRegistry.prototype, 'downloadTarball');
mock.data(NPMRegistry.prototype, 'getFullManifestsBuffer', {
data: Buffer.from(
Expand Down Expand Up @@ -2389,13 +2388,58 @@ describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => {
assert.ok(stream);
const log = await TestUtil.readStreamToLog(stream);
// console.log(log);
assert.match(log, /❌❌❌❌❌ cnpmcore-test-sync-deprecated ❌❌❌❌❌/);
assert.match(
log,
/Synced version 2.0.0 fail, large package version size: 104857601, allow size: 104857600, see https:\/\/github\.com\/cnpm\/unpkg-white-list/,
/Synced version 2.0.0 skipped, large package version size: 104857601, allow size: 104857600, see https:\/\/github\.com\/cnpm\/unpkg-white-list/,
);
});

it('should mock large package version size block when exceeding threshold', async () => {
mock.error(NPMRegistry.prototype, 'downloadTarball');
mock.data(NPMRegistry.prototype, 'getFullManifestsBuffer', {
data: Buffer.from(
JSON.stringify({
maintainers: [{ name: 'fengmk2', email: 'fengmk2@gmai.com' }],
versions: {
'1.0.0': {
version: '1.0.0',
dist: { tarball: 'http://foo.com/a.tgz', size: 100 * 1024 * 1024 + 1 },
},
'2.0.0': {
version: '2.0.0',
dist: { tarball: 'http://foo.com/b.tgz', size: 100 * 1024 * 1024 + 2 },
},
'3.0.0': {
version: '3.0.0',
dist: { tarball: 'http://foo.com/c.tgz', size: 100 * 1024 * 1024 + 3 },
},
'4.0.0': {
version: '4.0.0',
dist: { tarball: 'http://foo.com/d.tgz', size: 100 * 1024 * 1024 + 4 },
},
},
}),
),
res: {},
headers: {},
});
mock(app.config.cnpmcore, 'enableSyncUnpkgFilesWhiteList', true);
mock(app.config.cnpmcore, 'largePackageVersionSize', 100 * 1024 * 1024);
mock(app.config.cnpmcore, 'largePackageVersionBlockThreshold', 3);
const name = 'cnpmcore-test-sync-deprecated';
await packageSyncerService.createTask(name);
const task = await packageSyncerService.findExecuteTask();
assert.ok(task);
assert.equal(task.targetName, name);
await packageSyncerService.executeTask(task);
const stream = await packageSyncerService.findTaskLog(task);
assert.ok(stream);
const log = await TestUtil.readStreamToLog(stream);
// console.log(log);
assert.match(log, /❌❌❌❌❌ cnpmcore-test-sync-deprecated ❌❌❌❌❌/);
assert.match(log, /too many large versions/);
});

it('should mock large package version size allow', async () => {
app.mockHttpclient('https://registry.npmjs.org/Buffer/-/Buffer-0.0.0.tgz', 'GET', {
data: await TestUtil.readFixturesFile('registry.npmjs.org/foobar/-/foobar-1.0.0.tgz'),
Expand Down
78 changes: 62 additions & 16 deletions test/core/service/PackageSyncerService/executeTaskWithPackument.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2356,7 +2356,7 @@ describe('test/core/service/PackageSyncerService/executeTaskWithPackument.test.t
app.mockAgent().assertNoPendingInterceptors();
});

it('should mock large package version size block', async () => {
it('should mock large package version size skip when under threshold', async () => {
mock.error(NPMRegistry.prototype, 'downloadTarball');
mock.data(NPMRegistry.prototype, 'getFullManifestsBuffer', {
data: Buffer.from(
Expand Down Expand Up @@ -2385,14 +2385,13 @@ describe('test/core/service/PackageSyncerService/executeTaskWithPackument.test.t
assert.ok(stream);
const log = await TestUtil.readStreamToLog(stream);
// console.log(log);
assert.match(log, /❌❌❌❌❌ cnpmcore-test-sync-deprecated ❌❌❌❌❌/);
assert.match(
log,
/Synced version 2.0.0 fail, large package version size: 104857601, allow size: 104857600, see https:\/\/github\.com\/cnpm\/unpkg-white-list/,
/Synced version 2.0.0 skipped, large package version size: 104857601, allow size: 104857600, see https:\/\/github\.com\/cnpm\/unpkg-white-list/,
);
});

it('should mock large package version size block by unpackedSize', async () => {
it('should mock large package version size skip by unpackedSize when under threshold', async () => {
mock.error(NPMRegistry.prototype, 'downloadTarball');
mock.data(NPMRegistry.prototype, 'getFullManifestsBuffer', {
data: Buffer.from(
Expand Down Expand Up @@ -2421,13 +2420,58 @@ describe('test/core/service/PackageSyncerService/executeTaskWithPackument.test.t
assert.ok(stream);
const log = await TestUtil.readStreamToLog(stream);
// console.log(log);
assert.match(log, /❌❌❌❌❌ cnpmcore-test-sync-deprecated ❌❌❌❌❌/);
assert.match(
log,
/Synced version 2.0.0 fail, large package version size: 104857601, allow size: 104857600, see https:\/\/github\.com\/cnpm\/unpkg-white-list/,
/Synced version 2.0.0 skipped, large package version size: 104857601, allow size: 104857600, see https:\/\/github\.com\/cnpm\/unpkg-white-list/,
);
});

it('should mock large package version size block when exceeding threshold', async () => {
mock.error(NPMRegistry.prototype, 'downloadTarball');
mock.data(NPMRegistry.prototype, 'getFullManifestsBuffer', {
data: Buffer.from(
JSON.stringify({
maintainers: [{ name: 'fengmk2', email: 'fengmk2@gmai.com' }],
versions: {
'1.0.0': {
version: '1.0.0',
dist: { tarball: 'http://foo.com/a.tgz', size: 100 * 1024 * 1024 + 1 },
},
'2.0.0': {
version: '2.0.0',
dist: { tarball: 'http://foo.com/b.tgz', size: 100 * 1024 * 1024 + 2 },
},
'3.0.0': {
version: '3.0.0',
dist: { tarball: 'http://foo.com/c.tgz', size: 100 * 1024 * 1024 + 3 },
},
'4.0.0': {
version: '4.0.0',
dist: { tarball: 'http://foo.com/d.tgz', size: 100 * 1024 * 1024 + 4 },
},
},
}),
),
res: {},
headers: {},
});
mock(app.config.cnpmcore, 'enableSyncUnpkgFilesWhiteList', true);
mock(app.config.cnpmcore, 'largePackageVersionSize', 100 * 1024 * 1024);
mock(app.config.cnpmcore, 'largePackageVersionBlockThreshold', 3);
const name = 'cnpmcore-test-sync-deprecated';
await packageSyncerService.createTask(name);
const task = await packageSyncerService.findExecuteTask();
assert.ok(task);
assert.equal(task.targetName, name);
await packageSyncerService.executeTask(task);
const stream = await packageSyncerService.findTaskLog(task);
assert.ok(stream);
const log = await TestUtil.readStreamToLog(stream);
// console.log(log);
assert.match(log, /❌❌❌❌❌ cnpmcore-test-sync-deprecated ❌❌❌❌❌/);
assert.match(log, /too many large versions/);
});

it('should mock large package version size allow', async () => {
app.mockHttpclient('https://registry.npmjs.org/Buffer/-/Buffer-0.0.0.tgz', 'GET', {
data: await TestUtil.readFixturesFile('registry.npmjs.org/foobar/-/foobar-1.0.0.tgz'),
Expand Down Expand Up @@ -2466,7 +2510,7 @@ describe('test/core/service/PackageSyncerService/executeTaskWithPackument.test.t
app.mockAgent().assertNoPendingInterceptors();
});

it('issue-943: should block first then allow', async () => {
it('issue-943: should skip large version and sync rest, then allow all', async () => {
app.mockHttpclient('https://registry.npmjs.org/Buffer/-/Buffer-0.0.0.tgz', 'GET', {
data: await TestUtil.readFixturesFile('registry.npmjs.org/foobar/-/foobar-1.0.0.tgz'),
persist: false,
Expand Down Expand Up @@ -2556,17 +2600,19 @@ describe('test/core/service/PackageSyncerService/executeTaskWithPackument.test.t
assert.ok(stream);
log = await TestUtil.readStreamToLog(stream);
// console.log(log);
assert.match(log, /❌❌❌❌❌ cnpmcore-test-sync-deprecated ❌❌❌❌❌/);
assert.match(
log,
/Synced version 99.0.0-beta.0 fail, large package version size: 104857601, allow size: 104857600, see https:\/\/github\.com\/cnpm\/unpkg-white-list/,
);
// should at least one version success
// assert.match(log, /Synced version .+? success/);
// large version should be skipped, not fail the task
assert.match(log, /Synced version 99.0.0-beta.0 skipped, large package version size: 104857601/);
// other versions should be synced successfully
data = await packageManagerService.listPackageFullManifests('', name);
assert(data.data?.versions['0.0.0']);

// again should allow
assert(data.data?.versions['1.0.0']);
assert(data.data?.versions['1.0.1']);
assert(data.data?.versions['1.1.0']);
assert(data.data?.versions['1.2.0']);
// 99.0.0-beta.0 was skipped
assert(!data.data?.versions['99.0.0-beta.0']);

// again should allow with whitelist
mock.data(NPMRegistry.prototype, 'getFullManifestsBuffer', {
data: Buffer.from(
JSON.stringify({
Expand Down
Loading