diff --git a/scripts/package-inspector-nightly-scan.test.ts b/scripts/package-inspector-nightly-scan.test.ts index 927f2c91..04b93690 100644 --- a/scripts/package-inspector-nightly-scan.test.ts +++ b/scripts/package-inspector-nightly-scan.test.ts @@ -209,6 +209,63 @@ describe("package-inspector-nightly-scan", () => { await expect(access(path.join(pluginRoot, "package", "package.json"))).rejects.toThrow(); }); + it.each([ + ["delay seconds", "0"], + ["HTTP date", new Date(0).toUTCString()], + ])( + "retries a transient rate-limit contention response with Retry-After %s", + async (_label, retryAfter) => { + const workRoot = await mkdtemp(path.join(tmpdir(), "clawhub-inspector-legacy-retry-")); + temporaryRoots.push(workRoot); + const pluginRoot = path.join(workRoot, "plugin"); + await mkdir(pluginRoot, { recursive: true }); + const payload = "payload"; + const fetchMock = vi + .fn() + .mockResolvedValueOnce(new Response("Server Error", { status: 500 })) + .mockResolvedValueOnce( + Response.json({ + version: { + files: [ + { + path: "package.json", + size: 7, + sha256: "239f59ed55e737c77147cf55ad0c1b030b6d7ee748a7426952f9b852d5a935e5", + }, + ], + }, + }), + ) + .mockResolvedValueOnce( + new Response("Rate limit temporarily unavailable", { + status: 503, + headers: { "Retry-After": retryAfter }, + }), + ) + .mockResolvedValueOnce(new Response(payload)); + vi.stubGlobal("fetch", fetchMock); + + await expect( + downloadPackageArtifactForScan( + { + packageId: "packages:demo", + releaseId: "packageReleases:demo-1", + packageName: "demo", + version: "1.0.0", + artifactKind: "legacy-zip", + downloadUrl: "https://clawhub.ai/api/v1/package-inspector/artifact", + }, + workRoot, + pluginRoot, + ), + ).resolves.toBe("legacy-zip"); + expect(fetchMock).toHaveBeenCalledTimes(4); + await expect( + readFile(path.join(pluginRoot, "package", "package.json"), "utf8"), + ).resolves.toBe(payload); + }, + ); + it("fails closed when a reconstructed legacy manifest contains an unsafe path", async () => { const workRoot = await mkdtemp(path.join(tmpdir(), "clawhub-inspector-legacy-path-")); temporaryRoots.push(workRoot); diff --git a/scripts/package-inspector-nightly-scan.ts b/scripts/package-inspector-nightly-scan.ts index 21dafcbd..d4b047a8 100644 --- a/scripts/package-inspector-nightly-scan.ts +++ b/scripts/package-inspector-nightly-scan.ts @@ -100,6 +100,7 @@ const dryRunMaxBatches = Math.max( const artifactRoot = process.env.PLUGIN_INSPECTOR_ARTIFACT_DIR ?? "plugin-inspector-bulk-scan-reports"; const scanRunId = resolveScanRunId(process.env); +const legacyFileDownloadAttempts = 4; export function resolveScanRunId(env: Record) { return env.PLUGIN_INSPECTOR_RUN_ID?.trim() || env.GITHUB_RUN_ID?.trim() || randomUUID(); @@ -408,7 +409,7 @@ async function downloadLegacyPackageFiles(item: ClaimItem, pluginRoot: string) { ); fileUrl.searchParams.set("path", filePath); fileUrl.searchParams.set("version", item.version); - const response = await fetch(fileUrl); + const response = await fetchLegacyPackageFile(fileUrl, filePath); if (!response.ok) { throw new Error( `legacy package file download failed for ${filePath} ${response.status}: ${await response.text()}`, @@ -429,6 +430,31 @@ async function downloadLegacyPackageFiles(item: ClaimItem, pluginRoot: string) { } } +async function fetchLegacyPackageFile(fileUrl: URL, filePath: string) { + for (let attempt = 1; attempt <= legacyFileDownloadAttempts; attempt += 1) { + const response = await fetch(fileUrl); + const retryAfter = response.headers.get("Retry-After"); + if (response.ok || response.status !== 503 || retryAfter === null) return response; + if (attempt === legacyFileDownloadAttempts) return response; + const delayMs = parseRetryAfterDelayMs(retryAfter); + if (delayMs === undefined) return response; + await response.body?.cancel(); + console.warn( + `Legacy package file download temporarily unavailable for ${filePath}; retrying in ${delayMs}ms (attempt ${attempt + 1}/${legacyFileDownloadAttempts}).`, + ); + await new Promise((resolve) => setTimeout(resolve, delayMs)); + } + throw new Error(`legacy package file retry exhausted unexpectedly for ${filePath}`); +} + +function parseRetryAfterDelayMs(value: string) { + const seconds = Number(value); + if (Number.isSafeInteger(seconds) && seconds >= 0) return Math.min(seconds * 1000, 5000); + const retryAt = Date.parse(value); + if (!Number.isFinite(retryAt)) return undefined; + return Math.min(Math.max(0, retryAt - Date.now()), 5000); +} + function resolveLegacyScanFilePath(packageRoot: string, filePath: string) { if ( filePath.length > 500 ||