diff --git a/scripts/package-inspector-nightly-scan.test.ts b/scripts/package-inspector-nightly-scan.test.ts index c4413021..927f2c91 100644 --- a/scripts/package-inspector-nightly-scan.test.ts +++ b/scripts/package-inspector-nightly-scan.test.ts @@ -6,6 +6,7 @@ import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { acknowledgeBatch, + downloadPackageArtifactForScan, prepareExtractedPluginRoot, prepareBulkOpenClawTarget, resolveNightlyOpenClawTarget, @@ -20,6 +21,7 @@ import { const temporaryRoots: string[] = []; afterEach(async () => { + vi.unstubAllGlobals(); await Promise.all(temporaryRoots.splice(0).map((root) => rm(root, { recursive: true }))); }); @@ -108,6 +110,140 @@ describe("package-inspector-nightly-scan", () => { expect(resolveArtifactKind("npm-pack", new Headers())).toBe("npm-pack"); }); + it("reconstructs a historical legacy package after the protected archive exhausts memory", async () => { + const workRoot = await mkdtemp(path.join(tmpdir(), "clawhub-inspector-large-legacy-")); + temporaryRoots.push(workRoot); + const pluginRoot = path.join(workRoot, "plugin"); + await mkdir(pluginRoot, { recursive: true }); + const packageJson = "demo-json\n"; + const payload = "payload"; + const fetchMock = vi + .fn() + .mockResolvedValueOnce(new Response("Server Error", { status: 500 })) + .mockResolvedValueOnce( + Response.json({ + version: { + files: [ + { + path: "package.json", + size: 10, + sha256: "2386ce4f9ff896e68d02f8d831311d17f966d6f91bdbf2c8b9085bbf2f84417d", + }, + { + path: "output/payload.bin", + size: 7, + sha256: "239f59ed55e737c77147cf55ad0c1b030b6d7ee748a7426952f9b852d5a935e5", + }, + ], + }, + }), + ) + .mockResolvedValueOnce(new Response(packageJson)) + .mockResolvedValueOnce(new Response(payload)); + vi.stubGlobal("fetch", fetchMock); + + await expect( + downloadPackageArtifactForScan( + { + packageId: "packages:demo", + releaseId: "packageReleases:demo-1", + packageName: "@demo/large-plugin", + version: "1.0.0", + artifactKind: "legacy-zip", + downloadUrl: + "https://clawhub.ai/api/v1/package-inspector/artifact?releaseId=packageReleases%3Ademo-1", + }, + workRoot, + pluginRoot, + ), + ).resolves.toBe("legacy-zip"); + + await expect(readFile(path.join(pluginRoot, "package", "package.json"), "utf8")).resolves.toBe( + packageJson, + ); + await expect( + readFile(path.join(pluginRoot, "package", "output", "payload.bin"), "utf8"), + ).resolves.toBe(payload); + expect(fetchMock.mock.calls.map(([url]) => String(url))).toEqual([ + "https://clawhub.ai/api/v1/package-inspector/artifact?releaseId=packageReleases%3Ademo-1", + "https://clawhub.ai/api/v1/packages/%40demo%2Flarge-plugin/versions/1.0.0", + "https://clawhub.ai/api/v1/packages/%40demo%2Flarge-plugin/file?path=package.json&version=1.0.0", + "https://clawhub.ai/api/v1/packages/%40demo%2Flarge-plugin/file?path=output%2Fpayload.bin&version=1.0.0", + ]); + }); + + it("fails closed when a reconstructed legacy file does not match its manifest checksum", async () => { + const workRoot = await mkdtemp(path.join(tmpdir(), "clawhub-inspector-legacy-checksum-")); + temporaryRoots.push(workRoot); + const pluginRoot = path.join(workRoot, "plugin"); + await mkdir(pluginRoot, { recursive: true }); + vi.stubGlobal( + "fetch", + vi + .fn() + .mockResolvedValueOnce(new Response("Server Error", { status: 500 })) + .mockResolvedValueOnce( + Response.json({ + version: { + files: [{ path: "package.json", size: 7, sha256: "0".repeat(64) }], + }, + }), + ) + .mockResolvedValueOnce(new Response("payload")), + ); + + await expect( + downloadPackageArtifactForScan( + { + packageId: "packages:demo", + releaseId: "packageReleases:demo-1", + packageName: "demo", + version: "1.0.0", + artifactKind: "legacy-zip", + downloadUrl: "https://clawhub.ai/api/v1/package-inspector/artifact", + }, + workRoot, + pluginRoot, + ), + ).rejects.toThrow("legacy package file checksum mismatch for package.json"); + await expect(access(path.join(pluginRoot, "package", "package.json"))).rejects.toThrow(); + }); + + it("fails closed when a reconstructed legacy manifest contains an unsafe path", async () => { + const workRoot = await mkdtemp(path.join(tmpdir(), "clawhub-inspector-legacy-path-")); + temporaryRoots.push(workRoot); + const pluginRoot = path.join(workRoot, "plugin"); + await mkdir(pluginRoot, { recursive: true }); + const fetchMock = vi + .fn() + .mockResolvedValueOnce(new Response("Server Error", { status: 500 })) + .mockResolvedValueOnce( + Response.json({ + version: { + files: [{ path: "../outside", size: 7, sha256: "0".repeat(64) }], + }, + }), + ); + vi.stubGlobal("fetch", fetchMock); + + await expect( + downloadPackageArtifactForScan( + { + packageId: "packages:demo", + releaseId: "packageReleases:demo-1", + packageName: "demo", + version: "1.0.0", + artifactKind: "legacy-zip", + downloadUrl: "https://clawhub.ai/api/v1/package-inspector/artifact", + }, + workRoot, + pluginRoot, + ), + ).rejects.toThrow("legacy package contains unsafe file path: ../outside"); + expect(fetchMock).toHaveBeenCalledTimes(2); + await expect(access(path.join(workRoot, "outside"))).rejects.toThrow(); + }); + it("removes only verified POSIX archive metadata before inspecting a legacy plugin", async () => { const extractedRoot = await mkdtemp(path.join(tmpdir(), "clawhub-inspector-pax-")); temporaryRoots.push(extractedRoot); diff --git a/scripts/package-inspector-nightly-scan.ts b/scripts/package-inspector-nightly-scan.ts index 90bdba47..21dafcbd 100644 --- a/scripts/package-inspector-nightly-scan.ts +++ b/scripts/package-inspector-nightly-scan.ts @@ -1,5 +1,5 @@ import { spawnSync } from "node:child_process"; -import { randomUUID } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { existsSync, readFileSync } from "node:fs"; import { mkdir, readFile, readdir, rm, rmdir, stat, writeFile } from "node:fs/promises"; import { createRequire } from "node:module"; @@ -271,23 +271,7 @@ async function inspectPackageItem( await mkdir(pluginRoot, { recursive: true }); await mkdir(reportDir, { recursive: true }); try { - const artifact = await fetch(item.downloadUrl, { - headers: { Authorization: `Bearer ${token}` }, - }); - if (!artifact.ok) { - throw new Error(`download failed ${artifact.status}: ${await artifact.text()}`); - } - const artifactKind = resolveArtifactKind(item.artifactKind, artifact.headers); - const artifactPath = path.join( - workRoot, - artifactKind === "npm-pack" ? "plugin.tgz" : "plugin.zip", - ); - await writeFile(artifactPath, Buffer.from(await artifact.arrayBuffer())); - if (artifactKind === "npm-pack") { - run("tar", ["-xzf", artifactPath, "-C", pluginRoot, "--strip-components=1"]); - } else { - run("unzip", ["-q", artifactPath, "-d", pluginRoot]); - } + const artifactKind = await downloadPackageArtifactForScan(item, workRoot, pluginRoot); const scanRoot = await prepareExtractedPluginRoot(pluginRoot, artifactKind, item.packageName); if (!inspectorModule.pluginRoot || !inspectorModule.ci || !inspectorModule.reports) { throw new Error("The bundled Plugin Inspector bulk APIs are unavailable"); @@ -356,6 +340,112 @@ async function inspectPackageItem( } } +export async function downloadPackageArtifactForScan( + item: ClaimItem, + workRoot: string, + pluginRoot: string, +) { + const artifact = await fetch(item.downloadUrl, { + headers: { Authorization: `Bearer ${token}` }, + }); + if (!artifact.ok) { + const detail = await artifact.text(); + if (artifact.status === 500 && item.artifactKind === "legacy-zip") { + await downloadLegacyPackageFiles(item, pluginRoot); + console.warn( + `Protected legacy archive failed for ${item.packageName}@${item.version}; reconstructed verified files instead.`, + ); + return "legacy-zip" as const; + } + throw new Error(`download failed ${artifact.status}: ${detail}`); + } + + const artifactKind = resolveArtifactKind(item.artifactKind, artifact.headers); + const artifactPath = path.join( + workRoot, + artifactKind === "npm-pack" ? "plugin.tgz" : "plugin.zip", + ); + await writeFile(artifactPath, Buffer.from(await artifact.arrayBuffer())); + if (artifactKind === "npm-pack") { + run("tar", ["-xzf", artifactPath, "-C", pluginRoot, "--strip-components=1"]); + } else { + run("unzip", ["-q", artifactPath, "-d", pluginRoot]); + } + return artifactKind; +} + +async function downloadLegacyPackageFiles(item: ClaimItem, pluginRoot: string) { + const versionUrl = new URL( + `/api/v1/packages/${encodeURIComponent(item.packageName)}/versions/${encodeURIComponent(item.version)}`, + siteUrl, + ); + const detail = await fetch(versionUrl); + if (!detail.ok) { + throw new Error(`legacy package lookup failed ${detail.status}: ${await detail.text()}`); + } + const payload = (await detail.json()) as unknown; + if (!isPlainObject(payload) || !isPlainObject(payload.version)) { + throw new Error("legacy package lookup returned no version"); + } + const files = payload.version.files; + if (!Array.isArray(files) || files.length === 0) { + throw new Error("legacy package lookup returned no files"); + } + + const packageRoot = path.join(pluginRoot, "package"); + for (const value of files) { + if (!isPlainObject(value)) throw new Error("legacy package lookup returned an invalid file"); + const filePath = stringValue(value.path); + const expectedSha256 = stringValue(value.sha256); + const expectedSize = typeof value.size === "number" ? value.size : undefined; + if (!filePath || !expectedSha256 || expectedSize === undefined) { + throw new Error("legacy package lookup returned incomplete file metadata"); + } + const destination = resolveLegacyScanFilePath(packageRoot, filePath); + const fileUrl = new URL( + `/api/v1/packages/${encodeURIComponent(item.packageName)}/file`, + siteUrl, + ); + fileUrl.searchParams.set("path", filePath); + fileUrl.searchParams.set("version", item.version); + const response = await fetch(fileUrl); + if (!response.ok) { + throw new Error( + `legacy package file download failed for ${filePath} ${response.status}: ${await response.text()}`, + ); + } + const bytes = Buffer.from(await response.arrayBuffer()); + if (bytes.byteLength !== expectedSize) { + throw new Error( + `legacy package file size mismatch for ${filePath}: expected ${expectedSize}, got ${bytes.byteLength}`, + ); + } + const actualSha256 = createHash("sha256").update(bytes).digest("hex"); + if (actualSha256 !== expectedSha256) { + throw new Error(`legacy package file checksum mismatch for ${filePath}`); + } + await mkdir(path.dirname(destination), { recursive: true }); + await writeFile(destination, bytes); + } +} + +function resolveLegacyScanFilePath(packageRoot: string, filePath: string) { + if ( + filePath.length > 500 || + filePath !== filePath.trim() || + filePath.startsWith("/") || + filePath.includes("\\") || + filePath.includes("\0") + ) { + throw new Error(`legacy package contains unsafe file path: ${filePath}`); + } + const segments = filePath.split("/"); + if (segments.some((segment) => segment === "" || segment === "." || segment === "..")) { + throw new Error(`legacy package contains unsafe file path: ${filePath}`); + } + return path.join(packageRoot, ...segments); +} + export function resolveArtifactKind(value: string | undefined, headers: Headers) { if (value === "npm-pack" || value === "legacy-zip") return value; const header = headers.get("X-ClawHub-Artifact-Type")?.trim(); @@ -570,7 +660,7 @@ function hasInspectorConfig(value: unknown) { return isPlainObject(record.pluginInspector) || isPlainObject(record["plugin-inspector"]); } -function isPlainObject(value: unknown) { +function isPlainObject(value: unknown): value is Record { return Boolean(value && typeof value === "object" && !Array.isArray(value)); }