fix: reconstruct large legacy scan packages (#3345)

2026-08-14 00:47:57 +00:00 · 2026-07-31 01:33:25 -07:00
parent 110f92a0ef
commit 536674f49b
2 changed files with 245 additions and 19 deletions
@@ -6,6 +6,7 @@ import path from "node:path";
 import { afterEach, describe, expect, it, vi } from "vitest";
 import {
  acknowledgeBatch,
+  downloadPackageArtifactForScan,
  prepareExtractedPluginRoot,
  prepareBulkOpenClawTarget,
  resolveNightlyOpenClawTarget,
@@ -20,6 +21,7 @@ import {
 const temporaryRoots: string[] = [];

 afterEach(async () => {
+  vi.unstubAllGlobals();
  await Promise.all(temporaryRoots.splice(0).map((root) => rm(root, { recursive: true })));
 });

@@ -108,6 +110,140 @@ describe("package-inspector-nightly-scan", () => {
    expect(resolveArtifactKind("npm-pack", new Headers())).toBe("npm-pack");
  });

+  it("reconstructs a historical legacy package after the protected archive exhausts memory", async () => {
+    const workRoot = await mkdtemp(path.join(tmpdir(), "clawhub-inspector-large-legacy-"));
+    temporaryRoots.push(workRoot);
+    const pluginRoot = path.join(workRoot, "plugin");
+    await mkdir(pluginRoot, { recursive: true });
+    const packageJson = "demo-json\n";
+    const payload = "payload";
+    const fetchMock = vi
+      .fn()
+      .mockResolvedValueOnce(new Response("Server Error", { status: 500 }))
+      .mockResolvedValueOnce(
+        Response.json({
+          version: {
+            files: [
+              {
+                path: "package.json",
+                size: 10,
+                sha256: "2386ce4f9ff896e68d02f8d831311d17f966d6f91bdbf2c8b9085bbf2f84417d",
+              },
+              {
+                path: "output/payload.bin",
+                size: 7,
+                sha256: "239f59ed55e737c77147cf55ad0c1b030b6d7ee748a7426952f9b852d5a935e5",
+              },
+            ],
+          },
+        }),
+      )
+      .mockResolvedValueOnce(new Response(packageJson))
+      .mockResolvedValueOnce(new Response(payload));
+    vi.stubGlobal("fetch", fetchMock);
+
+    await expect(
+      downloadPackageArtifactForScan(
+        {
+          packageId: "packages:demo",
+          releaseId: "packageReleases:demo-1",
+          packageName: "@demo/large-plugin",
+          version: "1.0.0",
+          artifactKind: "legacy-zip",
+          downloadUrl:
+            "https://clawhub.ai/api/v1/package-inspector/artifact?releaseId=packageReleases%3Ademo-1",
+        },
+        workRoot,
+        pluginRoot,
+      ),
+    ).resolves.toBe("legacy-zip");
+
+    await expect(readFile(path.join(pluginRoot, "package", "package.json"), "utf8")).resolves.toBe(
+      packageJson,
+    );
+    await expect(
+      readFile(path.join(pluginRoot, "package", "output", "payload.bin"), "utf8"),
+    ).resolves.toBe(payload);
+    expect(fetchMock.mock.calls.map(([url]) => String(url))).toEqual([
+      "https://clawhub.ai/api/v1/package-inspector/artifact?releaseId=packageReleases%3Ademo-1",
+      "https://clawhub.ai/api/v1/packages/%40demo%2Flarge-plugin/versions/1.0.0",
+      "https://clawhub.ai/api/v1/packages/%40demo%2Flarge-plugin/file?path=package.json&version=1.0.0",
+      "https://clawhub.ai/api/v1/packages/%40demo%2Flarge-plugin/file?path=output%2Fpayload.bin&version=1.0.0",
+    ]);
+  });
+
+  it("fails closed when a reconstructed legacy file does not match its manifest checksum", async () => {
+    const workRoot = await mkdtemp(path.join(tmpdir(), "clawhub-inspector-legacy-checksum-"));
+    temporaryRoots.push(workRoot);
+    const pluginRoot = path.join(workRoot, "plugin");
+    await mkdir(pluginRoot, { recursive: true });
+    vi.stubGlobal(
+      "fetch",
+      vi
+        .fn()
+        .mockResolvedValueOnce(new Response("Server Error", { status: 500 }))
+        .mockResolvedValueOnce(
+          Response.json({
+            version: {
+              files: [{ path: "package.json", size: 7, sha256: "0".repeat(64) }],
+            },
+          }),
+        )
+        .mockResolvedValueOnce(new Response("payload")),
+    );
+
+    await expect(
+      downloadPackageArtifactForScan(
+        {
+          packageId: "packages:demo",
+          releaseId: "packageReleases:demo-1",
+          packageName: "demo",
+          version: "1.0.0",
+          artifactKind: "legacy-zip",
+          downloadUrl: "https://clawhub.ai/api/v1/package-inspector/artifact",
+        },
+        workRoot,
+        pluginRoot,
+      ),
+    ).rejects.toThrow("legacy package file checksum mismatch for package.json");
+    await expect(access(path.join(pluginRoot, "package", "package.json"))).rejects.toThrow();
+  });
+
+  it("fails closed when a reconstructed legacy manifest contains an unsafe path", async () => {
+    const workRoot = await mkdtemp(path.join(tmpdir(), "clawhub-inspector-legacy-path-"));
+    temporaryRoots.push(workRoot);
+    const pluginRoot = path.join(workRoot, "plugin");
+    await mkdir(pluginRoot, { recursive: true });
+    const fetchMock = vi
+      .fn()
+      .mockResolvedValueOnce(new Response("Server Error", { status: 500 }))
+      .mockResolvedValueOnce(
+        Response.json({
+          version: {
+            files: [{ path: "../outside", size: 7, sha256: "0".repeat(64) }],
+          },
+        }),
+      );
+    vi.stubGlobal("fetch", fetchMock);
+
+    await expect(
+      downloadPackageArtifactForScan(
+        {
+          packageId: "packages:demo",
+          releaseId: "packageReleases:demo-1",
+          packageName: "demo",
+          version: "1.0.0",
+          artifactKind: "legacy-zip",
+          downloadUrl: "https://clawhub.ai/api/v1/package-inspector/artifact",
+        },
+        workRoot,
+        pluginRoot,
+      ),
+    ).rejects.toThrow("legacy package contains unsafe file path: ../outside");
+    expect(fetchMock).toHaveBeenCalledTimes(2);
+    await expect(access(path.join(workRoot, "outside"))).rejects.toThrow();
+  });
+
  it("removes only verified POSIX archive metadata before inspecting a legacy plugin", async () => {
    const extractedRoot = await mkdtemp(path.join(tmpdir(), "clawhub-inspector-pax-"));
    temporaryRoots.push(extractedRoot);
@@ -1,5 +1,5 @@
 import { spawnSync } from "node:child_process";
-import { randomUUID } from "node:crypto";
+import { createHash, randomUUID } from "node:crypto";
 import { existsSync, readFileSync } from "node:fs";
 import { mkdir, readFile, readdir, rm, rmdir, stat, writeFile } from "node:fs/promises";
 import { createRequire } from "node:module";
@@ -271,23 +271,7 @@ async function inspectPackageItem(
  await mkdir(pluginRoot, { recursive: true });
  await mkdir(reportDir, { recursive: true });
  try {
-    const artifact = await fetch(item.downloadUrl, {
-      headers: { Authorization: `Bearer ${token}` },
-    });
-    if (!artifact.ok) {
-      throw new Error(`download failed ${artifact.status}: ${await artifact.text()}`);
-    }
-    const artifactKind = resolveArtifactKind(item.artifactKind, artifact.headers);
-    const artifactPath = path.join(
-      workRoot,
-      artifactKind === "npm-pack" ? "plugin.tgz" : "plugin.zip",
-    );
-    await writeFile(artifactPath, Buffer.from(await artifact.arrayBuffer()));
-    if (artifactKind === "npm-pack") {
-      run("tar", ["-xzf", artifactPath, "-C", pluginRoot, "--strip-components=1"]);
-    } else {
-      run("unzip", ["-q", artifactPath, "-d", pluginRoot]);
-    }
+    const artifactKind = await downloadPackageArtifactForScan(item, workRoot, pluginRoot);
    const scanRoot = await prepareExtractedPluginRoot(pluginRoot, artifactKind, item.packageName);
    if (!inspectorModule.pluginRoot || !inspectorModule.ci || !inspectorModule.reports) {
      throw new Error("The bundled Plugin Inspector bulk APIs are unavailable");
@@ -356,6 +340,112 @@ async function inspectPackageItem(
  }
 }

+export async function downloadPackageArtifactForScan(
+  item: ClaimItem,
+  workRoot: string,
+  pluginRoot: string,
+) {
+  const artifact = await fetch(item.downloadUrl, {
+    headers: { Authorization: `Bearer ${token}` },
+  });
+  if (!artifact.ok) {
+    const detail = await artifact.text();
+    if (artifact.status === 500 && item.artifactKind === "legacy-zip") {
+      await downloadLegacyPackageFiles(item, pluginRoot);
+      console.warn(
+        `Protected legacy archive failed for ${item.packageName}@${item.version}; reconstructed verified files instead.`,
+      );
+      return "legacy-zip" as const;
+    }
+    throw new Error(`download failed ${artifact.status}: ${detail}`);
+  }
+
+  const artifactKind = resolveArtifactKind(item.artifactKind, artifact.headers);
+  const artifactPath = path.join(
+    workRoot,
+    artifactKind === "npm-pack" ? "plugin.tgz" : "plugin.zip",
+  );
+  await writeFile(artifactPath, Buffer.from(await artifact.arrayBuffer()));
+  if (artifactKind === "npm-pack") {
+    run("tar", ["-xzf", artifactPath, "-C", pluginRoot, "--strip-components=1"]);
+  } else {
+    run("unzip", ["-q", artifactPath, "-d", pluginRoot]);
+  }
+  return artifactKind;
+}
+
+async function downloadLegacyPackageFiles(item: ClaimItem, pluginRoot: string) {
+  const versionUrl = new URL(
+    `/api/v1/packages/${encodeURIComponent(item.packageName)}/versions/${encodeURIComponent(item.version)}`,
+    siteUrl,
+  );
+  const detail = await fetch(versionUrl);
+  if (!detail.ok) {
+    throw new Error(`legacy package lookup failed ${detail.status}: ${await detail.text()}`);
+  }
+  const payload = (await detail.json()) as unknown;
+  if (!isPlainObject(payload) || !isPlainObject(payload.version)) {
+    throw new Error("legacy package lookup returned no version");
+  }
+  const files = payload.version.files;
+  if (!Array.isArray(files) || files.length === 0) {
+    throw new Error("legacy package lookup returned no files");
+  }
+
+  const packageRoot = path.join(pluginRoot, "package");
+  for (const value of files) {
+    if (!isPlainObject(value)) throw new Error("legacy package lookup returned an invalid file");
+    const filePath = stringValue(value.path);
+    const expectedSha256 = stringValue(value.sha256);
+    const expectedSize = typeof value.size === "number" ? value.size : undefined;
+    if (!filePath || !expectedSha256 || expectedSize === undefined) {
+      throw new Error("legacy package lookup returned incomplete file metadata");
+    }
+    const destination = resolveLegacyScanFilePath(packageRoot, filePath);
+    const fileUrl = new URL(
+      `/api/v1/packages/${encodeURIComponent(item.packageName)}/file`,
+      siteUrl,
+    );
+    fileUrl.searchParams.set("path", filePath);
+    fileUrl.searchParams.set("version", item.version);
+    const response = await fetch(fileUrl);
+    if (!response.ok) {
+      throw new Error(
+        `legacy package file download failed for ${filePath} ${response.status}: ${await response.text()}`,
+      );
+    }
+    const bytes = Buffer.from(await response.arrayBuffer());
+    if (bytes.byteLength !== expectedSize) {
+      throw new Error(
+        `legacy package file size mismatch for ${filePath}: expected ${expectedSize}, got ${bytes.byteLength}`,
+      );
+    }
+    const actualSha256 = createHash("sha256").update(bytes).digest("hex");
+    if (actualSha256 !== expectedSha256) {
+      throw new Error(`legacy package file checksum mismatch for ${filePath}`);
+    }
+    await mkdir(path.dirname(destination), { recursive: true });
+    await writeFile(destination, bytes);
+  }
+}
+
+function resolveLegacyScanFilePath(packageRoot: string, filePath: string) {
+  if (
+    filePath.length > 500 ||
+    filePath !== filePath.trim() ||
+    filePath.startsWith("/") ||
+    filePath.includes("\\") ||
+    filePath.includes("\0")
+  ) {
+    throw new Error(`legacy package contains unsafe file path: ${filePath}`);
+  }
+  const segments = filePath.split("/");
+  if (segments.some((segment) => segment === "" || segment === "." || segment === "..")) {
+    throw new Error(`legacy package contains unsafe file path: ${filePath}`);
+  }
+  return path.join(packageRoot, ...segments);
+}
+
 export function resolveArtifactKind(value: string | undefined, headers: Headers) {
  if (value === "npm-pack" || value === "legacy-zip") return value;
  const header = headers.get("X-ClawHub-Artifact-Type")?.trim();
@@ -570,7 +660,7 @@ function hasInspectorConfig(value: unknown) {
  return isPlainObject(record.pluginInspector) || isPlainObject(record["plugin-inspector"]);
 }

-function isPlainObject(value: unknown) {
+function isPlainObject(value: unknown): value is Record<string, unknown> {
  return Boolean(value && typeof value === "object" && !Array.isArray(value));
 }