fix: preserve legacy package ZIP hashes (#2636)

* test: fix indexed row helper lint

* fix: preserve legacy package zip hashes
This commit is contained in:
Patrick Erichsen
2026-06-14 18:56:31 -07:00
committed by GitHub
parent 6ae7cb5345
commit 03f97349b0
24 changed files with 843 additions and 83 deletions
+4
View File
@@ -76,6 +76,7 @@ import type * as lib_moderationReasonCodes from "../lib/moderationReasonCodes.js
import type * as lib_observabilityEvents from "../lib/observabilityEvents.js";
import type * as lib_officialPublishers from "../lib/officialPublishers.js";
import type * as lib_openaiResponse from "../lib/openaiResponse.js";
import type * as lib_packageArtifacts from "../lib/packageArtifacts.js";
import type * as lib_packageRegistry from "../lib/packageRegistry.js";
import type * as lib_packageSearchDigest from "../lib/packageSearchDigest.js";
import type * as lib_packageSecurity from "../lib/packageSecurity.js";
@@ -118,6 +119,7 @@ import type * as maintenance from "../maintenance.js";
import type * as managementDevSeed from "../managementDevSeed.js";
import type * as packageInspectorHttp from "../packageInspectorHttp.js";
import type * as packageInspectorNode from "../packageInspectorNode.js";
import type * as packageLegacyZipHashBackfill from "../packageLegacyZipHashBackfill.js";
import type * as packagePublishTokens from "../packagePublishTokens.js";
import type * as packages from "../packages.js";
import type * as publisherAbuse from "../publisherAbuse.js";
@@ -220,6 +222,7 @@ declare const fullApi: ApiFromModules<{
"lib/observabilityEvents": typeof lib_observabilityEvents;
"lib/officialPublishers": typeof lib_officialPublishers;
"lib/openaiResponse": typeof lib_openaiResponse;
"lib/packageArtifacts": typeof lib_packageArtifacts;
"lib/packageRegistry": typeof lib_packageRegistry;
"lib/packageSearchDigest": typeof lib_packageSearchDigest;
"lib/packageSecurity": typeof lib_packageSecurity;
@@ -262,6 +265,7 @@ declare const fullApi: ApiFromModules<{
managementDevSeed: typeof managementDevSeed;
packageInspectorHttp: typeof packageInspectorHttp;
packageInspectorNode: typeof packageInspectorNode;
packageLegacyZipHashBackfill: typeof packageLegacyZipHashBackfill;
packagePublishTokens: typeof packagePublishTokens;
packages: typeof packages;
publisherAbuse: typeof publisherAbuse;
+3 -2
View File
@@ -9081,7 +9081,7 @@ describe("httpApiV1 handlers", () => {
});
});
it("package security endpoint returns exact release trust and blocked reasons", async () => {
it("package security endpoint uses the canonical npm artifact hash", async () => {
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
if ("name" in args && !("version" in args)) {
return {
@@ -9161,7 +9161,7 @@ describe("httpApiV1 handlers", () => {
releaseId: "packageReleases:1",
version: "1.0.0",
artifactKind: "npm-pack",
artifactSha256: "c".repeat(64),
artifactSha256: "e".repeat(64),
npmIntegrity: "sha512-demo",
npmShasum: "d".repeat(40),
npmTarballName: "demo-plugin-1.0.0.tgz",
@@ -9248,6 +9248,7 @@ describe("httpApiV1 handlers", () => {
files: [],
artifactKind: "npm-pack",
integritySha256: "a".repeat(64),
sha256hash: "b".repeat(64),
npmIntegrity: "sha512-demo",
npmShasum: "d".repeat(40),
npmTarballName: "demo-plugin-1.0.0.tgz",
+2 -8
View File
@@ -36,6 +36,7 @@ import {
} from "../lib/githubActionsOidc";
import { corsHeaders, mergeHeaders } from "../lib/httpHeaders";
import { applyRateLimit } from "../lib/httpRateLimit";
import { getPackageReleaseArtifactSha256 } from "../lib/packageArtifacts";
import { tryNormalizePackageName } from "../lib/packageRegistry";
import {
getPackageDownloadSecurityBlock,
@@ -612,19 +613,12 @@ function toReleaseArtifact(release: ReleaseLike, packageName?: string) {
};
}
function packageReleaseArtifactSha256(release: ReleaseLike) {
if (release.artifactKind === "npm-pack") {
return release.sha256hash ?? release.clawpackSha256 ?? null;
}
return release.sha256hash ?? null;
}
function toPackageReleaseSecurityResponse(params: {
pkg: PublicPackageDocLike;
release: ReleaseLike;
}) {
const scanStatus = resolvePackageReleaseScanStatus(params.release);
const artifactSha256 = packageReleaseArtifactSha256(params.release);
const artifactSha256 = getPackageReleaseArtifactSha256(params.release);
const packageBlockedFromDownload = params.pkg.publicDownloadBlocked === true;
const reasons = getPackageTrustReasons(params.release, scanStatus);
if (packageBlockedFromDownload) reasons.push("package:malicious");
+32
View File
@@ -0,0 +1,32 @@
import { describe, expect, it } from "vitest";
import { getPackageReleaseArtifactSha256 } from "./packageArtifacts";
describe("getPackageReleaseArtifactSha256", () => {
it("uses the exact npm-pack artifact hash instead of the legacy ZIP hash", () => {
expect(
getPackageReleaseArtifactSha256({
artifactKind: "npm-pack",
clawpackSha256: "tgz-sha",
sha256hash: "legacy-zip-sha",
}),
).toBe("tgz-sha");
});
it("does not fall back to the legacy ZIP hash for npm-pack releases", () => {
expect(
getPackageReleaseArtifactSha256({
artifactKind: "npm-pack",
sha256hash: "legacy-zip-sha",
}),
).toBeNull();
});
it("uses the ZIP hash for legacy releases", () => {
expect(
getPackageReleaseArtifactSha256({
artifactKind: "legacy-zip",
sha256hash: "legacy-zip-sha",
}),
).toBe("legacy-zip-sha");
});
});
+15
View File
@@ -0,0 +1,15 @@
import type { Doc } from "../_generated/dataModel";
type PackageReleaseArtifactHashFields = Pick<
Doc<"packageReleases">,
"artifactKind" | "clawpackSha256" | "sha256hash"
>;
export function getPackageReleaseArtifactSha256(
release: PackageReleaseArtifactHashFields,
): string | null {
if (release.artifactKind === "npm-pack") {
return release.clawpackSha256 ?? null;
}
return release.sha256hash ?? null;
}
+318
View File
@@ -0,0 +1,318 @@
/* @vitest-environment node */
import { describe, expect, it, vi } from "vitest";
import { sha256Hex } from "./lib/clawpack";
import { buildDeterministicPackageZip } from "./lib/skillZip";
import {
applyLegacyPackageZipHashBackfillInternal,
backfillLegacyPackageZipHashesInternal,
getLegacyPackageZipHashBackfillBatchInternal,
} from "./packageLegacyZipHashBackfill";
type WrappedHandler<TArgs, TResult> = {
_handler: (ctx: unknown, args: TArgs) => Promise<TResult>;
};
type BackfillResult = {
dryRun: boolean;
done: boolean;
cursor: string | null;
scanned: number;
candidates: number;
matched: number;
wouldPatch: number;
patched: number;
skipped: number;
errors: number;
samples: Array<{
releaseId: string;
packageName: string;
version: string;
currentSha256hash: string | null;
expectedSha256hash?: string;
action: string;
}>;
};
const backfillHandler = (
backfillLegacyPackageZipHashesInternal as unknown as WrappedHandler<
{
dryRun: boolean;
batchSize?: number;
cursor?: string | null;
confirmationToken?: string;
},
BackfillResult
>
)._handler;
const applyHandler = (
applyLegacyPackageZipHashBackfillInternal as unknown as WrappedHandler<
{
releaseId: string;
expectedCurrentSha256hash?: string;
sha256hash: string;
confirmationToken: string;
},
{ status: "patched" | "matched" | "stale" | "skipped" }
>
)._handler;
const getBatchHandler = (
getLegacyPackageZipHashBackfillBatchInternal as unknown as WrappedHandler<
{ batchSize?: number; cursor?: string | null },
{
page: Array<{
releaseId: string;
packageName: string;
version: string;
currentSha256hash: string | null;
}>;
scanned: number;
continueCursor: string;
isDone: boolean;
}
>
)._handler;
const packageJson = '{"name":"demo-plugin"}';
const pluginManifest = '{"id":"demo.plugin"}';
function batch(currentSha256hash = "old-tgz-hash") {
return {
page: [
{
releaseId: "packageReleases:demo",
packageName: "demo-plugin",
version: "1.0.0",
currentSha256hash,
files: [
{ path: "package.json", storageId: "storage:package" },
{ path: "openclaw.plugin.json", storageId: "storage:manifest" },
],
},
],
scanned: 1,
continueCursor: "next-page",
isDone: false,
};
}
function storage() {
return {
get: vi.fn(async (storageId: string) => {
if (storageId === "storage:package") return new Blob([packageJson]);
if (storageId === "storage:manifest") return new Blob([pluginManifest]);
return null;
}),
};
}
async function expectedLegacyZipSha256() {
return await sha256Hex(
buildDeterministicPackageZip([
{ path: "package.json", bytes: new TextEncoder().encode(packageJson) },
{ path: "openclaw.plugin.json", bytes: new TextEncoder().encode(pluginManifest) },
]),
);
}
describe("legacy package ZIP hash backfill", () => {
it("reads a bounded resumable page and selects only npm-pack releases", async () => {
const paginate = vi.fn().mockResolvedValue({
page: [
{
_id: "packageReleases:npm",
packageId: "packages:npm",
version: "1.0.0",
artifactKind: "npm-pack",
files: [],
sha256hash: "old-tgz-hash",
},
{
_id: "packageReleases:soft-deleted-npm",
packageId: "packages:soft-deleted-npm",
version: "2.0.0",
artifactKind: "npm-pack",
files: [],
sha256hash: "old-soft-deleted-tgz-hash",
softDeletedAt: 123,
},
{
_id: "packageReleases:legacy",
packageId: "packages:legacy",
version: "1.0.0",
artifactKind: "legacy-zip",
files: [],
},
],
continueCursor: "next-page",
isDone: false,
});
const order = vi.fn(() => ({ paginate }));
const result = await getBatchHandler(
{
db: {
query: vi.fn(() => ({ order })),
get: vi.fn().mockResolvedValue({ name: "demo-plugin" }),
},
},
{ batchSize: 999, cursor: "current-page" },
);
expect(order).toHaveBeenCalledWith("asc");
expect(paginate).toHaveBeenCalledWith({ cursor: "current-page", numItems: 10 });
expect(result).toMatchObject({
scanned: 3,
continueCursor: "next-page",
isDone: false,
page: [
{
releaseId: "packageReleases:npm",
packageName: "demo-plugin",
currentSha256hash: "old-tgz-hash",
},
{
releaseId: "packageReleases:soft-deleted-npm",
packageName: "demo-plugin",
currentSha256hash: "old-soft-deleted-tgz-hash",
},
],
});
});
it("dry-runs a resumable batch without writing", async () => {
const runMutation = vi.fn();
const result = await backfillHandler(
{
runQuery: vi.fn().mockResolvedValue(batch()),
runMutation,
storage: storage(),
},
{ dryRun: true, batchSize: 1 },
);
expect(result).toMatchObject({
dryRun: true,
done: false,
cursor: "next-page",
scanned: 1,
candidates: 1,
matched: 0,
wouldPatch: 1,
patched: 0,
errors: 0,
});
expect(result.samples).toContainEqual(
expect.objectContaining({
releaseId: "packageReleases:demo",
currentSha256hash: "old-tgz-hash",
expectedSha256hash: await expectedLegacyZipSha256(),
action: "would-patch",
}),
);
expect(runMutation).not.toHaveBeenCalled();
});
it("requires an explicit confirmation token before apply", async () => {
const ctx = {
runQuery: vi.fn(),
runMutation: vi.fn(),
storage: storage(),
};
await expect(backfillHandler(ctx, { dryRun: false })).rejects.toThrow("confirmationToken");
expect(ctx.runQuery).not.toHaveBeenCalled();
expect(ctx.runMutation).not.toHaveBeenCalled();
});
it("applies a mismatched hash through the guarded mutation", async () => {
const expectedSha256hash = await expectedLegacyZipSha256();
const runMutation = vi.fn().mockResolvedValue({ status: "patched" });
const result = await backfillHandler(
{
runQuery: vi.fn().mockResolvedValue(batch()),
runMutation,
storage: storage(),
},
{
dryRun: false,
batchSize: 1,
confirmationToken: "BACKFILL_LEGACY_PACKAGE_ZIP_HASHES",
},
);
expect(runMutation).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
releaseId: "packageReleases:demo",
expectedCurrentSha256hash: "old-tgz-hash",
sha256hash: expectedSha256hash,
confirmationToken: "BACKFILL_LEGACY_PACKAGE_ZIP_HASHES",
}),
);
expect(result).toMatchObject({
dryRun: false,
wouldPatch: 0,
patched: 1,
errors: 0,
});
});
it("does not overwrite a release that changed after the batch read", async () => {
const patch = vi.fn();
const result = await applyHandler(
{
db: {
get: vi.fn().mockResolvedValue({
_id: "packageReleases:demo",
artifactKind: "npm-pack",
sha256hash: "newer-value",
}),
query: vi.fn(),
normalizeId: vi.fn(() => null),
patch,
},
},
{
releaseId: "packageReleases:demo",
expectedCurrentSha256hash: "old-tgz-hash",
sha256hash: await expectedLegacyZipSha256(),
confirmationToken: "BACKFILL_LEGACY_PACKAGE_ZIP_HASHES",
},
);
expect(result).toEqual({ status: "stale" });
expect(patch).not.toHaveBeenCalled();
});
it("patches soft-deleted npm-pack releases so restored downloads stay compatible", async () => {
const patch = vi.fn();
const sha256hash = await expectedLegacyZipSha256();
const result = await applyHandler(
{
db: {
get: vi.fn().mockResolvedValue({
_id: "packageReleases:demo",
artifactKind: "npm-pack",
sha256hash: "old-tgz-hash",
softDeletedAt: 123,
}),
query: vi.fn(),
normalizeId: vi.fn(() => null),
patch,
},
},
{
releaseId: "packageReleases:demo",
expectedCurrentSha256hash: "old-tgz-hash",
sha256hash,
confirmationToken: "BACKFILL_LEGACY_PACKAGE_ZIP_HASHES",
},
);
expect(result).toEqual({ status: "patched" });
expect(patch).toHaveBeenCalledWith("packageReleases:demo", { sha256hash });
});
});
+225
View File
@@ -0,0 +1,225 @@
import { v } from "convex/values";
import { internal } from "./_generated/api";
import type { Id } from "./_generated/dataModel";
import { internalAction, internalMutation, internalQuery } from "./functions";
import { sha256Hex } from "./lib/clawpack";
import { buildDeterministicPackageZip } from "./lib/skillZip";
// Temporary migration exception: computing the compatibility hash requires storage reads and
// deterministic ZIP construction in an action, which @convex-dev/migrations mutations cannot do.
const APPLY_CONFIRMATION_TOKEN = "BACKFILL_LEGACY_PACKAGE_ZIP_HASHES";
const DEFAULT_BATCH_SIZE = 5;
const MAX_BATCH_SIZE = 10;
const MAX_SAMPLES = 25;
const internalRefs = internal as unknown as {
packageLegacyZipHashBackfill: {
getLegacyPackageZipHashBackfillBatchInternal: unknown;
applyLegacyPackageZipHashBackfillInternal: unknown;
};
};
type BackfillTarget = {
releaseId: Id<"packageReleases">;
packageName: string;
version: string;
currentSha256hash: string | null;
files: Array<{ path: string; storageId: Id<"_storage"> }>;
};
type BackfillBatch = {
page: BackfillTarget[];
scanned: number;
continueCursor: string;
isDone: boolean;
};
function effectiveBatchSize(batchSize?: number) {
return Math.max(1, Math.min(Math.floor(batchSize ?? DEFAULT_BATCH_SIZE), MAX_BATCH_SIZE));
}
export const getLegacyPackageZipHashBackfillBatchInternal = internalQuery({
args: {
batchSize: v.optional(v.number()),
cursor: v.optional(v.union(v.string(), v.null())),
},
handler: async (ctx, args): Promise<BackfillBatch> => {
const page = await ctx.db
.query("packageReleases")
.order("asc")
.paginate({
cursor: args.cursor ?? null,
numItems: effectiveBatchSize(args.batchSize),
});
const targets: BackfillTarget[] = [];
for (const release of page.page) {
if (release.artifactKind !== "npm-pack") continue;
const pkg = await ctx.db.get(release.packageId);
targets.push({
releaseId: release._id,
packageName: pkg?.name ?? `<missing:${release.packageId}>`,
version: release.version,
currentSha256hash: release.sha256hash ?? null,
files: release.files.map((file) => ({
path: file.path,
storageId: file.storageId,
})),
});
}
return {
page: targets,
scanned: page.page.length,
continueCursor: page.continueCursor,
isDone: page.isDone,
};
},
});
export const applyLegacyPackageZipHashBackfillInternal = internalMutation({
args: {
releaseId: v.id("packageReleases"),
expectedCurrentSha256hash: v.optional(v.string()),
sha256hash: v.string(),
confirmationToken: v.string(),
},
handler: async (ctx, args) => {
if (args.confirmationToken !== APPLY_CONFIRMATION_TOKEN) {
throw new Error(`Apply requires confirmationToken=${APPLY_CONFIRMATION_TOKEN}`);
}
const release = await ctx.db.get(args.releaseId);
if (!release || release.artifactKind !== "npm-pack") {
return { status: "skipped" as const };
}
if (release.sha256hash === args.sha256hash) {
return { status: "matched" as const };
}
if ((release.sha256hash ?? null) !== (args.expectedCurrentSha256hash ?? null)) {
return { status: "stale" as const };
}
await ctx.db.patch(args.releaseId, { sha256hash: args.sha256hash });
return { status: "patched" as const };
},
});
export const backfillLegacyPackageZipHashesInternal = internalAction({
args: {
dryRun: v.boolean(),
batchSize: v.optional(v.number()),
cursor: v.optional(v.union(v.string(), v.null())),
confirmationToken: v.optional(v.string()),
},
handler: async (ctx, args) => {
if (!args.dryRun && args.confirmationToken !== APPLY_CONFIRMATION_TOKEN) {
throw new Error(`Apply requires confirmationToken=${APPLY_CONFIRMATION_TOKEN}`);
}
const batch = (await ctx.runQuery(
internalRefs.packageLegacyZipHashBackfill
.getLegacyPackageZipHashBackfillBatchInternal as never,
{
batchSize: effectiveBatchSize(args.batchSize),
cursor: args.cursor ?? null,
} as never,
)) as BackfillBatch;
const result = {
dryRun: args.dryRun,
done: batch.isDone,
cursor: batch.isDone ? null : batch.continueCursor,
scanned: batch.scanned,
candidates: batch.page.length,
matched: 0,
wouldPatch: 0,
patched: 0,
skipped: 0,
errors: 0,
samples: [] as Array<{
releaseId: string;
packageName: string;
version: string;
currentSha256hash: string | null;
expectedSha256hash?: string;
action: "matched" | "would-patch" | "patched" | "stale" | "skipped" | "error";
message?: string;
}>,
};
for (const target of batch.page) {
const sampleBase = {
releaseId: target.releaseId,
packageName: target.packageName,
version: target.version,
currentSha256hash: target.currentSha256hash,
};
const entries: Array<{ path: string; bytes: Uint8Array }> = [];
const missingPaths: string[] = [];
for (const file of target.files) {
const blob = await ctx.storage.get(file.storageId);
if (!blob) {
missingPaths.push(file.path);
continue;
}
entries.push({
path: file.path,
bytes: new Uint8Array(await blob.arrayBuffer()),
});
}
if (target.files.length === 0 || missingPaths.length > 0) {
result.errors += 1;
if (result.samples.length < MAX_SAMPLES) {
result.samples.push({
...sampleBase,
action: "error",
message:
target.files.length === 0
? "Release has no stored files"
: `Missing stored files: ${missingPaths.join(", ")}`,
});
}
continue;
}
const expectedSha256hash = await sha256Hex(buildDeterministicPackageZip(entries));
if (target.currentSha256hash === expectedSha256hash) {
result.matched += 1;
if (result.samples.length < MAX_SAMPLES) {
result.samples.push({ ...sampleBase, expectedSha256hash, action: "matched" });
}
continue;
}
if (args.dryRun) {
result.wouldPatch += 1;
if (result.samples.length < MAX_SAMPLES) {
result.samples.push({ ...sampleBase, expectedSha256hash, action: "would-patch" });
}
continue;
}
const mutationResult = (await ctx.runMutation(
internalRefs.packageLegacyZipHashBackfill
.applyLegacyPackageZipHashBackfillInternal as never,
{
releaseId: target.releaseId,
...(target.currentSha256hash
? { expectedCurrentSha256hash: target.currentSha256hash }
: {}),
sha256hash: expectedSha256hash,
confirmationToken: APPLY_CONFIRMATION_TOKEN,
} as never,
)) as { status: "patched" | "matched" | "stale" | "skipped" };
result[mutationResult.status === "stale" ? "skipped" : mutationResult.status] += 1;
if (result.samples.length < MAX_SAMPLES) {
result.samples.push({
...sampleBase,
expectedSha256hash,
action: mutationResult.status,
});
}
}
return result;
},
});
+129 -28
View File
@@ -2,11 +2,13 @@
import { getAuthUserId } from "@convex-dev/auth/server";
import { afterEach, describe, expect, it, vi } from "vitest";
import { sha256Hex } from "./lib/clawpack";
import { MAX_PUBLISH_FILE_BYTES } from "./lib/publishLimits";
import {
computeRecommendationScore,
RECOMMENDATION_SCORE_VERSION,
} from "./lib/recommendationScore";
import { buildDeterministicPackageZip } from "./lib/skillZip";
import {
backfillLatestPackageScanStatusInternal,
backfillPackageReleaseScansInternal,
@@ -217,6 +219,7 @@ const insertReleaseInternalHandler = (
contentType?: string;
}>;
integritySha256: string;
sha256hash?: string;
sourceRepo?: string;
runtimeId?: string;
channel?: "official" | "community" | "private";
@@ -5623,6 +5626,7 @@ describe("packages public queries", () => {
summary: "demo",
files: [],
integritySha256: "abc123",
sha256hash: "legacy-zip-sha",
artifactKind: "npm-pack",
clawpackStorageId: "storage:clawpack",
clawpackSha256: "a".repeat(64),
@@ -5651,6 +5655,7 @@ describe("packages public queries", () => {
capabilities: expect.objectContaining({ capabilityTags: expectedTags }),
artifact: expect.objectContaining({
kind: "npm-pack",
sha256: "a".repeat(64),
npmIntegrity: "sha512-demo",
npmShasum: "b".repeat(40),
}),
@@ -5659,6 +5664,45 @@ describe("packages public queries", () => {
);
});
it("uses the exact legacy ZIP hash in promoted legacy artifact summaries", async () => {
const ctx = makeInsertReleaseCtx(
makePackageDoc({
tags: { latest: "packageReleases:demo-1" },
latestReleaseId: "packageReleases:demo-1",
stats: { downloads: 0, installs: 0, stars: 0, versions: 1 },
}),
);
await insertReleaseInternalHandler(ctx, {
actorUserId: "users:owner",
ownerUserId: "users:owner",
name: "demo-plugin",
displayName: "Demo Plugin",
family: "code-plugin",
version: "1.1.0",
changelog: "legacy zip",
tags: ["latest"],
summary: "demo",
files: [],
integritySha256: "file-set-sha",
sha256hash: "legacy-zip-sha",
artifactKind: "legacy-zip",
});
expect(ctx.patch).toHaveBeenCalledWith(
"packages:demo",
expect.objectContaining({
latestVersionSummary: expect.objectContaining({
artifact: {
kind: "legacy-zip",
sha256: "legacy-zip-sha",
format: "zip",
},
}),
}),
);
});
it("keeps package summary pinned to the promoted release for non-latest publishes", async () => {
const ctx = makeInsertReleaseCtx(
makePackageDoc({
@@ -6481,6 +6525,33 @@ describe("packages public queries", () => {
it("scans plugin publishes and forwards scan status to insertReleaseInternal", async () => {
const runMutation = vi.fn(async (_ref: unknown, args: unknown) => args);
const storedFiles = new Map<string, string>([
[
"storage:package",
JSON.stringify({
name: "demo-plugin",
openclaw: {
extensions: ["./dist/index.js"],
hostTargets: ["darwin-arm64", "linux-x64"],
environment: {},
compat: { pluginApi: "^1.0.0" },
build: { openclawVersion: "2026.3.14" },
configSchema: { type: "object" },
},
}),
],
[
"storage:manifest",
JSON.stringify({
id: "demo.plugin",
tools: [{ name: "demoTool" }],
}),
],
[
"storage:code",
"import { execSync } from 'node:child_process';\nexecSync('curl http://x');\n",
],
]);
const ctx = {
runQuery: vi
.fn()
@@ -6500,34 +6571,7 @@ describe("packages public queries", () => {
},
storage: {
get: vi.fn(async (storageId: string) => {
const files = new Map<string, string>([
[
"storage:package",
JSON.stringify({
name: "demo-plugin",
openclaw: {
extensions: ["./dist/index.js"],
hostTargets: ["darwin-arm64", "linux-x64"],
environment: {},
compat: { pluginApi: "^1.0.0" },
build: { openclawVersion: "2026.3.14" },
configSchema: { type: "object" },
},
}),
],
[
"storage:manifest",
JSON.stringify({
id: "demo.plugin",
tools: [{ name: "demoTool" }],
}),
],
[
"storage:code",
"import { execSync } from 'node:child_process';\nexecSync('curl http://x');\n",
],
]);
const content = files.get(storageId);
const content = storedFiles.get(storageId);
return content ? new Blob([content]) : null;
}),
},
@@ -6585,8 +6629,25 @@ describe("packages public queries", () => {
],
},
})) as Record<string, unknown>;
const expectedLegacyZipSha256 = await sha256Hex(
buildDeterministicPackageZip([
{
path: "package.json",
bytes: new TextEncoder().encode(storedFiles.get("storage:package")),
},
{
path: "openclaw.plugin.json",
bytes: new TextEncoder().encode(storedFiles.get("storage:manifest")),
},
{
path: "dist/index.js",
bytes: new TextEncoder().encode(storedFiles.get("storage:code")),
},
]),
);
expect(runMutation).toHaveBeenCalled();
expect(result.sha256hash).toBe(expectedLegacyZipSha256);
expect(result.verification).toEqual(expect.objectContaining({ scanStatus: "pending" }));
expect(result.staticScan).toEqual(
expect.objectContaining({
@@ -10171,6 +10232,46 @@ describe("package scan backfill", () => {
]);
});
it("does not repeatedly rescan a release solely because its artifact hash is missing", async () => {
const result = await getPackageReleaseScanBackfillBatchInternalHandler(
{
db: {
query: vi.fn((table: string) => {
if (table !== "packageReleases") throw new Error(`Unexpected table ${table}`);
return {
order: vi.fn(() => ({
take: vi.fn().mockResolvedValue([]),
})),
withIndex: vi.fn(() => ({
order: vi.fn(() => ({
take: vi.fn().mockResolvedValue([
{
_id: "packageReleases:npm-missing-artifact-hash",
_creationTime: 10,
packageId: "packages:demo",
artifactKind: "npm-pack",
sha256hash: "legacy-zip-hash",
vtAnalysis: { status: "clean" },
llmAnalysis: { status: "clean" },
staticScan: { status: "clean" },
},
]),
})),
})),
};
}),
get: vi.fn(async (id: string) => {
if (id === "packages:demo") return makePackageDoc();
return null;
}),
},
} as never,
{ batchSize: 10 },
);
expect(result.releases).toEqual([]);
});
it("prioritizes recent releases before draining older backlog", async () => {
const result = await getPackageReleaseScanBackfillBatchInternalHandler(
{
+28 -13
View File
@@ -51,6 +51,7 @@ import { requireGitHubAccountAge } from "./lib/githubAccount";
import { normalizeGitHubRepository } from "./lib/githubActionsOidc";
import { readGlobalPublicPluginsCount } from "./lib/globalStats";
import { isOfficialPublisher } from "./lib/officialPublishers";
import { getPackageReleaseArtifactSha256 } from "./lib/packageArtifacts";
import {
assertPackageVersion,
ensurePluginNameMatchesPackage,
@@ -95,6 +96,7 @@ import {
import { MAX_ACTIVE_REPORTS_PER_USER, MAX_REPORT_REASON_LENGTH } from "./lib/reporting";
import { matchesAllTokens, matchesExploratoryTokenPrefixes, tokenize } from "./lib/searchText";
import { hashSkillFiles } from "./lib/skills";
import { buildDeterministicPackageZip } from "./lib/skillZip";
import { runStaticPublishScan } from "./lib/staticPublishScan";
import schema from "./schema";
@@ -1056,8 +1058,8 @@ function packageArtifactSummary(
release: Pick<
Doc<"packageReleases">,
| "artifactKind"
| "integritySha256"
| "clawpackSha256"
| "sha256hash"
| "clawpackSize"
| "clawpackFormat"
| "npmIntegrity"
@@ -1070,7 +1072,7 @@ function packageArtifactSummary(
if (release.artifactKind === "npm-pack") {
return {
kind: "npm-pack",
sha256: release.clawpackSha256,
sha256: getPackageReleaseArtifactSha256(release) ?? undefined,
size: release.clawpackSize,
format: release.clawpackFormat ?? "tgz",
npmIntegrity: release.npmIntegrity,
@@ -1082,7 +1084,7 @@ function packageArtifactSummary(
}
return {
kind: "legacy-zip",
sha256: release.integritySha256,
sha256: getPackageReleaseArtifactSha256(release) ?? undefined,
format: "zip",
};
}
@@ -5964,7 +5966,7 @@ export const getPackageReleaseScanBackfillBatchInternal = internalQuery({
const pkg = await ctx.db.get(release.packageId);
if (!pkg || pkg.softDeletedAt || pkg.family === "skill") continue;
const needsVt = !release.sha256hash || !release.vtAnalysis;
const needsVt = !release.vtAnalysis;
const needsLlm = !release.llmAnalysis || release.llmAnalysis.status === "error";
const needsStatic = !release.staticScan;
if (!needsVt && !needsLlm && !needsStatic) continue;
@@ -6103,19 +6105,26 @@ async function verifyPublishFileStorageMetadata(
ctx: Pick<ActionCtx, "storage">,
files: ReturnType<typeof normalizePublishFiles>,
) {
return await Promise.all(
const verified = await Promise.all(
files.map(async (file) => {
const blob = await ctx.storage.get(file.storageId as Id<"_storage">);
if (!blob) throw new ConvexError(`Uploaded file no longer exists: ${file.path}`);
const bytes = new Uint8Array(await blob.arrayBuffer());
return {
...file,
size: blob.size,
sha256: await sha256Hex(bytes),
contentType: file.contentType?.trim() || blob.type || undefined,
file: {
...file,
size: blob.size,
sha256: await sha256Hex(bytes),
contentType: file.contentType?.trim() || blob.type || undefined,
},
zipEntry: { path: file.path, bytes },
};
}),
);
return {
files: verified.map(({ file }) => file),
legacyZipEntries: verified.map(({ zipEntry }) => zipEntry),
};
}
async function publishPackageImpl(
@@ -6247,7 +6256,10 @@ async function publishPackageImpl(
}
const displayName = payload.displayName?.trim() || name;
const files = await verifyPublishFileStorageMetadata(ctx, normalizePublishFiles(payload.files));
const { files, legacyZipEntries } = await verifyPublishFileStorageMetadata(
ctx,
normalizePublishFiles(payload.files),
);
if (payload.artifact?.kind !== "npm-pack") {
const oversizedFile = findOversizedPublishFile(files);
if (oversizedFile) {
@@ -6258,6 +6270,7 @@ async function publishPackageImpl(
if (totalBytes > MAX_PUBLISH_TOTAL_BYTES) {
throw new ConvexError(getPublishTotalSizeError("package"));
}
const legacyZipSha256 = await sha256Hex(buildDeterministicPackageZip(legacyZipEntries));
const existingSkill = await runQueryRef(ctx, internalRefs.skills.getSkillBySlugInternal, {
slug: name,
@@ -6434,6 +6447,7 @@ async function publishPackageImpl(
staticScan,
files,
integritySha256,
sha256hash: legacyZipSha256,
artifactKind: payload.artifact?.kind ?? "legacy-zip",
clawpackStorageId: payload.artifact?.storageId as Id<"_storage"> | undefined,
clawpackSha256: payload.artifact?.sha256,
@@ -7606,6 +7620,7 @@ export const insertReleaseInternal = internalMutation({
}),
),
integritySha256: v.string(),
sha256hash: v.string(),
artifactKind: v.optional(v.union(v.literal("legacy-zip"), v.literal("npm-pack"))),
clawpackStorageId: v.optional(v.id("_storage")),
clawpackSha256: v.optional(v.string()),
@@ -7812,6 +7827,7 @@ export const insertReleaseInternal = internalMutation({
distTags: effectiveTags,
files: args.files,
integritySha256: args.integritySha256,
sha256hash: args.sha256hash,
artifactKind: args.artifactKind,
clawpackStorageId: args.clawpackStorageId,
clawpackSha256: args.clawpackSha256,
@@ -7906,6 +7922,7 @@ async function recordMaliciousPluginReleaseFinding(
release: Doc<"packageReleases">,
trigger: string,
) {
const artifactSha256 = getPackageReleaseArtifactSha256(release);
await ctx.scheduler.runAfter(0, internal.users.recordMaliciousArtifactFindingInternal, {
ownerUserId: release.createdBy,
artifactKind: "plugin",
@@ -7913,7 +7930,7 @@ async function recordMaliciousPluginReleaseFinding(
version: release.version,
trigger,
...(release.llmAnalysis?.summary ? { findingSummary: release.llmAnalysis.summary } : {}),
...(release.sha256hash ? { sha256hash: release.sha256hash } : {}),
...(artifactSha256 ? { sha256hash: artifactSha256 } : {}),
});
}
@@ -8125,7 +8142,6 @@ async function syncLatestPackageVerification(
export const updateReleaseScanResultsInternal = internalMutation({
args: {
releaseId: v.id("packageReleases"),
sha256hash: v.optional(v.string()),
vtAnalysis: v.optional(vtAnalysisValidator),
},
handler: async (ctx, args) => {
@@ -8133,7 +8149,6 @@ export const updateReleaseScanResultsInternal = internalMutation({
if (!release || release.softDeletedAt) return;
const patch: Partial<Doc<"packageReleases">> = {};
if (args.sha256hash !== undefined) patch.sha256hash = args.sha256hash;
if (args.vtAnalysis !== undefined) {
patch.vtAnalysis = args.vtAnalysis;
}
+1 -1
View File
@@ -301,7 +301,7 @@ const resolvePublishTargetForUserInternalHandler = (
>
)._handler;
function indexedRows<T>(rows: T[]) {
function indexedRows(rows: unknown[]) {
return {
collect: vi.fn(async () => rows),
order: vi.fn(() => ({
+1
View File
@@ -1230,6 +1230,7 @@ const packageReleases = defineTable({
runtimeId: v.optional(v.string()),
sourceRepo: v.optional(v.string()),
verification: packageVerificationValidator,
// Deprecated compatibility hash for exact /download ZIP bytes; use artifact.sha256 for installs.
sha256hash: v.optional(v.string()),
vtAnalysis: v.optional(vtAnalysisValidator),
skillSpectorAnalysis: v.optional(skillSpectorAnalysisValidator),
+2 -1
View File
@@ -4,6 +4,7 @@ import { internal } from "./_generated/api";
import type { Doc } from "./_generated/dataModel";
import type { ActionCtx, QueryCtx } from "./_generated/server";
import { internalAction, internalQuery } from "./functions";
import { getPackageReleaseArtifactSha256 } from "./lib/packageArtifacts";
import { getOwnerPublisher } from "./lib/publishers";
const MAX_EXPORT_PAGE_SIZE = 50;
@@ -270,7 +271,7 @@ async function packageReleasePageToExportRows(
publicOwnerHandle,
publicSlug: pkg.name,
version: release.version,
artifactSha256: release.sha256hash ?? release.integritySha256,
artifactSha256: getPackageReleaseArtifactSha256(release),
createdAt: release.createdAt,
softDeletedAt: release.softDeletedAt ?? null,
files: sanitizeFiles(release.files),
+34 -16
View File
@@ -440,13 +440,7 @@ describe("package VT retries", () => {
{ releaseId: "packageReleases:demo" },
);
expect(runMutation).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
releaseId: "packageReleases:demo",
sha256hash: expect.any(String),
}),
);
expect(mutationPayloads(runMutation).every((payload) => !("sha256hash" in payload))).toBe(true);
expect(scheduler.runAfter).toHaveBeenCalledWith(5 * 60 * 1000, expect.anything(), {
releaseId: "packageReleases:demo",
attempt: 2,
@@ -503,13 +497,7 @@ describe("package VT retries", () => {
{ releaseId: "packageReleases:demo" },
);
expect(runMutation).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
releaseId: "packageReleases:demo",
sha256hash: clawpackSha256,
}),
);
expect(mutationPayloads(runMutation).every((payload) => !("sha256hash" in payload))).toBe(true);
expect(fetchMock).toHaveBeenNthCalledWith(
1,
`https://www.virustotal.com/api/v3/files/${clawpackSha256}`,
@@ -807,7 +795,8 @@ describe("package VT retries", () => {
it("retries package poll when VT lookup throws", async () => {
process.env.VT_API_KEY = "test-key";
vi.stubGlobal("fetch", vi.fn().mockRejectedValue(new Error("network error")));
const fetchMock = vi.fn().mockRejectedValue(new Error("network error"));
vi.stubGlobal("fetch", fetchMock);
const scheduler = { runAfter: vi.fn(async () => null) };
await pollPackageReleaseScanResultsHandler(
@@ -816,7 +805,9 @@ describe("package VT retries", () => {
_id: "packageReleases:demo",
packageId: "packages:demo",
version: "1.0.0",
sha256hash: "abc123",
artifactKind: "npm-pack",
sha256hash: "legacy-zip-sha",
clawpackSha256: "tgz-sha",
}),
runMutation: vi.fn(async () => null),
scheduler,
@@ -824,12 +815,39 @@ describe("package VT retries", () => {
{ releaseId: "packageReleases:demo", attempt: 3 },
);
expect(fetchMock).toHaveBeenCalledWith(
"https://www.virustotal.com/api/v3/files/tgz-sha",
expect.objectContaining({ method: "GET" }),
);
expect(scheduler.runAfter).toHaveBeenCalledWith(5 * 60 * 1000, expect.anything(), {
releaseId: "packageReleases:demo",
attempt: 4,
});
});
it("does not poll a legacy ZIP hash when an npm-pack artifact hash is missing", async () => {
process.env.VT_API_KEY = "test-key";
const fetchMock = vi.fn();
vi.stubGlobal("fetch", fetchMock);
await pollPackageReleaseScanResultsHandler(
{
runQuery: vi.fn().mockResolvedValue({
_id: "packageReleases:demo",
packageId: "packages:demo",
version: "1.0.0",
artifactKind: "npm-pack",
sha256hash: "legacy-zip-sha",
}),
runMutation: vi.fn(async () => null),
scheduler: { runAfter: vi.fn(async () => null) },
} as never,
{ releaseId: "packageReleases:demo", attempt: 3 },
);
expect(fetchMock).not.toHaveBeenCalled();
});
it("stores undetected-only package VT telemetry during polling even when static scan is suspicious", async () => {
process.env.VT_API_KEY = "test-key";
vi.stubGlobal(
+6 -8
View File
@@ -2,6 +2,7 @@ import { v } from "convex/values";
import { internal } from "./_generated/api";
import type { Doc, Id } from "./_generated/dataModel";
import { internalAction, internalMutation } from "./functions";
import { getPackageReleaseArtifactSha256 } from "./lib/packageArtifacts";
import { sourceSkillVersionFiles } from "./lib/skillCards";
import { buildDeterministicPackageZip, buildDeterministicZip } from "./lib/skillZip";
@@ -722,11 +723,6 @@ export const scanPackageReleaseWithVirusTotal = internalAction({
return;
}
await runMutationRef(ctx, internalRefs.packages.updateReleaseScanResultsInternal, {
releaseId: args.releaseId,
sha256hash: artifact.sha256hash,
});
try {
const existingFile = await checkExistingFile(apiKey, artifact.sha256hash);
const vtAnalysis = existingFile
@@ -812,7 +808,9 @@ export const pollPackageReleaseScanResults = internalAction({
const release = (await runQueryRef(ctx, internalRefs.packages.getReleaseByIdInternal, {
releaseId: args.releaseId,
})) as Doc<"packageReleases"> | null;
if (!release || release.softDeletedAt || !release.sha256hash) return;
if (!release || release.softDeletedAt) return;
const artifactSha256 = getPackageReleaseArtifactSha256(release);
if (!artifactSha256) return;
const pkg = (await runQueryRef(ctx, internalRefs.packages.getPackageByIdInternal, {
packageId: release.packageId,
})) as Doc<"packages"> | null;
@@ -820,7 +818,7 @@ export const pollPackageReleaseScanResults = internalAction({
const attempt = args.attempt ?? 1;
try {
const vtResult = await checkExistingFile(apiKey, release.sha256hash);
const vtResult = await checkExistingFile(apiKey, artifactSha256);
if (!vtResult) {
if (attempt < PACKAGE_SCAN_MAX_ATTEMPTS) {
await runAfterRef(
@@ -870,7 +868,7 @@ export const pollPackageReleaseScanResults = internalAction({
await enqueuePackageCodexForVtSignal(ctx, args.releaseId);
}
} catch (error) {
console.error(`[vt:package] Error polling ${release.sha256hash}:`, error);
console.error(`[vt:package] Error polling ${artifactSha256}:`, error);
if (attempt < PACKAGE_SCAN_MAX_ATTEMPTS) {
await runAfterRef(
ctx,
+6 -1
View File
@@ -722,7 +722,12 @@ Notes:
`npm-pack` for ClawPack-backed releases.
- ClawPack releases include npm-compatible `npmIntegrity`, `npmShasum`, and
`npmTarballName` fields.
- `version.sha256hash`, `version.vtAnalysis`, `version.llmAnalysis`, and `version.staticScan` are included when scan data exists.
- `version.sha256hash` is deprecated compatibility metadata for old clients. It
hashes the exact ZIP bytes returned by `/api/v1/packages/{name}/download`.
Modern clients should use `version.artifact.sha256`, which identifies the
canonical release artifact.
- `version.vtAnalysis`, `version.llmAnalysis`, and `version.staticScan` are
included when scan data exists.
- Private packages return `404` unless the caller can read the owning publisher.
### `GET /api/v1/packages/{name}/versions/{version}/security`
+1
View File
@@ -422,6 +422,7 @@ export const ApiV1PackageVersionResponseSchema = type({
capabilities: PackageCapabilitySummarySchema.or("null").optional(),
verification: PackageVerificationSummarySchema.or("null").optional(),
artifact: PackageArtifactSummarySchema.or("null").optional(),
// Deprecated compatibility hash for exact /download ZIP bytes; use artifact.sha256 for installs.
sha256hash: "string|null?",
vtAnalysis: PackageVtAnalysisSchema.or("null").optional(),
skillSpectorAnalysis: PackageSkillSpectorAnalysisSchema.or("null").optional(),
+1
View File
@@ -344,6 +344,7 @@ export const ApiV1PackageVersionResponseSchema = type({
capabilities: PackageCapabilitySummarySchema.or("null").optional(),
verification: PackageVerificationSummarySchema.or("null").optional(),
artifact: PackageArtifactSummarySchema.or("null").optional(),
// Deprecated compatibility hash for exact /download ZIP bytes; use artifact.sha256 for installs.
sha256hash: "string|null?",
vtAnalysis: PackageVtAnalysisSchema.or("null").optional(),
skillSpectorAnalysis: PackageSkillSpectorAnalysisSchema.or("null").optional(),
File diff suppressed because one or more lines are too long
+1
View File
@@ -451,6 +451,7 @@ export const ApiV1PackageVersionResponseSchema = type({
capabilities: PackageCapabilitySummarySchema.or("null").optional(),
verification: PackageVerificationSummarySchema.or("null").optional(),
artifact: PackageArtifactSummarySchema.or("null").optional(),
// Deprecated compatibility hash for exact /download ZIP bytes; use artifact.sha256 for installs.
sha256hash: "string|null?",
vtAnalysis: PackageVtAnalysisSchema.or("null").optional(),
skillSpectorAnalysis: PackageSkillSpectorAnalysisSchema.or("null").optional(),
@@ -143,7 +143,10 @@ describe("Convex export dataset ingestion", () => {
packageId: "packages:public",
version: "2.0.0",
createdAt: 1,
integritySha256: "pkg-sha",
artifactKind: "npm-pack",
clawpackSha256: "pkg-tgz-sha",
sha256hash: "pkg-zip-sha",
integritySha256: "pkg-file-set-sha",
files: [{ path: "package/index.js", size: 24, sha256: "pkg-file-sha" }],
staticScan: {
status: "clean",
@@ -172,7 +175,7 @@ describe("Convex export dataset ingestion", () => {
expect(rows[0]).toMatchObject({
sourceKind: "package",
sourceDocId: "packageReleases:1",
artifactSha256: "pkg-sha",
artifactSha256: "pkg-tgz-sha",
publicOwnerHandle: "alice",
packageChannel: "community",
sourceRepoHost: "github.com",
+8 -1
View File
@@ -228,7 +228,7 @@ function packageReleaseToExportRow(
),
publicSlug: stringOrNull(pkg.name),
version: requiredString(release.version, "packageReleases.version"),
artifactSha256: stringOrNull(release.sha256hash) ?? stringOrNull(release.integritySha256),
artifactSha256: packageReleaseArtifactSha256(release),
createdAt: numberValue(release.createdAt, "packageReleases.createdAt"),
softDeletedAt: numberOrNull(release.softDeletedAt),
files: filesFromExport(release.files),
@@ -246,6 +246,13 @@ function packageReleaseToExportRow(
];
}
function packageReleaseArtifactSha256(release: ConvexDoc) {
if (stringOrNull(release.artifactKind) === "npm-pack") {
return stringOrNull(release.clawpackSha256);
}
return stringOrNull(release.sha256hash);
}
function publicOwnerHandleFromExport(
source: ConvexDoc,
usersById: Map<string, ConvexDoc>,
@@ -80,4 +80,22 @@ describe("plugin security audit route", () => {
expect(screen.queryByRole("button", { name: "Rescan" })).toBeNull();
expect(screen.queryByRole("button", { name: "Download security audit" })).toBeNull();
});
it("links VirusTotal to the canonical npm-pack artifact hash", () => {
const loaderData = makeLoaderData();
loaderData.version.version = {
...loaderData.version.version,
sha256hash: "legacy-zip-sha",
artifact: {
kind: "npm-pack",
sha256: "tgz-sha",
},
} as never;
render(<PluginSecurityAuditPage name="demo-plugin" loaderData={loaderData as never} />);
expect(screen.getByRole("link", { name: "View on VirusTotal" }).getAttribute("href")).toBe(
"https://www.virustotal.com/gui/file/tgz-sha",
);
});
});
+1
View File
@@ -64,6 +64,7 @@ export type PackageVersionDetail = {
npmUnpackedSize?: number;
npmFileCount?: number;
} | null;
/** @deprecated Compatibility hash for exact /download ZIP bytes. Use artifact.sha256. */
sha256hash?: string | null;
vtAnalysis?: {
status: string;
+1 -1
View File
@@ -145,7 +145,7 @@ export function PluginSecurityAuditPage({
ownerPublisherId: null,
detailPath: buildPluginDetailHref(name),
}}
sha256hash={release.sha256hash ?? null}
sha256hash={release.artifact?.sha256 ?? null}
vtAnalysis={release.vtAnalysis ?? null}
llmAnalysis={release.llmAnalysis ?? null}
skillSpectorAnalysis={release.skillSpectorAnalysis ?? null}