diff --git a/convex/packages.public.test.ts b/convex/packages.public.test.ts index 2444966f..0d2a5f06 100644 --- a/convex/packages.public.test.ts +++ b/convex/packages.public.test.ts @@ -1,6 +1,7 @@ /* @vitest-environment node */ import { getAuthUserId } from "@convex-dev/auth/server"; +import { gzipSync } from "fflate"; import { afterEach, describe, expect, it, vi } from "vitest"; import { sha256Hex } from "./lib/clawpack"; import { MAX_PUBLISH_FILE_BYTES } from "./lib/publishLimits"; @@ -12,6 +13,8 @@ import { buildDeterministicPackageZip } from "./lib/skillZip"; import { backfillLatestPackageScanStatusInternal, backfillPackageReleaseScansInternal, + repairHistoricalClawPackReleaseFilesInternal, + getHistoricalClawPackRepairBatchInternal, getPackageReleaseScanBackfillBatchInternal, getByName, list, @@ -45,6 +48,7 @@ import { listPublicPage, listPageForViewerInternal, listVersions, + repairClawPackReleaseFilesInternal, updateReleaseLlmAnalysisInternal, updateReleaseStaticScanInternal, applyAccountDeletionToOwnedPackagesBatchInternal, @@ -669,6 +673,78 @@ const backfillPackageReleaseScansInternalHandler = ( { scheduled: number; nextCursor: number; done: boolean } > )._handler; +const repairClawPackReleaseFilesInternalHandler = ( + repairClawPackReleaseFilesInternal as unknown as WrappedHandler< + { releaseId: string; dryRun?: boolean }, + { + ok: boolean; + repaired: boolean; + dryRun?: boolean; + files?: number; + skipped?: string; + error?: string; + integritySha256?: string; + samplePaths?: string[]; + } + > +)._handler; +const getHistoricalClawPackRepairBatchInternalHandler = ( + getHistoricalClawPackRepairBatchInternal as unknown as WrappedHandler< + { + cursor?: string | null; + batchSize?: number; + releaseId?: string; + packageName?: string; + version?: string; + }, + { + candidates: Array<{ + releaseId: string; + packageId: string; + packageName: string; + version: string; + existingFileCount: number; + npmFileCount: number | null; + }>; + scanned: number; + cursor: string | null; + isDone: boolean; + } + > +)._handler; +const repairHistoricalClawPackReleaseFilesInternalHandler = ( + repairHistoricalClawPackReleaseFilesInternal as unknown as WrappedHandler< + { + cursor?: string | null; + batchSize?: number; + maxBatches?: number; + dryRun?: boolean; + releaseId?: string; + packageName?: string; + version?: string; + confirmation?: string; + scheduleNext?: boolean; + }, + { + ok: true; + dryRun: boolean; + cursor: string | null; + isDone: boolean; + stats: { + scanned: number; + candidates: number; + validated: number; + wouldRepair: number; + repaired: number; + skipped: number; + errors: number; + }; + samples: Array<{ releaseId: string; repairedFiles: number; samplePaths: string[] }>; + logLines: string[]; + writeConfirmation: string | null; + } + > +)._handler; const listOfficialPluginMigrationsInternalHandler = ( listOfficialPluginMigrationsInternal as unknown as WrappedHandler< { @@ -1065,6 +1141,63 @@ function makeReleaseDoc(overrides: Partial> = {}) { }; } +const TAR_BLOCK_SIZE = 512; + +function tarOctal(value: number, width: number) { + return `${value.toString(8).padStart(width - 1, "0")}\0`; +} + +function writeTarString(target: Uint8Array, offset: number, width: number, value: string) { + const encoded = new TextEncoder().encode(value); + target.set(encoded.subarray(0, width), offset); +} + +function tarFile(path: string, content: string | Uint8Array) { + const bytes = typeof content === "string" ? new TextEncoder().encode(content) : content; + const header = new Uint8Array(TAR_BLOCK_SIZE); + writeTarString(header, 0, 100, path); + writeTarString(header, 100, 8, tarOctal(0o644, 8)); + writeTarString(header, 108, 8, tarOctal(0, 8)); + writeTarString(header, 116, 8, tarOctal(0, 8)); + writeTarString(header, 124, 12, tarOctal(bytes.byteLength, 12)); + writeTarString(header, 136, 12, tarOctal(0, 12)); + header.fill(0x20, 148, 156); + header[156] = "0".charCodeAt(0); + writeTarString(header, 257, 6, "ustar"); + writeTarString(header, 263, 2, "00"); + + let checksum = 0; + for (const byte of header) checksum += byte; + writeTarString(header, 148, 8, tarOctal(checksum, 8)); + + const paddedSize = Math.ceil(bytes.byteLength / TAR_BLOCK_SIZE) * TAR_BLOCK_SIZE; + const body = new Uint8Array(paddedSize); + body.set(bytes); + return [header, body]; +} + +function npmPackFixture(files: Record) { + const parts: Uint8Array[] = []; + for (const [path, content] of Object.entries(files)) { + parts.push(...tarFile(path, content)); + } + parts.push(new Uint8Array(TAR_BLOCK_SIZE), new Uint8Array(TAR_BLOCK_SIZE)); + const size = parts.reduce((sum, part) => sum + part.byteLength, 0); + const tar = new Uint8Array(size); + let offset = 0; + for (const part of parts) { + tar.set(part, offset); + offset += part.byteLength; + } + return gzipSync(tar); +} + +function bytesToArrayBuffer(bytes: Uint8Array): ArrayBuffer { + const copy = new Uint8Array(bytes.byteLength); + copy.set(bytes); + return copy.buffer; +} + function makeDigestCtx(options: { pages?: Array<{ page: Array>; @@ -10758,6 +10891,56 @@ describe("package scan backfill", () => { ]); }); + it("does not hide historical ClawPack file repair inside the scan backfill batch", async () => { + const result = await getPackageReleaseScanBackfillBatchInternalHandler( + { + db: { + query: vi.fn((table: string) => { + if (table !== "packageReleases") throw new Error(`Unexpected table ${table}`); + return { + order: vi.fn(() => ({ + take: vi.fn().mockResolvedValue([]), + })), + withIndex: vi.fn(() => ({ + order: vi.fn(() => ({ + take: vi.fn().mockResolvedValue([ + { + _id: "packageReleases:truncated-clawpack", + _creationTime: 10, + packageId: "packages:demo", + artifactKind: "npm-pack", + clawpackStorageId: "storage:clawpack", + npmFileCount: 4, + files: [ + { path: "package.json", size: 10, storageId: "storage:package" }, + { + path: "openclaw.plugin.json", + size: 10, + storageId: "storage:plugin", + }, + ], + sha256hash: "hash", + vtAnalysis: { status: "clean" }, + llmAnalysis: { status: "clean" }, + staticScan: { status: "clean" }, + }, + ]), + })), + })), + }; + }), + get: vi.fn(async (id: string) => { + if (id === "packages:demo") return makePackageDoc(); + return null; + }), + }, + } as never, + { batchSize: 10 }, + ); + + expect(result.releases).toEqual([]); + }); + it("schedules static rescans for releases missing only static scan data", async () => { const originalVtApiKey = process.env.VT_API_KEY; process.env.VT_API_KEY = "vt-test-key"; @@ -10799,6 +10982,521 @@ describe("package scan backfill", () => { } }); + it("keeps scan backfill scoped to scan work for historical ClawPack releases", async () => { + const runAfter = vi.fn().mockResolvedValue(undefined); + const runMutation = vi.fn(); + const result = await backfillPackageReleaseScansInternalHandler( + { + runQuery: vi.fn().mockResolvedValue({ + releases: [ + { + releaseId: "packageReleases:truncated-clawpack", + needsVt: false, + needsLlm: true, + needsStatic: true, + }, + ], + nextCursor: 123, + done: true, + }), + runMutation, + scheduler: { runAfter }, + } as never, + { batchSize: 10 }, + ); + + expect(result).toEqual({ scheduled: 1, nextCursor: 123, done: true }); + expect(runAfter).toHaveBeenCalledTimes(1); + expect(runAfter).toHaveBeenCalledWith( + 0, + expect.anything(), + expect.objectContaining({ releaseId: "packageReleases:truncated-clawpack" }), + ); + expect(runMutation).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + releaseId: "packageReleases:truncated-clawpack", + source: "backfill", + }), + ); + }); + + it("lists historical ClawPack repair candidates with cursor progress and package targeting", async () => { + const packageWithIndex = vi.fn(() => ({ + unique: vi.fn().mockResolvedValue( + makePackageDoc({ + _id: "packages:demo", + name: "@scope/demo", + normalizedName: "@scope/demo", + }), + ), + })); + const paginate = vi.fn().mockResolvedValue({ + page: [ + makeReleaseDoc({ + _id: "packageReleases:truncated-clawpack", + _creationTime: 10, + packageId: "packages:demo", + version: "1.0.0", + artifactKind: "npm-pack", + clawpackStorageId: "storage:clawpack", + clawpackSha256: "a".repeat(64), + npmFileCount: 4, + files: [ + { path: "package.json", size: 10, storageId: "storage:package" }, + { path: "openclaw.plugin.json", size: 10, storageId: "storage:plugin" }, + ], + }), + makeReleaseDoc({ + _id: "packageReleases:complete-clawpack", + _creationTime: 11, + packageId: "packages:demo", + version: "1.0.1", + artifactKind: "npm-pack", + clawpackStorageId: "storage:clawpack-complete", + npmFileCount: 2, + files: [ + { path: "package.json", size: 10, storageId: "storage:package" }, + { path: "openclaw.plugin.json", size: 10, storageId: "storage:plugin" }, + ], + }), + ], + continueCursor: "cursor:next", + isDone: false, + }); + + const result = await getHistoricalClawPackRepairBatchInternalHandler( + { + db: { + query: vi.fn((table: string) => { + if (table === "packages") { + return { withIndex: packageWithIndex }; + } + if (table !== "packageReleases") throw new Error(`Unexpected table ${table}`); + return { + withIndex: vi.fn(() => ({ + order: vi.fn(() => ({ paginate })), + })), + }; + }), + get: vi.fn(), + }, + } as never, + { cursor: "cursor:start", batchSize: 2, packageName: "@scope/demo" }, + ); + + expect(packageWithIndex).toHaveBeenCalledWith("by_name", expect.any(Function)); + expect(paginate).toHaveBeenCalledWith({ cursor: "cursor:start", numItems: 2 }); + expect(result).toEqual({ + candidates: [ + expect.objectContaining({ + releaseId: "packageReleases:truncated-clawpack", + packageName: "@scope/demo", + version: "1.0.0", + existingFileCount: 2, + npmFileCount: 4, + }), + ], + scanned: 2, + cursor: "cursor:next", + isDone: false, + }); + }); + + it("uses package/version indexes for targeted ClawPack repair beyond the first global page", async () => { + const packageWithIndex = vi.fn(() => ({ + unique: vi.fn().mockResolvedValue( + makePackageDoc({ + _id: "packages:target", + name: "@scope/target", + normalizedName: "@scope/target", + }), + ), + })); + const releaseWithIndex = vi.fn(() => ({ + unique: vi.fn().mockResolvedValue( + makeReleaseDoc({ + _id: "packageReleases:target-2", + _creationTime: 9999, + packageId: "packages:target", + version: "2.0.0", + artifactKind: "npm-pack", + clawpackStorageId: "storage:clawpack", + npmFileCount: 4, + files: [ + { path: "package.json", size: 10, storageId: "storage:package" }, + { path: "openclaw.plugin.json", size: 10, storageId: "storage:plugin" }, + ], + }), + ), + })); + const globalPaginate = vi.fn().mockResolvedValue({ + page: [ + makeReleaseDoc({ + _id: "packageReleases:unrelated", + packageId: "packages:other", + version: "1.0.0", + }), + ], + continueCursor: "cursor:global-next", + isDone: false, + }); + + const result = await getHistoricalClawPackRepairBatchInternalHandler( + { + db: { + query: vi.fn((table: string) => { + if (table === "packages") { + return { withIndex: packageWithIndex }; + } + if (table !== "packageReleases") throw new Error(`Unexpected table ${table}`); + return { + withIndex: releaseWithIndex, + order: vi.fn(() => ({ paginate: globalPaginate })), + }; + }), + get: vi.fn(), + }, + } as never, + { batchSize: 1, packageName: "@scope/target", version: "2.0.0" }, + ); + + expect(packageWithIndex).toHaveBeenCalledWith("by_name", expect.any(Function)); + expect(releaseWithIndex).toHaveBeenCalledWith("by_package_version", expect.any(Function)); + expect(globalPaginate).not.toHaveBeenCalled(); + expect(result).toEqual({ + candidates: [ + expect.objectContaining({ + releaseId: "packageReleases:target-2", + packageName: "@scope/target", + version: "2.0.0", + existingFileCount: 2, + npmFileCount: 4, + }), + ], + scanned: 1, + cursor: null, + isDone: true, + }); + }); + + it("dry-runs historical ClawPack repair with artifact validation and no writes", async () => { + const pack = npmPackFixture({ + "package/package.json": JSON.stringify({ name: "@scope/demo", version: "1.0.0" }), + "package/openclaw.plugin.json": JSON.stringify({ id: "scope.demo" }), + "package/README.md": "# Demo\n", + "package/dist/index.js": "export const demo = true;\n", + }); + const storageGet = vi.fn( + async () => new Blob([bytesToArrayBuffer(pack)], { type: "application/octet-stream" }), + ); + const storageStore = vi.fn(); + const runMutation = vi.fn(); + const runAfter = vi.fn(); + + const result = await repairHistoricalClawPackReleaseFilesInternalHandler( + { + runQuery: vi.fn(async (_ref: unknown, args: { packageId?: string; releaseId?: string }) => { + if ("batchSize" in (args as Record)) { + return { + candidates: [ + { + releaseId: "packageReleases:truncated-clawpack", + packageId: "packages:demo", + packageName: "@scope/demo", + version: "1.0.0", + createdAt: 10, + existingFileCount: 2, + npmFileCount: 4, + clawpackSha256: null, + }, + ], + scanned: 1, + cursor: null, + isDone: true, + }; + } + if (args.releaseId) { + return makeReleaseDoc({ + _id: args.releaseId, + packageId: "packages:demo", + version: "1.0.0", + artifactKind: "npm-pack", + clawpackStorageId: "storage:clawpack", + npmFileCount: 4, + files: [ + { path: "package.json", size: 10, storageId: "storage:package" }, + { path: "openclaw.plugin.json", size: 10, storageId: "storage:plugin" }, + ], + }); + } + if (args.packageId === "packages:demo") { + return makePackageDoc({ + _id: "packages:demo", + name: "@scope/demo", + normalizedName: "@scope/demo", + }); + } + return null; + }), + runMutation, + scheduler: { runAfter }, + storage: { get: storageGet, store: storageStore }, + } as never, + { dryRun: true, batchSize: 1, maxBatches: 1, packageName: "@scope/demo" }, + ); + + expect(result.stats).toEqual({ + scanned: 1, + candidates: 1, + validated: 1, + wouldRepair: 1, + repaired: 0, + skipped: 0, + errors: 0, + }); + expect(result.writeConfirmation).toBe("repair-historical-clawpack-files"); + expect(result.samples[0]).toEqual( + expect.objectContaining({ + releaseId: "packageReleases:truncated-clawpack", + repairedFiles: 4, + samplePaths: ["package.json", "openclaw.plugin.json", "README.md", "dist/index.js"], + }), + ); + expect(result.logLines.join("\n")).toContain("would-repair package=@scope/demo"); + expect(storageStore).not.toHaveBeenCalled(); + expect(runMutation).not.toHaveBeenCalled(); + expect(runAfter).not.toHaveBeenCalled(); + }); + + it("requires explicit confirmation before applying historical ClawPack repair", async () => { + await expect( + repairHistoricalClawPackReleaseFilesInternalHandler( + { + runQuery: vi.fn(), + runMutation: vi.fn(), + scheduler: { runAfter: vi.fn() }, + storage: { get: vi.fn(), store: vi.fn() }, + } as never, + { dryRun: false, batchSize: 1 }, + ), + ).rejects.toThrow(/confirmation/); + }); + + it("applies historical ClawPack repair with confirmation and schedules resumable follow-up", async () => { + const pack = npmPackFixture({ + "package/package.json": JSON.stringify({ name: "@scope/demo", version: "1.0.0" }), + "package/openclaw.plugin.json": JSON.stringify({ id: "scope.demo" }), + "package/README.md": "# Demo\n", + "package/dist/index.js": "export const demo = true;\n", + }); + const storageGet = vi.fn( + async () => new Blob([bytesToArrayBuffer(pack)], { type: "application/octet-stream" }), + ); + const storageStore = vi.fn(async () => `storage:repaired-${storageStore.mock.calls.length}`); + const runMutation = vi.fn(async (_ref: unknown, args: Record) => { + if (Array.isArray(args.files)) return { repaired: true }; + return { jobId: "securityScanJobs:1" }; + }); + const runAfter = vi.fn().mockResolvedValue(undefined); + + const result = await repairHistoricalClawPackReleaseFilesInternalHandler( + { + runQuery: vi.fn(async (_ref: unknown, args: { packageId?: string; releaseId?: string }) => { + if ("batchSize" in (args as Record)) { + return { + candidates: [ + { + releaseId: "packageReleases:truncated-clawpack", + packageId: "packages:demo", + packageName: "@scope/demo", + version: "1.0.0", + createdAt: 10, + existingFileCount: 2, + npmFileCount: 4, + clawpackSha256: null, + }, + ], + scanned: 1, + cursor: "cursor:next", + isDone: false, + }; + } + if (args.releaseId) { + return makeReleaseDoc({ + _id: args.releaseId, + packageId: "packages:demo", + version: "1.0.0", + artifactKind: "npm-pack", + clawpackStorageId: "storage:clawpack", + npmFileCount: 4, + files: [ + { path: "package.json", size: 10, storageId: "storage:package" }, + { path: "openclaw.plugin.json", size: 10, storageId: "storage:plugin" }, + ], + }); + } + if (args.packageId === "packages:demo") { + return makePackageDoc({ + _id: "packages:demo", + name: "@scope/demo", + normalizedName: "@scope/demo", + }); + } + return null; + }), + runMutation, + scheduler: { runAfter }, + storage: { get: storageGet, store: storageStore }, + } as never, + { + dryRun: false, + batchSize: 1, + maxBatches: 1, + packageName: "@scope/demo", + confirmation: "repair-historical-clawpack-files", + scheduleNext: true, + }, + ); + + expect(result.stats).toEqual({ + scanned: 1, + candidates: 1, + validated: 1, + wouldRepair: 0, + repaired: 1, + skipped: 0, + errors: 0, + }); + expect(result.cursor).toBe("cursor:next"); + expect(result.isDone).toBe(false); + expect(storageStore).toHaveBeenCalledTimes(4); + expect(runMutation).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + releaseId: "packageReleases:truncated-clawpack", + files: expect.arrayContaining([ + expect.objectContaining({ path: "README.md", contentType: "text/markdown" }), + ]), + }), + ); + expect(runMutation).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + releaseId: "packageReleases:truncated-clawpack", + source: "manual", + }), + ); + expect(runAfter).toHaveBeenCalledWith( + 0, + expect.anything(), + expect.objectContaining({ releaseId: "packageReleases:truncated-clawpack" }), + ); + expect(runAfter).toHaveBeenCalledWith( + 0, + expect.anything(), + expect.objectContaining({ + cursor: "cursor:next", + dryRun: false, + confirmation: "repair-historical-clawpack-files", + scheduleNext: true, + }), + ); + }); + + it("repairs historical ClawPack release file rows from the stored npm-pack artifact", async () => { + const pack = npmPackFixture({ + "package/package.json": JSON.stringify({ name: "@scope/demo", version: "1.0.0" }), + "package/openclaw.plugin.json": JSON.stringify({ id: "scope.demo" }), + "package/README.md": "# Demo\n", + "package/dist/index.js": "export const demo = true;\n", + }); + const storageGet = vi.fn(async (storageId: string) => + storageId === "storage:clawpack" + ? new Blob([bytesToArrayBuffer(pack)], { type: "application/octet-stream" }) + : null, + ); + const storageStore = vi.fn(async () => `storage:repaired-${storageStore.mock.calls.length}`); + const runMutation = vi.fn(async (_ref: unknown, args: Record) => { + if (Array.isArray(args.files)) return { repaired: true }; + return { jobId: "securityScanJobs:1" }; + }); + const runAfter = vi.fn().mockResolvedValue(undefined); + + const result = await repairClawPackReleaseFilesInternalHandler( + { + runQuery: vi.fn(async (_ref: unknown, args: { packageId?: string; releaseId?: string }) => { + if (args.releaseId) { + return makeReleaseDoc({ + _id: args.releaseId, + packageId: "packages:demo", + version: "1.0.0", + artifactKind: "npm-pack", + clawpackStorageId: "storage:clawpack", + npmFileCount: 4, + files: [ + { path: "package.json", size: 10, storageId: "storage:package" }, + { + path: "openclaw.plugin.json", + size: 10, + storageId: "storage:plugin", + }, + ], + }); + } + if (args.packageId === "packages:demo") { + return makePackageDoc({ _id: "packages:demo", name: "@scope/demo" }); + } + return null; + }), + runMutation, + scheduler: { runAfter }, + storage: { get: storageGet, store: storageStore }, + } as never, + { releaseId: "packageReleases:truncated-clawpack" }, + ); + + expect(result).toEqual( + expect.objectContaining({ + ok: true, + repaired: true, + dryRun: false, + files: 4, + existingFiles: 2, + npmFileCount: 4, + integritySha256: expect.stringMatching(/^[a-f0-9]{64}$/), + samplePaths: ["package.json", "openclaw.plugin.json", "README.md", "dist/index.js"], + }), + ); + expect(storageStore).toHaveBeenCalledTimes(4); + const repairCall = runMutation.mock.calls.find(([, args]) => Array.isArray(args.files)); + expect(repairCall?.[1]).toEqual( + expect.objectContaining({ + releaseId: "packageReleases:truncated-clawpack", + npmFileCount: 4, + integritySha256: expect.stringMatching(/^[a-f0-9]{64}$/), + files: [ + expect.objectContaining({ path: "package.json" }), + expect.objectContaining({ path: "openclaw.plugin.json" }), + expect.objectContaining({ path: "README.md", contentType: "text/markdown" }), + expect.objectContaining({ path: "dist/index.js", contentType: "application/javascript" }), + ], + }), + ); + expect(runAfter).toHaveBeenCalledWith( + 0, + expect.anything(), + expect.objectContaining({ releaseId: "packageReleases:truncated-clawpack" }), + ); + expect(runMutation).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + releaseId: "packageReleases:truncated-clawpack", + source: "manual", + }), + ); + }); + it("backfills legacy static-only package scan status into the package search digest", async () => { const verification = { tier: "source-linked", diff --git a/convex/packages.ts b/convex/packages.ts index 9b4389bf..4a0987cb 100644 --- a/convex/packages.ts +++ b/convex/packages.ts @@ -1,6 +1,7 @@ import { ServerPackagePublishRequestSchema, derivePluginCategoryTags, + guessTextContentType, getPackageScopeOwnerMismatch, isPluginCategorySlug, parseArk, @@ -45,7 +46,7 @@ import { readArtifactReportStatus, appendPackageModerationEventLog, } from "./lib/artifactModeration"; -import { sha256Hex } from "./lib/clawpack"; +import { parseClawPack, sha256Hex } from "./lib/clawpack"; import { buildPackageInspectorFindingsEmail } from "./lib/emails"; import { requireGitHubAccountAge } from "./lib/githubAccount"; import { normalizeGitHubRepository } from "./lib/githubActionsOidc"; @@ -357,7 +358,10 @@ function isTrustedOpenClawPluginPackage(params: { const internalRefs = internal as unknown as { packages: { backfillPackageReleaseScansInternal: unknown; + repairClawPackReleaseFilesInternal: unknown; + repairHistoricalClawPackReleaseFilesInternal: unknown; scanPackageReleaseStaticallyInternal: unknown; + replaceReleaseFilesAfterClawPackRepairInternal: unknown; insertReleaseInternal: unknown; getPackageByNameInternal: unknown; getTrustedPublisherByPackageIdInternal: unknown; @@ -365,6 +369,7 @@ const internalRefs = internal as unknown as { getPackageByIdInternal: unknown; getReleaseByIdInternal: unknown; getPackageReleaseScanBackfillBatchInternal: unknown; + getHistoricalClawPackRepairBatchInternal: unknown; listVersionsForViewerInternal: unknown; getVersionByNameForViewerInternal: unknown; publishPackageForUserInternal: unknown; @@ -471,6 +476,86 @@ type PackageInspectorFinding = { decision?: string; }; +type PackageReleaseFilePatch = Array<{ + path: string; + size: number; + storageId: Id<"_storage">; + sha256: string; + contentType?: string; +}>; + +type ClawPackRepairCandidate = { + releaseId: Id<"packageReleases">; + packageId: Id<"packages">; + packageName: string; + version: string; + createdAt: number; + existingFileCount: number; + npmFileCount: number | null; + clawpackSha256: string | null; +}; + +type ClawPackRepairResult = + | { + ok: true; + repaired: boolean; + dryRun: boolean; + files: number; + existingFiles: number; + npmFileCount: number; + integritySha256: string; + samplePaths: string[]; + artifactSha256: string; + } + | { + ok: true; + repaired: false; + dryRun: boolean; + skipped: "missing_release" | "not_needed" | "missing_package"; + } + | { + ok: false; + repaired: false; + dryRun: boolean; + error: string; + }; + +type HistoricalClawPackRepairStats = { + scanned: number; + candidates: number; + validated: number; + wouldRepair: number; + repaired: number; + skipped: number; + errors: number; +}; + +const CLAWPACK_REPAIR_WRITE_CONFIRMATION = "repair-historical-clawpack-files"; + +function bytesToArrayBuffer(bytes: Uint8Array) { + const copy = new Uint8Array(bytes.byteLength); + copy.set(bytes); + return copy.buffer; +} + +function inferClawPackEntryContentType(path: string) { + return guessTextContentType(path) ?? "application/octet-stream"; +} + +function needsClawPackReleaseFileRepair( + release: Pick< + Doc<"packageReleases">, + "artifactKind" | "clawpackStorageId" | "npmFileCount" | "files" + >, +) { + return ( + release.artifactKind === "npm-pack" && + Boolean(release.clawpackStorageId) && + typeof release.npmFileCount === "number" && + release.npmFileCount > release.files.length + ); +} + type PackageInspectorPublishResult = { status: "pass" | "fail"; summary: { @@ -5809,6 +5894,128 @@ export const getPackageReleaseScanBackfillBatchInternal = internalQuery({ }, }); +export const getHistoricalClawPackRepairBatchInternal = internalQuery({ + args: { + cursor: v.optional(v.union(v.string(), v.null())), + batchSize: v.optional(v.number()), + releaseId: v.optional(v.id("packageReleases")), + packageName: v.optional(v.string()), + version: v.optional(v.string()), + }, + handler: async (ctx, args) => { + const batchSize = Math.max(1, Math.min(Math.floor(args.batchSize ?? 25), 100)); + const targetName = args.packageName ? normalizePackageName(args.packageName) : undefined; + + const toCandidateForPackage = ( + release: Doc<"packageReleases">, + pkg: Doc<"packages">, + ): ClawPackRepairCandidate | null => { + if (!isReleaseActive(release)) return null; + if (!needsClawPackReleaseFileRepair(release)) return null; + if (args.version && release.version !== args.version) return null; + if (!pkg || pkg.softDeletedAt || pkg.family === "skill") return null; + if (targetName && pkg.normalizedName !== targetName) return null; + + return { + releaseId: release._id, + packageId: release.packageId, + packageName: pkg.normalizedName, + version: release.version, + createdAt: release.createdAt, + existingFileCount: release.files.length, + npmFileCount: release.npmFileCount ?? null, + clawpackSha256: release.clawpackSha256 ?? null, + }; + }; + + const toCandidate = async (release: Doc<"packageReleases">) => { + const pkg = await ctx.db.get(release.packageId); + if (!pkg) return null; + return toCandidateForPackage(release, pkg); + }; + + if (args.releaseId) { + const release = await ctx.db.get(args.releaseId); + const candidate = release ? await toCandidate(release) : null; + return { + candidates: candidate ? [candidate] : [], + scanned: release ? 1 : 0, + cursor: null, + isDone: true, + }; + } + + if (targetName) { + const pkg = await getPackageByNormalizedName(ctx, targetName); + if (!pkg || pkg.softDeletedAt || pkg.family === "skill") { + return { + candidates: [], + scanned: 0, + cursor: null, + isDone: true, + }; + } + + if (args.version) { + const targetVersion = args.version; + const release = await ctx.db + .query("packageReleases") + .withIndex("by_package_version", (q) => + q.eq("packageId", pkg._id).eq("version", targetVersion), + ) + .unique(); + const candidate = release ? toCandidateForPackage(release, pkg) : null; + return { + candidates: candidate ? [candidate] : [], + scanned: release ? 1 : 0, + cursor: null, + isDone: true, + }; + } + + const { page, continueCursor, isDone } = await ctx.db + .query("packageReleases") + .withIndex("by_package_active_created", (q) => + q.eq("packageId", pkg._id).eq("softDeletedAt", undefined), + ) + .order("asc") + .paginate({ cursor: args.cursor ?? null, numItems: batchSize }); + + const candidates: ClawPackRepairCandidate[] = []; + for (const release of page) { + const candidate = toCandidateForPackage(release, pkg); + if (candidate) candidates.push(candidate); + } + + return { + candidates, + scanned: page.length, + cursor: isDone ? null : continueCursor, + isDone, + }; + } + + const { page, continueCursor, isDone } = await ctx.db + .query("packageReleases") + .withIndex("by_active_created", (q) => q.eq("softDeletedAt", undefined)) + .order("asc") + .paginate({ cursor: args.cursor ?? null, numItems: batchSize }); + + const candidates: ClawPackRepairCandidate[] = []; + for (const release of page) { + const candidate = await toCandidate(release); + if (candidate) candidates.push(candidate); + } + + return { + candidates, + scanned: page.length, + cursor: isDone ? null : continueCursor, + isDone, + }; + }, +}); + function buildGitHubActionsPublishActor( publishToken: Doc<"packagePublishTokens">, ): Extract { @@ -8161,6 +8368,368 @@ export const updateReleaseStaticScanInternal = internalMutation({ }, }); +export const replaceReleaseFilesAfterClawPackRepairInternal = internalMutation({ + args: { + releaseId: v.id("packageReleases"), + files: v.array( + v.object({ + path: v.string(), + size: v.number(), + storageId: v.id("_storage"), + sha256: v.string(), + contentType: v.optional(v.string()), + }), + ), + clawpackSha256: v.string(), + npmIntegrity: v.string(), + npmShasum: v.string(), + npmTarballName: v.string(), + npmUnpackedSize: v.number(), + npmFileCount: v.number(), + integritySha256: v.string(), + extractedPackageJson: v.any(), + extractedPluginManifest: v.any(), + checkedAt: v.number(), + }, + handler: async (ctx, args) => { + const release = await ctx.db.get(args.releaseId); + if (!isReleaseActive(release) || !needsClawPackReleaseFileRepair(release)) { + return { repaired: false as const }; + } + + const nextVerification = release.verification + ? { + ...release.verification, + scanStatus: "pending" as const, + } + : release.verification; + const patch: Partial> = { + files: args.files, + integritySha256: args.integritySha256, + clawpackSha256: args.clawpackSha256, + npmIntegrity: args.npmIntegrity, + npmShasum: args.npmShasum, + npmTarballName: args.npmTarballName, + npmUnpackedSize: args.npmUnpackedSize, + npmFileCount: args.npmFileCount, + extractedPackageJson: args.extractedPackageJson, + extractedPluginManifest: args.extractedPluginManifest, + verification: nextVerification, + staticScan: undefined, + skillSpectorAnalysis: undefined, + llmAnalysis: { + status: "pending", + summary: "ClawPack file listing repaired from the stored artifact; fresh review queued.", + checkedAt: args.checkedAt, + }, + }; + await ctx.db.patch(args.releaseId, patch); + + await syncLatestPackageVerification(ctx, { + ...release, + ...patch, + } as Doc<"packageReleases">); + + return { repaired: true as const }; + }, +}); + +async function repairClawPackReleaseFiles( + ctx: ActionCtx, + args: { releaseId: Id<"packageReleases">; dryRun?: boolean }, +): Promise { + const dryRun = args.dryRun === true; + const release = await runQueryRef | null>( + ctx, + internalRefs.packages.getReleaseByIdInternal, + { releaseId: args.releaseId }, + ); + if (!release || release.softDeletedAt) { + return { ok: true, repaired: false, dryRun, skipped: "missing_release" }; + } + if (!needsClawPackReleaseFileRepair(release) || !release.clawpackStorageId) { + return { ok: true, repaired: false, dryRun, skipped: "not_needed" }; + } + + const pkg = await runQueryRef | null>( + ctx, + internalRefs.packages.getPackageByIdInternal, + { packageId: release.packageId }, + ); + if (!pkg || pkg.softDeletedAt || pkg.family === "skill") { + return { ok: true, repaired: false, dryRun, skipped: "missing_package" }; + } + + const blob = await ctx.storage.get(release.clawpackStorageId); + if (!blob) { + return { + ok: false, + repaired: false, + dryRun, + error: "ClawPack artifact missing", + }; + } + + const artifactBytes = new Uint8Array(await blob.arrayBuffer()); + const parsed = await parseClawPack(artifactBytes); + if (release.clawpackSha256 && parsed.artifactSha256 !== release.clawpackSha256) { + throw new Error("Stored ClawPack artifact digest does not match package release metadata"); + } + const parsedPackageName = normalizePackageName(parsed.packageName); + const storedPackageNames = new Set([pkg.normalizedName, normalizePackageName(pkg.name)]); + if (!storedPackageNames.has(parsedPackageName) || parsed.packageVersion !== release.version) { + throw new Error("Stored ClawPack artifact identity does not match package release metadata"); + } + + const parsedFiles = []; + for (const entry of parsed.entries) { + parsedFiles.push({ + path: entry.path, + size: entry.bytes.byteLength, + sha256: await sha256Hex(entry.bytes), + contentType: inferClawPackEntryContentType(entry.path), + bytes: entry.bytes, + }); + } + const integritySha256 = await hashSkillFiles(parsedFiles); + + if (dryRun) { + return { + ok: true, + repaired: false, + dryRun: true, + files: parsedFiles.length, + existingFiles: release.files.length, + npmFileCount: parsed.fileCount, + integritySha256, + artifactSha256: parsed.artifactSha256, + samplePaths: parsedFiles.slice(0, 10).map((file) => file.path), + }; + } + + const files: PackageReleaseFilePatch = []; + for (const file of parsedFiles) { + const storageId = await ctx.storage.store( + new Blob([bytesToArrayBuffer(file.bytes)], { type: file.contentType }), + ); + files.push({ + path: file.path, + size: file.size, + storageId, + sha256: file.sha256, + contentType: file.contentType, + }); + } + + const result = await runMutationRef<{ repaired: boolean }>( + ctx, + internalRefs.packages.replaceReleaseFilesAfterClawPackRepairInternal, + { + releaseId: args.releaseId, + files, + clawpackSha256: parsed.artifactSha256, + npmIntegrity: parsed.npmIntegrity, + npmShasum: parsed.npmShasum, + npmTarballName: parsed.npmTarballName, + npmUnpackedSize: parsed.unpackedSize, + npmFileCount: parsed.fileCount, + integritySha256, + extractedPackageJson: parsed.packageJson, + extractedPluginManifest: parsed.pluginManifest, + checkedAt: Date.now(), + }, + ); + + if (!result.repaired) { + return { ok: true, repaired: false, dryRun: false, skipped: "not_needed" }; + } + + await runAfterRef(ctx, 0, internalRefs.packages.scanPackageReleaseStaticallyInternal, { + releaseId: args.releaseId, + }); + await runMutationRef(ctx, internalRefs.securityScan.enqueuePackageReleaseScanInternal, { + releaseId: args.releaseId, + source: "manual", + }); + + return { + ok: true, + repaired: true, + dryRun: false, + files: files.length, + existingFiles: release.files.length, + npmFileCount: parsed.fileCount, + integritySha256, + artifactSha256: parsed.artifactSha256, + samplePaths: files.slice(0, 10).map((file) => file.path), + }; +} + +export const repairClawPackReleaseFilesInternal = internalAction({ + args: { + releaseId: v.id("packageReleases"), + dryRun: v.optional(v.boolean()), + }, + handler: repairClawPackReleaseFiles, +}); + +export const repairHistoricalClawPackReleaseFilesInternal = internalAction({ + args: { + cursor: v.optional(v.union(v.string(), v.null())), + batchSize: v.optional(v.number()), + maxBatches: v.optional(v.number()), + dryRun: v.optional(v.boolean()), + releaseId: v.optional(v.id("packageReleases")), + packageName: v.optional(v.string()), + version: v.optional(v.string()), + confirmation: v.optional(v.string()), + scheduleNext: v.optional(v.boolean()), + }, + handler: async (ctx, args) => { + const dryRun = args.dryRun !== false; + if (!dryRun && args.confirmation !== CLAWPACK_REPAIR_WRITE_CONFIRMATION) { + throw new ConvexError( + `Set confirmation to "${CLAWPACK_REPAIR_WRITE_CONFIRMATION}" to repair historical ClawPack release files.`, + ); + } + + const batchSize = Math.max(1, Math.min(Math.floor(args.batchSize ?? 25), 100)); + const maxBatches = Math.max(1, Math.min(Math.floor(args.maxBatches ?? 1), 20)); + const stats: HistoricalClawPackRepairStats = { + scanned: 0, + candidates: 0, + validated: 0, + wouldRepair: 0, + repaired: 0, + skipped: 0, + errors: 0, + }; + const logLines = [ + `[clawpack-repair] mode=${dryRun ? "dry-run" : "apply"} batchSize=${batchSize} maxBatches=${maxBatches}`, + ]; + const samples: Array< + ClawPackRepairCandidate & { + repairedFiles: number; + integritySha256: string; + artifactSha256: string; + samplePaths: string[]; + } + > = []; + let cursor: string | null = args.cursor ?? null; + let isDone = false; + + for (let batchIndex = 0; batchIndex < maxBatches; batchIndex++) { + const batch = await runQueryRef<{ + candidates: ClawPackRepairCandidate[]; + scanned: number; + cursor: string | null; + isDone: boolean; + }>(ctx, internalRefs.packages.getHistoricalClawPackRepairBatchInternal, { + cursor, + batchSize, + releaseId: args.releaseId, + packageName: args.packageName, + version: args.version, + }); + + stats.scanned += batch.scanned; + stats.candidates += batch.candidates.length; + cursor = batch.cursor; + isDone = batch.isDone; + logLines.push( + `[clawpack-repair] batch=${batchIndex + 1} scanned=${batch.scanned} candidates=${batch.candidates.length} nextCursor=${cursor ?? ""}`, + ); + + for (const candidate of batch.candidates) { + try { + const result = await repairClawPackReleaseFiles(ctx, { + releaseId: candidate.releaseId, + dryRun, + }); + if (!result.ok) { + stats.errors++; + logLines.push( + `[clawpack-repair] error package=${candidate.packageName} version=${candidate.version} release=${candidate.releaseId} error=${result.error}`, + ); + continue; + } + if ("skipped" in result) { + stats.skipped++; + logLines.push( + `[clawpack-repair] skipped package=${candidate.packageName} version=${candidate.version} release=${candidate.releaseId} reason=${result.skipped}`, + ); + continue; + } + + stats.validated++; + if (dryRun) { + stats.wouldRepair++; + } else if (result.repaired) { + stats.repaired++; + } + if (samples.length < 20) { + samples.push({ + ...candidate, + repairedFiles: result.files, + integritySha256: result.integritySha256, + artifactSha256: result.artifactSha256, + samplePaths: result.samplePaths, + }); + } + logLines.push( + `[clawpack-repair] ${dryRun ? "would-repair" : "repaired"} package=${candidate.packageName} version=${candidate.version} release=${candidate.releaseId} files=${candidate.existingFileCount}->${result.files} npmFileCount=${result.npmFileCount} integrity=${result.integritySha256.slice(0, 12)}...`, + ); + } catch (error) { + stats.errors++; + const message = error instanceof Error ? error.message : String(error); + logLines.push( + `[clawpack-repair] error package=${candidate.packageName} version=${candidate.version} release=${candidate.releaseId} error=${message}`, + ); + } + } + + if (args.releaseId || isDone) break; + } + + if (!dryRun && args.scheduleNext === true && !isDone && cursor) { + await runAfterRef( + ctx, + 0, + internalRefs.packages.repairHistoricalClawPackReleaseFilesInternal, + { + cursor, + batchSize, + maxBatches, + dryRun: false, + releaseId: args.releaseId, + packageName: args.packageName, + version: args.version, + confirmation: args.confirmation, + scheduleNext: true, + }, + ); + logLines.push(`[clawpack-repair] scheduled-next cursor=${cursor}`); + } + + return { + ok: true as const, + dryRun, + confirmationRequired: !dryRun, + target: { + releaseId: args.releaseId ?? null, + packageName: args.packageName ?? null, + version: args.version ?? null, + }, + cursor, + isDone, + stats, + samples, + logLines, + writeConfirmation: dryRun ? CLAWPACK_REPAIR_WRITE_CONFIRMATION : null, + }; + }, +}); + export const scanPackageReleaseStaticallyInternal = internalAction({ args: { releaseId: v.id("packageReleases"), @@ -8286,6 +8855,32 @@ export const backfillPackageReleaseScans = action({ }, }); +export const repairHistoricalClawPackReleaseFiles = action({ + args: { + cursor: v.optional(v.union(v.string(), v.null())), + batchSize: v.optional(v.number()), + maxBatches: v.optional(v.number()), + dryRun: v.optional(v.boolean()), + releaseId: v.optional(v.id("packageReleases")), + packageName: v.optional(v.string()), + version: v.optional(v.string()), + confirmation: v.optional(v.string()), + scheduleNext: v.optional(v.boolean()), + }, + handler: async (ctx, args) => { + const { user } = await requireUserFromAction(ctx); + assertAdmin(user); + return await runActionRef( + ctx, + internalRefs.packages.repairHistoricalClawPackReleaseFilesInternal, + { + ...args, + dryRun: args.dryRun !== false, + }, + ); + }, +}); + export const setBatch = mutation({ args: { packageId: v.id("packages"), batch: v.optional(v.string()) }, handler: async (ctx, args) => { diff --git a/specs/security-moderation.md b/specs/security-moderation.md index 9e160117..e371c715 100644 --- a/specs/security-moderation.md +++ b/specs/security-moderation.md @@ -247,10 +247,24 @@ See also: [acceptable-usage.md](./acceptable-usage.md) for the marketplace polic `vt-only`, `both`) are retired. - Package/plugin scan backfills may recompute deterministic static scan results for older releases, but those results remain ClawScan context and are not public trust status. -- ClawPack package releases keep static/LLM scan inputs intentionally metadata-only for now: - `package.json`, `openclaw.plugin.json`, package/source metadata, and release facts. VirusTotal - scans the exact uploaded `.tgz`; ClawHub does not currently run deep static/LLM scans across every - tarball file. +- ClawPack package releases materialize parsed npm-pack artifact entries into the release file + surface. Static scan, LLM review, package inspect/file APIs, and Codex package ClawScan use those + stored artifact entries instead of metadata-only `package.json` / `openclaw.plugin.json` rows. + VirusTotal still scans the exact uploaded `.tgz`; ClawHub also retains the stored ClawPack artifact + so scanner workers can download/extract it for artifact-backed review. +- Historical ClawPack releases whose stored file rows are shorter than the artifact `npmFileCount` + are production data repair candidates, not normal scan-backfill work. Operators repair them through + the explicit `packages:repairHistoricalClawPackReleaseFiles` migration action: + - default to `dryRun: true` and inspect the returned `logLines`, candidate samples, parsed + artifact file counts, hashes, and resume `cursor` before writing. + - target narrowly with `releaseId`, or with `packageName` and optional `version`, before running + an all-release pass. + - pass `dryRun: false` plus `confirmation: "repair-historical-clawpack-files"` before any row + rewrite; the action replaces `release.files`, recomputes `integritySha256`, resets stale scan + analysis, and queues fresh static/Codex package scans for repaired releases. + - resume incomplete runs by passing the returned `cursor`; set `scheduleNext: true` only when an + operator intentionally wants the action to enqueue the next batch. + The ordinary package scan backfill must not schedule this repair implicitly. - Packages cache VirusTotal undetected-only engine results as clean VT telemetry. ClawHub does not request or consume VirusTotal AI/code-insight results; VT is engine/vendor telemetry only.