mirror of
https://github.com/openclaw/clawhub.git
synced 2026-08-14 08:52:21 +00:00
fix: repair historical ClawPack release files
Merge PR #2583 to add an admin-gated historical ClawPack release file repair action with dry-run, confirmation, cursor/resume support, regression coverage, and security-moderation docs.
This commit is contained in:
Andy Ye
@@ -1,6 +1,7 @@
|
||||
/* @vitest-environment node */
|
||||
|
||||
import { getAuthUserId } from "@convex-dev/auth/server";
|
||||
import { gzipSync } from "fflate";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { sha256Hex } from "./lib/clawpack";
|
||||
import { MAX_PUBLISH_FILE_BYTES } from "./lib/publishLimits";
|
||||
@@ -12,6 +13,8 @@ import { buildDeterministicPackageZip } from "./lib/skillZip";
|
||||
import {
|
||||
backfillLatestPackageScanStatusInternal,
|
||||
backfillPackageReleaseScansInternal,
|
||||
repairHistoricalClawPackReleaseFilesInternal,
|
||||
getHistoricalClawPackRepairBatchInternal,
|
||||
getPackageReleaseScanBackfillBatchInternal,
|
||||
getByName,
|
||||
list,
|
||||
@@ -45,6 +48,7 @@ import {
|
||||
listPublicPage,
|
||||
listPageForViewerInternal,
|
||||
listVersions,
|
||||
repairClawPackReleaseFilesInternal,
|
||||
updateReleaseLlmAnalysisInternal,
|
||||
updateReleaseStaticScanInternal,
|
||||
applyAccountDeletionToOwnedPackagesBatchInternal,
|
||||
@@ -669,6 +673,78 @@ const backfillPackageReleaseScansInternalHandler = (
|
||||
{ scheduled: number; nextCursor: number; done: boolean }
|
||||
>
|
||||
)._handler;
|
||||
const repairClawPackReleaseFilesInternalHandler = (
|
||||
repairClawPackReleaseFilesInternal as unknown as WrappedHandler<
|
||||
{ releaseId: string; dryRun?: boolean },
|
||||
{
|
||||
ok: boolean;
|
||||
repaired: boolean;
|
||||
dryRun?: boolean;
|
||||
files?: number;
|
||||
skipped?: string;
|
||||
error?: string;
|
||||
integritySha256?: string;
|
||||
samplePaths?: string[];
|
||||
}
|
||||
>
|
||||
)._handler;
|
||||
const getHistoricalClawPackRepairBatchInternalHandler = (
|
||||
getHistoricalClawPackRepairBatchInternal as unknown as WrappedHandler<
|
||||
{
|
||||
cursor?: string | null;
|
||||
batchSize?: number;
|
||||
releaseId?: string;
|
||||
packageName?: string;
|
||||
version?: string;
|
||||
},
|
||||
{
|
||||
candidates: Array<{
|
||||
releaseId: string;
|
||||
packageId: string;
|
||||
packageName: string;
|
||||
version: string;
|
||||
existingFileCount: number;
|
||||
npmFileCount: number | null;
|
||||
}>;
|
||||
scanned: number;
|
||||
cursor: string | null;
|
||||
isDone: boolean;
|
||||
}
|
||||
>
|
||||
)._handler;
|
||||
const repairHistoricalClawPackReleaseFilesInternalHandler = (
|
||||
repairHistoricalClawPackReleaseFilesInternal as unknown as WrappedHandler<
|
||||
{
|
||||
cursor?: string | null;
|
||||
batchSize?: number;
|
||||
maxBatches?: number;
|
||||
dryRun?: boolean;
|
||||
releaseId?: string;
|
||||
packageName?: string;
|
||||
version?: string;
|
||||
confirmation?: string;
|
||||
scheduleNext?: boolean;
|
||||
},
|
||||
{
|
||||
ok: true;
|
||||
dryRun: boolean;
|
||||
cursor: string | null;
|
||||
isDone: boolean;
|
||||
stats: {
|
||||
scanned: number;
|
||||
candidates: number;
|
||||
validated: number;
|
||||
wouldRepair: number;
|
||||
repaired: number;
|
||||
skipped: number;
|
||||
errors: number;
|
||||
};
|
||||
samples: Array<{ releaseId: string; repairedFiles: number; samplePaths: string[] }>;
|
||||
logLines: string[];
|
||||
writeConfirmation: string | null;
|
||||
}
|
||||
>
|
||||
)._handler;
|
||||
const listOfficialPluginMigrationsInternalHandler = (
|
||||
listOfficialPluginMigrationsInternal as unknown as WrappedHandler<
|
||||
{
|
||||
@@ -1065,6 +1141,63 @@ function makeReleaseDoc(overrides: Partial<Record<string, unknown>> = {}) {
|
||||
};
|
||||
}
|
||||
|
||||
const TAR_BLOCK_SIZE = 512;
|
||||
|
||||
function tarOctal(value: number, width: number) {
|
||||
return `${value.toString(8).padStart(width - 1, "0")}\0`;
|
||||
}
|
||||
|
||||
function writeTarString(target: Uint8Array, offset: number, width: number, value: string) {
|
||||
const encoded = new TextEncoder().encode(value);
|
||||
target.set(encoded.subarray(0, width), offset);
|
||||
}
|
||||
|
||||
function tarFile(path: string, content: string | Uint8Array) {
|
||||
const bytes = typeof content === "string" ? new TextEncoder().encode(content) : content;
|
||||
const header = new Uint8Array(TAR_BLOCK_SIZE);
|
||||
writeTarString(header, 0, 100, path);
|
||||
writeTarString(header, 100, 8, tarOctal(0o644, 8));
|
||||
writeTarString(header, 108, 8, tarOctal(0, 8));
|
||||
writeTarString(header, 116, 8, tarOctal(0, 8));
|
||||
writeTarString(header, 124, 12, tarOctal(bytes.byteLength, 12));
|
||||
writeTarString(header, 136, 12, tarOctal(0, 12));
|
||||
header.fill(0x20, 148, 156);
|
||||
header[156] = "0".charCodeAt(0);
|
||||
writeTarString(header, 257, 6, "ustar");
|
||||
writeTarString(header, 263, 2, "00");
|
||||
|
||||
let checksum = 0;
|
||||
for (const byte of header) checksum += byte;
|
||||
writeTarString(header, 148, 8, tarOctal(checksum, 8));
|
||||
|
||||
const paddedSize = Math.ceil(bytes.byteLength / TAR_BLOCK_SIZE) * TAR_BLOCK_SIZE;
|
||||
const body = new Uint8Array(paddedSize);
|
||||
body.set(bytes);
|
||||
return [header, body];
|
||||
}
|
||||
|
||||
function npmPackFixture(files: Record<string, string | Uint8Array>) {
|
||||
const parts: Uint8Array[] = [];
|
||||
for (const [path, content] of Object.entries(files)) {
|
||||
parts.push(...tarFile(path, content));
|
||||
}
|
||||
parts.push(new Uint8Array(TAR_BLOCK_SIZE), new Uint8Array(TAR_BLOCK_SIZE));
|
||||
const size = parts.reduce((sum, part) => sum + part.byteLength, 0);
|
||||
const tar = new Uint8Array(size);
|
||||
let offset = 0;
|
||||
for (const part of parts) {
|
||||
tar.set(part, offset);
|
||||
offset += part.byteLength;
|
||||
}
|
||||
return gzipSync(tar);
|
||||
}
|
||||
|
||||
function bytesToArrayBuffer(bytes: Uint8Array): ArrayBuffer {
|
||||
const copy = new Uint8Array(bytes.byteLength);
|
||||
copy.set(bytes);
|
||||
return copy.buffer;
|
||||
}
|
||||
|
||||
function makeDigestCtx(options: {
|
||||
pages?: Array<{
|
||||
page: Array<Record<string, unknown>>;
|
||||
@@ -10758,6 +10891,56 @@ describe("package scan backfill", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not hide historical ClawPack file repair inside the scan backfill batch", async () => {
|
||||
const result = await getPackageReleaseScanBackfillBatchInternalHandler(
|
||||
{
|
||||
db: {
|
||||
query: vi.fn((table: string) => {
|
||||
if (table !== "packageReleases") throw new Error(`Unexpected table ${table}`);
|
||||
return {
|
||||
order: vi.fn(() => ({
|
||||
take: vi.fn().mockResolvedValue([]),
|
||||
})),
|
||||
withIndex: vi.fn(() => ({
|
||||
order: vi.fn(() => ({
|
||||
take: vi.fn().mockResolvedValue([
|
||||
{
|
||||
_id: "packageReleases:truncated-clawpack",
|
||||
_creationTime: 10,
|
||||
packageId: "packages:demo",
|
||||
artifactKind: "npm-pack",
|
||||
clawpackStorageId: "storage:clawpack",
|
||||
npmFileCount: 4,
|
||||
files: [
|
||||
{ path: "package.json", size: 10, storageId: "storage:package" },
|
||||
{
|
||||
path: "openclaw.plugin.json",
|
||||
size: 10,
|
||||
storageId: "storage:plugin",
|
||||
},
|
||||
],
|
||||
sha256hash: "hash",
|
||||
vtAnalysis: { status: "clean" },
|
||||
llmAnalysis: { status: "clean" },
|
||||
staticScan: { status: "clean" },
|
||||
},
|
||||
]),
|
||||
})),
|
||||
})),
|
||||
};
|
||||
}),
|
||||
get: vi.fn(async (id: string) => {
|
||||
if (id === "packages:demo") return makePackageDoc();
|
||||
return null;
|
||||
}),
|
||||
},
|
||||
} as never,
|
||||
{ batchSize: 10 },
|
||||
);
|
||||
|
||||
expect(result.releases).toEqual([]);
|
||||
});
|
||||
|
||||
it("schedules static rescans for releases missing only static scan data", async () => {
|
||||
const originalVtApiKey = process.env.VT_API_KEY;
|
||||
process.env.VT_API_KEY = "vt-test-key";
|
||||
@@ -10799,6 +10982,521 @@ describe("package scan backfill", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps scan backfill scoped to scan work for historical ClawPack releases", async () => {
|
||||
const runAfter = vi.fn().mockResolvedValue(undefined);
|
||||
const runMutation = vi.fn();
|
||||
const result = await backfillPackageReleaseScansInternalHandler(
|
||||
{
|
||||
runQuery: vi.fn().mockResolvedValue({
|
||||
releases: [
|
||||
{
|
||||
releaseId: "packageReleases:truncated-clawpack",
|
||||
needsVt: false,
|
||||
needsLlm: true,
|
||||
needsStatic: true,
|
||||
},
|
||||
],
|
||||
nextCursor: 123,
|
||||
done: true,
|
||||
}),
|
||||
runMutation,
|
||||
scheduler: { runAfter },
|
||||
} as never,
|
||||
{ batchSize: 10 },
|
||||
);
|
||||
|
||||
expect(result).toEqual({ scheduled: 1, nextCursor: 123, done: true });
|
||||
expect(runAfter).toHaveBeenCalledTimes(1);
|
||||
expect(runAfter).toHaveBeenCalledWith(
|
||||
0,
|
||||
expect.anything(),
|
||||
expect.objectContaining({ releaseId: "packageReleases:truncated-clawpack" }),
|
||||
);
|
||||
expect(runMutation).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
releaseId: "packageReleases:truncated-clawpack",
|
||||
source: "backfill",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("lists historical ClawPack repair candidates with cursor progress and package targeting", async () => {
|
||||
const packageWithIndex = vi.fn(() => ({
|
||||
unique: vi.fn().mockResolvedValue(
|
||||
makePackageDoc({
|
||||
_id: "packages:demo",
|
||||
name: "@scope/demo",
|
||||
normalizedName: "@scope/demo",
|
||||
}),
|
||||
),
|
||||
}));
|
||||
const paginate = vi.fn().mockResolvedValue({
|
||||
page: [
|
||||
makeReleaseDoc({
|
||||
_id: "packageReleases:truncated-clawpack",
|
||||
_creationTime: 10,
|
||||
packageId: "packages:demo",
|
||||
version: "1.0.0",
|
||||
artifactKind: "npm-pack",
|
||||
clawpackStorageId: "storage:clawpack",
|
||||
clawpackSha256: "a".repeat(64),
|
||||
npmFileCount: 4,
|
||||
files: [
|
||||
{ path: "package.json", size: 10, storageId: "storage:package" },
|
||||
{ path: "openclaw.plugin.json", size: 10, storageId: "storage:plugin" },
|
||||
],
|
||||
}),
|
||||
makeReleaseDoc({
|
||||
_id: "packageReleases:complete-clawpack",
|
||||
_creationTime: 11,
|
||||
packageId: "packages:demo",
|
||||
version: "1.0.1",
|
||||
artifactKind: "npm-pack",
|
||||
clawpackStorageId: "storage:clawpack-complete",
|
||||
npmFileCount: 2,
|
||||
files: [
|
||||
{ path: "package.json", size: 10, storageId: "storage:package" },
|
||||
{ path: "openclaw.plugin.json", size: 10, storageId: "storage:plugin" },
|
||||
],
|
||||
}),
|
||||
],
|
||||
continueCursor: "cursor:next",
|
||||
isDone: false,
|
||||
});
|
||||
|
||||
const result = await getHistoricalClawPackRepairBatchInternalHandler(
|
||||
{
|
||||
db: {
|
||||
query: vi.fn((table: string) => {
|
||||
if (table === "packages") {
|
||||
return { withIndex: packageWithIndex };
|
||||
}
|
||||
if (table !== "packageReleases") throw new Error(`Unexpected table ${table}`);
|
||||
return {
|
||||
withIndex: vi.fn(() => ({
|
||||
order: vi.fn(() => ({ paginate })),
|
||||
})),
|
||||
};
|
||||
}),
|
||||
get: vi.fn(),
|
||||
},
|
||||
} as never,
|
||||
{ cursor: "cursor:start", batchSize: 2, packageName: "@scope/demo" },
|
||||
);
|
||||
|
||||
expect(packageWithIndex).toHaveBeenCalledWith("by_name", expect.any(Function));
|
||||
expect(paginate).toHaveBeenCalledWith({ cursor: "cursor:start", numItems: 2 });
|
||||
expect(result).toEqual({
|
||||
candidates: [
|
||||
expect.objectContaining({
|
||||
releaseId: "packageReleases:truncated-clawpack",
|
||||
packageName: "@scope/demo",
|
||||
version: "1.0.0",
|
||||
existingFileCount: 2,
|
||||
npmFileCount: 4,
|
||||
}),
|
||||
],
|
||||
scanned: 2,
|
||||
cursor: "cursor:next",
|
||||
isDone: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("uses package/version indexes for targeted ClawPack repair beyond the first global page", async () => {
|
||||
const packageWithIndex = vi.fn(() => ({
|
||||
unique: vi.fn().mockResolvedValue(
|
||||
makePackageDoc({
|
||||
_id: "packages:target",
|
||||
name: "@scope/target",
|
||||
normalizedName: "@scope/target",
|
||||
}),
|
||||
),
|
||||
}));
|
||||
const releaseWithIndex = vi.fn(() => ({
|
||||
unique: vi.fn().mockResolvedValue(
|
||||
makeReleaseDoc({
|
||||
_id: "packageReleases:target-2",
|
||||
_creationTime: 9999,
|
||||
packageId: "packages:target",
|
||||
version: "2.0.0",
|
||||
artifactKind: "npm-pack",
|
||||
clawpackStorageId: "storage:clawpack",
|
||||
npmFileCount: 4,
|
||||
files: [
|
||||
{ path: "package.json", size: 10, storageId: "storage:package" },
|
||||
{ path: "openclaw.plugin.json", size: 10, storageId: "storage:plugin" },
|
||||
],
|
||||
}),
|
||||
),
|
||||
}));
|
||||
const globalPaginate = vi.fn().mockResolvedValue({
|
||||
page: [
|
||||
makeReleaseDoc({
|
||||
_id: "packageReleases:unrelated",
|
||||
packageId: "packages:other",
|
||||
version: "1.0.0",
|
||||
}),
|
||||
],
|
||||
continueCursor: "cursor:global-next",
|
||||
isDone: false,
|
||||
});
|
||||
|
||||
const result = await getHistoricalClawPackRepairBatchInternalHandler(
|
||||
{
|
||||
db: {
|
||||
query: vi.fn((table: string) => {
|
||||
if (table === "packages") {
|
||||
return { withIndex: packageWithIndex };
|
||||
}
|
||||
if (table !== "packageReleases") throw new Error(`Unexpected table ${table}`);
|
||||
return {
|
||||
withIndex: releaseWithIndex,
|
||||
order: vi.fn(() => ({ paginate: globalPaginate })),
|
||||
};
|
||||
}),
|
||||
get: vi.fn(),
|
||||
},
|
||||
} as never,
|
||||
{ batchSize: 1, packageName: "@scope/target", version: "2.0.0" },
|
||||
);
|
||||
|
||||
expect(packageWithIndex).toHaveBeenCalledWith("by_name", expect.any(Function));
|
||||
expect(releaseWithIndex).toHaveBeenCalledWith("by_package_version", expect.any(Function));
|
||||
expect(globalPaginate).not.toHaveBeenCalled();
|
||||
expect(result).toEqual({
|
||||
candidates: [
|
||||
expect.objectContaining({
|
||||
releaseId: "packageReleases:target-2",
|
||||
packageName: "@scope/target",
|
||||
version: "2.0.0",
|
||||
existingFileCount: 2,
|
||||
npmFileCount: 4,
|
||||
}),
|
||||
],
|
||||
scanned: 1,
|
||||
cursor: null,
|
||||
isDone: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("dry-runs historical ClawPack repair with artifact validation and no writes", async () => {
|
||||
const pack = npmPackFixture({
|
||||
"package/package.json": JSON.stringify({ name: "@scope/demo", version: "1.0.0" }),
|
||||
"package/openclaw.plugin.json": JSON.stringify({ id: "scope.demo" }),
|
||||
"package/README.md": "# Demo\n",
|
||||
"package/dist/index.js": "export const demo = true;\n",
|
||||
});
|
||||
const storageGet = vi.fn(
|
||||
async () => new Blob([bytesToArrayBuffer(pack)], { type: "application/octet-stream" }),
|
||||
);
|
||||
const storageStore = vi.fn();
|
||||
const runMutation = vi.fn();
|
||||
const runAfter = vi.fn();
|
||||
|
||||
const result = await repairHistoricalClawPackReleaseFilesInternalHandler(
|
||||
{
|
||||
runQuery: vi.fn(async (_ref: unknown, args: { packageId?: string; releaseId?: string }) => {
|
||||
if ("batchSize" in (args as Record<string, unknown>)) {
|
||||
return {
|
||||
candidates: [
|
||||
{
|
||||
releaseId: "packageReleases:truncated-clawpack",
|
||||
packageId: "packages:demo",
|
||||
packageName: "@scope/demo",
|
||||
version: "1.0.0",
|
||||
createdAt: 10,
|
||||
existingFileCount: 2,
|
||||
npmFileCount: 4,
|
||||
clawpackSha256: null,
|
||||
},
|
||||
],
|
||||
scanned: 1,
|
||||
cursor: null,
|
||||
isDone: true,
|
||||
};
|
||||
}
|
||||
if (args.releaseId) {
|
||||
return makeReleaseDoc({
|
||||
_id: args.releaseId,
|
||||
packageId: "packages:demo",
|
||||
version: "1.0.0",
|
||||
artifactKind: "npm-pack",
|
||||
clawpackStorageId: "storage:clawpack",
|
||||
npmFileCount: 4,
|
||||
files: [
|
||||
{ path: "package.json", size: 10, storageId: "storage:package" },
|
||||
{ path: "openclaw.plugin.json", size: 10, storageId: "storage:plugin" },
|
||||
],
|
||||
});
|
||||
}
|
||||
if (args.packageId === "packages:demo") {
|
||||
return makePackageDoc({
|
||||
_id: "packages:demo",
|
||||
name: "@scope/demo",
|
||||
normalizedName: "@scope/demo",
|
||||
});
|
||||
}
|
||||
return null;
|
||||
}),
|
||||
runMutation,
|
||||
scheduler: { runAfter },
|
||||
storage: { get: storageGet, store: storageStore },
|
||||
} as never,
|
||||
{ dryRun: true, batchSize: 1, maxBatches: 1, packageName: "@scope/demo" },
|
||||
);
|
||||
|
||||
expect(result.stats).toEqual({
|
||||
scanned: 1,
|
||||
candidates: 1,
|
||||
validated: 1,
|
||||
wouldRepair: 1,
|
||||
repaired: 0,
|
||||
skipped: 0,
|
||||
errors: 0,
|
||||
});
|
||||
expect(result.writeConfirmation).toBe("repair-historical-clawpack-files");
|
||||
expect(result.samples[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
releaseId: "packageReleases:truncated-clawpack",
|
||||
repairedFiles: 4,
|
||||
samplePaths: ["package.json", "openclaw.plugin.json", "README.md", "dist/index.js"],
|
||||
}),
|
||||
);
|
||||
expect(result.logLines.join("\n")).toContain("would-repair package=@scope/demo");
|
||||
expect(storageStore).not.toHaveBeenCalled();
|
||||
expect(runMutation).not.toHaveBeenCalled();
|
||||
expect(runAfter).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("requires explicit confirmation before applying historical ClawPack repair", async () => {
|
||||
await expect(
|
||||
repairHistoricalClawPackReleaseFilesInternalHandler(
|
||||
{
|
||||
runQuery: vi.fn(),
|
||||
runMutation: vi.fn(),
|
||||
scheduler: { runAfter: vi.fn() },
|
||||
storage: { get: vi.fn(), store: vi.fn() },
|
||||
} as never,
|
||||
{ dryRun: false, batchSize: 1 },
|
||||
),
|
||||
).rejects.toThrow(/confirmation/);
|
||||
});
|
||||
|
||||
it("applies historical ClawPack repair with confirmation and schedules resumable follow-up", async () => {
|
||||
const pack = npmPackFixture({
|
||||
"package/package.json": JSON.stringify({ name: "@scope/demo", version: "1.0.0" }),
|
||||
"package/openclaw.plugin.json": JSON.stringify({ id: "scope.demo" }),
|
||||
"package/README.md": "# Demo\n",
|
||||
"package/dist/index.js": "export const demo = true;\n",
|
||||
});
|
||||
const storageGet = vi.fn(
|
||||
async () => new Blob([bytesToArrayBuffer(pack)], { type: "application/octet-stream" }),
|
||||
);
|
||||
const storageStore = vi.fn(async () => `storage:repaired-${storageStore.mock.calls.length}`);
|
||||
const runMutation = vi.fn(async (_ref: unknown, args: Record<string, unknown>) => {
|
||||
if (Array.isArray(args.files)) return { repaired: true };
|
||||
return { jobId: "securityScanJobs:1" };
|
||||
});
|
||||
const runAfter = vi.fn().mockResolvedValue(undefined);
|
||||
|
||||
const result = await repairHistoricalClawPackReleaseFilesInternalHandler(
|
||||
{
|
||||
runQuery: vi.fn(async (_ref: unknown, args: { packageId?: string; releaseId?: string }) => {
|
||||
if ("batchSize" in (args as Record<string, unknown>)) {
|
||||
return {
|
||||
candidates: [
|
||||
{
|
||||
releaseId: "packageReleases:truncated-clawpack",
|
||||
packageId: "packages:demo",
|
||||
packageName: "@scope/demo",
|
||||
version: "1.0.0",
|
||||
createdAt: 10,
|
||||
existingFileCount: 2,
|
||||
npmFileCount: 4,
|
||||
clawpackSha256: null,
|
||||
},
|
||||
],
|
||||
scanned: 1,
|
||||
cursor: "cursor:next",
|
||||
isDone: false,
|
||||
};
|
||||
}
|
||||
if (args.releaseId) {
|
||||
return makeReleaseDoc({
|
||||
_id: args.releaseId,
|
||||
packageId: "packages:demo",
|
||||
version: "1.0.0",
|
||||
artifactKind: "npm-pack",
|
||||
clawpackStorageId: "storage:clawpack",
|
||||
npmFileCount: 4,
|
||||
files: [
|
||||
{ path: "package.json", size: 10, storageId: "storage:package" },
|
||||
{ path: "openclaw.plugin.json", size: 10, storageId: "storage:plugin" },
|
||||
],
|
||||
});
|
||||
}
|
||||
if (args.packageId === "packages:demo") {
|
||||
return makePackageDoc({
|
||||
_id: "packages:demo",
|
||||
name: "@scope/demo",
|
||||
normalizedName: "@scope/demo",
|
||||
});
|
||||
}
|
||||
return null;
|
||||
}),
|
||||
runMutation,
|
||||
scheduler: { runAfter },
|
||||
storage: { get: storageGet, store: storageStore },
|
||||
} as never,
|
||||
{
|
||||
dryRun: false,
|
||||
batchSize: 1,
|
||||
maxBatches: 1,
|
||||
packageName: "@scope/demo",
|
||||
confirmation: "repair-historical-clawpack-files",
|
||||
scheduleNext: true,
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.stats).toEqual({
|
||||
scanned: 1,
|
||||
candidates: 1,
|
||||
validated: 1,
|
||||
wouldRepair: 0,
|
||||
repaired: 1,
|
||||
skipped: 0,
|
||||
errors: 0,
|
||||
});
|
||||
expect(result.cursor).toBe("cursor:next");
|
||||
expect(result.isDone).toBe(false);
|
||||
expect(storageStore).toHaveBeenCalledTimes(4);
|
||||
expect(runMutation).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
releaseId: "packageReleases:truncated-clawpack",
|
||||
files: expect.arrayContaining([
|
||||
expect.objectContaining({ path: "README.md", contentType: "text/markdown" }),
|
||||
]),
|
||||
}),
|
||||
);
|
||||
expect(runMutation).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
releaseId: "packageReleases:truncated-clawpack",
|
||||
source: "manual",
|
||||
}),
|
||||
);
|
||||
expect(runAfter).toHaveBeenCalledWith(
|
||||
0,
|
||||
expect.anything(),
|
||||
expect.objectContaining({ releaseId: "packageReleases:truncated-clawpack" }),
|
||||
);
|
||||
expect(runAfter).toHaveBeenCalledWith(
|
||||
0,
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
cursor: "cursor:next",
|
||||
dryRun: false,
|
||||
confirmation: "repair-historical-clawpack-files",
|
||||
scheduleNext: true,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("repairs historical ClawPack release file rows from the stored npm-pack artifact", async () => {
|
||||
const pack = npmPackFixture({
|
||||
"package/package.json": JSON.stringify({ name: "@scope/demo", version: "1.0.0" }),
|
||||
"package/openclaw.plugin.json": JSON.stringify({ id: "scope.demo" }),
|
||||
"package/README.md": "# Demo\n",
|
||||
"package/dist/index.js": "export const demo = true;\n",
|
||||
});
|
||||
const storageGet = vi.fn(async (storageId: string) =>
|
||||
storageId === "storage:clawpack"
|
||||
? new Blob([bytesToArrayBuffer(pack)], { type: "application/octet-stream" })
|
||||
: null,
|
||||
);
|
||||
const storageStore = vi.fn(async () => `storage:repaired-${storageStore.mock.calls.length}`);
|
||||
const runMutation = vi.fn(async (_ref: unknown, args: Record<string, unknown>) => {
|
||||
if (Array.isArray(args.files)) return { repaired: true };
|
||||
return { jobId: "securityScanJobs:1" };
|
||||
});
|
||||
const runAfter = vi.fn().mockResolvedValue(undefined);
|
||||
|
||||
const result = await repairClawPackReleaseFilesInternalHandler(
|
||||
{
|
||||
runQuery: vi.fn(async (_ref: unknown, args: { packageId?: string; releaseId?: string }) => {
|
||||
if (args.releaseId) {
|
||||
return makeReleaseDoc({
|
||||
_id: args.releaseId,
|
||||
packageId: "packages:demo",
|
||||
version: "1.0.0",
|
||||
artifactKind: "npm-pack",
|
||||
clawpackStorageId: "storage:clawpack",
|
||||
npmFileCount: 4,
|
||||
files: [
|
||||
{ path: "package.json", size: 10, storageId: "storage:package" },
|
||||
{
|
||||
path: "openclaw.plugin.json",
|
||||
size: 10,
|
||||
storageId: "storage:plugin",
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
if (args.packageId === "packages:demo") {
|
||||
return makePackageDoc({ _id: "packages:demo", name: "@scope/demo" });
|
||||
}
|
||||
return null;
|
||||
}),
|
||||
runMutation,
|
||||
scheduler: { runAfter },
|
||||
storage: { get: storageGet, store: storageStore },
|
||||
} as never,
|
||||
{ releaseId: "packageReleases:truncated-clawpack" },
|
||||
);
|
||||
|
||||
expect(result).toEqual(
|
||||
expect.objectContaining({
|
||||
ok: true,
|
||||
repaired: true,
|
||||
dryRun: false,
|
||||
files: 4,
|
||||
existingFiles: 2,
|
||||
npmFileCount: 4,
|
||||
integritySha256: expect.stringMatching(/^[a-f0-9]{64}$/),
|
||||
samplePaths: ["package.json", "openclaw.plugin.json", "README.md", "dist/index.js"],
|
||||
}),
|
||||
);
|
||||
expect(storageStore).toHaveBeenCalledTimes(4);
|
||||
const repairCall = runMutation.mock.calls.find(([, args]) => Array.isArray(args.files));
|
||||
expect(repairCall?.[1]).toEqual(
|
||||
expect.objectContaining({
|
||||
releaseId: "packageReleases:truncated-clawpack",
|
||||
npmFileCount: 4,
|
||||
integritySha256: expect.stringMatching(/^[a-f0-9]{64}$/),
|
||||
files: [
|
||||
expect.objectContaining({ path: "package.json" }),
|
||||
expect.objectContaining({ path: "openclaw.plugin.json" }),
|
||||
expect.objectContaining({ path: "README.md", contentType: "text/markdown" }),
|
||||
expect.objectContaining({ path: "dist/index.js", contentType: "application/javascript" }),
|
||||
],
|
||||
}),
|
||||
);
|
||||
expect(runAfter).toHaveBeenCalledWith(
|
||||
0,
|
||||
expect.anything(),
|
||||
expect.objectContaining({ releaseId: "packageReleases:truncated-clawpack" }),
|
||||
);
|
||||
expect(runMutation).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
releaseId: "packageReleases:truncated-clawpack",
|
||||
source: "manual",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("backfills legacy static-only package scan status into the package search digest", async () => {
|
||||
const verification = {
|
||||
tier: "source-linked",
|
||||
|
||||
+596
-1
@@ -1,6 +1,7 @@
|
||||
import {
|
||||
ServerPackagePublishRequestSchema,
|
||||
derivePluginCategoryTags,
|
||||
guessTextContentType,
|
||||
getPackageScopeOwnerMismatch,
|
||||
isPluginCategorySlug,
|
||||
parseArk,
|
||||
@@ -45,7 +46,7 @@ import {
|
||||
readArtifactReportStatus,
|
||||
appendPackageModerationEventLog,
|
||||
} from "./lib/artifactModeration";
|
||||
import { sha256Hex } from "./lib/clawpack";
|
||||
import { parseClawPack, sha256Hex } from "./lib/clawpack";
|
||||
import { buildPackageInspectorFindingsEmail } from "./lib/emails";
|
||||
import { requireGitHubAccountAge } from "./lib/githubAccount";
|
||||
import { normalizeGitHubRepository } from "./lib/githubActionsOidc";
|
||||
@@ -357,7 +358,10 @@ function isTrustedOpenClawPluginPackage(params: {
|
||||
const internalRefs = internal as unknown as {
|
||||
packages: {
|
||||
backfillPackageReleaseScansInternal: unknown;
|
||||
repairClawPackReleaseFilesInternal: unknown;
|
||||
repairHistoricalClawPackReleaseFilesInternal: unknown;
|
||||
scanPackageReleaseStaticallyInternal: unknown;
|
||||
replaceReleaseFilesAfterClawPackRepairInternal: unknown;
|
||||
insertReleaseInternal: unknown;
|
||||
getPackageByNameInternal: unknown;
|
||||
getTrustedPublisherByPackageIdInternal: unknown;
|
||||
@@ -365,6 +369,7 @@ const internalRefs = internal as unknown as {
|
||||
getPackageByIdInternal: unknown;
|
||||
getReleaseByIdInternal: unknown;
|
||||
getPackageReleaseScanBackfillBatchInternal: unknown;
|
||||
getHistoricalClawPackRepairBatchInternal: unknown;
|
||||
listVersionsForViewerInternal: unknown;
|
||||
getVersionByNameForViewerInternal: unknown;
|
||||
publishPackageForUserInternal: unknown;
|
||||
@@ -471,6 +476,86 @@ type PackageInspectorFinding = {
|
||||
decision?: string;
|
||||
};
|
||||
|
||||
type PackageReleaseFilePatch = Array<{
|
||||
path: string;
|
||||
size: number;
|
||||
storageId: Id<"_storage">;
|
||||
sha256: string;
|
||||
contentType?: string;
|
||||
}>;
|
||||
|
||||
type ClawPackRepairCandidate = {
|
||||
releaseId: Id<"packageReleases">;
|
||||
packageId: Id<"packages">;
|
||||
packageName: string;
|
||||
version: string;
|
||||
createdAt: number;
|
||||
existingFileCount: number;
|
||||
npmFileCount: number | null;
|
||||
clawpackSha256: string | null;
|
||||
};
|
||||
|
||||
type ClawPackRepairResult =
|
||||
| {
|
||||
ok: true;
|
||||
repaired: boolean;
|
||||
dryRun: boolean;
|
||||
files: number;
|
||||
existingFiles: number;
|
||||
npmFileCount: number;
|
||||
integritySha256: string;
|
||||
samplePaths: string[];
|
||||
artifactSha256: string;
|
||||
}
|
||||
| {
|
||||
ok: true;
|
||||
repaired: false;
|
||||
dryRun: boolean;
|
||||
skipped: "missing_release" | "not_needed" | "missing_package";
|
||||
}
|
||||
| {
|
||||
ok: false;
|
||||
repaired: false;
|
||||
dryRun: boolean;
|
||||
error: string;
|
||||
};
|
||||
|
||||
type HistoricalClawPackRepairStats = {
|
||||
scanned: number;
|
||||
candidates: number;
|
||||
validated: number;
|
||||
wouldRepair: number;
|
||||
repaired: number;
|
||||
skipped: number;
|
||||
errors: number;
|
||||
};
|
||||
|
||||
const CLAWPACK_REPAIR_WRITE_CONFIRMATION = "repair-historical-clawpack-files";
|
||||
|
||||
function bytesToArrayBuffer(bytes: Uint8Array) {
|
||||
const copy = new Uint8Array(bytes.byteLength);
|
||||
copy.set(bytes);
|
||||
return copy.buffer;
|
||||
}
|
||||
|
||||
function inferClawPackEntryContentType(path: string) {
|
||||
return guessTextContentType(path) ?? "application/octet-stream";
|
||||
}
|
||||
|
||||
function needsClawPackReleaseFileRepair(
|
||||
release: Pick<
|
||||
Doc<"packageReleases">,
|
||||
"artifactKind" | "clawpackStorageId" | "npmFileCount" | "files"
|
||||
>,
|
||||
) {
|
||||
return (
|
||||
release.artifactKind === "npm-pack" &&
|
||||
Boolean(release.clawpackStorageId) &&
|
||||
typeof release.npmFileCount === "number" &&
|
||||
release.npmFileCount > release.files.length
|
||||
);
|
||||
}
|
||||
|
||||
type PackageInspectorPublishResult = {
|
||||
status: "pass" | "fail";
|
||||
summary: {
|
||||
@@ -5809,6 +5894,128 @@ export const getPackageReleaseScanBackfillBatchInternal = internalQuery({
|
||||
},
|
||||
});
|
||||
|
||||
export const getHistoricalClawPackRepairBatchInternal = internalQuery({
|
||||
args: {
|
||||
cursor: v.optional(v.union(v.string(), v.null())),
|
||||
batchSize: v.optional(v.number()),
|
||||
releaseId: v.optional(v.id("packageReleases")),
|
||||
packageName: v.optional(v.string()),
|
||||
version: v.optional(v.string()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const batchSize = Math.max(1, Math.min(Math.floor(args.batchSize ?? 25), 100));
|
||||
const targetName = args.packageName ? normalizePackageName(args.packageName) : undefined;
|
||||
|
||||
const toCandidateForPackage = (
|
||||
release: Doc<"packageReleases">,
|
||||
pkg: Doc<"packages">,
|
||||
): ClawPackRepairCandidate | null => {
|
||||
if (!isReleaseActive(release)) return null;
|
||||
if (!needsClawPackReleaseFileRepair(release)) return null;
|
||||
if (args.version && release.version !== args.version) return null;
|
||||
if (!pkg || pkg.softDeletedAt || pkg.family === "skill") return null;
|
||||
if (targetName && pkg.normalizedName !== targetName) return null;
|
||||
|
||||
return {
|
||||
releaseId: release._id,
|
||||
packageId: release.packageId,
|
||||
packageName: pkg.normalizedName,
|
||||
version: release.version,
|
||||
createdAt: release.createdAt,
|
||||
existingFileCount: release.files.length,
|
||||
npmFileCount: release.npmFileCount ?? null,
|
||||
clawpackSha256: release.clawpackSha256 ?? null,
|
||||
};
|
||||
};
|
||||
|
||||
const toCandidate = async (release: Doc<"packageReleases">) => {
|
||||
const pkg = await ctx.db.get(release.packageId);
|
||||
if (!pkg) return null;
|
||||
return toCandidateForPackage(release, pkg);
|
||||
};
|
||||
|
||||
if (args.releaseId) {
|
||||
const release = await ctx.db.get(args.releaseId);
|
||||
const candidate = release ? await toCandidate(release) : null;
|
||||
return {
|
||||
candidates: candidate ? [candidate] : [],
|
||||
scanned: release ? 1 : 0,
|
||||
cursor: null,
|
||||
isDone: true,
|
||||
};
|
||||
}
|
||||
|
||||
if (targetName) {
|
||||
const pkg = await getPackageByNormalizedName(ctx, targetName);
|
||||
if (!pkg || pkg.softDeletedAt || pkg.family === "skill") {
|
||||
return {
|
||||
candidates: [],
|
||||
scanned: 0,
|
||||
cursor: null,
|
||||
isDone: true,
|
||||
};
|
||||
}
|
||||
|
||||
if (args.version) {
|
||||
const targetVersion = args.version;
|
||||
const release = await ctx.db
|
||||
.query("packageReleases")
|
||||
.withIndex("by_package_version", (q) =>
|
||||
q.eq("packageId", pkg._id).eq("version", targetVersion),
|
||||
)
|
||||
.unique();
|
||||
const candidate = release ? toCandidateForPackage(release, pkg) : null;
|
||||
return {
|
||||
candidates: candidate ? [candidate] : [],
|
||||
scanned: release ? 1 : 0,
|
||||
cursor: null,
|
||||
isDone: true,
|
||||
};
|
||||
}
|
||||
|
||||
const { page, continueCursor, isDone } = await ctx.db
|
||||
.query("packageReleases")
|
||||
.withIndex("by_package_active_created", (q) =>
|
||||
q.eq("packageId", pkg._id).eq("softDeletedAt", undefined),
|
||||
)
|
||||
.order("asc")
|
||||
.paginate({ cursor: args.cursor ?? null, numItems: batchSize });
|
||||
|
||||
const candidates: ClawPackRepairCandidate[] = [];
|
||||
for (const release of page) {
|
||||
const candidate = toCandidateForPackage(release, pkg);
|
||||
if (candidate) candidates.push(candidate);
|
||||
}
|
||||
|
||||
return {
|
||||
candidates,
|
||||
scanned: page.length,
|
||||
cursor: isDone ? null : continueCursor,
|
||||
isDone,
|
||||
};
|
||||
}
|
||||
|
||||
const { page, continueCursor, isDone } = await ctx.db
|
||||
.query("packageReleases")
|
||||
.withIndex("by_active_created", (q) => q.eq("softDeletedAt", undefined))
|
||||
.order("asc")
|
||||
.paginate({ cursor: args.cursor ?? null, numItems: batchSize });
|
||||
|
||||
const candidates: ClawPackRepairCandidate[] = [];
|
||||
for (const release of page) {
|
||||
const candidate = await toCandidate(release);
|
||||
if (candidate) candidates.push(candidate);
|
||||
}
|
||||
|
||||
return {
|
||||
candidates,
|
||||
scanned: page.length,
|
||||
cursor: isDone ? null : continueCursor,
|
||||
isDone,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
function buildGitHubActionsPublishActor(
|
||||
publishToken: Doc<"packagePublishTokens">,
|
||||
): Extract<PackagePublishActor, { kind: "github-actions" }> {
|
||||
@@ -8161,6 +8368,368 @@ export const updateReleaseStaticScanInternal = internalMutation({
|
||||
},
|
||||
});
|
||||
|
||||
export const replaceReleaseFilesAfterClawPackRepairInternal = internalMutation({
|
||||
args: {
|
||||
releaseId: v.id("packageReleases"),
|
||||
files: v.array(
|
||||
v.object({
|
||||
path: v.string(),
|
||||
size: v.number(),
|
||||
storageId: v.id("_storage"),
|
||||
sha256: v.string(),
|
||||
contentType: v.optional(v.string()),
|
||||
}),
|
||||
),
|
||||
clawpackSha256: v.string(),
|
||||
npmIntegrity: v.string(),
|
||||
npmShasum: v.string(),
|
||||
npmTarballName: v.string(),
|
||||
npmUnpackedSize: v.number(),
|
||||
npmFileCount: v.number(),
|
||||
integritySha256: v.string(),
|
||||
extractedPackageJson: v.any(),
|
||||
extractedPluginManifest: v.any(),
|
||||
checkedAt: v.number(),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const release = await ctx.db.get(args.releaseId);
|
||||
if (!isReleaseActive(release) || !needsClawPackReleaseFileRepair(release)) {
|
||||
return { repaired: false as const };
|
||||
}
|
||||
|
||||
const nextVerification = release.verification
|
||||
? {
|
||||
...release.verification,
|
||||
scanStatus: "pending" as const,
|
||||
}
|
||||
: release.verification;
|
||||
const patch: Partial<Doc<"packageReleases">> = {
|
||||
files: args.files,
|
||||
integritySha256: args.integritySha256,
|
||||
clawpackSha256: args.clawpackSha256,
|
||||
npmIntegrity: args.npmIntegrity,
|
||||
npmShasum: args.npmShasum,
|
||||
npmTarballName: args.npmTarballName,
|
||||
npmUnpackedSize: args.npmUnpackedSize,
|
||||
npmFileCount: args.npmFileCount,
|
||||
extractedPackageJson: args.extractedPackageJson,
|
||||
extractedPluginManifest: args.extractedPluginManifest,
|
||||
verification: nextVerification,
|
||||
staticScan: undefined,
|
||||
skillSpectorAnalysis: undefined,
|
||||
llmAnalysis: {
|
||||
status: "pending",
|
||||
summary: "ClawPack file listing repaired from the stored artifact; fresh review queued.",
|
||||
checkedAt: args.checkedAt,
|
||||
},
|
||||
};
|
||||
await ctx.db.patch(args.releaseId, patch);
|
||||
|
||||
await syncLatestPackageVerification(ctx, {
|
||||
...release,
|
||||
...patch,
|
||||
} as Doc<"packageReleases">);
|
||||
|
||||
return { repaired: true as const };
|
||||
},
|
||||
});
|
||||
|
||||
async function repairClawPackReleaseFiles(
|
||||
ctx: ActionCtx,
|
||||
args: { releaseId: Id<"packageReleases">; dryRun?: boolean },
|
||||
): Promise<ClawPackRepairResult> {
|
||||
const dryRun = args.dryRun === true;
|
||||
const release = await runQueryRef<Doc<"packageReleases"> | null>(
|
||||
ctx,
|
||||
internalRefs.packages.getReleaseByIdInternal,
|
||||
{ releaseId: args.releaseId },
|
||||
);
|
||||
if (!release || release.softDeletedAt) {
|
||||
return { ok: true, repaired: false, dryRun, skipped: "missing_release" };
|
||||
}
|
||||
if (!needsClawPackReleaseFileRepair(release) || !release.clawpackStorageId) {
|
||||
return { ok: true, repaired: false, dryRun, skipped: "not_needed" };
|
||||
}
|
||||
|
||||
const pkg = await runQueryRef<Doc<"packages"> | null>(
|
||||
ctx,
|
||||
internalRefs.packages.getPackageByIdInternal,
|
||||
{ packageId: release.packageId },
|
||||
);
|
||||
if (!pkg || pkg.softDeletedAt || pkg.family === "skill") {
|
||||
return { ok: true, repaired: false, dryRun, skipped: "missing_package" };
|
||||
}
|
||||
|
||||
const blob = await ctx.storage.get(release.clawpackStorageId);
|
||||
if (!blob) {
|
||||
return {
|
||||
ok: false,
|
||||
repaired: false,
|
||||
dryRun,
|
||||
error: "ClawPack artifact missing",
|
||||
};
|
||||
}
|
||||
|
||||
const artifactBytes = new Uint8Array(await blob.arrayBuffer());
|
||||
const parsed = await parseClawPack(artifactBytes);
|
||||
if (release.clawpackSha256 && parsed.artifactSha256 !== release.clawpackSha256) {
|
||||
throw new Error("Stored ClawPack artifact digest does not match package release metadata");
|
||||
}
|
||||
const parsedPackageName = normalizePackageName(parsed.packageName);
|
||||
const storedPackageNames = new Set([pkg.normalizedName, normalizePackageName(pkg.name)]);
|
||||
if (!storedPackageNames.has(parsedPackageName) || parsed.packageVersion !== release.version) {
|
||||
throw new Error("Stored ClawPack artifact identity does not match package release metadata");
|
||||
}
|
||||
|
||||
const parsedFiles = [];
|
||||
for (const entry of parsed.entries) {
|
||||
parsedFiles.push({
|
||||
path: entry.path,
|
||||
size: entry.bytes.byteLength,
|
||||
sha256: await sha256Hex(entry.bytes),
|
||||
contentType: inferClawPackEntryContentType(entry.path),
|
||||
bytes: entry.bytes,
|
||||
});
|
||||
}
|
||||
const integritySha256 = await hashSkillFiles(parsedFiles);
|
||||
|
||||
if (dryRun) {
|
||||
return {
|
||||
ok: true,
|
||||
repaired: false,
|
||||
dryRun: true,
|
||||
files: parsedFiles.length,
|
||||
existingFiles: release.files.length,
|
||||
npmFileCount: parsed.fileCount,
|
||||
integritySha256,
|
||||
artifactSha256: parsed.artifactSha256,
|
||||
samplePaths: parsedFiles.slice(0, 10).map((file) => file.path),
|
||||
};
|
||||
}
|
||||
|
||||
const files: PackageReleaseFilePatch = [];
|
||||
for (const file of parsedFiles) {
|
||||
const storageId = await ctx.storage.store(
|
||||
new Blob([bytesToArrayBuffer(file.bytes)], { type: file.contentType }),
|
||||
);
|
||||
files.push({
|
||||
path: file.path,
|
||||
size: file.size,
|
||||
storageId,
|
||||
sha256: file.sha256,
|
||||
contentType: file.contentType,
|
||||
});
|
||||
}
|
||||
|
||||
const result = await runMutationRef<{ repaired: boolean }>(
|
||||
ctx,
|
||||
internalRefs.packages.replaceReleaseFilesAfterClawPackRepairInternal,
|
||||
{
|
||||
releaseId: args.releaseId,
|
||||
files,
|
||||
clawpackSha256: parsed.artifactSha256,
|
||||
npmIntegrity: parsed.npmIntegrity,
|
||||
npmShasum: parsed.npmShasum,
|
||||
npmTarballName: parsed.npmTarballName,
|
||||
npmUnpackedSize: parsed.unpackedSize,
|
||||
npmFileCount: parsed.fileCount,
|
||||
integritySha256,
|
||||
extractedPackageJson: parsed.packageJson,
|
||||
extractedPluginManifest: parsed.pluginManifest,
|
||||
checkedAt: Date.now(),
|
||||
},
|
||||
);
|
||||
|
||||
if (!result.repaired) {
|
||||
return { ok: true, repaired: false, dryRun: false, skipped: "not_needed" };
|
||||
}
|
||||
|
||||
await runAfterRef(ctx, 0, internalRefs.packages.scanPackageReleaseStaticallyInternal, {
|
||||
releaseId: args.releaseId,
|
||||
});
|
||||
await runMutationRef(ctx, internalRefs.securityScan.enqueuePackageReleaseScanInternal, {
|
||||
releaseId: args.releaseId,
|
||||
source: "manual",
|
||||
});
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
repaired: true,
|
||||
dryRun: false,
|
||||
files: files.length,
|
||||
existingFiles: release.files.length,
|
||||
npmFileCount: parsed.fileCount,
|
||||
integritySha256,
|
||||
artifactSha256: parsed.artifactSha256,
|
||||
samplePaths: files.slice(0, 10).map((file) => file.path),
|
||||
};
|
||||
}
|
||||
|
||||
export const repairClawPackReleaseFilesInternal = internalAction({
|
||||
args: {
|
||||
releaseId: v.id("packageReleases"),
|
||||
dryRun: v.optional(v.boolean()),
|
||||
},
|
||||
handler: repairClawPackReleaseFiles,
|
||||
});
|
||||
|
||||
export const repairHistoricalClawPackReleaseFilesInternal = internalAction({
|
||||
args: {
|
||||
cursor: v.optional(v.union(v.string(), v.null())),
|
||||
batchSize: v.optional(v.number()),
|
||||
maxBatches: v.optional(v.number()),
|
||||
dryRun: v.optional(v.boolean()),
|
||||
releaseId: v.optional(v.id("packageReleases")),
|
||||
packageName: v.optional(v.string()),
|
||||
version: v.optional(v.string()),
|
||||
confirmation: v.optional(v.string()),
|
||||
scheduleNext: v.optional(v.boolean()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const dryRun = args.dryRun !== false;
|
||||
if (!dryRun && args.confirmation !== CLAWPACK_REPAIR_WRITE_CONFIRMATION) {
|
||||
throw new ConvexError(
|
||||
`Set confirmation to "${CLAWPACK_REPAIR_WRITE_CONFIRMATION}" to repair historical ClawPack release files.`,
|
||||
);
|
||||
}
|
||||
|
||||
const batchSize = Math.max(1, Math.min(Math.floor(args.batchSize ?? 25), 100));
|
||||
const maxBatches = Math.max(1, Math.min(Math.floor(args.maxBatches ?? 1), 20));
|
||||
const stats: HistoricalClawPackRepairStats = {
|
||||
scanned: 0,
|
||||
candidates: 0,
|
||||
validated: 0,
|
||||
wouldRepair: 0,
|
||||
repaired: 0,
|
||||
skipped: 0,
|
||||
errors: 0,
|
||||
};
|
||||
const logLines = [
|
||||
`[clawpack-repair] mode=${dryRun ? "dry-run" : "apply"} batchSize=${batchSize} maxBatches=${maxBatches}`,
|
||||
];
|
||||
const samples: Array<
|
||||
ClawPackRepairCandidate & {
|
||||
repairedFiles: number;
|
||||
integritySha256: string;
|
||||
artifactSha256: string;
|
||||
samplePaths: string[];
|
||||
}
|
||||
> = [];
|
||||
let cursor: string | null = args.cursor ?? null;
|
||||
let isDone = false;
|
||||
|
||||
for (let batchIndex = 0; batchIndex < maxBatches; batchIndex++) {
|
||||
const batch = await runQueryRef<{
|
||||
candidates: ClawPackRepairCandidate[];
|
||||
scanned: number;
|
||||
cursor: string | null;
|
||||
isDone: boolean;
|
||||
}>(ctx, internalRefs.packages.getHistoricalClawPackRepairBatchInternal, {
|
||||
cursor,
|
||||
batchSize,
|
||||
releaseId: args.releaseId,
|
||||
packageName: args.packageName,
|
||||
version: args.version,
|
||||
});
|
||||
|
||||
stats.scanned += batch.scanned;
|
||||
stats.candidates += batch.candidates.length;
|
||||
cursor = batch.cursor;
|
||||
isDone = batch.isDone;
|
||||
logLines.push(
|
||||
`[clawpack-repair] batch=${batchIndex + 1} scanned=${batch.scanned} candidates=${batch.candidates.length} nextCursor=${cursor ?? "<done>"}`,
|
||||
);
|
||||
|
||||
for (const candidate of batch.candidates) {
|
||||
try {
|
||||
const result = await repairClawPackReleaseFiles(ctx, {
|
||||
releaseId: candidate.releaseId,
|
||||
dryRun,
|
||||
});
|
||||
if (!result.ok) {
|
||||
stats.errors++;
|
||||
logLines.push(
|
||||
`[clawpack-repair] error package=${candidate.packageName} version=${candidate.version} release=${candidate.releaseId} error=${result.error}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if ("skipped" in result) {
|
||||
stats.skipped++;
|
||||
logLines.push(
|
||||
`[clawpack-repair] skipped package=${candidate.packageName} version=${candidate.version} release=${candidate.releaseId} reason=${result.skipped}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
stats.validated++;
|
||||
if (dryRun) {
|
||||
stats.wouldRepair++;
|
||||
} else if (result.repaired) {
|
||||
stats.repaired++;
|
||||
}
|
||||
if (samples.length < 20) {
|
||||
samples.push({
|
||||
...candidate,
|
||||
repairedFiles: result.files,
|
||||
integritySha256: result.integritySha256,
|
||||
artifactSha256: result.artifactSha256,
|
||||
samplePaths: result.samplePaths,
|
||||
});
|
||||
}
|
||||
logLines.push(
|
||||
`[clawpack-repair] ${dryRun ? "would-repair" : "repaired"} package=${candidate.packageName} version=${candidate.version} release=${candidate.releaseId} files=${candidate.existingFileCount}->${result.files} npmFileCount=${result.npmFileCount} integrity=${result.integritySha256.slice(0, 12)}...`,
|
||||
);
|
||||
} catch (error) {
|
||||
stats.errors++;
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
logLines.push(
|
||||
`[clawpack-repair] error package=${candidate.packageName} version=${candidate.version} release=${candidate.releaseId} error=${message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (args.releaseId || isDone) break;
|
||||
}
|
||||
|
||||
if (!dryRun && args.scheduleNext === true && !isDone && cursor) {
|
||||
await runAfterRef(
|
||||
ctx,
|
||||
0,
|
||||
internalRefs.packages.repairHistoricalClawPackReleaseFilesInternal,
|
||||
{
|
||||
cursor,
|
||||
batchSize,
|
||||
maxBatches,
|
||||
dryRun: false,
|
||||
releaseId: args.releaseId,
|
||||
packageName: args.packageName,
|
||||
version: args.version,
|
||||
confirmation: args.confirmation,
|
||||
scheduleNext: true,
|
||||
},
|
||||
);
|
||||
logLines.push(`[clawpack-repair] scheduled-next cursor=${cursor}`);
|
||||
}
|
||||
|
||||
return {
|
||||
ok: true as const,
|
||||
dryRun,
|
||||
confirmationRequired: !dryRun,
|
||||
target: {
|
||||
releaseId: args.releaseId ?? null,
|
||||
packageName: args.packageName ?? null,
|
||||
version: args.version ?? null,
|
||||
},
|
||||
cursor,
|
||||
isDone,
|
||||
stats,
|
||||
samples,
|
||||
logLines,
|
||||
writeConfirmation: dryRun ? CLAWPACK_REPAIR_WRITE_CONFIRMATION : null,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
export const scanPackageReleaseStaticallyInternal = internalAction({
|
||||
args: {
|
||||
releaseId: v.id("packageReleases"),
|
||||
@@ -8286,6 +8855,32 @@ export const backfillPackageReleaseScans = action({
|
||||
},
|
||||
});
|
||||
|
||||
export const repairHistoricalClawPackReleaseFiles = action({
|
||||
args: {
|
||||
cursor: v.optional(v.union(v.string(), v.null())),
|
||||
batchSize: v.optional(v.number()),
|
||||
maxBatches: v.optional(v.number()),
|
||||
dryRun: v.optional(v.boolean()),
|
||||
releaseId: v.optional(v.id("packageReleases")),
|
||||
packageName: v.optional(v.string()),
|
||||
version: v.optional(v.string()),
|
||||
confirmation: v.optional(v.string()),
|
||||
scheduleNext: v.optional(v.boolean()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const { user } = await requireUserFromAction(ctx);
|
||||
assertAdmin(user);
|
||||
return await runActionRef(
|
||||
ctx,
|
||||
internalRefs.packages.repairHistoricalClawPackReleaseFilesInternal,
|
||||
{
|
||||
...args,
|
||||
dryRun: args.dryRun !== false,
|
||||
},
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
export const setBatch = mutation({
|
||||
args: { packageId: v.id("packages"), batch: v.optional(v.string()) },
|
||||
handler: async (ctx, args) => {
|
||||
|
||||
@@ -247,10 +247,24 @@ See also: [acceptable-usage.md](./acceptable-usage.md) for the marketplace polic
|
||||
`vt-only`, `both`) are retired.
|
||||
- Package/plugin scan backfills may recompute deterministic static scan results for older releases,
|
||||
but those results remain ClawScan context and are not public trust status.
|
||||
- ClawPack package releases keep static/LLM scan inputs intentionally metadata-only for now:
|
||||
`package.json`, `openclaw.plugin.json`, package/source metadata, and release facts. VirusTotal
|
||||
scans the exact uploaded `.tgz`; ClawHub does not currently run deep static/LLM scans across every
|
||||
tarball file.
|
||||
- ClawPack package releases materialize parsed npm-pack artifact entries into the release file
|
||||
surface. Static scan, LLM review, package inspect/file APIs, and Codex package ClawScan use those
|
||||
stored artifact entries instead of metadata-only `package.json` / `openclaw.plugin.json` rows.
|
||||
VirusTotal still scans the exact uploaded `.tgz`; ClawHub also retains the stored ClawPack artifact
|
||||
so scanner workers can download/extract it for artifact-backed review.
|
||||
- Historical ClawPack releases whose stored file rows are shorter than the artifact `npmFileCount`
|
||||
are production data repair candidates, not normal scan-backfill work. Operators repair them through
|
||||
the explicit `packages:repairHistoricalClawPackReleaseFiles` migration action:
|
||||
- default to `dryRun: true` and inspect the returned `logLines`, candidate samples, parsed
|
||||
artifact file counts, hashes, and resume `cursor` before writing.
|
||||
- target narrowly with `releaseId`, or with `packageName` and optional `version`, before running
|
||||
an all-release pass.
|
||||
- pass `dryRun: false` plus `confirmation: "repair-historical-clawpack-files"` before any row
|
||||
rewrite; the action replaces `release.files`, recomputes `integritySha256`, resets stale scan
|
||||
analysis, and queues fresh static/Codex package scans for repaired releases.
|
||||
- resume incomplete runs by passing the returned `cursor`; set `scheduleNext: true` only when an
|
||||
operator intentionally wants the action to enqueue the next batch.
|
||||
The ordinary package scan backfill must not schedule this repair implicitly.
|
||||
- Packages cache VirusTotal undetected-only engine results as clean VT telemetry.
|
||||
ClawHub does not request or consume VirusTotal AI/code-insight results; VT is
|
||||
engine/vendor telemetry only.
|
||||
|
||||
Reference in New Issue
Block a user