admin/clawhub
1
0
Fork 0
mirror of https://github.com/openclaw/clawhub.git synced 2026-08-14 08:52:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add VT pending repair command (#2441)

Browse Source
This commit is contained in:
Patrick Erichsen
2026-05-29 15:07:29 -05:00
committed by GitHub
parent a20e2efd68
commit f71139e9ae
11 changed files with 548 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
convex/httpApiV1.handlers.test.ts
+69
View File
@@ -4999,6 +4999,75 @@ describe("httpApiV1 handlers", () => {
);
});
it("VT pending repair requires admin role", async () => {
vi.mocked(requireApiTokenUser).mockResolvedValue({
userId: "users:moderator",
user: { _id: "users:moderator", role: "moderator" },
} as never);
const runMutation = vi.fn(async (_mutation: unknown, args: Record<string, unknown>) => {
if (isRateLimitArgs(args)) return okRate();
throw new Error("should not repair");
});
const response = await __handlers.skillsPostRouterV1Handler(
makeCtx({ runMutation }),
new Request("https://example.com/api/v1/skills/-/repair-vt-pending", {
method: "POST",
headers: { Authorization: "Bearer clh_test" },
body: JSON.stringify({ batchSize: 25, dryRun: true }),
}),
);
expect(response.status).toBe(403);
await expect(response.text()).resolves.toBe("Admin role required.");
expect(runMutation).toHaveBeenCalledTimes(1);
});
it("VT pending repair invokes the internal repair action via admin API", async () => {
vi.mocked(requireApiTokenUser).mockResolvedValue({
userId: "users:admin",
user: { _id: "users:admin", role: "admin" },
} as never);
const runAction = vi.fn(async () => ({
dryRun: true,
total: 2,
wouldUpdate: 2,
updated: 0,
noResults: 0,
noDecisiveStats: 0,
errors: 0,
done: false,
cursor: "cursor-2",
statusCounts: { clean: 2 },
sampleUpdated: [{ slug: "demo", status: "clean" }],
}));
const response = await __handlers.skillsPostRouterV1Handler(
makeCtx({ runAction }),
new Request("https://example.com/api/v1/skills/-/repair-vt-pending", {
method: "POST",
headers: { Authorization: "Bearer clh_test" },
body: JSON.stringify({ batchSize: 25, cursor: null, dryRun: true }),
}),
);
expect(response.status).toBe(200);
await expect(response.json()).resolves.toMatchObject({
ok: true,
dryRun: true,
wouldUpdate: 2,
cursor: "cursor-2",
});
expect(runAction).toHaveBeenCalledWith(
(internal as unknown as { vt: Record<string, unknown> }).vt.repairPendingSkillVtAnalysis,
{
dryRun: true,
cursor: null,
batchSize: 25,
},
);
});
it("package rescan enqueues owner-authorized ClawScan jobs", async () => {
vi.mocked(requireApiTokenUser).mockResolvedValue({
userId: "users:1",
convex/httpApiV1/skillsV1.ts
+57
View File
@@ -1,6 +1,7 @@
import {
ApiV1SkillBulkRescanBatchRequestSchema,
ApiV1SkillBulkRescanStatusRequestSchema,
ApiV1SkillRepairVtPendingRequestSchema,
SkillAppealRequestSchema,
SkillAppealResolveRequestSchema,
SkillReportTriageRequestSchema,
@@ -282,6 +283,9 @@ const internalRefs = internal as unknown as {
getBulkSkillRescanBatchStatusForAdminInternal: unknown;
requestSkillRescanForUserInternal: unknown;
};
vt: {
repairPendingSkillVtAnalysis: unknown;
};
skills: {
getSecurityVerdictTargetInternal: unknown;
reportSkillForUserInternal: unknown;
@@ -301,6 +305,10 @@ async function runMutationRef<T>(ctx: ActionCtx, ref: unknown, args: unknown): P
return (await ctx.runMutation(ref as never, args as never)) as T;
}
async function runActionRef<T>(ctx: ActionCtx, ref: unknown, args: unknown): Promise<T> {
return (await ctx.runAction(ref as never, args as never)) as T;
}
function isDefinitiveSecurityStatus(
status: NormalizedSecurityStatus | null | undefined,
): status is "clean" | "suspicious" | "malicious" {
@@ -2059,6 +2067,55 @@ export async function skillsPostRouterV1Handler(ctx: ActionCtx, request: Request
const action = segments[1] ?? "";
const slug = segments[0]?.trim().toLowerCase() ?? "";
if (segments[0] === "-" && segments[1] === "repair-vt-pending" && segments.length === 2) {
const auth = await requireApiTokenUserOrResponse(ctx, request, rate.headers);
if (!auth.ok) return auth.response;
const admin = requireAdminOrResponse(auth.user, rate.headers);
if (!admin.ok) return admin.response;
try {
const body = parseArk(
ApiV1SkillRepairVtPendingRequestSchema,
await request.json(),
"Skill VT pending repair payload",
) as {
cursor?: string | null;
batchSize?: number;
concurrency?: number;
dryRun?: boolean;
};
const result = await runActionRef<
| {
dryRun: boolean;
total: number;
wouldUpdate: number;
updated: number;
noResults: number;
noDecisiveStats: number;
errors: number;
done: boolean;
cursor: string | null;
statusCounts: Record<string, number>;
sampleUpdated: Array<{ slug: string; status: string }>;
}
| { error: string }
>(ctx, internalRefs.vt.repairPendingSkillVtAnalysis, {
dryRun: body.dryRun !== false,
cursor: body.cursor ?? null,
...(body.batchSize !== undefined ? { batchSize: body.batchSize } : {}),
...(body.concurrency !== undefined ? { concurrency: body.concurrency } : {}),
});
if ("error" in result) return text(result.error, 400, rate.headers);
return json({ ok: true, ...result }, 200, rate.headers);
} catch (error) {
if (error instanceof SyntaxError) return text("Invalid JSON", 400, rate.headers);
return text(
error instanceof Error ? error.message : "Skill VT pending repair failed",
400,
rate.headers,
);
}
}
if (segments[0] === "-" && segments[1] === "rescan-batch" && segments.length === 2) {
const auth = await requireApiTokenUserOrResponse(ctx, request, rate.headers);
if (!auth.ok) return auth.response;
convex/vt.ts
+12 -5
View File
@@ -1030,6 +1030,7 @@ export const repairPendingSkillVtAnalysis = internalAction({
args: {
dryRun: v.boolean(),
batchSize: v.optional(v.number()),
concurrency: v.optional(v.number()),
cursor: v.optional(v.union(v.string(), v.null())),
},
handler: async (ctx, args): Promise<RepairPendingSkillVtAnalysisResult> => {
@@ -1038,8 +1039,10 @@ export const repairPendingSkillVtAnalysis = internalAction({
console.log("[vt:repairPendingSkillVt] VT_API_KEY not configured");
return { error: "VT_API_KEY not configured" };
}
const vtApiKey = apiKey;
const batchSize = Math.max(1, Math.min(Math.floor(args.batchSize ?? 100), 500));
const concurrency = Math.max(1, Math.min(Math.floor(args.concurrency ?? 16), 32));
const pendingPage: {
skills: PendingVTSkill[];
cursor: string | null;
@@ -1058,25 +1061,25 @@ export const repairPendingSkillVtAnalysis = internalAction({
const statusCounts: Record<string, number> = {};
const sampleUpdated: Array<{ slug: string; status: string }> = [];
for (const { skillId, versionId, sha256hash, slug } of skills) {
async function repairSkill({ skillId, versionId, sha256hash, slug }: PendingVTSkill) {
try {
const vtResult = await checkExistingFile(apiKey, sha256hash);
const vtResult = await checkExistingFile(vtApiKey, sha256hash);
if (!vtResult) {
noResults++;
continue;
return;
}
const stats = vtResult.data.attributes.last_analysis_stats;
const status = statusFromAvStats(stats);
if (!status) {
noDecisiveStats++;
continue;
return;
}
wouldUpdate++;
statusCounts[status] = (statusCounts[status] ?? 0) + 1;
if (sampleUpdated.length < 20) sampleUpdated.push({ slug, status });
if (args.dryRun) continue;
if (args.dryRun) return;
await ctx.runMutation(internal.skills.updateVersionScanResultsInternal, {
versionId,
@@ -1102,6 +1105,10 @@ export const repairPendingSkillVtAnalysis = internalAction({
}
}
for (let index = 0; index < skills.length; index += concurrency) {
await Promise.all(skills.slice(index, index + concurrency).map(repairSkill));
}
return {
dryRun: args.dryRun,
total: skills.length,
packages/clawhub-mod/src/cli.ts
+22
View File
@@ -26,6 +26,7 @@ import {
cmdBanUser,
cmdReclassifyBan,
cmdRemediateAutobans,
cmdRepairVtPendingSkills,
cmdRescanAllSkills,
cmdRescanSkill,
cmdSetRole,
@@ -642,6 +643,27 @@ function registerSkillModerationCommands(command: Command) {
await cmdRescanAllSkills(opts, options, isInputAllowed());
});
command
.command("repair-vt-pending")
.description("Repair stale pending VirusTotal skill cache by rechecking hashes")
.option("--batch-size <n>", "Batch size; backend caps at 500", (value) =>
Number.parseInt(value, 10),
)
.option(
"--concurrency <n>",
"Per-batch VirusTotal lookup concurrency; backend caps at 32",
(value) => Number.parseInt(value, 10),
)
.option("--cursor <cursor>", "Resume from a backend pagination cursor")
.option("--dry-run", "Check pending rows without writing VT cache updates")
.option("--all", "Continue paging until the backend reports done")
.option("--yes", "Skip confirmation for write runs")
.option("--json", "Output JSON progress events")
.action(async (options) => {
const opts = await resolveGlobalOpts();
await cmdRepairVtPendingSkills(opts, options, isInputAllowed());
});
command
.command("reports")
.description("List skill reports for moderator review")
packages/clawhub-mod/src/commands/moderation.test.ts
+137
View File
@@ -23,6 +23,7 @@ const {
cmdBanUser,
cmdReclassifyBan,
cmdRemediateAutobans,
cmdRepairVtPendingSkills,
cmdRescanAllSkills,
cmdRescanSkill,
cmdSetRole,
@@ -408,6 +409,142 @@ describe("cmdRescanAllSkills", () => {
});
});
describe("cmdRepairVtPendingSkills", () => {
it("requires --yes for real runs when input is disabled", async () => {
await expect(cmdRepairVtPendingSkills(makeGlobalOpts(), {}, false)).rejects.toThrow(/--yes/i);
expect(httpMocks.apiRequest).not.toHaveBeenCalled();
});
it("pages dry-runs without confirmation", async () => {
httpMocks.apiRequest
.mockResolvedValueOnce({
ok: true,
dryRun: true,
total: 2,
wouldUpdate: 2,
updated: 0,
noResults: 0,
noDecisiveStats: 0,
errors: 0,
done: false,
cursor: "cursor-2",
statusCounts: { clean: 2 },
sampleUpdated: [{ slug: "one", status: "clean" }],
})
.mockResolvedValueOnce({
ok: true,
dryRun: true,
total: 1,
wouldUpdate: 1,
updated: 0,
noResults: 0,
noDecisiveStats: 0,
errors: 0,
done: true,
cursor: null,
statusCounts: { suspicious: 1 },
sampleUpdated: [{ slug: "two", status: "suspicious" }],
});
const result = await cmdRepairVtPendingSkills(
makeGlobalOpts(),
{ dryRun: true, batchSize: 2, all: true },
false,
);
expect(result).toMatchObject({
ok: true,
dryRun: true,
batches: 2,
total: 3,
wouldUpdate: 3,
updated: 0,
statusCounts: { clean: 2, suspicious: 1 },
});
expect(httpMocks.apiRequest).toHaveBeenNthCalledWith(
1,
"https://clawhub.ai",
expect.objectContaining({
method: "POST",
path: "/api/v1/skills/-/repair-vt-pending",
token: "tkn",
body: {
cursor: null,
batchSize: 2,
dryRun: true,
},
}),
expect.anything(),
);
expect(httpMocks.apiRequest).toHaveBeenNthCalledWith(
2,
"https://clawhub.ai",
expect.objectContaining({
method: "POST",
path: "/api/v1/skills/-/repair-vt-pending",
token: "tkn",
body: {
cursor: "cursor-2",
batchSize: 2,
dryRun: true,
},
}),
expect.anything(),
);
});
it("writes one batch when confirmed", async () => {
httpMocks.apiRequest.mockResolvedValueOnce({
ok: true,
dryRun: false,
total: 2,
wouldUpdate: 2,
updated: 2,
noResults: 0,
noDecisiveStats: 0,
errors: 0,
done: false,
cursor: "cursor-2",
statusCounts: { clean: 1, malicious: 1 },
sampleUpdated: [
{ slug: "one", status: "clean" },
{ slug: "two", status: "malicious" },
],
});
const result = await cmdRepairVtPendingSkills(
makeGlobalOpts(),
{ yes: true, batchSize: 2 },
false,
);
expect(result).toMatchObject({
ok: true,
dryRun: false,
batches: 1,
total: 2,
wouldUpdate: 2,
updated: 2,
nextCursor: "cursor-2",
done: false,
});
expect(httpMocks.apiRequest).toHaveBeenCalledWith(
"https://clawhub.ai",
expect.objectContaining({
method: "POST",
path: "/api/v1/skills/-/repair-vt-pending",
token: "tkn",
body: {
cursor: null,
batchSize: 2,
dryRun: false,
},
}),
expect.anything(),
);
});
});
describe("cmdSetRole", () => {
it("requires --yes when input is disabled", async () => {
await expect(cmdSetRole(makeGlobalOpts(), "demo", "moderator", {}, false)).rejects.toThrow(
packages/clawhub-mod/src/commands/moderation.ts
+142
View File
@@ -18,6 +18,7 @@ import {
ApiV1SetRoleResponseSchema,
ApiV1SkillBulkRescanBatchResponseSchema,
ApiV1SkillBulkRescanStatusResponseSchema,
ApiV1SkillRepairVtPendingResponseSchema,
ApiV1SkillRescanResponseSchema,
ApiV1UnbanUserResponseSchema,
ApiV1UserSearchResponseSchema,
@@ -346,6 +347,121 @@ export async function cmdRescanAllSkills(
return summary;
}
export async function cmdRepairVtPendingSkills(
opts: GlobalOpts,
options: {
batchSize?: number;
concurrency?: number;
cursor?: string;
dryRun?: boolean;
all?: boolean;
yes?: boolean;
json?: boolean;
},
inputAllowed: boolean,
) {
const batchSize = normalizePositiveInt(options.batchSize, 500);
const concurrency =
options.concurrency === undefined ? undefined : normalizePositiveInt(options.concurrency, 16);
const dryRun = options.dryRun === true;
const allowPrompt = isInteractive() && inputAllowed !== false;
if (!dryRun && !options.yes) {
if (!allowPrompt) fail("Pass --yes (no input)");
const ok = await promptConfirm(
`Repair pending VirusTotal cache in batches of ${batchSize} with concurrency ${concurrency ?? 16}? (admin)`,
);
if (!ok) return undefined;
}
const token = await requireAuthToken();
const registry = await getRegistry(opts, { cache: true });
let cursor = options.cursor?.trim() || null;
let batches = 0;
let total = 0;
let wouldUpdate = 0;
let updated = 0;
let noResults = 0;
let noDecisiveStats = 0;
let errors = 0;
const statusCounts: Record<string, number> = {};
const sampleUpdated: Array<{ slug: string; status: string }> = [];
let done = false;
while (!done) {
const result = await apiRequest(
registry,
{
method: "POST",
path: `${ApiRoutes.skills}/-/repair-vt-pending`,
token,
body: {
cursor,
batchSize,
...(concurrency !== undefined ? { concurrency } : {}),
dryRun,
},
},
ApiV1SkillRepairVtPendingResponseSchema,
);
const batch = parseArk(
ApiV1SkillRepairVtPendingResponseSchema,
result,
"Skill VT pending repair response",
);
batches++;
total += batch.total;
wouldUpdate += batch.wouldUpdate;
updated += batch.updated;
noResults += batch.noResults;
noDecisiveStats += batch.noDecisiveStats;
errors += batch.errors;
for (const [status, count] of Object.entries(batch.statusCounts)) {
statusCounts[status] = (statusCounts[status] ?? 0) + count;
}
for (const sample of batch.sampleUpdated) {
if (sampleUpdated.length < 20) sampleUpdated.push(sample);
}
done = batch.done;
emitVtRepairProgress(options, {
type: "batch",
batch: batches,
cursor,
nextCursor: batch.cursor,
total: batch.total,
wouldUpdate: batch.wouldUpdate,
updated: batch.updated,
noResults: batch.noResults,
noDecisiveStats: batch.noDecisiveStats,
errors: batch.errors,
statusCounts: batch.statusCounts,
done: batch.done,
dryRun,
});
cursor = batch.cursor;
if (!options.all || !cursor) break;
}
const summary = {
ok: errors === 0,
dryRun,
batches,
total,
wouldUpdate,
updated,
noResults,
noDecisiveStats,
errors,
statusCounts,
sampleUpdated,
nextCursor: cursor,
done,
};
emitVtRepairProgress(options, { type: "summary", ...summary });
if (errors > 0) fail(`VT pending repair finished with ${errors} error(s)`);
return summary;
}
async function pollBulkRescanStatus(
registry: string,
token: string,
@@ -410,6 +526,32 @@ function emitBulkRescanProgress(options: { json?: boolean }, event: Record<strin
}
}
function emitVtRepairProgress(options: { json?: boolean }, event: Record<string, unknown>) {
if (options.json) {
process.stdout.write(`${JSON.stringify(event)}\n`);
return;
}
if (event.type === "batch") {
const nextCursor = typeof event.nextCursor === "string" ? event.nextCursor : null;
const suffix = nextCursor ? ` Next cursor: ${nextCursor}.` : "";
console.log(
`VT repair batch ${readEventNumber(event, "batch")}: scanned ${readEventNumber(event, "total")}, ${event.dryRun ? "would update" : "updated"} ${readEventNumber(event, event.dryRun ? "wouldUpdate" : "updated")}, no results ${readEventNumber(event, "noResults")}, errors ${readEventNumber(event, "errors")}.${suffix}`,
);
return;
}
if (event.type === "summary") {
const label = event.dryRun ? "VT repair dry run" : "VT repair";
console.log(
`${label} finished: ${readEventNumber(event, "batches")} batch(es), scanned ${readEventNumber(event, "total")}, ${event.dryRun ? "would update" : "updated"} ${readEventNumber(event, event.dryRun ? "wouldUpdate" : "updated")}, no results ${readEventNumber(event, "noResults")}, errors ${readEventNumber(event, "errors")}.`,
);
if (typeof event.nextCursor === "string" && event.nextCursor) {
console.log(`Resume cursor: ${event.nextCursor}`);
}
}
}
function readEventNumber(event: Record<string, unknown>, key: string) {
const value = event[key];
return typeof value === "number" && Number.isFinite(value) ? value : 0;
packages/clawhub/src/schema/schemas.ts
+29
View File
@@ -470,6 +470,35 @@ export const ApiV1SkillBulkRescanStatusResponseSchema = type({
export type ApiV1SkillBulkRescanStatusResponse =
(typeof ApiV1SkillBulkRescanStatusResponseSchema)[inferred];
export const ApiV1SkillRepairVtPendingRequestSchema = type({
cursor: "string|null?",
batchSize: "number?",
concurrency: "number?",
dryRun: "boolean?",
});
export type ApiV1SkillRepairVtPendingRequest =
(typeof ApiV1SkillRepairVtPendingRequestSchema)[inferred];
export const ApiV1SkillRepairVtPendingResponseSchema = type({
ok: "true",
dryRun: "boolean",
total: "number",
wouldUpdate: "number",
updated: "number",
noResults: "number",
noDecisiveStats: "number",
errors: "number",
done: "boolean",
cursor: "string|null",
statusCounts: { "[string]": "number" },
sampleUpdated: type({
slug: "string",
status: "string",
}).array(),
});
export type ApiV1SkillRepairVtPendingResponse =
(typeof ApiV1SkillRepairVtPendingResponseSchema)[inferred];
export const ApiV1SkillVersionListResponseSchema = type({
items: type({
version: "string",
packages/schema/dist/schemas.d.ts
Vendored
+27
View File
@@ -406,6 +406,33 @@ export declare const ApiV1SkillBulkRescanStatusResponseSchema: import("arktype/i
failedJobIds: string[];
}, {}>;
export type ApiV1SkillBulkRescanStatusResponse = (typeof ApiV1SkillBulkRescanStatusResponseSchema)[inferred];
export declare const ApiV1SkillRepairVtPendingRequestSchema: import("arktype/internal/variants/object.ts").ObjectType<{
cursor?: string | null | undefined;
batchSize?: number | undefined;
concurrency?: number | undefined;
dryRun?: boolean | undefined;
}, {}>;
export type ApiV1SkillRepairVtPendingRequest = (typeof ApiV1SkillRepairVtPendingRequestSchema)[inferred];
export declare const ApiV1SkillRepairVtPendingResponseSchema: import("arktype/internal/variants/object.ts").ObjectType<{
ok: true;
dryRun: boolean;
total: number;
wouldUpdate: number;
updated: number;
noResults: number;
noDecisiveStats: number;
errors: number;
done: boolean;
cursor: string | null;
statusCounts: {
[x: string]: number;
};
sampleUpdated: {
slug: string;
status: string;
}[];
}, {}>;
export type ApiV1SkillRepairVtPendingResponse = (typeof ApiV1SkillRepairVtPendingResponseSchema)[inferred];
export declare const ApiV1SkillVersionListResponseSchema: import("arktype/internal/variants/object.ts").ObjectType<{
items: {
version: string;
packages/schema/dist/schemas.js
Vendored
+23
View File
@@ -365,6 +365,29 @@ export const ApiV1SkillBulkRescanStatusResponseSchema = type({
done: "boolean",
failedJobIds: "string[]",
});
export const ApiV1SkillRepairVtPendingRequestSchema = type({
cursor: "string|null?",
batchSize: "number?",
concurrency: "number?",
dryRun: "boolean?",
});
export const ApiV1SkillRepairVtPendingResponseSchema = type({
ok: "true",
dryRun: "boolean",
total: "number",
wouldUpdate: "number",
updated: "number",
noResults: "number",
noDecisiveStats: "number",
errors: "number",
done: "boolean",
cursor: "string|null",
statusCounts: { "[string]": "number" },
sampleUpdated: type({
slug: "string",
status: "string",
}).array(),
});
export const ApiV1SkillVersionListResponseSchema = type({
items: type({
version: "string",
packages/schema/dist/schemas.js.map
Vendored
+1 -1
View File
File diff suppressed because one or more lines are too long
packages/schema/src/schemas.ts
+29
View File
@@ -442,6 +442,35 @@ export const ApiV1SkillBulkRescanStatusResponseSchema = type({
export type ApiV1SkillBulkRescanStatusResponse =
(typeof ApiV1SkillBulkRescanStatusResponseSchema)[inferred];
export const ApiV1SkillRepairVtPendingRequestSchema = type({
cursor: "string|null?",
batchSize: "number?",
concurrency: "number?",
dryRun: "boolean?",
});
export type ApiV1SkillRepairVtPendingRequest =
(typeof ApiV1SkillRepairVtPendingRequestSchema)[inferred];
export const ApiV1SkillRepairVtPendingResponseSchema = type({
ok: "true",
dryRun: "boolean",
total: "number",
wouldUpdate: "number",
updated: "number",
noResults: "number",
noDecisiveStats: "number",
errors: "number",
done: "boolean",
cursor: "string|null",
statusCounts: { "[string]": "number" },
sampleUpdated: type({
slug: "string",
status: "string",
}).array(),
});
export type ApiV1SkillRepairVtPendingResponse =
(typeof ApiV1SkillRepairVtPendingResponseSchema)[inferred];
export const ApiV1SkillVersionListResponseSchema = type({
items: type({
version: "string",