feat: show skill scan queue progress

This commit is contained in:
Patrick Erichsen
2026-06-05 11:35:08 -07:00
parent 2d719feef5
commit bd30b182d7
10 changed files with 449 additions and 6 deletions
+270
View File
@@ -9,6 +9,7 @@ import {
enqueueBulkSkillRescanBatchForAdminInternal,
failCodexScanJob,
getBulkSkillRescanBatchStatusForAdminInternal,
getSkillScanRequestForUserInternal,
pruneExpiredSkillScanRequestsInternal,
requestPackageRescanForUserInternal,
requestPackageRescan,
@@ -102,6 +103,7 @@ type ScanJob = {
targetKind: string;
skillVersionId?: string;
packageReleaseId?: string;
skillScanRequestId?: string;
source: string;
priority: number;
hasMaliciousSignal: boolean;
@@ -199,6 +201,26 @@ const getBulkSkillRescanBatchStatusForAdminInternalHandler = (
>
)._handler;
const getSkillScanRequestForUserInternalHandler = (
getSkillScanRequestForUserInternal as unknown as WrappedHandler<
{ actorUserId: string; scanId: string },
{
ok: true;
scanId: string;
jobId?: string;
status: string;
queue: {
queuedAhead: number;
queuedAheadIsEstimate?: boolean;
position: number | null;
running: number;
runningIsEstimate?: boolean;
note: string;
};
}
>
)._handler;
const claimedJob = {
_id: "securityScanJobs:1",
_creationTime: 1,
@@ -586,6 +608,83 @@ function makeClaimCtx(jobs: ScanJob[]) {
};
}
function makeSkillScanStatusCtx(options: {
actor: Record<string, unknown>;
request: Record<string, unknown>;
jobs: ScanJob[];
}) {
const docs = new Map<string, Record<string, unknown>>([
[String(options.actor._id), options.actor],
[String(options.request._id), options.request],
...options.jobs.map((job) => [job._id, job] as const),
]);
const get = vi.fn(async (id: string) => docs.get(id) ?? null);
const query = vi.fn((tableName: string) => {
expect(tableName).toBe("securityScanJobs");
return {
withIndex: vi.fn(
(
indexName: string,
buildRange: (q: {
eq: (field: string, value: unknown) => unknown;
lte: (field: string, value: number) => unknown;
}) => unknown,
) => {
const eqFilters = new Map<string, unknown>();
const lteFilters = new Map<string, number>();
const indexBuilder = {
eq(field: string, value: unknown) {
eqFilters.set(field, value);
return indexBuilder;
},
lte(field: string, value: number) {
lteFilters.set(field, value);
return indexBuilder;
},
};
buildRange(indexBuilder);
const select = () =>
options.jobs
.filter((job) => {
for (const [field, value] of eqFilters) {
if ((job as unknown as Record<string, unknown>)[field] !== value) return false;
}
for (const [field, value] of lteFilters) {
const fieldValue = (job as unknown as Record<string, unknown>)[field];
if (typeof fieldValue !== "number" || fieldValue > value) return false;
}
return true;
})
.sort((a, b) => {
if (indexName.includes("next_run_at")) {
if (a.nextRunAt !== b.nextRunAt) return a.nextRunAt - b.nextRunAt;
if (a._creationTime !== b._creationTime) {
return a._creationTime - b._creationTime;
}
return a._id.localeCompare(b._id);
}
return a.createdAt - b.createdAt;
});
const collect = vi.fn(async () => select());
const take = vi.fn(async (limit: number) => select().slice(0, limit));
return {
collect,
take,
order: vi.fn(() => ({ collect, take })),
};
},
),
};
});
return {
db: {
get,
query,
},
};
}
describe("securityScan", () => {
afterEach(() => {
vi.unstubAllEnvs();
@@ -1488,6 +1587,177 @@ describe("securityScan", () => {
]);
});
it("reports queued scan position for manual scan requests", async () => {
const targetJob = makeScanJob({
_id: "securityScanJobs:target",
targetKind: "skillScanRequest",
skillScanRequestId: "skillScanRequests:target",
source: "manual",
createdAt: 300,
nextRunAt: 300,
});
const ctx = makeSkillScanStatusCtx({
actor: { _id: "users:owner", role: "user" },
request: {
_id: "skillScanRequests:target",
actorUserId: "users:owner",
sourceKind: "upload",
update: false,
writtenBack: false,
status: "queued",
securityScanJobId: targetJob._id,
files: [],
expiresAt: 1000,
createdAt: 300,
updatedAt: 300,
},
jobs: [
makeScanJob({
_id: "securityScanJobs:older",
source: "manual",
createdAt: 100,
nextRunAt: 100,
}),
makeScanJob({
_id: "securityScanJobs:running",
status: "running",
source: "manual",
createdAt: 200,
nextRunAt: 200,
}),
targetJob,
makeScanJob({
_id: "securityScanJobs:bulk",
source: "bulk-rescan",
createdAt: 1,
nextRunAt: 1,
}),
],
});
const status = await getSkillScanRequestForUserInternalHandler(ctx, {
actorUserId: "users:owner",
scanId: "skillScanRequests:target",
});
expect(status.queue).toEqual({
queuedAhead: 1,
queuedAheadIsEstimate: false,
position: 2,
running: 1,
runningIsEstimate: false,
note: "Scans are asynchronous and may take time to complete.",
});
});
it("uses claim-order tie-breaks for same-timestamp queued scan positions", async () => {
const targetJob = makeScanJob({
_id: "securityScanJobs:target",
_creationTime: 2,
targetKind: "skillScanRequest",
skillScanRequestId: "skillScanRequests:target",
source: "manual",
createdAt: 300,
nextRunAt: 300,
});
const ctx = makeSkillScanStatusCtx({
actor: { _id: "users:owner", role: "user" },
request: {
_id: "skillScanRequests:target",
actorUserId: "users:owner",
sourceKind: "upload",
update: false,
writtenBack: false,
status: "queued",
securityScanJobId: targetJob._id,
files: [],
expiresAt: 1000,
createdAt: 300,
updatedAt: 300,
},
jobs: [
makeScanJob({
_id: "securityScanJobs:first",
_creationTime: 1,
source: "manual",
createdAt: 300,
nextRunAt: 300,
}),
targetJob,
makeScanJob({
_id: "securityScanJobs:last",
_creationTime: 3,
source: "manual",
createdAt: 300,
nextRunAt: 300,
}),
],
});
const status = await getSkillScanRequestForUserInternalHandler(ctx, {
actorUserId: "users:owner",
scanId: "skillScanRequests:target",
});
expect(status.queue).toMatchObject({
queuedAhead: 1,
queuedAheadIsEstimate: false,
position: 2,
});
});
it("bounds large queue position scans and marks the count as estimated", async () => {
const targetJob = makeScanJob({
_id: "securityScanJobs:target",
targetKind: "skillScanRequest",
skillScanRequestId: "skillScanRequests:target",
source: "manual",
createdAt: 1_000,
nextRunAt: 1_000,
});
const ctx = makeSkillScanStatusCtx({
actor: { _id: "users:owner", role: "user" },
request: {
_id: "skillScanRequests:target",
actorUserId: "users:owner",
sourceKind: "upload",
update: false,
writtenBack: false,
status: "queued",
securityScanJobId: targetJob._id,
files: [],
expiresAt: 1000,
createdAt: 1_000,
updatedAt: 1_000,
},
jobs: [
...Array.from({ length: 300 }, (_, index) =>
makeScanJob({
_id: `securityScanJobs:older-${index}`,
source: "manual",
createdAt: index,
nextRunAt: index,
}),
),
targetJob,
],
});
const status = await getSkillScanRequestForUserInternalHandler(ctx, {
actorUserId: "users:owner",
scanId: "skillScanRequests:target",
});
expect(status.queue).toEqual({
queuedAhead: 250,
queuedAheadIsEstimate: true,
position: null,
running: 0,
runningIsEstimate: false,
note: "Scans are asynchronous and may take time to complete.",
});
});
it("caps SkillSpector findings before storing completed scan results", async () => {
vi.stubEnv("SECURITY_SCAN_WORKER_TOKEN", "worker-secret");
const longSnippet = "sensitive SkillSpector artifact text ".repeat(200);
+85 -2
View File
@@ -28,6 +28,9 @@ const MAX_STORED_SKILLSPECTOR_ISSUES = 25;
const MAX_STORED_SKILLSPECTOR_TEXT_CHARS = 2_000;
const MAX_STORED_SKILLSPECTOR_SHORT_TEXT_CHARS = 512;
const DEFAULT_SKILL_SCAN_REQUEST_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
const MAX_SKILL_SCAN_QUEUE_POSITION_READS = 250;
const MAX_SKILL_SCAN_RUNNING_COUNT_READS = 512;
const SKILL_SCAN_ASYNC_NOTE = "Scans are asynchronous and may take time to complete.";
const finalLlmAnalysisStatuses = new Set(["clean", "suspicious", "malicious"]);
const artifactBackedLlmAnalysisStatuses = new Set(["clean", "benign", "suspicious", "malicious"]);
@@ -800,7 +803,84 @@ function skillScanArtifactFromRequest(request: Doc<"skillScanRequests">) {
};
}
function skillScanStatusResponse(
async function countSecurityScanJobs(
ctx: QueryCtx | MutationCtx,
status: Doc<"securityScanJobs">["status"],
source: SecurityScanJobSource,
) {
const jobs = await ctx.db
.query("securityScanJobs")
.withIndex("by_status_source_created_at", (q) => q.eq("status", status).eq("source", source))
.take(MAX_SKILL_SCAN_RUNNING_COUNT_READS + 1);
return {
count: Math.min(jobs.length, MAX_SKILL_SCAN_RUNNING_COUNT_READS),
isEstimate: jobs.length > MAX_SKILL_SCAN_RUNNING_COUNT_READS,
};
}
function compareQueuedScanClaimOrder(a: Doc<"securityScanJobs">, b: Doc<"securityScanJobs">) {
if (a.nextRunAt !== b.nextRunAt) return a.nextRunAt - b.nextRunAt;
if (a._creationTime !== b._creationTime) return a._creationTime - b._creationTime;
return a._id.localeCompare(b._id);
}
async function countQueuedJobsAhead(ctx: QueryCtx | MutationCtx, job: Doc<"securityScanJobs">) {
const candidates = await ctx.db
.query("securityScanJobs")
.withIndex("by_status_source_next_run_at", (q) =>
q.eq("status", "queued").eq("source", job.source).lte("nextRunAt", job.nextRunAt),
)
.order("asc")
.take(MAX_SKILL_SCAN_QUEUE_POSITION_READS + 1);
const queuedAhead = candidates.reduce((count, candidate) => {
if (candidate._id === job._id) return count;
return compareQueuedScanClaimOrder(candidate, job) < 0 ? count + 1 : count;
}, 0);
const sawTarget = candidates.some((candidate) => candidate._id === job._id);
const isEstimate =
!sawTarget ||
candidates.length > MAX_SKILL_SCAN_QUEUE_POSITION_READS ||
queuedAhead > MAX_SKILL_SCAN_QUEUE_POSITION_READS;
return {
queuedAhead: Math.min(queuedAhead, MAX_SKILL_SCAN_QUEUE_POSITION_READS),
isEstimate,
};
}
async function skillScanQueueState(
ctx: QueryCtx | MutationCtx,
job: Doc<"securityScanJobs"> | null,
) {
if (!job) {
return {
queuedAhead: 0,
position: null,
running: 0,
note: SKILL_SCAN_ASYNC_NOTE,
};
}
const running = await countSecurityScanJobs(ctx, "running", job.source);
const queuedAhead =
job.status === "queued"
? await countQueuedJobsAhead(ctx, job)
: { queuedAhead: 0, isEstimate: false };
return {
queuedAhead: queuedAhead.queuedAhead,
queuedAheadIsEstimate: queuedAhead.isEstimate,
position:
job.status === "queued" && !queuedAhead.isEstimate ? queuedAhead.queuedAhead + 1 : null,
running: running.count,
runningIsEstimate: running.isEstimate,
note: SKILL_SCAN_ASYNC_NOTE,
};
}
async function skillScanStatusResponse(
ctx: QueryCtx | MutationCtx,
request: Doc<"skillScanRequests">,
job: Doc<"securityScanJobs"> | null,
) {
@@ -818,6 +898,7 @@ function skillScanStatusResponse(
writtenBack: request.writtenBack,
artifact: skillScanArtifactFromRequest(request),
report: skillScanReportFromRequest(request),
queue: await skillScanQueueState(ctx, job),
lastError: request.lastError ?? job?.lastError,
createdAt: request.createdAt,
updatedAt: Math.max(request.updatedAt, job?.updatedAt ?? request.updatedAt),
@@ -904,6 +985,7 @@ export const createUploadedSkillScanRequestInternal = internalMutation({
sourceKind: "upload" as const,
update: false,
alreadyQueued: false,
queue: await skillScanQueueState(ctx, await ctx.db.get(jobId)),
};
},
});
@@ -1006,6 +1088,7 @@ export const createPublishedSkillScanRequestInternal = internalMutation({
sourceKind: "published" as const,
update,
alreadyQueued: false,
queue: await skillScanQueueState(ctx, await ctx.db.get(jobId)),
};
},
});
@@ -1024,7 +1107,7 @@ export const getSkillScanRequestForUserInternal = internalQuery({
throw new ConvexError("Forbidden");
}
const job = request.securityScanJobId ? await ctx.db.get(request.securityScanJobId) : null;
return skillScanStatusResponse(request, job);
return await skillScanStatusResponse(ctx, request, job);
},
});
+1
View File
@@ -193,6 +193,7 @@ clawhub skill publish ./my-skill --version 1.0.0
- Requires `clawhub login`.
- Runs ClawHub ClawScan through `POST /api/v1/skills/-/scan`, then polls until the scan is terminal.
- Scans are asynchronous and may take time to complete. While queued, the terminal spinner shows the current prioritized scan position and how many scans are ahead.
- Local path scans are always ephemeral. They upload the local skill bundle for scanning, print the security report, and never create or update a published skill/version.
- Published scans require ownership or publisher management access. Moderators/admins can use the same backend through `clawhub-mod`.
- `--update` is valid only with `--slug`; it writes successful published scan results back to the selected version.
+3 -1
View File
@@ -394,13 +394,15 @@ Notes:
- Local upload scans reject `update: true`.
- Published scans require owner/publisher management access, or platform moderator/admin authority.
- Published scans write back only when `update: true` and the scan completes successfully.
- Response is `202` with `{ "ok": true, "scanId": "...", "jobId": "...", "status": "queued", "sourceKind": "upload|published", "update": false }`.
- Response is `202` with `{ "ok": true, "scanId": "...", "jobId": "...", "status": "queued", "sourceKind": "upload|published", "update": false, "queue": { "queuedAhead": 0, "queuedAheadIsEstimate": false, "position": 1, "running": 0, "runningIsEstimate": false, "note": "Scans are asynchronous and may take time to complete." } }`.
- Scan jobs are asynchronous. Manual scan requests are prioritized ahead of normal publish/backfill work, but completion still depends on worker availability.
### `GET /api/v1/skills/-/scan/{scanId}`
Authenticated poll endpoint for a submitted scan.
- Returns queued/running/succeeded/failed status.
- Returns `queue.queuedAhead` and `queue.position` while queued so clients can show how many prioritized manual scans are ahead of the request. Very large queues are bounded and reported with `queuedAheadIsEstimate: true`.
- When available, `report` contains `clawscan`, `skillspector`, `staticAnalysis`, and `virustotal` sections.
- Failed scan jobs return `status: "failed"` with `lastError`.
+30 -2
View File
@@ -38,7 +38,7 @@ export async function cmdScan(opts: GlobalOpts, pathArg: string | undefined, opt
? await submitLocalScan(opts, registry, token, pathArg)
: await submitPublishedScan(registry, token, options);
spinner.text = `Scan queued (${submitted.scanId})`;
spinner.text = formatScanProgress("queued", submitted.scanId, submitted.queue);
const status = await pollScan(registry, token, submitted.scanId, spinner);
if (status.status === "failed") {
@@ -145,7 +145,7 @@ async function pollScan(
},
ApiV1SkillScanStatusResponseSchema,
);
spinner.text = `Scan ${status.status} (${scanId})`;
spinner.text = formatScanProgress(status.status, scanId, status.queue);
if (status.status === "succeeded" || status.status === "failed") return status;
await sleep(DEFAULT_POLL_INTERVAL_MS);
}
@@ -156,6 +156,34 @@ function sleep(ms: number) {
return new Promise((resolvePromise) => setTimeout(resolvePromise, ms));
}
function formatScanProgress(
status: ApiV1SkillScanStatusResponse["status"],
scanId: string,
queue: ApiV1SkillScanStatusResponse["queue"],
) {
const queueProgress = formatQueueProgress(status, queue);
return `Scan ${status} (${scanId})${queueProgress ? ` - ${queueProgress}` : ""}`;
}
function formatQueueProgress(
status: ApiV1SkillScanStatusResponse["status"],
queue: ApiV1SkillScanStatusResponse["queue"],
) {
if (!queue) return undefined;
if (status === "queued") {
const aheadCount = `${queue.queuedAhead}${queue.queuedAheadIsEstimate ? "+" : ""}`;
const ahead =
queue.queuedAhead === 0
? "no scans ahead"
: `${aheadCount} scan${queue.queuedAhead === 1 && !queue.queuedAheadIsEstimate ? "" : "s"} ahead`;
const position = typeof queue.position === "number" ? `position ${queue.position}` : undefined;
const running = `${queue.running}${queue.runningIsEstimate ? "+" : ""} running`;
return [position, ahead, running, queue.note].filter(Boolean).join("; ");
}
if (status === "running") return `${queue.running}${queue.runningIsEstimate ? "+" : ""} running`;
return undefined;
}
function printJson(status: ApiV1SkillScanStatusResponse) {
console.log(JSON.stringify(status, null, 2));
}
+12
View File
@@ -507,6 +507,16 @@ export const ApiV1SkillScanSubmitRequestSchema = type({
});
export type ApiV1SkillScanSubmitRequest = (typeof ApiV1SkillScanSubmitRequestSchema)[inferred];
export const ApiV1SkillScanQueueSchema = type({
queuedAhead: "number",
queuedAheadIsEstimate: "boolean?",
position: "number|null",
running: "number",
runningIsEstimate: "boolean?",
note: "string",
});
export type ApiV1SkillScanQueue = (typeof ApiV1SkillScanQueueSchema)[inferred];
export const ApiV1SkillScanSubmitResponseSchema = type({
ok: "true",
scanId: "string",
@@ -515,6 +525,7 @@ export const ApiV1SkillScanSubmitResponseSchema = type({
sourceKind: '"upload"|"published"',
update: "boolean",
alreadyQueued: "boolean?",
queue: ApiV1SkillScanQueueSchema.optional(),
});
export type ApiV1SkillScanSubmitResponse = (typeof ApiV1SkillScanSubmitResponseSchema)[inferred];
@@ -528,6 +539,7 @@ export const ApiV1SkillScanStatusResponseSchema = type({
writtenBack: "boolean?",
artifact: "unknown?",
report: "unknown?",
queue: ApiV1SkillScanQueueSchema.optional(),
lastError: "string?",
createdAt: "number",
updatedAt: "number",
+25
View File
@@ -419,6 +419,15 @@ export declare const ApiV1SkillScanSubmitRequestSchema: import("arktype/internal
update?: boolean | undefined;
}, {}>;
export type ApiV1SkillScanSubmitRequest = (typeof ApiV1SkillScanSubmitRequestSchema)[inferred];
export declare const ApiV1SkillScanQueueSchema: import("arktype/internal/variants/object.ts").ObjectType<{
queuedAhead: number;
position: number | null;
running: number;
note: string;
queuedAheadIsEstimate?: boolean | undefined;
runningIsEstimate?: boolean | undefined;
}, {}>;
export type ApiV1SkillScanQueue = (typeof ApiV1SkillScanQueueSchema)[inferred];
export declare const ApiV1SkillScanSubmitResponseSchema: import("arktype/internal/variants/object.ts").ObjectType<{
ok: true;
scanId: string;
@@ -427,6 +436,14 @@ export declare const ApiV1SkillScanSubmitResponseSchema: import("arktype/interna
update: boolean;
jobId?: string | undefined;
alreadyQueued?: boolean | undefined;
queue?: {
queuedAhead: number;
position: number | null;
running: number;
note: string;
queuedAheadIsEstimate?: boolean | undefined;
runningIsEstimate?: boolean | undefined;
} | undefined;
}, {}>;
export type ApiV1SkillScanSubmitResponse = (typeof ApiV1SkillScanSubmitResponseSchema)[inferred];
export declare const ApiV1SkillScanStatusResponseSchema: import("arktype/internal/variants/object.ts").ObjectType<{
@@ -441,6 +458,14 @@ export declare const ApiV1SkillScanStatusResponseSchema: import("arktype/interna
writtenBack?: boolean | undefined;
artifact?: unknown;
report?: unknown;
queue?: {
queuedAhead: number;
position: number | null;
running: number;
note: string;
queuedAheadIsEstimate?: boolean | undefined;
runningIsEstimate?: boolean | undefined;
} | undefined;
lastError?: string | undefined;
completedAt?: number | undefined;
}, {}>;
+10
View File
@@ -374,6 +374,14 @@ export const ApiV1SkillScanSubmitRequestSchema = type({
source: ApiV1SkillScanSourceSchema,
update: "boolean?",
});
export const ApiV1SkillScanQueueSchema = type({
queuedAhead: "number",
queuedAheadIsEstimate: "boolean?",
position: "number|null",
running: "number",
runningIsEstimate: "boolean?",
note: "string",
});
export const ApiV1SkillScanSubmitResponseSchema = type({
ok: "true",
scanId: "string",
@@ -382,6 +390,7 @@ export const ApiV1SkillScanSubmitResponseSchema = type({
sourceKind: '"upload"|"published"',
update: "boolean",
alreadyQueued: "boolean?",
queue: ApiV1SkillScanQueueSchema.optional(),
});
export const ApiV1SkillScanStatusResponseSchema = type({
ok: "true",
@@ -393,6 +402,7 @@ export const ApiV1SkillScanStatusResponseSchema = type({
writtenBack: "boolean?",
artifact: "unknown?",
report: "unknown?",
queue: ApiV1SkillScanQueueSchema.optional(),
lastError: "string?",
createdAt: "number",
updatedAt: "number",
File diff suppressed because one or more lines are too long
+12
View File
@@ -449,6 +449,16 @@ export const ApiV1SkillScanSubmitRequestSchema = type({
});
export type ApiV1SkillScanSubmitRequest = (typeof ApiV1SkillScanSubmitRequestSchema)[inferred];
export const ApiV1SkillScanQueueSchema = type({
queuedAhead: "number",
queuedAheadIsEstimate: "boolean?",
position: "number|null",
running: "number",
runningIsEstimate: "boolean?",
note: "string",
});
export type ApiV1SkillScanQueue = (typeof ApiV1SkillScanQueueSchema)[inferred];
export const ApiV1SkillScanSubmitResponseSchema = type({
ok: "true",
scanId: "string",
@@ -457,6 +467,7 @@ export const ApiV1SkillScanSubmitResponseSchema = type({
sourceKind: '"upload"|"published"',
update: "boolean",
alreadyQueued: "boolean?",
queue: ApiV1SkillScanQueueSchema.optional(),
});
export type ApiV1SkillScanSubmitResponse = (typeof ApiV1SkillScanSubmitResponseSchema)[inferred];
@@ -470,6 +481,7 @@ export const ApiV1SkillScanStatusResponseSchema = type({
writtenBack: "boolean?",
artifact: "unknown?",
report: "unknown?",
queue: ApiV1SkillScanQueueSchema.optional(),
lastError: "string?",
createdAt: "number",
updatedAt: "number",