diff --git a/convex/publishAttempts.test.ts b/convex/publishAttempts.test.ts index 203d0daa..4708f055 100644 --- a/convex/publishAttempts.test.ts +++ b/convex/publishAttempts.test.ts @@ -844,6 +844,7 @@ describe("publishAttempts", () => { checkClaimedAt: undefined, checkClaimExpiresAt: expect.any(Number), checkClaimLastError: "scanner unavailable", + checkFailureCount: 1, failedAt: undefined, }), ); @@ -851,6 +852,69 @@ describe("publishAttempts", () => { expect(patch.checkClaimExpiresAt).toBeGreaterThan(now); }); + it("terminalizes an attempt after three consecutive scanner execution failures", async () => { + const now = Date.now(); + const ctx = { + db: { + get: vi.fn(async () => ({ + _id: "publishAttempts:poison", + kind: "skill", + status: "pending_checks", + artifactFingerprint: "fingerprint", + checkClaimId: "checks:claim", + checkClaimExpiresAt: now + 60_000, + checkFailureCount: 2, + checks: { + trufflehog: { status: "clean", checkedAt: now - 300_000 }, + clawscan: { + status: "failed", + checkedAt: now - 300_000, + summary: "ClawScan scanner did not complete: skillspector=failed", + }, + }, + })), + patch: vi.fn(), + insert: vi.fn(), + replace: vi.fn(), + delete: vi.fn(), + query: vi.fn(), + normalizeId: vi.fn(), + system: {}, + }, + storage: { + delete: vi.fn(), + }, + }; + + await expect( + completePendingChecksHandler(ctx, { + attemptId: "publishAttempts:poison", + claimId: "checks:claim", + artifactFingerprint: "fingerprint", + trufflehog: { status: "clean" }, + clawscan: { + status: "failed", + summary: "ClawScan scanner did not complete: skillspector=failed", + }, + }), + ).resolves.toEqual({ + attemptId: "publishAttempts:poison", + kind: "skill", + status: "failed", + }); + + expect(ctx.db.patch).toHaveBeenCalledWith( + "publishAttempts:poison", + expect.objectContaining({ + status: "failed", + checkFailureCount: 3, + checkClaimExpiresAt: undefined, + checkClaimLastError: "ClawScan scanner did not complete: skillspector=failed", + failedAt: expect.any(Number), + }), + ); + }); + it("terminalizes an attempt when its staged target disappears during scanning", async () => { const now = Date.now(); const ctx = { diff --git a/convex/publishAttempts.ts b/convex/publishAttempts.ts index 1c8ac411..95ccc132 100644 --- a/convex/publishAttempts.ts +++ b/convex/publishAttempts.ts @@ -9,6 +9,7 @@ import { requestPublishAttemptDispatch } from "./publishAttemptDispatch"; const THIRTY_DAYS_MS = 30 * 24 * 60 * 60 * 1000; const CHECK_CLAIM_LEASE_MS = 30 * 60 * 1000; const CHECK_RETRY_BACKOFF_MS = 5 * 60 * 1000; +const MAX_CONSECUTIVE_SCANNER_FAILURES = 3; const FINALIZATION_CLAIM_LEASE_MS = 10 * 60 * 1000; const PUBLISH_ATTEMPT_STATUSES = [ "pending_checks", @@ -107,6 +108,14 @@ function scannerFailureSummary(args: { return "Pre-publication scanner failed before returning a verdict."; } +function previousScannerFailureCount(attempt: Doc<"publishAttempts">) { + if (attempt.checkFailureCount !== undefined) return attempt.checkFailureCount; + return attempt.checks.trufflehog.status === "failed" || + attempt.checks.clawscan.status === "failed" + ? 1 + : 0; +} + function isTerminalFinalizationConflict(error: string | undefined) { return ( typeof error === "string" && @@ -136,6 +145,7 @@ function releaseFinalizationClaimPatch(error: string | undefined, now: number) { checkClaimedAt: undefined, checkClaimExpiresAt: undefined, checkClaimLastError: undefined, + checkFailureCount: undefined, finalizationClaimId: undefined, finalizationClaimedAt: undefined, finalizationClaimExpiresAt: undefined, @@ -683,6 +693,7 @@ export const completePendingPublishAttemptChecksInternal = internalMutation({ checkClaimedAt: undefined, checkClaimExpiresAt: undefined, checkClaimLastError: undefined, + checkFailureCount: undefined, blockedAt: now, updatedAt: now, }); @@ -725,6 +736,7 @@ export const completePendingPublishAttemptChecksInternal = internalMutation({ checkClaimedAt: undefined, checkClaimExpiresAt: undefined, checkClaimLastError: undefined, + checkFailureCount: undefined, blockedAt: now, updatedAt: now, }); @@ -736,17 +748,24 @@ export const completePendingPublishAttemptChecksInternal = internalMutation({ } if (args.trufflehog.status === "failed" || args.clawscan.status === "failed") { + const checkFailureCount = previousScannerFailureCount(attempt) + 1; + const terminal = checkFailureCount >= MAX_CONSECUTIVE_SCANNER_FAILURES; await ctx.db.patch(attempt._id, { - status: "pending_checks", + status: terminal ? "failed" : "pending_checks", checks, checkClaimId: undefined, checkClaimedAt: undefined, - checkClaimExpiresAt: now + CHECK_RETRY_BACKOFF_MS, + checkClaimExpiresAt: terminal ? undefined : now + CHECK_RETRY_BACKOFF_MS, checkClaimLastError: scannerFailureSummary(args), - failedAt: undefined, + checkFailureCount, + failedAt: terminal ? now : undefined, updatedAt: now, }); - return { attemptId: attempt._id, kind: attempt.kind, status: "pending_checks" as const }; + return { + attemptId: attempt._id, + kind: attempt.kind, + status: terminal ? ("failed" as const) : ("pending_checks" as const), + }; } if (attempt.kind === "skill" && attempt.skillVersionId && args.clawscanAnalysis) { @@ -777,6 +796,7 @@ export const completePendingPublishAttemptChecksInternal = internalMutation({ checkClaimedAt: undefined, checkClaimExpiresAt: undefined, checkClaimLastError: undefined, + checkFailureCount: undefined, updatedAt: now, }); return { attemptId: attempt._id, kind: attempt.kind, status: "ready_to_finalize" as const }; diff --git a/convex/schema.ts b/convex/schema.ts index 9457f6a6..91e1bdbe 100644 --- a/convex/schema.ts +++ b/convex/schema.ts @@ -1304,6 +1304,7 @@ const publishAttempts = defineTable({ checkClaimedAt: v.optional(v.number()), checkClaimExpiresAt: v.optional(v.number()), checkClaimLastError: v.optional(v.string()), + checkFailureCount: v.optional(v.number()), finalizationClaimId: v.optional(v.string()), finalizationClaimedAt: v.optional(v.number()), finalizationClaimExpiresAt: v.optional(v.number()),