fix: bound prepublication scanner retries (#3353)

convex/publishAttempts.test.ts

@@ -844,6 +844,7 @@ describe("publishAttempts", () => {
         checkClaimedAt: undefined,
         checkClaimExpiresAt: expect.any(Number),
         checkClaimLastError: "scanner unavailable",
+        checkFailureCount: 1,
         failedAt: undefined,
       }),
     );
@@ -851,6 +852,69 @@ describe("publishAttempts", () => {
     expect(patch.checkClaimExpiresAt).toBeGreaterThan(now);
   });
 
+  it("terminalizes an attempt after three consecutive scanner execution failures", async () => {
+    const now = Date.now();
+    const ctx = {
+      db: {
+        get: vi.fn(async () => ({
+          _id: "publishAttempts:poison",
+          kind: "skill",
+          status: "pending_checks",
+          artifactFingerprint: "fingerprint",
+          checkClaimId: "checks:claim",
+          checkClaimExpiresAt: now + 60_000,
+          checkFailureCount: 2,
+          checks: {
+            trufflehog: { status: "clean", checkedAt: now - 300_000 },
+            clawscan: {
+              status: "failed",
+              checkedAt: now - 300_000,
+              summary: "ClawScan scanner did not complete: skillspector=failed",
+            },
+          },
+        })),
+        patch: vi.fn(),
+        insert: vi.fn(),
+        replace: vi.fn(),
+        delete: vi.fn(),
+        query: vi.fn(),
+        normalizeId: vi.fn(),
+        system: {},
+      },
+      storage: {
+        delete: vi.fn(),
+      },
+    };
+
+    await expect(
+      completePendingChecksHandler(ctx, {
+        attemptId: "publishAttempts:poison",
+        claimId: "checks:claim",
+        artifactFingerprint: "fingerprint",
+        trufflehog: { status: "clean" },
+        clawscan: {
+          status: "failed",
+          summary: "ClawScan scanner did not complete: skillspector=failed",
+        },
+      }),
+    ).resolves.toEqual({
+      attemptId: "publishAttempts:poison",
+      kind: "skill",
+      status: "failed",
+    });
+
+    expect(ctx.db.patch).toHaveBeenCalledWith(
+      "publishAttempts:poison",
+      expect.objectContaining({
+        status: "failed",
+        checkFailureCount: 3,
+        checkClaimExpiresAt: undefined,
+        checkClaimLastError: "ClawScan scanner did not complete: skillspector=failed",
+        failedAt: expect.any(Number),
+      }),
+    );
+  });
+
   it("terminalizes an attempt when its staged target disappears during scanning", async () => {
     const now = Date.now();
     const ctx = {

convex/publishAttempts.ts

@@ -9,6 +9,7 @@ import { requestPublishAttemptDispatch } from "./publishAttemptDispatch";
 const THIRTY_DAYS_MS = 30 * 24 * 60 * 60 * 1000;
 const CHECK_CLAIM_LEASE_MS = 30 * 60 * 1000;
 const CHECK_RETRY_BACKOFF_MS = 5 * 60 * 1000;
+const MAX_CONSECUTIVE_SCANNER_FAILURES = 3;
 const FINALIZATION_CLAIM_LEASE_MS = 10 * 60 * 1000;
 const PUBLISH_ATTEMPT_STATUSES = [
   "pending_checks",
@@ -107,6 +108,14 @@ function scannerFailureSummary(args: {
   return "Pre-publication scanner failed before returning a verdict.";
 }
 
+function previousScannerFailureCount(attempt: Doc<"publishAttempts">) {
+  if (attempt.checkFailureCount !== undefined) return attempt.checkFailureCount;
+  return attempt.checks.trufflehog.status === "failed" ||
+    attempt.checks.clawscan.status === "failed"
+    ? 1
+    : 0;
+}
+
 function isTerminalFinalizationConflict(error: string | undefined) {
   return (
     typeof error === "string" &&
@@ -136,6 +145,7 @@ function releaseFinalizationClaimPatch(error: string | undefined, now: number) {
     checkClaimedAt: undefined,
     checkClaimExpiresAt: undefined,
     checkClaimLastError: undefined,
+    checkFailureCount: undefined,
     finalizationClaimId: undefined,
     finalizationClaimedAt: undefined,
     finalizationClaimExpiresAt: undefined,
@@ -683,6 +693,7 @@ export const completePendingPublishAttemptChecksInternal = internalMutation({
         checkClaimedAt: undefined,
         checkClaimExpiresAt: undefined,
         checkClaimLastError: undefined,
+        checkFailureCount: undefined,
         blockedAt: now,
         updatedAt: now,
       });
@@ -725,6 +736,7 @@ export const completePendingPublishAttemptChecksInternal = internalMutation({
         checkClaimedAt: undefined,
         checkClaimExpiresAt: undefined,
         checkClaimLastError: undefined,
+        checkFailureCount: undefined,
         blockedAt: now,
         updatedAt: now,
       });
@@ -736,17 +748,24 @@ export const completePendingPublishAttemptChecksInternal = internalMutation({
     }
 
     if (args.trufflehog.status === "failed" || args.clawscan.status === "failed") {
+      const checkFailureCount = previousScannerFailureCount(attempt) + 1;
+      const terminal = checkFailureCount >= MAX_CONSECUTIVE_SCANNER_FAILURES;
       await ctx.db.patch(attempt._id, {
-        status: "pending_checks",
+        status: terminal ? "failed" : "pending_checks",
         checks,
         checkClaimId: undefined,
         checkClaimedAt: undefined,
-        checkClaimExpiresAt: now + CHECK_RETRY_BACKOFF_MS,
+        checkClaimExpiresAt: terminal ? undefined : now + CHECK_RETRY_BACKOFF_MS,
         checkClaimLastError: scannerFailureSummary(args),
-        failedAt: undefined,
+        checkFailureCount,
+        failedAt: terminal ? now : undefined,
         updatedAt: now,
       });
-      return { attemptId: attempt._id, kind: attempt.kind, status: "pending_checks" as const };
+      return {
+        attemptId: attempt._id,
+        kind: attempt.kind,
+        status: terminal ? ("failed" as const) : ("pending_checks" as const),
+      };
     }
 
     if (attempt.kind === "skill" && attempt.skillVersionId && args.clawscanAnalysis) {
@@ -777,6 +796,7 @@ export const completePendingPublishAttemptChecksInternal = internalMutation({
       checkClaimedAt: undefined,
       checkClaimExpiresAt: undefined,
       checkClaimLastError: undefined,
+      checkFailureCount: undefined,
       updatedAt: now,
     });
     return { attemptId: attempt._id, kind: attempt.kind, status: "ready_to_finalize" as const };

convex/schema.ts

@@ -1304,6 +1304,7 @@ const publishAttempts = defineTable({
   checkClaimedAt: v.optional(v.number()),
   checkClaimExpiresAt: v.optional(v.number()),
   checkClaimLastError: v.optional(v.string()),
+  checkFailureCount: v.optional(v.number()),
   finalizationClaimId: v.optional(v.string()),
   finalizationClaimedAt: v.optional(v.number()),
   finalizationClaimExpiresAt: v.optional(v.number()),