From 906428a557f76dc12d3c6723af6e23d2c9758135 Mon Sep 17 00:00:00 2001 From: Patrick Erichsen Date: Fri, 24 Jul 2026 21:36:16 -0500 Subject: [PATCH] test: isolate GitHub Skill Sync proof jobs (#3257) --- convex/securityScan.test.ts | 118 +++++++++++++++++++++++++++++++++- convex/securityScan.ts | 31 ++++++++- specs/github-backed-skills.md | 5 ++ 3 files changed, 152 insertions(+), 2 deletions(-) diff --git a/convex/securityScan.test.ts b/convex/securityScan.test.ts index 275b4691..1aebe763 100644 --- a/convex/securityScan.test.ts +++ b/convex/securityScan.test.ts @@ -61,6 +61,7 @@ const claimCodexScanJobLeasesHandler = ( lane?: "priority" | "shared" | "catalog"; limit?: number; leaseMs?: number; + targetedJobIds?: string[]; }, Array > @@ -77,7 +78,13 @@ const hydrateCodexScanJobHandler = ( const claimQueuedJobsInternalHandler = ( claimQueuedJobsInternal as unknown as WrappedHandler< - { workerId: string; lane?: "priority" | "shared" | "catalog"; limit: number; leaseMs?: number }, + { + workerId: string; + lane?: "priority" | "shared" | "catalog"; + limit: number; + leaseMs?: number; + targetedJobIds?: string[]; + }, Array > )._handler; @@ -376,6 +383,7 @@ type ScanJob = { skillVersionId?: string; packageReleaseId?: string; skillScanRequestId?: string; + rolloutGate?: "github-skill-sync"; source: string; priority: number; hasMaliciousSignal: boolean; @@ -3223,6 +3231,36 @@ describe("securityScan", () => { expect(getUrl).not.toHaveBeenCalled(); }); + it("forwards exact Test GitHub Skill Sync job IDs to the lease mutation", async () => { + vi.stubEnv("SECURITY_SCAN_WORKER_TOKEN", "worker-secret"); + const leases = [ + { + ...claimedJob, + _id: "securityScanJobs:github-sync", + leaseToken: "lease-github-sync", + }, + ]; + const runMutation = vi.fn(async () => leases); + + const result = await claimCodexScanJobLeasesHandler( + { runMutation, runQuery: vi.fn(), storage: { getUrl: vi.fn() } }, + { + token: "worker-secret", + workerId: "worker-1", + limit: 1, + targetedJobIds: ["securityScanJobs:github-sync"], + }, + ); + + expect(runMutation).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + targetedJobIds: ["securityScanJobs:github-sync"], + }), + ); + expect(result).toEqual(leases); + }); + it("refuses to hydrate a lease owned by a different worker", async () => { vi.stubEnv("SECURITY_SCAN_WORKER_TOKEN", "worker-secret"); const runQuery = vi.fn(async () => ({ @@ -3751,6 +3789,84 @@ describe("securityScan", () => { expect(patches.map((entry) => entry.id)).toEqual(claimed.map((job) => job._id)); }); + it("claims only requested GitHub Skill Sync jobs in the permanent Test environment", async () => { + vi.stubEnv("CLAWHUB_ENV", "test"); + vi.stubEnv("CLAWHUB_DEPLOYMENT_NAME", "academic-chihuahua-392"); + vi.stubEnv("CLAWHUB_GITHUB_SKILL_SYNC_ROLLOUT_MODE", "test"); + vi.stubEnv("CONVEX_CLOUD_URL", "https://academic-chihuahua-392.convex.cloud"); + const { ctx, patches } = makeClaimCtx([ + makeScanJob({ + _id: "securityScanJobs:unrelated", + source: "publish", + createdAt: 1, + nextRunAt: 1, + }), + makeScanJob({ + _id: "securityScanJobs:github-sync", + source: "publish", + rolloutGate: "github-skill-sync", + createdAt: 2, + nextRunAt: 2, + }), + ]); + + const claimed = await claimQueuedJobsInternalHandler(ctx, { + workerId: "targeted-test-worker", + limit: 1, + targetedJobIds: ["securityScanJobs:github-sync"], + }); + + expect(claimed.map((job) => job._id)).toEqual(["securityScanJobs:github-sync"]); + expect(patches.map((entry) => entry.id)).toEqual(["securityScanJobs:github-sync"]); + }); + + it("does not turn an empty exact claim into a broad queue claim", async () => { + vi.stubEnv("CLAWHUB_ENV", "test"); + vi.stubEnv("CLAWHUB_DEPLOYMENT_NAME", "academic-chihuahua-392"); + vi.stubEnv("CLAWHUB_GITHUB_SKILL_SYNC_ROLLOUT_MODE", "test"); + vi.stubEnv("CONVEX_CLOUD_URL", "https://academic-chihuahua-392.convex.cloud"); + const { ctx, patches } = makeClaimCtx([ + makeScanJob({ + _id: "securityScanJobs:unrelated", + source: "publish", + createdAt: 1, + nextRunAt: 1, + }), + ]); + + const claimed = await claimQueuedJobsInternalHandler(ctx, { + workerId: "targeted-test-worker", + limit: 1, + targetedJobIds: [], + }); + + expect(claimed).toEqual([]); + expect(patches).toEqual([]); + }); + + it("rejects exact GitHub Skill Sync job claims outside the permanent Test rollout", async () => { + vi.stubEnv("CLAWHUB_ENV", "production"); + vi.stubEnv("CLAWHUB_DEPLOYMENT_NAME", "wry-manatee-359"); + vi.stubEnv("CLAWHUB_GITHUB_SKILL_SYNC_ROLLOUT_MODE", "off"); + vi.stubEnv("CONVEX_CLOUD_URL", "https://wry-manatee-359.convex.cloud"); + const { ctx } = makeClaimCtx([ + makeScanJob({ + _id: "securityScanJobs:github-sync", + source: "publish", + rolloutGate: "github-skill-sync", + nextRunAt: 1, + }), + ]); + + await expect( + claimQueuedJobsInternalHandler(ctx, { + workerId: "targeted-test-worker", + limit: 1, + targetedJobIds: ["securityScanJobs:github-sync"], + }), + ).rejects.toThrow("Exact GitHub Skill Sync job claims are Test-only"); + }); + it("claims bulk rescans after every supported source", async () => { const { ctx } = makeClaimCtx([ makeScanJob({ diff --git a/convex/securityScan.ts b/convex/securityScan.ts index ef1f1887..de888305 100644 --- a/convex/securityScan.ts +++ b/convex/securityScan.ts @@ -35,6 +35,7 @@ import { requestSecurityScanDispatch } from "./securityScanDispatch"; const DEFAULT_VT_WAIT_MS = 10 * 60 * 1000; const DEFAULT_LEASE_MS = 60 * 60 * 1000; +const MAX_TARGETED_TEST_GITHUB_SYNC_JOBS = 32; const MAX_ATTEMPTS = 3; const DEFAULT_CODEX_SCAN_CLAIM_LIMIT = 64; const MAX_CODEX_SCAN_CLAIM_LIMIT = 512; @@ -2842,6 +2843,7 @@ export const claimQueuedJobsInternal = internalMutation({ lane: v.optional(codexScanWorkerLaneValidator), limit: v.number(), leaseMs: v.optional(v.number()), + targetedJobIds: v.optional(v.array(v.id("securityScanJobs"))), }, handler: async (ctx, args) => { const now = Date.now(); @@ -2943,7 +2945,32 @@ export const claimQueuedJobsInternal = internalMutation({ return eligible; }; - if (args.lane === "catalog") { + const targetedJobIds = args.targetedJobIds; + if (targetedJobIds !== undefined) { + const rollout = getRuntimeRolloutCapabilities(); + if ( + rollout.environment !== "test" || + rollout.githubSkillSync.mode !== "test" || + !rollout.githubSkillSync.runtimeEnabled + ) { + throw new ConvexError("Exact GitHub Skill Sync job claims are Test-only"); + } + if (targetedJobIds.length > MAX_TARGETED_TEST_GITHUB_SYNC_JOBS) { + throw new ConvexError("Too many exact GitHub Skill Sync jobs requested"); + } + const targetedJobs: Doc<"securityScanJobs">[] = []; + for (const jobId of new Set(targetedJobIds)) { + const job = await ctx.db.get(jobId); + if ( + job?.status === "queued" && + job.rolloutGate === "github-skill-sync" && + job.nextRunAt <= now + ) { + targetedJobs.push(job); + } + } + addReadyJobs(targetedJobs); + } else if (args.lane === "catalog") { addReadyJobs(await takeReadySourceJobs("skills-sh-catalog-test"), false); } else { addReadyJobs(await takeReadySourceJobs("manual")); @@ -3697,6 +3724,7 @@ export const claimCodexScanJobLeases = action({ lane: v.optional(codexScanWorkerLaneValidator), limit: v.optional(v.number()), leaseMs: v.optional(v.number()), + targetedJobIds: v.optional(v.array(v.id("securityScanJobs"))), }, handler: async (ctx, args) => { assertWorkerToken(args.token); @@ -3708,6 +3736,7 @@ export const claimCodexScanJobLeases = action({ lane: args.lane ?? "shared", limit: normalizeLimit(args.limit), leaseMs: args.leaseMs, + targetedJobIds: args.targetedJobIds, }, ); }, diff --git a/specs/github-backed-skills.md b/specs/github-backed-skills.md index cb7aa6ea..a2cf3351 100644 --- a/specs/github-backed-skills.md +++ b/specs/github-backed-skills.md @@ -335,6 +335,11 @@ returns the same commit-pinned GitHub descriptor used by native GitHub-backed skills. Catalog pause, kill, publication disable, and exact-attempt rollback must fail closed without disabling or mutating native scan work. +Permanent-Test proof workers may claim explicit security job IDs only when the +runtime is the permanent Test environment, GitHub Skill Sync is in `test` mode, +and every selected job carries the `github-skill-sync` rollout gate. Production +and broad queue claims must not accept this proof-only targeting path. + Pending verification keeps the skill visible in ClawHub search and detail UI, but normal install/update returns a structured block: