diff --git a/convex/httpApiV1/skillsShCatalogV1.test.ts b/convex/httpApiV1/skillsShCatalogV1.test.ts index ed760702..5043c703 100644 --- a/convex/httpApiV1/skillsShCatalogV1.test.ts +++ b/convex/httpApiV1/skillsShCatalogV1.test.ts @@ -10,6 +10,14 @@ vi.mock("../lib/githubAuth", () => ({ buildGitHubApiHeaders: vi.fn(async () => ({ Authorization: "Bearer placeholder" })), })); +vi.mock("../lib/githubSkillSync", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + computeGitHubSkillFolderContentHash: vi.fn(original.computeGitHubSkillFolderContentHash), + }; +}); + vi.mock("./shared", async (importOriginal) => { const original = await importOriginal(); return { @@ -21,7 +29,9 @@ vi.mock("./shared", async (importOriginal) => { const { requireAdminOrResponse, requireApiTokenUserOrResponse } = await import("./shared"); const { buildGitHubApiHeaders } = await import("../lib/githubAuth"); -const { skillsShCatalogTestV1Handler } = await import("./skillsShCatalogV1"); +const { computeGitHubSkillFolderContentHash } = await import("../lib/githubSkillSync"); +const { skillsShCatalogTestV1Handler, verifyControlledCanaryGitHubSource } = + await import("./skillsShCatalogV1"); function sha256(value: string) { return createHash("sha256").update(value).digest("hex"); @@ -374,4 +384,185 @@ describe("skills.sh catalog Test HTTP API", () => { ); expect(githubFetch).toHaveBeenCalledTimes(1); }); + + it("verifies the controlled canary against immutable public GitHub content", async () => { + const skillMarkdown = "# HTML Artifact Chooser\n"; + const fileHash = sha256(skillMarkdown); + const contentHash = sha256(`SKILL.md\0${Buffer.byteLength(skillMarkdown)}\0${fileHash}`); + const githubFetch = vi.fn(async (url: string, init?: RequestInit) => { + expect(init?.headers).toMatchObject({ Authorization: "Bearer placeholder" }); + if (url.endsWith("/repos/patrick-erichsen/skills")) { + return new Response( + JSON.stringify({ + private: false, + full_name: "Patrick-Erichsen/skills", + owner: { id: 20_157_849, login: "Patrick-Erichsen" }, + }), + ); + } + if (url.includes("/git/commits/")) { + return new Response( + JSON.stringify({ + sha: "050daba89f6b6636470add5cb300aac46a412cf8", + tree: { sha: "tree-sha" }, + }), + ); + } + if (url.endsWith("/git/trees/tree-sha?recursive=1")) { + return new Response( + JSON.stringify({ + truncated: false, + tree: [ + { + path: "skills/html/SKILL.md", + type: "blob", + sha: "blob-sha", + size: Buffer.byteLength(skillMarkdown), + }, + ], + }), + ); + } + if (url.endsWith("/git/blobs/blob-sha")) { + return new Response( + JSON.stringify({ + encoding: "base64", + content: Buffer.from(skillMarkdown).toString("base64"), + }), + ); + } + return new Response("not found", { status: 404 }); + }); + vi.stubGlobal("fetch", githubFetch); + const proof = await verifyControlledCanaryGitHubSource({ + fetchImpl: githubFetch as typeof fetch, + checkedAt: "2026-07-22T05:00:00.000Z", + expected: { + externalId: "patrick-erichsen/skills/html", + githubOwnerId: 20_157_849, + owner: "patrick-erichsen", + repo: "skills", + slug: "html", + displayName: "HTML Artifact Chooser", + sourceUrl: "https://www.skills.sh/patrick-erichsen/skills/html", + githubRepoUrl: "https://github.com/Patrick-Erichsen/skills", + githubPath: "skills/html", + githubCommit: "050daba89f6b6636470add5cb300aac46a412cf8", + githubContentHash: contentHash, + sourceContentHash: contentHash, + installs: 0, + }, + }); + + expect(proof).toMatchObject({ + authentication: "clawhub-github-authenticated", + fixtureId: "patrick-html-canary-v1", + externalId: "patrick-erichsen/skills/html", + githubOwnerId: 20_157_849, + githubPath: "skills/html", + githubCommit: "050daba89f6b6636470add5cb300aac46a412cf8", + githubContentHash: contentHash, + githubFetches: 4, + }); + expect(githubFetch).toHaveBeenCalledTimes(4); + expect(buildGitHubApiHeaders).toHaveBeenCalledWith({ + userAgent: "clawhub/skills-sh-catalog-canary", + allowAnonymous: false, + useGitHubApp: false, + }); + }); + + it("passes only server-fetched GitHub verification when starting the controlled canary", async () => { + const expectedContentHash = "a47adb2c1ac33c088f664b5187971b63d2b958a7b9f01516d26005ca941a108f"; + vi.mocked(computeGitHubSkillFolderContentHash).mockResolvedValueOnce(expectedContentHash); + vi.stubGlobal( + "fetch", + vi.fn(async (url: string) => { + if (url.endsWith("/repos/patrick-erichsen/skills")) { + return new Response( + JSON.stringify({ + private: false, + full_name: "Patrick-Erichsen/skills", + owner: { id: 20_157_849, login: "Patrick-Erichsen" }, + }), + ); + } + if (url.includes("/git/commits/")) { + return new Response( + JSON.stringify({ + sha: "050daba89f6b6636470add5cb300aac46a412cf8", + tree: { sha: "tree-sha" }, + }), + ); + } + if (url.endsWith("/git/trees/tree-sha?recursive=1")) { + return new Response( + JSON.stringify({ + truncated: false, + tree: [ + { + path: "skills/html/SKILL.md", + type: "blob", + sha: "blob-sha", + }, + ], + }), + ); + } + if (url.endsWith("/git/blobs/blob-sha")) { + return new Response( + JSON.stringify({ + encoding: "base64", + content: Buffer.from("# Server-fetched content\n").toString("base64"), + }), + ); + } + return new Response("not found", { status: 404 }); + }), + ); + const runMutation = vi.fn(async (_ref: unknown, _args: Record) => ({ + runId: "skillsShCatalogRuns:canary", + })); + const ctx = { + runQuery: vi.fn(async () => ({ + environment: "test", + deploymentName: "academic-chihuahua-392", + buildSha: "test-sha", + control: {}, + })), + runMutation, + } as never; + const request = new Request("https://academic-chihuahua-392.convex.site/api/v1/ops", { + method: "POST", + body: JSON.stringify({ + operation: "start-canary", + reason: "CLAW-557 hidden metadata canary", + sourceVerification: { + githubOwnerId: 1, + githubCommit: "client-controlled", + githubContentHash: "client-controlled", + githubCheckedAt: "client-controlled", + githubFetches: 99_999, + }, + }), + }); + + const response = await skillsShCatalogTestV1Handler(ctx, request); + + expect(response.status).toBe(200); + expect(runMutation).toHaveBeenCalledOnce(); + expect(runMutation.mock.calls[0]?.[1]).toEqual({ + fixtureId: "patrick-html-canary-v1", + actor: "catalog-operator", + reason: "CLAW-557 hidden metadata canary", + sourceVerification: { + githubOwnerId: 20_157_849, + githubCommit: "050daba89f6b6636470add5cb300aac46a412cf8", + githubContentHash: expectedContentHash, + githubCheckedAt: expect.any(String), + githubFetches: 4, + }, + }); + expect(JSON.stringify(runMutation.mock.calls[0]?.[1])).not.toContain("client-controlled"); + }); }); diff --git a/convex/httpApiV1/skillsShCatalogV1.ts b/convex/httpApiV1/skillsShCatalogV1.ts index 37e2a0bc..5a8d206a 100644 --- a/convex/httpApiV1/skillsShCatalogV1.ts +++ b/convex/httpApiV1/skillsShCatalogV1.ts @@ -2,21 +2,32 @@ import { internal } from "../_generated/api"; import type { Id } from "../_generated/dataModel"; import type { ActionCtx } from "../_generated/server"; import { buildGitHubApiHeaders } from "../lib/githubAuth"; +import { computeGitHubSkillFolderContentHash } from "../lib/githubSkillSync"; import { applyRateLimit } from "../lib/httpRateLimit"; +import { + getSkillsShCatalogFixture, + type SkillsShCatalogFixtureRow, +} from "../lib/skillsShCatalogFixtures"; import { json, requireAdminOrResponse, requireApiTokenUserOrResponse, text } from "./shared"; const internalRefs = internal as unknown as { skillsShCatalog: { admitRealScansInternal: unknown; assertFreshGitHubOwnerAssignmentsInternal: unknown; + getRunReconciliationInternal: unknown; getStagingLiveControlInternal: unknown; + processFixtureBatchInternal: unknown; processStagingLiveBatchInternal: unknown; resolveKnownGitHubOwnersInternal: unknown; + rollbackFixtureRunInternal: unknown; + startFixtureRunInternal: unknown; startStagingLiveRunInternal: unknown; }; }; const MAX_GITHUB_OWNER_RESOLUTIONS = 500; const GITHUB_OWNER_RESOLUTION_CONCURRENCY = 8; +const CONTROLLED_CANARY_FIXTURE_ID = "patrick-html-canary-v1"; +const MAX_CONTROLLED_CANARY_FILES = 100; async function runMutationRef( ctx: ActionCtx, @@ -124,6 +135,130 @@ async function fetchAuthenticatedGitHubOwners(owners: string[]) { return resolved; } +async function fetchGitHubJson( + url: string, + headers: Record, + label: string, + fetchImpl: typeof fetch, +) { + const response = await fetchImpl(url, { headers }); + if (!response.ok) { + throw new Error(`${label} failed with HTTP ${response.status}`); + } + return (await response.json()) as Record; +} + +function decodeGitHubBlob(payload: Record, path: string) { + if (payload.encoding !== "base64" || typeof payload.content !== "string") { + throw new Error(`Controlled canary returned invalid GitHub blob content: ${path}`); + } + return decodeBase64(payload.content.replace(/\s+/g, "")); +} + +export async function verifyControlledCanaryGitHubSource(options: { + expected?: SkillsShCatalogFixtureRow; + fetchImpl?: typeof fetch; + checkedAt?: string; +}) { + const expected = + options.expected ?? + getSkillsShCatalogFixture(CONTROLLED_CANARY_FIXTURE_ID).findByExternalId( + "patrick-erichsen/skills/html", + ); + if (!expected || !expected.githubPath || !expected.githubCommit || !expected.githubContentHash) { + throw new Error("Controlled canary fixture lacks immutable GitHub provenance"); + } + const fetchImpl = options.fetchImpl ?? fetch; + const headers = await buildGitHubApiHeaders({ + userAgent: "clawhub/skills-sh-catalog-canary", + allowAnonymous: false, + useGitHubApp: false, + }); + const repository = await fetchGitHubJson( + `https://api.github.com/repos/${encodeURIComponent(expected.owner)}/${encodeURIComponent(expected.repo)}`, + headers, + "Controlled canary GitHub repository lookup", + fetchImpl, + ); + const repositoryOwner = asRecord(repository.owner); + if ( + repository.private !== false || + requireString(repository, "full_name").toLowerCase() !== + `${expected.owner}/${expected.repo}`.toLowerCase() || + requireNumber(repositoryOwner ?? {}, "id") !== expected.githubOwnerId || + requireString(repositoryOwner ?? {}, "login").toLowerCase() !== expected.owner.toLowerCase() + ) { + throw new Error("Controlled canary GitHub repository identity mismatch"); + } + const commit = await fetchGitHubJson( + `https://api.github.com/repos/${encodeURIComponent(expected.owner)}/${encodeURIComponent(expected.repo)}/git/commits/${encodeURIComponent(expected.githubCommit)}`, + headers, + "Controlled canary GitHub commit lookup", + fetchImpl, + ); + const commitSha = requireString(commit, "sha").toLowerCase(); + const commitTree = asRecord(commit.tree); + const treeSha = requireString(commitTree ?? {}, "sha"); + if (commitSha !== expected.githubCommit.toLowerCase()) { + throw new Error("Controlled canary GitHub commit mismatch"); + } + const tree = await fetchGitHubJson( + `https://api.github.com/repos/${encodeURIComponent(expected.owner)}/${encodeURIComponent(expected.repo)}/git/trees/${encodeURIComponent(treeSha)}?recursive=1`, + headers, + "Controlled canary GitHub tree lookup", + fetchImpl, + ); + if (tree.truncated !== false || !Array.isArray(tree.tree)) { + throw new Error("Controlled canary GitHub tree is incomplete"); + } + const prefix = `${expected.githubPath.replace(/\/+$/g, "")}/`; + const blobs = tree.tree + .map(asRecord) + .filter( + (entry): entry is Record => + entry !== null && + entry.type === "blob" && + typeof entry.path === "string" && + entry.path.startsWith(prefix), + ) + .sort((left, right) => String(left.path).localeCompare(String(right.path))); + if (blobs.length < 1 || blobs.length > MAX_CONTROLLED_CANARY_FILES) { + throw new Error("Controlled canary GitHub folder has an invalid file count"); + } + const entries: Record = {}; + for (const blob of blobs) { + const path = requireString(blob, "path"); + const blobSha = requireString(blob, "sha"); + const payload = await fetchGitHubJson( + `https://api.github.com/repos/${encodeURIComponent(expected.owner)}/${encodeURIComponent(expected.repo)}/git/blobs/${encodeURIComponent(blobSha)}`, + headers, + `Controlled canary GitHub blob lookup: ${path}`, + fetchImpl, + ); + entries[path] = decodeGitHubBlob(payload, path); + } + const contentHash = await computeGitHubSkillFolderContentHash(entries, expected.githubPath); + if (contentHash !== expected.githubContentHash.toLowerCase()) { + throw new Error("Controlled canary GitHub content hash mismatch"); + } + const checkedAt = options.checkedAt ?? new Date().toISOString(); + if (Number.isNaN(Date.parse(checkedAt))) { + throw new Error("Controlled canary GitHub checked time is invalid"); + } + return { + authentication: "clawhub-github-authenticated" as const, + fixtureId: CONTROLLED_CANARY_FIXTURE_ID, + externalId: expected.externalId, + githubOwnerId: expected.githubOwnerId, + githubRepo: `${expected.owner}/${expected.repo}`, + githubPath: expected.githubPath, + githubCommit: commitSha, + githubContentHash: contentHash, + githubCheckedAt: checkedAt, + githubFetches: 3 + blobs.length, + }; +} + async function resolveAuthenticatedGitHubOwners(ctx: ActionCtx, ownersValue: unknown) { const owners = normalizeGitHubOwners(ownersValue); const known = await runQueryRef<{ @@ -264,6 +399,60 @@ export async function skillsShCatalogTestV1Handler(ctx: ActionCtx, request: Requ const body = asRecord(await request.json()); if (!body) return text("Invalid JSON", 400, rate.headers); const operation = requireString(body, "operation"); + if (operation === "verify-canary") { + return json(await verifyControlledCanaryGitHubSource({}), 200, rate.headers); + } + if (operation === "start-canary") { + const verification = await verifyControlledCanaryGitHubSource({}); + const sourceVerification = { + githubOwnerId: verification.githubOwnerId, + githubCommit: verification.githubCommit, + githubContentHash: verification.githubContentHash, + githubCheckedAt: verification.githubCheckedAt, + githubFetches: verification.githubFetches, + }; + const result = await runMutationRef<{ runId: string }>( + ctx, + internalRefs.skillsShCatalog.startFixtureRunInternal, + { + fixtureId: CONTROLLED_CANARY_FIXTURE_ID, + actor: auth.user.handle, + reason: requireString(body, "reason"), + sourceVerification, + }, + ); + return json({ ...result, sourceVerification: verification }, 200, rate.headers); + } + if (operation === "process-fixture") { + return json( + await runMutationRef(ctx, internalRefs.skillsShCatalog.processFixtureBatchInternal, { + runId: requireString(body, "runId"), + }), + 200, + rate.headers, + ); + } + if (operation === "reconcile") { + return json( + await runQueryRef(ctx, internalRefs.skillsShCatalog.getRunReconciliationInternal, { + runId: requireString(body, "runId"), + }), + 200, + rate.headers, + ); + } + if (operation === "rollback-canary") { + return json( + await runMutationRef(ctx, internalRefs.skillsShCatalog.rollbackFixtureRunInternal, { + runId: requireString(body, "runId"), + actor: auth.user.handle, + reason: requireString(body, "reason"), + confirm: requireString(body, "confirm"), + }), + 200, + rate.headers, + ); + } if (operation === "resolve-owners") { return json(await resolveAuthenticatedGitHubOwners(ctx, body.owners), 200, rate.headers); } diff --git a/convex/lib/skillsShCatalogFixtures.ts b/convex/lib/skillsShCatalogFixtures.ts index 02fceb57..ffe20269 100644 --- a/convex/lib/skillsShCatalogFixtures.ts +++ b/convex/lib/skillsShCatalogFixtures.ts @@ -9,6 +9,10 @@ export type SkillsShCatalogFixtureRow = { displayName: string; sourceUrl: string; githubRepoUrl: string; + githubPath?: string; + githubCommit?: string; + githubContentHash?: string; + claimPublisherHandle?: string; sourceContentHash: string; installs: number; }; @@ -43,6 +47,23 @@ const AIQ_DEPLOY_V2: SkillsShCatalogFixtureRow = { installs: 101, }; +const PATRICK_HTML_CANARY: SkillsShCatalogFixtureRow = { + externalId: "patrick-erichsen/skills/html", + githubOwnerId: 20_157_849, + owner: "patrick-erichsen", + repo: "skills", + slug: "html", + displayName: "HTML Artifact Chooser", + sourceUrl: "https://www.skills.sh/patrick-erichsen/skills/html", + githubRepoUrl: "https://github.com/Patrick-Erichsen/skills", + githubPath: "skills/html", + githubCommit: "050daba89f6b6636470add5cb300aac46a412cf8", + githubContentHash: "a47adb2c1ac33c088f664b5187971b63d2b958a7b9f01516d26005ca941a108f", + claimPublisherHandle: "patrick-erichsen", + sourceContentHash: "a47adb2c1ac33c088f664b5187971b63d2b958a7b9f01516d26005ca941a108f", + installs: 0, +}; + const FROZEN_ROWS = frozenSnapshot.rows satisfies SkillsShCatalogFixtureRow[]; function fromRows( @@ -108,6 +129,15 @@ function syntheticDiscoveryRow(index: number): SkillsShCatalogFixtureRow { } const FIXTURES = { + "patrick-html-canary-v1": fromRows( + { + snapshotId: "patrick-html-canary-v1", + sourceKind: "fixture", + capturedAt: null, + snapshotCaptureFetches: 0, + }, + [PATRICK_HTML_CANARY], + ), "nvidia-small-v1": fromRows( { snapshotId: "nvidia-small-v1", diff --git a/convex/schema.ts b/convex/schema.ts index e415678c..42a786fe 100644 --- a/convex/schema.ts +++ b/convex/schema.ts @@ -2788,6 +2788,10 @@ const skillsShCatalogRunCountsValidator = v.object({ updated: v.number(), unchanged: v.number(), rejected: v.number(), + newExternal: v.optional(v.number()), + exactNativeMatches: v.optional(v.number()), + routeCollisions: v.optional(v.number()), + claimOpportunities: v.optional(v.number()), scansPlanned: v.number(), scansAdmitted: v.number(), scansCompleted: v.number(), @@ -2796,6 +2800,7 @@ const skillsShCatalogRunCountsValidator = v.object({ const skillsShCatalogRuns = defineTable({ fixtureId: v.union( + v.literal("patrick-html-canary-v1"), v.literal("nvidia-small-v1"), v.literal("nvidia-small-v2"), v.literal("skills-sh-500-2026-07-21"), @@ -2811,6 +2816,15 @@ const skillsShCatalogRuns = defineTable({ ), sourceCapturedAt: v.optional(v.string()), snapshotCaptureFetches: v.number(), + githubVerification: v.optional( + v.object({ + ownerId: v.number(), + commit: v.string(), + contentHash: v.string(), + checkedAt: v.string(), + fetches: v.number(), + }), + ), dryRun: v.boolean(), status: v.union( v.literal("running"), @@ -2867,9 +2881,23 @@ const skillsShCatalogEntries = defineTable({ displayName: v.string(), sourceUrl: v.string(), githubRepoUrl: v.string(), + githubPath: v.optional(v.string()), + githubCommit: v.optional(v.string()), + githubContentHash: v.optional(v.string()), sourceContentHash: v.string(), installs: v.number(), sourceSnapshotId: v.string(), + reconciliation: v.optional( + v.object({ + kind: v.union(v.literal("new"), v.literal("exact-native"), v.literal("route-collision")), + nativeSkillId: v.optional(v.id("skills")), + nativeSlug: v.optional(v.string()), + nativeStatsDownloads: v.optional(v.number()), + claimOpportunity: v.boolean(), + claimPublisherHandle: v.optional(v.string()), + observedAt: v.number(), + }), + ), publicVisible: v.boolean(), scanStatus: v.union( v.literal("not-planned"), diff --git a/convex/skillsShCatalog.test.ts b/convex/skillsShCatalog.test.ts index 981ec00c..db0c8bc1 100644 --- a/convex/skillsShCatalog.test.ts +++ b/convex/skillsShCatalog.test.ts @@ -833,6 +833,10 @@ describe("skills.sh catalog overload control plane", () => { updated: 0, unchanged: 500, rejected: 0, + newExternal: 500, + exactNativeMatches: 0, + routeCollisions: 0, + claimOpportunities: 0, scansPlanned: 0, scansAdmitted: 0, scansCompleted: 0, @@ -858,6 +862,10 @@ describe("skills.sh catalog overload control plane", () => { updated: 2, unchanged: 498, rejected: 0, + newExternal: 500, + exactNativeMatches: 0, + routeCollisions: 0, + claimOpportunities: 0, scansPlanned: 1, scansAdmitted: 0, scansCompleted: 0, @@ -2091,6 +2099,10 @@ describe("skills.sh catalog overload control plane", () => { updated: 0, unchanged: 0, rejected: 0, + newExternal: 1, + exactNativeMatches: 0, + routeCollisions: 0, + claimOpportunities: 0, scansPlanned: 1, scansAdmitted: 1, scansCompleted: 0, @@ -2486,6 +2498,10 @@ describe("skills.sh catalog overload control plane", () => { updated: 0, unchanged: 0, rejected: 0, + newExternal: 1, + exactNativeMatches: 0, + routeCollisions: 0, + claimOpportunities: 0, scansPlanned: 1, scansAdmitted: 1, scansCompleted: 0, diff --git a/convex/skillsShCatalog.ts b/convex/skillsShCatalog.ts index e57a8fb4..366a1dde 100644 --- a/convex/skillsShCatalog.ts +++ b/convex/skillsShCatalog.ts @@ -19,6 +19,8 @@ import { enqueueSkillsShCatalogScanRequest } from "./securityScan"; const CONTROL_KEY = "global"; const ENABLE_FIXTURE_CONFIRM = "enable-skills-sh-fixture-control"; const DISABLE_CATALOG_CONFIRM = "disable-skills-sh-catalog"; +const ROLLBACK_CONTROLLED_CANARY_CONFIRM = "rollback-skills-sh-controlled-canary"; +const CONTROLLED_CANARY_FIXTURE_ID = "patrick-html-canary-v1"; const STATUS_LIMIT = 50; const MAX_DISCOVERY_ROWS = 20_000; const MAX_ENTRIES_PER_BATCH = 250; @@ -29,6 +31,7 @@ const MAX_REAL_TEST_ADMISSIONS = 10; const MAX_DETERMINISTIC_COMPLETIONS_PER_BATCH = 50; const fixtureIdValidator = v.union( + v.literal("patrick-html-canary-v1"), v.literal("nvidia-small-v1"), v.literal("nvidia-small-v2"), v.literal("skills-sh-500-2026-07-21"), @@ -51,9 +54,20 @@ const stagingLiveRowValidator = v.object({ displayName: v.string(), sourceUrl: v.string(), githubRepoUrl: v.string(), + githubPath: v.optional(v.string()), + githubCommit: v.optional(v.string()), + githubContentHash: v.optional(v.string()), + claimPublisherHandle: v.optional(v.string()), sourceContentHash: v.string(), installs: v.number(), }); +const sourceVerificationValidator = v.object({ + githubOwnerId: v.number(), + githubCommit: v.string(), + githubContentHash: v.string(), + githubCheckedAt: v.string(), + githubFetches: v.number(), +}); const scanRequestFileValidator = v.object({ path: v.string(), size: v.number(), @@ -109,10 +123,95 @@ function normalizeIdentity(row: SkillsShCatalogFixtureRow) { repo, slug, externalId: `${owner}/${repo}/${slug}`, + githubPath: row.githubPath?.trim(), + githubCommit: row.githubCommit?.trim().toLowerCase(), + githubContentHash: row.githubContentHash?.trim().toLowerCase(), + claimPublisherHandle: row.claimPublisherHandle?.trim().toLowerCase(), sourceContentHash: row.sourceContentHash.trim().toLowerCase(), }; } +async function reconcileNativeSkill( + ctx: MutationCtx, + row: ReturnType, + observedAt: number, +) { + const nativeSkills = ( + await ctx.db + .query("skills") + .withIndex("by_slug", (q) => q.eq("slug", row.slug)) + .collect() + ).filter((skill) => skill.softDeletedAt === undefined); + let reads = 1; + for (const skill of nativeSkills) { + if ( + skill.installKind !== "github" || + !skill.githubSourceId || + !row.githubPath || + !row.githubCommit || + !row.githubContentHash || + skill.githubPath !== row.githubPath || + skill.githubCurrentCommit?.toLowerCase() !== row.githubCommit || + skill.githubCurrentContentHash?.toLowerCase() !== row.githubContentHash + ) { + continue; + } + const source = await ctx.db.get(skill.githubSourceId); + reads += 1; + if (source?.repo.trim().toLowerCase() !== `${row.owner}/${row.repo}`) continue; + return { + reads, + reconciliation: { + kind: "exact-native" as const, + nativeSkillId: skill._id, + nativeSlug: skill.slug, + nativeStatsDownloads: skill.statsDownloads ?? skill.stats.downloads, + claimOpportunity: Boolean(row.claimPublisherHandle), + ...(row.claimPublisherHandle ? { claimPublisherHandle: row.claimPublisherHandle } : {}), + observedAt, + }, + }; + } + const collision = nativeSkills[0]; + return { + reads, + reconciliation: { + kind: collision ? ("route-collision" as const) : ("new" as const), + ...(collision + ? { + nativeSkillId: collision._id, + nativeSlug: collision.slug, + nativeStatsDownloads: collision.statsDownloads ?? collision.stats.downloads, + } + : {}), + claimOpportunity: Boolean(row.claimPublisherHandle), + ...(row.claimPublisherHandle ? { claimPublisherHandle: row.claimPublisherHandle } : {}), + observedAt, + }, + }; +} + +function incrementReconciliationCounts( + counts: ReturnType, + reconciliation: Doc<"skillsShCatalogEntries">["reconciliation"], +) { + if (!reconciliation) return; + if (reconciliation.kind === "new") counts.newExternal += 1; + if (reconciliation.kind === "exact-native") counts.exactNativeMatches += 1; + if (reconciliation.kind === "route-collision") counts.routeCollisions += 1; + if (reconciliation.claimOpportunity) counts.claimOpportunities += 1; +} + +function normalizedCounts(counts: Doc<"skillsShCatalogRuns">["counts"]) { + return { + ...counts, + newExternal: counts.newExternal ?? 0, + exactNativeMatches: counts.exactNativeMatches ?? 0, + routeCollisions: counts.routeCollisions ?? 0, + claimOpportunities: counts.claimOpportunities ?? 0, + }; +} + async function getControlDoc(ctx: Pick) { return await ctx.db .query("skillsShCatalogControls") @@ -377,18 +476,74 @@ export const startFixtureRunInternal = internalMutation({ actor: v.string(), reason: v.string(), dryRun: v.optional(v.boolean()), + sourceVerification: v.optional(sourceVerificationValidator), }, handler: async (ctx, args) => { assertSkillsShFixtureEnvironmentAllowed(); const control = assertFixtureMode(assertDiscoveryEnabled(await getControlDoc(ctx))); const fixture = getSkillsShCatalogFixture(args.fixtureId); + let githubVerification: + | { + ownerId: number; + commit: string; + contentHash: string; + checkedAt: string; + fetches: number; + } + | undefined; + if (args.fixtureId === CONTROLLED_CANARY_FIXTURE_ID) { + if ( + control.scanAdmissionEnabled || + control.publicVisibilityEnabled || + control.maxEntriesPerRun !== 1 || + control.maxEntriesPerBatch !== 1 || + control.maxPlannedScans !== 1 || + control.maxScanAdmissionsPerBatch !== 0 || + control.maxScanAdmissionsPerRun !== 0 || + control.maxScanAdmissionsPerDay !== 0 || + control.maxCatalogQueued !== 0 || + control.maxCatalogInFlight !== 0 + ) { + throw new ConvexError( + "controlled skills.sh canary requires the exact one-row hidden no-admission control", + ); + } + const row = normalizeIdentity(fixture.rowAt(0)); + const verification = args.sourceVerification; + if ( + !verification || + verification.githubOwnerId !== row.githubOwnerId || + verification.githubCommit.trim().toLowerCase() !== row.githubCommit || + verification.githubContentHash.trim().toLowerCase() !== row.githubContentHash || + !Number.isInteger(verification.githubFetches) || + verification.githubFetches < 1 || + Number.isNaN(Date.parse(verification.githubCheckedAt)) + ) { + throw new ConvexError( + "controlled skills.sh canary requires exact authenticated GitHub source verification", + ); + } + githubVerification = { + ownerId: verification.githubOwnerId, + commit: verification.githubCommit.trim().toLowerCase(), + contentHash: verification.githubContentHash.trim().toLowerCase(), + checkedAt: verification.githubCheckedAt, + fetches: verification.githubFetches, + }; + } else if (args.sourceVerification) { + throw new ConvexError("GitHub source verification is reserved for the controlled canary"); + } const now = Date.now(); const runId = await ctx.db.insert("skillsShCatalogRuns", { fixtureId: args.fixtureId, snapshotId: fixture.snapshotId, sourceKind: fixture.sourceKind, - ...(fixture.capturedAt ? { sourceCapturedAt: fixture.capturedAt } : {}), - snapshotCaptureFetches: fixture.snapshotCaptureFetches, + ...(githubVerification + ? { sourceCapturedAt: githubVerification.checkedAt, githubVerification } + : fixture.capturedAt + ? { sourceCapturedAt: fixture.capturedAt } + : {}), + snapshotCaptureFetches: githubVerification?.fetches ?? fixture.snapshotCaptureFetches, dryRun: args.dryRun ?? false, status: "running", cursor: 0, @@ -515,7 +670,7 @@ export const processStagingLiveBatchInternal = internalMutation({ let cursor = run.cursor; let writesUsed = 0; let readsUsed = 2; - const counts = { ...run.counts }; + const counts = normalizedCounts(run.counts); const now = Date.now(); for (const inputRow of args.rows) { if (counts.observed >= run.budgets.maxEntriesPerRun) break; @@ -534,7 +689,11 @@ export const processStagingLiveBatchInternal = internalMutation({ cursor += 1; continue; } - const observationUnchanged = existing ? sameFixtureObservation(existing, row) : false; + const native = await reconcileNativeSkill(ctx, row, now); + readsUsed += native.reads; + const observationUnchanged = existing + ? sameFixtureObservation(existing, row, native.reconciliation) + : false; const contentChanged = existing ? existing.sourceContentHash !== row.sourceContentHash : false; @@ -559,6 +718,7 @@ export const processStagingLiveBatchInternal = internalMutation({ (!existing || contentChanged || existing.scanStatus !== "planned"); if (writesUsed + 2 > run.budgets.maxWritesPerBatch) break; + incrementReconciliationCounts(counts, native.reconciliation); counts.observed += 1; cursor += 1; if (shouldPlanScan) counts.scansPlanned += 1; @@ -577,9 +737,13 @@ export const processStagingLiveBatchInternal = internalMutation({ displayName: row.displayName, sourceUrl: row.sourceUrl, githubRepoUrl: row.githubRepoUrl, + githubPath: row.githubPath, + githubCommit: row.githubCommit, + githubContentHash: row.githubContentHash, sourceContentHash: row.sourceContentHash, installs: row.installs, sourceSnapshotId: run.snapshotId, + reconciliation: native.reconciliation, publicVisible: false, scanStatus: shouldPlanScan ? "planned" @@ -604,9 +768,13 @@ export const processStagingLiveBatchInternal = internalMutation({ displayName: row.displayName, sourceUrl: row.sourceUrl, githubRepoUrl: row.githubRepoUrl, + githubPath: row.githubPath, + githubCommit: row.githubCommit, + githubContentHash: row.githubContentHash, sourceContentHash: row.sourceContentHash, installs: row.installs, sourceSnapshotId: run.snapshotId, + reconciliation: native.reconciliation, publicVisible: false, scanStatus: shouldPlanScan ? "planned" : "not-planned", firstObservedAt: now, @@ -672,7 +840,7 @@ export const processFixtureBatchInternal = internalMutation({ let writesUsed = 0; let readsUsed = 2; let entriesProcessed = 0; - const counts = { ...run.counts }; + const counts = normalizedCounts(run.counts); const now = Date.now(); while ( @@ -693,8 +861,12 @@ export const processFixtureBatchInternal = internalMutation({ entriesProcessed += 1; continue; } + const native = await reconcileNativeSkill(ctx, row, now); + readsUsed += native.reads; - const observationUnchanged = existing ? sameFixtureObservation(existing, row) : false; + const observationUnchanged = existing + ? sameFixtureObservation(existing, row, native.reconciliation) + : false; const contentChanged = existing ? existing.sourceContentHash !== row.sourceContentHash : false; @@ -720,6 +892,7 @@ export const processFixtureBatchInternal = internalMutation({ const entryWriteRequired = !run.dryRun; if (entryWriteRequired && writesUsed + 2 > run.budgets.maxWritesPerBatch) break; + incrementReconciliationCounts(counts, native.reconciliation); counts.observed += 1; cursor += 1; entriesProcessed += 1; @@ -740,9 +913,13 @@ export const processFixtureBatchInternal = internalMutation({ displayName: row.displayName, sourceUrl: row.sourceUrl, githubRepoUrl: row.githubRepoUrl, + githubPath: row.githubPath, + githubCommit: row.githubCommit, + githubContentHash: row.githubContentHash, sourceContentHash: row.sourceContentHash, installs: row.installs, sourceSnapshotId: fixture.snapshotId, + reconciliation: native.reconciliation, // This gate has no publication seam; every catalog write reasserts dark visibility. publicVisible: false, scanStatus: shouldPlanScan @@ -772,9 +949,13 @@ export const processFixtureBatchInternal = internalMutation({ displayName: row.displayName, sourceUrl: row.sourceUrl, githubRepoUrl: row.githubRepoUrl, + githubPath: row.githubPath, + githubCommit: row.githubCommit, + githubContentHash: row.githubContentHash, sourceContentHash: row.sourceContentHash, installs: row.installs, sourceSnapshotId: fixture.snapshotId, + reconciliation: native.reconciliation, publicVisible: false, scanStatus: shouldPlanScan ? "planned" : "not-planned", firstObservedAt: now, @@ -1781,6 +1962,127 @@ export const getRunInternal = internalQuery({ }, }); +export const getRunReconciliationInternal = internalQuery({ + args: { + runId: v.id("skillsShCatalogRuns"), + }, + handler: async (ctx, args) => { + const run = await ctx.db.get(args.runId); + if (!run) throw new ConvexError("skills.sh catalog run not found"); + if (run.sourceKind === "staging-live" || run.fixtureId === "skills-sh-test-live-500") { + throw new ConvexError("skills.sh reconciliation readback requires a fixture run"); + } + const fixture = getSkillsShCatalogFixture(run.fixtureId); + const entries = []; + const mismatches: string[] = []; + for (let index = 0; index < fixture.length; index += 1) { + const expected = normalizeIdentity(fixture.rowAt(index)); + const entry = await ctx.db + .query("skillsShCatalogEntries") + .withIndex("by_external_id", (q) => q.eq("externalId", expected.externalId)) + .unique(); + if (!entry) { + mismatches.push(`missing:${expected.externalId}`); + continue; + } + if (entry.sourceSnapshotId !== fixture.snapshotId) { + mismatches.push(`snapshot:${expected.externalId}`); + } + if ( + entry.githubOwnerId !== expected.githubOwnerId || + entry.githubPath !== expected.githubPath || + entry.githubCommit !== expected.githubCommit || + entry.githubContentHash !== expected.githubContentHash || + entry.sourceContentHash !== expected.sourceContentHash + ) { + mismatches.push(`provenance:${expected.externalId}`); + } + if (entry.publicVisible || !entry.reconciliation) { + mismatches.push(`dark-state:${expected.externalId}`); + } + entries.push({ + ...entry, + resolution: { + externalRoute: `/skills-sh/${entry.externalId}`, + installRef: `skills-sh:${entry.externalId}`, + installable: false, + }, + }); + } + return { + run: summarizeRun(run), + reconciled: mismatches.length === 0 && entries.length === fixture.length, + mismatches, + entries, + limits: { + fixtureRows: fixture.length, + maxEntriesPerRun: run.budgets.maxEntriesPerRun, + maxEntriesPerBatch: run.budgets.maxEntriesPerBatch, + maxWritesPerBatch: run.budgets.maxWritesPerBatch, + maxPlannedScans: run.budgets.maxPlannedScans, + maxScanAdmissionsPerRun: run.budgets.maxScanAdmissionsPerRun, + }, + }; + }, +}); + +export const rollbackFixtureRunInternal = internalMutation({ + args: { + runId: v.id("skillsShCatalogRuns"), + actor: v.string(), + reason: v.string(), + confirm: v.string(), + }, + handler: async (ctx, args) => { + assertSkillsShFixtureEnvironmentAllowed(); + assertFixtureMode(await getControlDoc(ctx)); + if (args.confirm !== ROLLBACK_CONTROLLED_CANARY_CONFIRM) { + throw new ConvexError( + `Pass confirm="${ROLLBACK_CONTROLLED_CANARY_CONFIRM}" to roll back the controlled canary.`, + ); + } + const run = await ctx.db.get(args.runId); + if (!run) throw new ConvexError("skills.sh catalog run not found"); + if (run.fixtureId !== CONTROLLED_CANARY_FIXTURE_ID || run.sourceKind !== "fixture") { + throw new ConvexError("Only the controlled skills.sh canary fixture can be rolled back"); + } + const fixture = getSkillsShCatalogFixture(run.fixtureId); + let deletedEntries = 0; + for (let index = 0; index < fixture.length; index += 1) { + const expected = normalizeIdentity(fixture.rowAt(index)); + const entry = await ctx.db + .query("skillsShCatalogEntries") + .withIndex("by_external_id", (q) => q.eq("externalId", expected.externalId)) + .unique(); + if (!entry) continue; + if (entry.sourceKind !== "fixture" || entry.sourceSnapshotId !== fixture.snapshotId) { + throw new ConvexError(`Controlled canary no longer owns ${expected.externalId}`); + } + const attempt = await ctx.db + .query("skillsShCatalogScanAttempts") + .withIndex("by_entry_and_source_content_hash", (q) => + q.eq("entryId", entry._id).eq("sourceContentHash", entry.sourceContentHash), + ) + .filter((q) => q.neq(q.field("status"), "canceled")) + .first(); + if (attempt) { + throw new ConvexError( + `Controlled canary has retained scan history: ${expected.externalId}`, + ); + } + await ctx.db.delete(entry._id); + deletedEntries += 1; + } + return { + fixtureId: run.fixtureId, + actor: args.actor.trim(), + reason: args.reason.trim(), + deletedEntries, + nativeSkillsChanged: 0, + }; + }, +}); + export const getStagingLiveControlInternal = internalQuery({ args: {}, handler: async (ctx) => { @@ -1922,6 +2224,10 @@ function emptyCounts() { updated: 0, unchanged: 0, rejected: 0, + newExternal: 0, + exactNativeMatches: 0, + routeCollisions: 0, + claimOpportunities: 0, scansPlanned: 0, scansAdmitted: 0, scansCompleted: 0, @@ -1932,7 +2238,9 @@ function emptyCounts() { function sameFixtureObservation( existing: Doc<"skillsShCatalogEntries">, row: ReturnType, + reconciliation: NonNullable["reconciliation"]>, ) { + const existingReconciliation = existing.reconciliation; return ( existing.githubOwnerId === row.githubOwnerId && existing.owner === row.owner && @@ -1941,8 +2249,17 @@ function sameFixtureObservation( existing.displayName === row.displayName && existing.sourceUrl === row.sourceUrl && existing.githubRepoUrl === row.githubRepoUrl && + existing.githubPath === row.githubPath && + existing.githubCommit === row.githubCommit && + existing.githubContentHash === row.githubContentHash && existing.sourceContentHash === row.sourceContentHash && - existing.installs === row.installs + existing.installs === row.installs && + existingReconciliation?.kind === reconciliation.kind && + existingReconciliation.nativeSkillId === reconciliation.nativeSkillId && + existingReconciliation.nativeSlug === reconciliation.nativeSlug && + existingReconciliation.nativeStatsDownloads === reconciliation.nativeStatsDownloads && + existingReconciliation.claimOpportunity === reconciliation.claimOpportunity && + existingReconciliation.claimPublisherHandle === reconciliation.claimPublisherHandle ); } @@ -2163,12 +2480,13 @@ function summarizeRun(run: Doc<"skillsShCatalogRuns">) { sourceKind: run.sourceKind, sourceCapturedAt: run.sourceCapturedAt, snapshotCaptureFetches: run.snapshotCaptureFetches, + githubVerification: run.githubVerification, dryRun: run.dryRun, status: run.status, cursor: run.cursor, scanCursor: run.scanCursor, fixtureLength: run.fixtureLength, - counts: run.counts, + counts: normalizedCounts(run.counts), budgets: run.budgets, operations: run.operations, actor: run.actor, diff --git a/convex/skillsShCatalogCanary.test.ts b/convex/skillsShCatalogCanary.test.ts new file mode 100644 index 00000000..37c94f99 --- /dev/null +++ b/convex/skillsShCatalogCanary.test.ts @@ -0,0 +1,296 @@ +/// +/* @vitest-environment edge-runtime */ +import { convexTest } from "convex-test"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { internal } from "./_generated/api"; +import type { Id } from "./_generated/dataModel"; +import schema from "./schema"; + +const modules = import.meta.glob("./**/*.ts"); + +const LOCAL_ENV = { + CONVEX_CLOUD_URL: "http://127.0.0.1:3210", +}; + +const CANARY_EXTERNAL_ID = "patrick-erichsen/skills/html"; +const CANARY_COMMIT = "050daba89f6b6636470add5cb300aac46a412cf8"; +const CANARY_CONTENT_HASH = "a47adb2c1ac33c088f664b5187971b63d2b958a7b9f01516d26005ca941a108f"; + +const CANARY_CONTROL = { + actor: "codex-test", + reason: "exercise the controlled hidden metadata canary", + confirm: "enable-skills-sh-fixture-control", + mode: "fixture" as const, + discoveryEnabled: true, + writesEnabled: true, + scanPlanningEnabled: true, + scanAdmissionEnabled: false, + maxEntriesPerRun: 1, + maxEntriesPerBatch: 1, + maxWritesPerBatch: 2, + maxPlannedScans: 1, + maxScanAdmissionsPerBatch: 0, + maxScanAdmissionsPerRun: 0, + maxScanAdmissionsPerDay: 0, + maxCatalogQueued: 0, + maxCatalogInFlight: 0, + maxNativeQueued: 0, + maxNativeInFlight: 0, + realScanAllowlist: [] as string[], +}; + +const SOURCE_VERIFICATION = { + githubOwnerId: 20_157_849, + githubCommit: CANARY_COMMIT, + githubContentHash: CANARY_CONTENT_HASH, + githubCheckedAt: "2026-07-22T05:00:00.000Z", + githubFetches: 4, +}; + +type CatalogTest = ReturnType; + +function useLocalEnvironment() { + for (const [name, value] of Object.entries(LOCAL_ENV)) vi.stubEnv(name, value); +} + +async function configureCanary(t: CatalogTest) { + return await t.mutation(internal.skillsShCatalog.configureFixtureControlInternal, CANARY_CONTROL); +} + +async function runCanary(t: CatalogTest) { + const started = await t.mutation(internal.skillsShCatalog.startFixtureRunInternal, { + fixtureId: "patrick-html-canary-v1", + actor: "codex-test", + reason: "apply one controlled hidden metadata canary", + sourceVerification: SOURCE_VERIFICATION, + }); + const run = await t.mutation(internal.skillsShCatalog.processFixtureBatchInternal, { + runId: started.runId, + }); + return { runId: started.runId, run }; +} + +async function seedNativeSkill( + t: CatalogTest, + options: { + exactSource: boolean; + downloads: number; + }, +) { + return await t.run(async (ctx) => { + const userId = await ctx.db.insert("users", { + handle: "native-owner", + displayName: "Native Owner", + role: "user", + }); + let githubSourceId: Id<"githubSkillSources"> | undefined; + if (options.exactSource) { + githubSourceId = await ctx.db.insert("githubSkillSources", { + repo: "Patrick-Erichsen/skills", + lastSyncStatus: "ok", + createdAt: 1, + updatedAt: 1, + }); + } + const skillId = await ctx.db.insert("skills", { + slug: "html", + displayName: options.exactSource ? "HTML Artifact Chooser" : "Native HTML", + ownerUserId: userId, + ...(githubSourceId + ? { + installKind: "github" as const, + githubSourceId, + githubPath: "skills/html", + githubCurrentCommit: CANARY_COMMIT, + githubCurrentContentHash: CANARY_CONTENT_HASH, + githubCurrentStatus: "present" as const, + githubCurrentCheckedAt: 1, + githubScanStatus: "clean" as const, + } + : {}), + tags: {}, + moderationStatus: "active", + statsDownloads: options.downloads, + statsStars: 0, + statsInstallsCurrent: 0, + statsInstallsAllTime: 0, + stats: { + downloads: options.downloads, + stars: 0, + installsCurrent: 0, + installsAllTime: 0, + versions: 0, + comments: 0, + }, + createdAt: 1, + updatedAt: 1, + }); + return skillId; + }); +} + +describe("skills.sh controlled hidden metadata canary", () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it("records a new external skill without creating native state", async () => { + useLocalEnvironment(); + const t = convexTest(schema, modules); + await configureCanary(t); + + const { runId, run } = await runCanary(t); + const readback = await t.query(internal.skillsShCatalog.getRunReconciliationInternal, { + runId, + }); + + expect(run).toMatchObject({ + status: "completed", + counts: { + observed: 1, + inserted: 1, + newExternal: 1, + exactNativeMatches: 0, + routeCollisions: 0, + claimOpportunities: 1, + scansPlanned: 1, + scansAdmitted: 0, + }, + }); + expect(readback).toMatchObject({ + reconciled: true, + mismatches: [], + entries: [ + { + externalId: CANARY_EXTERNAL_ID, + githubOwnerId: 20_157_849, + githubPath: "skills/html", + githubCommit: CANARY_COMMIT, + githubContentHash: CANARY_CONTENT_HASH, + publicVisible: false, + reconciliation: { + kind: "new", + claimOpportunity: true, + claimPublisherHandle: "patrick-erichsen", + }, + resolution: { + installable: false, + }, + }, + ], + }); + expect(await t.run(async (ctx) => await ctx.db.query("skills").collect())).toHaveLength(0); + expect( + await t.run(async (ctx) => await ctx.db.query("securityScanJobs").collect()), + ).toHaveLength(0); + }); + + it("records an exact native match and preserves its downloads", async () => { + useLocalEnvironment(); + const t = convexTest(schema, modules); + const nativeSkillId = await seedNativeSkill(t, { exactSource: true, downloads: 143 }); + await configureCanary(t); + + const { runId, run } = await runCanary(t); + const readback = await t.query(internal.skillsShCatalog.getRunReconciliationInternal, { + runId, + }); + const native = await t.run(async (ctx) => await ctx.db.get(nativeSkillId)); + + expect(run.counts).toMatchObject({ + newExternal: 0, + exactNativeMatches: 1, + routeCollisions: 0, + }); + expect(readback.entries[0]).toMatchObject({ + reconciliation: { + kind: "exact-native", + nativeSkillId, + nativeStatsDownloads: 143, + claimOpportunity: true, + }, + }); + expect(native).toMatchObject({ + _id: nativeSkillId, + statsDownloads: 143, + stats: { downloads: 143 }, + githubCurrentCommit: CANARY_COMMIT, + githubCurrentContentHash: CANARY_CONTENT_HASH, + }); + }); + + it("records a route collision without changing or attaching the native skill", async () => { + useLocalEnvironment(); + const t = convexTest(schema, modules); + const nativeSkillId = await seedNativeSkill(t, { exactSource: false, downloads: 77 }); + await configureCanary(t); + + const { runId, run } = await runCanary(t); + const readback = await t.query(internal.skillsShCatalog.getRunReconciliationInternal, { + runId, + }); + const native = await t.run(async (ctx) => await ctx.db.get(nativeSkillId)); + + expect(run.counts).toMatchObject({ + newExternal: 0, + exactNativeMatches: 0, + routeCollisions: 1, + }); + expect(readback.entries[0]).toMatchObject({ + reconciliation: { + kind: "route-collision", + nativeSkillId, + nativeStatsDownloads: 77, + claimOpportunity: true, + }, + }); + expect(native).toMatchObject({ + _id: nativeSkillId, + statsDownloads: 77, + stats: { downloads: 77 }, + }); + expect(native?.ownerPublisherId).toBeUndefined(); + }); + + it("reruns idempotently and rolls back only the hidden canary metadata", async () => { + useLocalEnvironment(); + const t = convexTest(schema, modules); + const nativeSkillId = await seedNativeSkill(t, { exactSource: false, downloads: 91 }); + await configureCanary(t); + + const first = await runCanary(t); + const repeated = await runCanary(t); + expect(repeated.run.counts).toMatchObject({ + observed: 1, + inserted: 0, + updated: 0, + unchanged: 1, + scansPlanned: 0, + routeCollisions: 1, + }); + + const rollback = await t.mutation(internal.skillsShCatalog.rollbackFixtureRunInternal, { + runId: repeated.runId, + actor: "codex-test", + reason: "remove only the controlled canary metadata", + confirm: "rollback-skills-sh-controlled-canary", + }); + const native = await t.run(async (ctx) => await ctx.db.get(nativeSkillId)); + const catalogEntries = await t.run( + async (ctx) => await ctx.db.query("skillsShCatalogEntries").collect(), + ); + + expect(rollback).toMatchObject({ + fixtureId: "patrick-html-canary-v1", + deletedEntries: 1, + nativeSkillsChanged: 0, + }); + expect(catalogEntries).toHaveLength(0); + expect(native).toMatchObject({ + _id: nativeSkillId, + statsDownloads: 91, + stats: { downloads: 91 }, + }); + expect(first.runId).not.toBe(repeated.runId); + }); +}); diff --git a/package.json b/package.json index acfd2183..e87536db 100644 --- a/package.json +++ b/package.json @@ -72,6 +72,7 @@ "skill-cards:worker": "bun scripts/skill-cards/run-skill-card-worker.ts", "skills-sh:capture-500": "bun scripts/skills-sh-catalog/capture-frozen-snapshot.ts", "skills-sh:prove-500": "bun scripts/skills-sh-catalog/prove-500.ts", + "skills-sh:prove-canary": "CLAWHUB_TEST_CATALOG_MODE=controlled-canary bun scripts/skills-sh-catalog/run-test-gate.ts", "skills:install": "npx --yes skills@1.5.16 add openclaw/design-system --skill openclaw-design openclaw-brand openclaw-design-system openclaw-marketing-pages openclaw-design-audit --agent codex --copy --yes", "test": "vitest run", "test:e2e": "vitest run -c vitest.e2e.config.ts", diff --git a/scripts/skills-sh-catalog/run-test-gate.ts b/scripts/skills-sh-catalog/run-test-gate.ts index a19029a9..29b5614e 100644 --- a/scripts/skills-sh-catalog/run-test-gate.ts +++ b/scripts/skills-sh-catalog/run-test-gate.ts @@ -8,11 +8,18 @@ function requireEnv(name: string) { const targetUrl = requireEnv("CLAWHUB_TEST_CATALOG_GATE_URL"); const operatorAuthorization = requireEnv("CLAWHUB_TEST_OPERATOR_TOKEN"); +const mode = process.env.CLAWHUB_TEST_CATALOG_MODE?.trim() || "live-500"; +if (mode !== "live-500" && mode !== "controlled-canary") { + throw new Error("CLAWHUB_TEST_CATALOG_MODE must be live-500 or controlled-canary"); +} const allowlist = (process.env.CLAWHUB_TEST_CATALOG_ALLOWLIST ?? "") .split(",") .map((externalId) => externalId.trim().toLowerCase()) .filter(Boolean); if (allowlist.length > 10) throw new Error("CLAWHUB_TEST_CATALOG_ALLOWLIST cannot exceed 10"); +if (mode === "controlled-canary" && allowlist.length > 0) { + throw new Error("The controlled canary does not admit scans"); +} async function callGate(body: Record) { const response = await fetch(targetUrl, { @@ -28,7 +35,11 @@ async function callGate(body: Record) { const request = { allowlist, - reason: "CLAW-556 bounded permanent Test proof", + mode, + reason: + mode === "controlled-canary" + ? "CLAW-557 controlled hidden metadata canary" + : "CLAW-556 bounded permanent Test proof", }; const execution = await callGate(request); if (!execution.response.ok) { diff --git a/server/routes/ops/skills-sh/catalog-test.post.ts b/server/routes/ops/skills-sh/catalog-test.post.ts index 7372d02c..b5cb7139 100644 --- a/server/routes/ops/skills-sh/catalog-test.post.ts +++ b/server/routes/ops/skills-sh/catalog-test.post.ts @@ -12,6 +12,7 @@ const MAX_BATCH_SIZE = 50; type CatalogTestRequest = { allowlist?: string[]; + mode?: "live-500" | "controlled-canary"; reason?: string; }; @@ -22,12 +23,14 @@ function parseCatalogTestRequest(value: unknown): CatalogTestRequest | null { (body.allowlist !== undefined && (!Array.isArray(body.allowlist) || !body.allowlist.every((externalId) => typeof externalId === "string"))) || + (body.mode !== undefined && body.mode !== "live-500" && body.mode !== "controlled-canary") || (body.reason !== undefined && typeof body.reason !== "string") ) { return null; } return { ...(body.allowlist !== undefined ? { allowlist: body.allowlist as string[] } : {}), + ...(body.mode !== undefined ? { mode: body.mode as "live-500" | "controlled-canary" } : {}), ...(body.reason !== undefined ? { reason: body.reason as string } : {}), }; } @@ -105,6 +108,95 @@ async function executeSnapshotRun( return { runId, run }; } +async function executeControlledCanaryRun(authorization: string, reason: string) { + const start = await callConvexOperator(authorization, { + method: "POST", + body: { + operation: "start-canary", + reason, + }, + }); + const runId = start.runId; + if (typeof runId !== "string") throw new Error("Convex Test operator did not return a run id"); + const run = await callConvexOperator(authorization, { + method: "POST", + body: { + operation: "process-fixture", + runId, + }, + }); + if (run.status !== "completed" || run.cursor !== 1) { + throw new Error("Convex Test controlled canary did not complete exactly one row"); + } + const reconciliation = await callConvexOperator(authorization, { + method: "POST", + body: { + operation: "reconcile", + runId, + }, + }); + if ( + reconciliation.reconciled !== true || + !Array.isArray(reconciliation.mismatches) || + reconciliation.mismatches.length !== 0 || + !Array.isArray(reconciliation.entries) || + reconciliation.entries.length !== 1 + ) { + throw new Error("Convex Test controlled canary reconciliation failed"); + } + return { + runId, + sourceVerification: start.sourceVerification, + run, + reconciliation, + }; +} + +function assertControlledCanaryControl(control: Record) { + const expected = { + mode: "fixture", + discoveryEnabled: true, + writesEnabled: true, + scanPlanningEnabled: true, + scanAdmissionEnabled: false, + publicVisibilityEnabled: false, + maxEntriesPerRun: 1, + maxEntriesPerBatch: 1, + maxWritesPerBatch: 2, + maxPlannedScans: 1, + maxScanAdmissionsPerBatch: 0, + maxScanAdmissionsPerRun: 0, + maxScanAdmissionsPerDay: 0, + maxCatalogQueued: 0, + maxCatalogInFlight: 0, + } as const; + for (const [key, value] of Object.entries(expected)) { + if (control[key] !== value) { + throw new Error(`Convex Test controlled canary requires ${key}=${String(value)}`); + } + } +} + +function assertControlledCanaryIdenticalRerun(run: Record) { + const counts = run.counts; + if (!counts || typeof counts !== "object" || Array.isArray(counts)) { + throw new Error("Convex Test controlled canary identical rerun lacked counters"); + } + const expected = { + observed: 1, + inserted: 0, + updated: 0, + unchanged: 1, + scansPlanned: 0, + scansAdmitted: 0, + } as const; + for (const [key, value] of Object.entries(expected)) { + if ((counts as Record)[key] !== value) { + throw new Error("Convex Test controlled canary identical rerun was not idempotent"); + } + } +} + function batchSizeFromControl(control: Record) { const maxEntriesPerBatch = control.maxEntriesPerBatch; const maxWritesPerBatch = control.maxWritesPerBatch; @@ -142,6 +234,10 @@ export default defineEventHandler(async (event) => { if (allowlist.length > policy.maxRealScanAdmissions) { return jsonResponse({ error: "allowlist_exceeds_test_ceiling" }, 400); } + const mode = body.mode ?? "live-500"; + if (mode === "controlled-canary" && allowlist.length > 0) { + return jsonResponse({ error: "controlled_canary_does_not_admit_scans" }, 400); + } const memoryStart = process.memoryUsage(); const startedAt = Date.now(); try { @@ -150,6 +246,44 @@ export default defineEventHandler(async (event) => { if (!control || typeof control !== "object") { throw new Error("Convex Test operator did not return catalog controls"); } + if (mode === "controlled-canary") { + assertControlledCanaryControl(control as Record); + const reason = body.reason?.trim() || "CLAW-557 controlled hidden metadata canary"; + const firstRun = await executeControlledCanaryRun(authorization, reason); + const identicalRerun = await executeControlledCanaryRun( + authorization, + `${reason} identical rerun`, + ); + assertControlledCanaryIdenticalRerun(identicalRerun.run); + const memoryEnd = process.memoryUsage(); + return jsonResponse({ + ok: true, + mode, + source: firstRun.sourceVerification, + convex: { + deploymentName: staging.deploymentName, + buildSha: staging.buildSha, + firstRun, + identicalRerun, + }, + runtime: { + elapsedMs: Date.now() - startedAt, + rssStartBytes: memoryStart.rss, + rssEndBytes: memoryEnd.rss, + heapUsedStartBytes: memoryStart.heapUsed, + heapUsedEndBytes: memoryEnd.heapUsed, + }, + controls: { + publicVisibilityEnabled: false, + installabilityEnabled: false, + claimExecutionEnabled: false, + publisherAttachmentEnabled: false, + schedulesEnabled: false, + scanAdmissionEnabled: false, + maxEntriesPerRun: 1, + }, + }); + } const batchSize = batchSizeFromControl(control as Record); const snapshot = await captureSkillsShCatalogTestSnapshot({ env: process.env, @@ -205,6 +339,7 @@ export default defineEventHandler(async (event) => { const memoryEnd = process.memoryUsage(); return jsonResponse({ ok: true, + mode, source: { project: "openclaw-foundation/clawhub", vercelSourceSha: process.env.VERCEL_GIT_COMMIT_SHA ?? null, diff --git a/server/skillsShCatalogTestRoute.test.ts b/server/skillsShCatalogTestRoute.test.ts index 17373952..41d658ce 100644 --- a/server/skillsShCatalogTestRoute.test.ts +++ b/server/skillsShCatalogTestRoute.test.ts @@ -23,6 +23,99 @@ vi.mock("./skillsShCatalogSource", () => ({ getSkillsShCatalogTestSourcePolicy: (...args: unknown[]) => sourcePolicyMock(...args), })); +function controlledCanaryFetch( + identicalCounts: Record = { + observed: 1, + inserted: 0, + updated: 0, + unchanged: 1, + scansPlanned: 0, + scansAdmitted: 0, + }, +) { + return vi.fn(async (_url: string, init: RequestInit) => { + if (init.method === "GET") { + return new Response( + JSON.stringify({ + deploymentName: "academic-chihuahua-392", + buildSha: "canary-sha", + control: { + mode: "fixture", + discoveryEnabled: true, + writesEnabled: true, + scanPlanningEnabled: true, + scanAdmissionEnabled: false, + publicVisibilityEnabled: false, + maxEntriesPerRun: 1, + maxEntriesPerBatch: 1, + maxWritesPerBatch: 2, + maxPlannedScans: 1, + maxScanAdmissionsPerBatch: 0, + maxScanAdmissionsPerRun: 0, + maxScanAdmissionsPerDay: 0, + maxCatalogQueued: 0, + maxCatalogInFlight: 0, + }, + }), + ); + } + const body = JSON.parse(String(init.body)) as { + operation: string; + runId?: string; + reason?: string; + }; + if (body.operation === "start-canary") { + return new Response( + JSON.stringify({ + runId: `skillsShCatalogRuns:${body.reason?.includes("identical") ? "repeat" : "first"}`, + sourceVerification: { + externalId: "patrick-erichsen/skills/html", + githubOwnerId: 20_157_849, + githubCommit: "050daba89f6b6636470add5cb300aac46a412cf8", + githubContentHash: "a47adb2c1ac33c088f664b5187971b63d2b958a7b9f01516d26005ca941a108f", + githubFetches: 4, + }, + }), + ); + } + if (body.operation === "process-fixture") { + const firstRun = body.runId?.endsWith("first"); + return new Response( + JSON.stringify({ + status: "completed", + cursor: 1, + counts: firstRun + ? { + observed: 1, + inserted: 1, + updated: 0, + unchanged: 0, + scansPlanned: 1, + scansAdmitted: 0, + } + : identicalCounts, + }), + ); + } + if (body.operation === "reconcile") { + return new Response( + JSON.stringify({ + reconciled: true, + mismatches: [], + entries: [ + { + externalId: "patrick-erichsen/skills/html", + publicVisible: false, + resolution: { installable: false }, + }, + ], + }), + ); + } + return new Response(JSON.stringify({ error: "unsupported_operation" }), { status: 400 }); + }); +} + describe("skills.sh permanent Test operator route", () => { beforeEach(() => { getHeaderMock.mockReset(); @@ -204,6 +297,76 @@ describe("skills.sh permanent Test operator route", () => { expect(fetch).not.toHaveBeenCalled(); }); + it("applies and reconciles the one-row controlled canary without OIDC or scan admission", async () => { + readBodyMock.mockResolvedValue({ + mode: "controlled-canary", + reason: "CLAW-557 hidden metadata canary", + }); + vi.stubGlobal("fetch", controlledCanaryFetch()); + + const handler = (await import("./routes/ops/skills-sh/catalog-test.post")).default; + const response = (await handler({} as never)) as Response; + + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ + ok: true, + mode: "controlled-canary", + source: { + externalId: "patrick-erichsen/skills/html", + githubOwnerId: 20_157_849, + githubFetches: 4, + }, + convex: { + firstRun: { + run: { counts: { inserted: 1, scansAdmitted: 0 } }, + reconciliation: { reconciled: true, mismatches: [] }, + }, + identicalRerun: { + run: { counts: { unchanged: 1, scansPlanned: 0, scansAdmitted: 0 } }, + reconciliation: { reconciled: true, mismatches: [] }, + }, + }, + controls: { + publicVisibilityEnabled: false, + installabilityEnabled: false, + claimExecutionEnabled: false, + publisherAttachmentEnabled: false, + schedulesEnabled: false, + scanAdmissionEnabled: false, + }, + }); + expect(getVercelOidcTokenMock).not.toHaveBeenCalled(); + expect(captureSnapshotMock).not.toHaveBeenCalled(); + expect(fetch).toHaveBeenCalledTimes(7); + }); + + it("fails closed when the controlled canary rerun is not idempotent", async () => { + readBodyMock.mockResolvedValue({ + mode: "controlled-canary", + reason: "CLAW-557 hidden metadata canary", + }); + vi.stubGlobal( + "fetch", + controlledCanaryFetch({ + observed: 1, + inserted: 0, + updated: 1, + unchanged: 0, + scansPlanned: 1, + scansAdmitted: 0, + }), + ); + + const handler = (await import("./routes/ops/skills-sh/catalog-test.post")).default; + const response = (await handler({} as never)) as Response; + + expect(response.status).toBe(502); + expect(await response.json()).toEqual({ + error: "skills_sh_catalog_test_failed", + message: "Convex Test controlled canary identical rerun was not idempotent", + }); + }); + it.each([ { body: [], label: "array body" }, { body: { allowlist: {} }, label: "non-array allowlist" }, diff --git a/specs/github-backed-skills.md b/specs/github-backed-skills.md index b9667d3c..243f2060 100644 --- a/specs/github-backed-skills.md +++ b/specs/github-backed-skills.md @@ -50,6 +50,28 @@ skills: This table is not an install artifact store. OpenClaw must not install from `githubSkillContents`. +## Dark skills.sh discovery metadata + +The skills.sh discovery pipeline has a separate hidden metadata table and does +not create or mutate `skills` rows during its planning gates. + +- A discovery row keeps the immutable GitHub owner ID plus exact repository, + path, commit, folder content hash, source URL, and source snapshot. +- Native reconciliation is advisory metadata only. It classifies an observation + as new, an exact native GitHub-source match, or a slug/route collision. +- Exact matches may record the native download count for readback, but discovery + must not patch that count or any other native field. +- A verified GitHub owner may create a claim opportunity marker. That marker is + not publisher attachment, ownership, profile content, or claim execution. +- Hidden discovery rows are never installable and cannot become public through + the discovery control plane. +- The permanent Test canary uses one committed skills.sh observation fixture, + while GitHub owner, repository, commit, path, and content verification remain + live and authenticated. +- Canary rollback may delete only the controlled hidden fixture row. It must + leave native skills, download history, scan jobs, publishers, and aliases + unchanged. + `skills` stores the public catalog row and install state: - `installKind: "github"`