diff --git a/convex/agentSkillsHttp.test.ts b/convex/agentSkillsHttp.test.ts index 9f64dac5..f9014e83 100644 --- a/convex/agentSkillsHttp.test.ts +++ b/convex/agentSkillsHttp.test.ts @@ -326,12 +326,12 @@ describe("Agent Skills discovery HTTP handler", () => { }) .mockResolvedValueOnce({ githubSourceId: "githubSkillSources:demo", + repo: "openclaw/openclaw", contentHash, - commit: "def456", + commit: "abc123", path: "skills/demo", status: "clean", - }) - .mockResolvedValueOnce({ repo: "openclaw/openclaw", defaultBranch: "main" }); + }); const response = await agentSkillsHttpHandler( makeCtx({ runQuery, storage: { get: vi.fn() } }), diff --git a/convex/agentSkillsHttp.ts b/convex/agentSkillsHttp.ts index 8d469dd8..b6964d9a 100644 --- a/convex/agentSkillsHttp.ts +++ b/convex/agentSkillsHttp.ts @@ -202,10 +202,12 @@ async function resolveSkill( internal.githubSkillSync.getArchiveScanBySkillAndContentHashInternal, { skillId: skill._id, + commit: archivePin.commit, contentHash: archivePin.contentHash, }, )) as { githubSourceId: Id<"githubSkillSources">; + repo: string; contentHash: string; commit: string; path: string; @@ -213,17 +215,12 @@ async function resolveSkill( } | null; if ( !scan || + scan.commit !== archivePin.commit || scan.contentHash !== archivePin.contentHash || (scan.status !== "clean" && scan.status !== "suspicious") ) { return { ok: false, status: 404, message: "GitHub skill archive not available" }; } - const source = (await ctx.runQuery(internal.githubSkillSources.getByIdInternal, { - sourceId: scan.githubSourceId, - })) as InstallResolverSource | null; - if (!source) { - return { ok: false, status: 404, message: "GitHub skill archive not available" }; - } const moderationBlock = getPublicSkillFileAccessBlock(publicResult.moderationInfo); if (moderationBlock) { return { @@ -237,11 +234,11 @@ async function resolveSkill( slug: skill.slug, installKind: "github", github: { - repo: source.repo, + repo: scan.repo, path: scan.path, commit: archivePin.commit, contentHash: scan.contentHash, - sourceUrl: `https://github.com/${source.repo}/tree/${archivePin.commit}/${scan.path}`, + sourceUrl: `https://github.com/${scan.repo}/tree/${archivePin.commit}/${scan.path}`, }, }; } else { diff --git a/convex/catalogFeed.test.ts b/convex/catalogFeed.test.ts index 1f5116cb..4f1a649d 100644 --- a/convex/catalogFeed.test.ts +++ b/convex/catalogFeed.test.ts @@ -469,10 +469,19 @@ describe("catalog feed projection", () => { it("projects current GitHub-backed skills into public GitHub install candidates", async () => { const result = (await listOfficialSkillEntriesHandler( - makeCtx([makeGitHubSkill({ slug: "aiq-deploy", displayName: "AIQ Deploy" })], { - "publishers:1": { _id: "publishers:1", kind: "org", handle: "nvidia" }, - "githubSkillSources:1": makeGitHubSource(), - }), + makeCtx( + [ + makeGitHubSkill({ + slug: "aiq-deploy", + displayName: "AIQ Deploy", + githubCurrentRepo: "NVIDIA/skills-archive", + }), + ], + { + "publishers:1": { _id: "publishers:1", kind: "org", handle: "nvidia" }, + "githubSkillSources:1": makeGitHubSource({ repo: "NVIDIA/renamed-skills" }), + }, + ), { publisherId: "publishers:1", cursor: null }, )) as { entries: unknown[]; isDone: boolean }; @@ -494,7 +503,7 @@ describe("catalog feed projection", () => { version: "1111111111111111111111111111111111111111", integrity: "sha256:hash-aiq-deploy", github: { - repo: "NVIDIA/skills", + repo: "NVIDIA/skills-archive", path: "skills/aiq-deploy", commit: "1111111111111111111111111111111111111111", contentHash: "hash-aiq-deploy", diff --git a/convex/catalogFeed.ts b/convex/catalogFeed.ts index 96efa389..9049a759 100644 --- a/convex/catalogFeed.ts +++ b/convex/catalogFeed.ts @@ -295,7 +295,7 @@ async function buildSkillEntry( const source = await ctx.db.get(skill.githubSourceId); if (!source || source.ownerPublisherId !== skill.ownerPublisherId) return null; - const repo = source.repo.trim(); + const repo = (skill.githubCurrentRepo ?? source.repo).trim(); const path = skill.githubPath.trim(); const commit = skill.githubCurrentCommit.trim(); const contentHash = skill.githubCurrentContentHash.trim(); diff --git a/convex/githubSkillSources.test.ts b/convex/githubSkillSources.test.ts index 9c9f52c0..b229c220 100644 --- a/convex/githubSkillSources.test.ts +++ b/convex/githubSkillSources.test.ts @@ -40,7 +40,11 @@ type WrappedHandler = { const listForManageableOfficialPublishersHandler = ( listForManageableOfficialPublishers as unknown as WrappedHandler< Record, - Array<{ _id: string; repo: string; ownerPublisher: { handle: string } | null }> + Array<{ + _id: string; + repo: string; + ownerPublisher: { handle: string } | null; + }> > )._handler; @@ -210,7 +214,9 @@ function makeAliasSkill(id: string, githubPath = "skills/html"): Row { describe("githubSkillSources.deleteForPublisherHandler", () => { beforeEach(() => { - vi.mocked(requireUser).mockResolvedValue({ userId: "users:owner" } as never); + vi.mocked(requireUser).mockResolvedValue({ + userId: "users:owner", + } as never); vi.mocked(requirePublisherRole).mockResolvedValue(undefined as never); }); @@ -240,7 +246,7 @@ describe("githubSkillSources.deleteForPublisherHandler", () => { expect(scheduler.runAfter).not.toHaveBeenCalled(); }); - it("deletes a source and removes only GitHub-backed skills from that source", async () => { + it("disconnects a source and removes only GitHub-backed skills from that source", async () => { const { db, tables } = createDb({ githubSkillSources: [ { @@ -248,7 +254,7 @@ describe("githubSkillSources.deleteForPublisherHandler", () => { repo: "mattpocock/skills", ownerPublisherId: "publishers:openclaw", createdAt: 1, - updatedAt: 2, + updatedAt: 123, }, ], githubSkillContents: [ @@ -280,7 +286,9 @@ describe("githubSkillSources.deleteForPublisherHandler", () => { githubPath: "skills/hosted-candidate", githubCommit: "c".repeat(40), githubContentHash: "hash-hosted-candidate", - scanStatus: "pending", + scanStatus: "failed", + lifecycleStatus: "failed", + failedAt: 100, }, ], skills: [ @@ -367,13 +375,27 @@ describe("githubSkillSources.deleteForPublisherHandler", () => { allowed: ["admin"], }), ); - expect(tables.githubSkillSources).toHaveLength(0); + expect(tables.githubSkillSources).toEqual([ + expect.objectContaining({ + _id: "githubSkillSources:matt", + disconnectedOwnerPublisherId: "publishers:openclaw", + authorizationStatus: "revoked", + authorizationCheckedAt: 123, + updatedAt: 124, + }), + ]); + expect(tables.githubSkillSources[0]).not.toHaveProperty("ownerPublisherId"); expect(tables.githubSkillContents).toHaveLength(0); - expect(tables.githubSkillCandidates).toHaveLength(0); + expect(tables.githubSkillCandidates).toEqual([ + expect.objectContaining({ + _id: "githubSkillCandidates:hosted", + lifecycleStatus: "canceled", + canceledAt: 123, + cancellationReason: "github.source.disconnected", + }), + ]); expect(tables.githubSkillScans).toHaveLength(2); - expect(scheduler.runAfter).toHaveBeenCalledWith(0, expect.anything(), { - sourceId: "githubSkillSources:matt", - }); + expect(scheduler.runAfter).not.toHaveBeenCalled(); const deletedSkill = tables.skills.find((skill) => skill._id === "skills:github"); expect(deletedSkill).toMatchObject({ softDeletedAt: 123, @@ -469,7 +491,9 @@ describe("githubSkillSources.deleteForPublisherHandler", () => { expect(tables.skillScanRequests?.[0]).not.toHaveProperty("githubSkillScanId"); expect(tables.skillScanRequests?.[0]).not.toHaveProperty("securityScanJobId"); expect(tables.skillScanRequests?.[0]?.expiresAt).toBeLessThan(Number.MAX_SAFE_INTEGER); - expect(scheduler.runAfter).toHaveBeenCalledWith(0, expect.anything(), { batchSize: 10 }); + expect(scheduler.runAfter).toHaveBeenCalledWith(0, expect.anything(), { + batchSize: 10, + }); }); it("rejects deleting a source from another publisher", async () => { diff --git a/convex/githubSkillSources.ts b/convex/githubSkillSources.ts index 6718f41d..1be0b698 100644 --- a/convex/githubSkillSources.ts +++ b/convex/githubSkillSources.ts @@ -71,6 +71,7 @@ export const getSkillsShAliasTargetInternal = internalQuery({ const matches = skills.filter( (skill) => skill.githubPath === path && + (skill.githubCurrentRepo ?? source.repo).toLowerCase() === repo && skill.installKind === "github" && skill.githubCurrentStatus === "present" && (skill.githubScanStatus === "clean" || skill.githubScanStatus === "suspicious") && @@ -231,6 +232,7 @@ export async function deleteForPublisherHandler( assertGenericGitHubSkillSyncEnabled(source.repo); const now = args.now ?? Date.now(); + const sourceUpdatedAt = Math.max(now, source.updatedAt + 1); const contents = await ctx.db .query("githubSkillContents") .withIndex("by_github_source", (q) => q.eq("githubSourceId", args.sourceId)) @@ -238,10 +240,27 @@ export async function deleteForPublisherHandler( for (const content of contents) { await ctx.db.delete(content._id); } - const candidates = await ctx.db - .query("githubSkillCandidates") - .withIndex("by_github_source", (q) => q.eq("githubSourceId", args.sourceId)) - .collect(); + const [pendingCandidates, failedCandidates, legacyCandidates] = await Promise.all([ + ctx.db + .query("githubSkillCandidates") + .withIndex("by_github_source_and_lifecycle_status", (q) => + q.eq("githubSourceId", args.sourceId).eq("lifecycleStatus", "pending"), + ) + .collect(), + ctx.db + .query("githubSkillCandidates") + .withIndex("by_github_source_and_lifecycle_status", (q) => + q.eq("githubSourceId", args.sourceId).eq("lifecycleStatus", "failed"), + ) + .collect(), + ctx.db + .query("githubSkillCandidates") + .withIndex("by_github_source_and_lifecycle_status", (q) => + q.eq("githubSourceId", args.sourceId).eq("lifecycleStatus", undefined), + ) + .collect(), + ]); + const candidates = [...pendingCandidates, ...failedCandidates, ...legacyCandidates]; for (const candidate of candidates) { const skill = await ctx.db.get(candidate.skillId); if (skill?.githubPendingCandidateId === candidate._id) { @@ -249,13 +268,14 @@ export async function deleteForPublisherHandler( githubPendingCandidateId: undefined, updatedAt: now, }); + await ctx.db.patch(candidate._id, { + lifecycleStatus: "canceled", + canceledAt: now, + cancellationReason: "github.source.disconnected", + updatedAt: now, + }); } - await ctx.db.delete(candidate._id); } - await ctx.scheduler.runAfter(0, internal.githubSkillSources.cleanupDeletedSourceScansInternal, { - sourceId: args.sourceId, - }); - const skills = await ctx.db .query("skills") .withIndex("by_github_source", (q) => q.eq("githubSourceId", args.sourceId)) @@ -286,7 +306,14 @@ export async function deleteForPublisherHandler( if (publicSkillDelta !== 0) { await adjustGlobalPublicSkillsCount(ctx, publicSkillDelta, now); } - await ctx.db.delete(args.sourceId); + await ctx.db.patch(args.sourceId, { + ownerPublisherId: undefined, + disconnectedOwnerPublisherId: args.ownerPublisherId, + authorizationStatus: "revoked", + authorizationCheckedAt: now, + authorizationError: "GitHub source disconnected by publisher.", + updatedAt: sourceUpdatedAt, + }); return { ok: true as const, deletedSkills }; } diff --git a/convex/githubSkillSync.test.ts b/convex/githubSkillSync.test.ts index 39480708..ebf7da0e 100644 --- a/convex/githubSkillSync.test.ts +++ b/convex/githubSkillSync.test.ts @@ -8,9 +8,11 @@ import { applyGitHubSkillSourceSyncHandler, applyGitHubSkillVerificationResultHandler, configurePublicGitHubSkillSourceHandler, + getArchiveScanBySkillAndContentHashHandler, listSourcesForSyncHandler, recordGitHubSkillSourceSyncAttemptHandler, revokeGitHubSkillSourceAuthorizationHandler, + rollbackGitHubSkillCandidateHandler, resolveOwnerUserIdForPublisherHandler, syncGitHubSkillSourcesHandler, upsertGitHubSkillCandidateContentHandler, @@ -182,7 +184,9 @@ function createFakeGitHubSkillsRepo() { ]), ); const zip = zipSync(prefixedEntries); - return new Response(zip, { headers: { "content-length": String(zip.byteLength) } }); + return new Response(zip, { + headers: { "content-length": String(zip.byteLength) }, + }); } return new Response("not found", { status: 404 }); @@ -381,7 +385,10 @@ describe("configurePublicGitHubSkillSourceHandler", () => { existingSource: null, }; }); - const runMutation = vi.fn(async () => ({ ok: true, stats: { discovered: 1 } })); + const runMutation = vi.fn(async () => ({ + ok: true, + stats: { discovered: 1 }, + })); const fetchMock = vi .fn() .mockResolvedValueOnce({ @@ -578,7 +585,10 @@ describe("configurePublicGitHubSkillSourceHandler", () => { ownerUserId: "users:publisher-owner", existingSource: null, })); - const runMutation = vi.fn(async () => ({ ok: true, stats: { discovered: 1 } })); + const runMutation = vi.fn(async () => ({ + ok: true, + stats: { discovered: 1 }, + })); const fetchMock = vi .fn() .mockResolvedValueOnce({ @@ -884,7 +894,10 @@ describe("syncGitHubSkillSourcesHandler", () => { }); await expect( - listSourcesForSyncHandler({ db } as never, { batchSize: 20, legacyOnly: true }), + listSourcesForSyncHandler({ db } as never, { + batchSize: 20, + legacyOnly: true, + }), ).resolves.toEqual({ sources: [ expect.objectContaining({ @@ -920,9 +933,11 @@ describe("syncGitHubSkillSourcesHandler", () => { it("emits structured sync lifecycle events", async () => { const consoleLog = vi.spyOn(console, "log").mockImplementation(() => {}); - const runQuery = vi - .fn() - .mockResolvedValueOnce({ sources: [], continueCursor: null, isDone: true }); + const runQuery = vi.fn().mockResolvedValueOnce({ + sources: [], + continueCursor: null, + isDone: true, + }); const runMutation = vi.fn(); try { @@ -932,7 +947,12 @@ describe("syncGitHubSkillSourcesHandler", () => { vi.fn() as never, ); - expect(result).toMatchObject({ ok: true, synced: 0, skipped: 0, errors: 0 }); + expect(result).toMatchObject({ + ok: true, + synced: 0, + skipped: 0, + errors: 0, + }); const events = consoleLog.mock.calls.map(([message]) => JSON.parse(String(message))); expect(events).toEqual( expect.arrayContaining([ @@ -961,9 +981,11 @@ describe("syncGitHubSkillSourcesHandler", () => { undefined, ), }; - const runQuery = vi - .fn() - .mockResolvedValueOnce({ sources: [], continueCursor: "next-page", isDone: false }); + const runQuery = vi.fn().mockResolvedValueOnce({ + sources: [], + continueCursor: "next-page", + isDone: false, + }); const result = await syncGitHubSkillSourcesHandler( { runQuery, runMutation: vi.fn(), scheduler } as never, @@ -971,7 +993,11 @@ describe("syncGitHubSkillSourcesHandler", () => { vi.fn() as never, ); - expect(result).toMatchObject({ scheduledNext: true, cursor: "next-page", isDone: false }); + expect(result).toMatchObject({ + scheduledNext: true, + cursor: "next-page", + isDone: false, + }); const scheduledFunction = scheduler.runAfter.mock.calls[0]?.[1]; expect(getFunctionName(scheduledFunction as Parameters[0])).toBe( "githubSkillSyncNode:syncGitHubSkillSourcesInternal", @@ -1136,7 +1162,11 @@ description: Invalid because the folder name is too long. }, ); - expect(result.stats).toMatchObject({ discovered: 1, inserted: 0, invalid: 1 }); + expect(result.stats).toMatchObject({ + discovered: 1, + inserted: 0, + invalid: 1, + }); expect(result.invalidSkills).toEqual([ { slug: longSlug, @@ -1427,7 +1457,9 @@ description: Install from a GitHub-backed source. ); expect(contentSyncCalls).toHaveLength(1); expect(contentSyncCalls[0]?.[1]).toMatchObject({ - discovered: { skillMarkdown: expect.stringContaining("# Demo Source A") }, + discovered: { + skillMarkdown: expect.stringContaining("# Demo Source A"), + }, }); let skill = getSkill(tables, "demo-source"); @@ -1539,7 +1571,10 @@ description: Install from a GitHub-backed source. skillMarkdown: expect.stringContaining("# Demo Source A"), githubCommit: "a".repeat(40), }); - const candidate = tables.githubSkillCandidates[0]; + const candidate = tables.githubSkillCandidates.find( + (row) => row._id === skill.githubPendingCandidateId, + ); + if (!candidate) throw new Error("expected pending GitHub candidate"); expect(candidate).toMatchObject({ skillId: skill._id, githubCommit: "b".repeat(40), @@ -1566,9 +1601,14 @@ description: Install from a GitHub-backed source. ok: true, github: { commit: "a".repeat(40), contentHash: commitAContentHash }, }); + const completedScan = tables.githubSkillScans.find( + (row) => row._id === candidate.verdictSourceScanId, + ); + Object.assign(completedScan ?? {}, { status: "clean" }); await applyGitHubSkillVerificationResultHandler({ db } as never, { skillId: skill._id as never, contentHash: candidate.githubContentHash as string, + githubSkillScanId: candidate.verdictSourceScanId as never, scanStatus: "clean", now, }); @@ -1641,11 +1681,11 @@ description: Install from a GitHub-backed source. const reappeared = getSkill(tables, "demo-source"); expect(reappeared).toMatchObject({ _id: skill._id, - githubCurrentCommit: "d".repeat(40), - githubCurrentStatus: "present", - githubScanStatus: "pending", - moderationStatus: "active", - moderationReason: "pending.scan", + githubCurrentCommit: "b".repeat(40), + githubCurrentStatus: "missing", + githubScanStatus: "clean", + moderationStatus: "hidden", + moderationReason: "github.upstream.removed", statsDownloads: 41, statsStars: 7, statsInstallsCurrent: 3, @@ -1653,23 +1693,35 @@ description: Install from a GitHub-backed source. statsSkillsShInstalls: 29, statsGithubStars: 701, }); - expect(reappeared).not.toHaveProperty("softDeletedAt"); + const reappearanceCandidate = tables.githubSkillCandidates.find( + (row) => row._id === reappeared.githubPendingCandidateId, + ); + expect(reappearanceCandidate).toMatchObject({ + githubCommit: "d".repeat(40), + lifecycleStatus: "pending", + }); expect(resolveInstallFromTables(tables, "demo-source")).toMatchObject({ ok: false, - reason: "github_verification_pending", - status: 423, + reason: "github_upstream_removed", + status: 410, }); + const completedReappearanceScan = tables.githubSkillScans.find( + (row) => row._id === reappearanceCandidate?.verdictSourceScanId, + ); + Object.assign(completedReappearanceScan ?? {}, { status: "clean" }); await applyGitHubSkillVerificationResultHandler({ db } as never, { skillId: reappeared._id as never, - contentHash: reappeared.githubCurrentContentHash as string, + contentHash: reappearanceCandidate?.githubContentHash as string, + githubSkillScanId: reappearanceCandidate?.verdictSourceScanId as never, scanStatus: "clean", now: 410, }); + expect(getSkill(tables, "demo-source")).not.toHaveProperty("softDeletedAt"); expect(resolveInstallFromTables(tables, "demo-source")).toMatchObject({ ok: true, github: { commit: "d".repeat(40), - contentHash: reappeared.githubCurrentContentHash, + contentHash: reappearanceCandidate?.githubContentHash, }, }); } finally { @@ -1860,7 +1912,9 @@ describe("applyGitHubSkillSourceSyncHandler", () => { githubPath: "skills/html", githubCommit: "5".repeat(40), githubContentHash: "next-hash", - scanStatus: "pending", + scanStatus: "failed", + lifecycleStatus: "failed", + failedAt: 150, }, ], skills: [ @@ -1901,7 +1955,14 @@ describe("applyGitHubSkillSourceSyncHandler", () => { now: 200, }); - expect(tables.githubSkillCandidates).toEqual([]); + expect(tables.githubSkillCandidates).toEqual([ + expect.objectContaining({ + _id: "githubSkillCandidates:html", + lifecycleStatus: "canceled", + canceledAt: 200, + cancellationReason: "github.authorization.revoked", + }), + ]); expect(tables.githubSkillSources[0]).toMatchObject({ authorizationStatus: "revoked", authorizationCheckedAt: 200, @@ -1990,7 +2051,10 @@ describe("applyGitHubSkillSourceSyncHandler", () => { }); const candidate = tables.githubSkillCandidates[0]; - expect(candidate).toMatchObject({ scanStatus: "clean", githubContentHash: contentHash }); + expect(candidate).toMatchObject({ + scanStatus: "clean", + githubContentHash: contentHash, + }); expect(scheduler.runAfter).not.toHaveBeenCalled(); await upsertGitHubSkillCandidateContentHandler({ db } as never, { @@ -2000,13 +2064,1495 @@ describe("applyGitHubSkillSourceSyncHandler", () => { now: 110, }); - expect(tables.githubSkillCandidates).toEqual([]); + expect(tables.githubSkillCandidates).toEqual([ + expect.objectContaining({ + _id: candidate?._id, + lifecycleStatus: "promoted", + promotedAt: 110, + verdictSourceScanId: "githubSkillScans:html", + }), + ]); expect(tables.skills[0]).toMatchObject({ _id: "skills:html", installKind: "github", githubCurrentCommit: snapshot.commit, githubCurrentContentHash: contentHash, githubScanStatus: "clean", + githubCurrentCandidateId: candidate?._id, + }); + }); + + it("records pointer-only GitHub changes as immutable candidates without rescanning", async () => { + const snapshot = await buildGitHubSkillSourceSnapshot({ + repo: "patrick-erichsen/skills", + defaultBranch: "main", + commit: "b".repeat(40), + entries: { + "skills/html/SKILL.md": new TextEncoder().encode("# HTML\n"), + }, + }); + const contentHash = snapshot.skills[0]?.contentHash ?? ""; + const { db, tables } = createDb({ + githubSkillSources: [ + { + _id: "githubSkillSources:current", + repo: "patrick-erichsen/skills", + ownerPublisherId: "publishers:patrick", + githubRepositoryId: "100", + githubOwnerId: "200", + authorizationStatus: "active", + createdAt: 1, + updatedAt: 1, + }, + ], + publishers: [ + { + _id: "publishers:patrick", + kind: "user", + handle: "patrick", + displayName: "Patrick", + linkedUserId: "users:patrick", + createdAt: 1, + updatedAt: 1, + }, + ], + skills: [ + { + _id: "skills:html", + slug: "html", + displayName: "HTML", + ownerUserId: "users:patrick", + ownerPublisherId: "publishers:patrick", + installKind: "github", + githubSourceId: "githubSkillSources:current", + githubPath: "skills/html", + githubCurrentCommit: "a".repeat(40), + githubCurrentContentHash: contentHash, + githubCurrentStatus: "present", + githubScanStatus: "clean", + tags: {}, + stats: { downloads: 0, stars: 0, versions: 0, comments: 0 }, + moderationStatus: "active", + moderationFlags: [], + isSuspicious: false, + createdAt: 1, + updatedAt: 1, + }, + ], + githubSkillScans: [ + { + _id: "githubSkillScans:html", + skillId: "skills:html", + githubSourceId: "githubSkillSources:current", + contentHash, + commit: "a".repeat(40), + path: "skills/html", + status: "clean", + createdAt: 1, + updatedAt: 1, + }, + ], + }); + const scheduler = { runAfter: vi.fn(async () => undefined) }; + + await applyGitHubSkillSourceSyncHandler({ db, scheduler } as never, { + sourceId: "githubSkillSources:current" as never, + repo: "patrick-erichsen/skills", + ownerUserId: "users:patrick" as never, + ownerPublisherId: "publishers:patrick" as never, + githubRepositoryId: "100", + githubOwnerId: "200", + snapshot, + now: 2, + }); + + const candidate = tables.githubSkillCandidates.find( + (row) => row._id === tables.skills[0]?.githubPendingCandidateId, + ); + const retainedCurrent = tables.githubSkillCandidates.find( + (row) => row.githubCommit === "a".repeat(40), + ); + expect(tables.skills[0]).toMatchObject({ + githubCurrentCommit: "a".repeat(40), + githubCurrentCandidateId: retainedCurrent?._id, + githubPendingCandidateId: candidate?._id, + }); + expect(retainedCurrent).toMatchObject({ + githubRepo: "patrick-erichsen/skills", + githubPath: "skills/html", + githubContentHash: contentHash, + scanStatus: "clean", + lifecycleStatus: "promoted", + verdictSourceScanId: "githubSkillScans:html", + }); + expect(candidate).toMatchObject({ + githubRepo: "patrick-erichsen/skills", + githubCommit: "b".repeat(40), + githubContentHash: contentHash, + lifecycleStatus: "pending", + verdictSourceScanId: "githubSkillScans:html", + }); + expect(scheduler.runAfter).not.toHaveBeenCalled(); + + await upsertGitHubSkillCandidateContentHandler({ db } as never, { + candidateId: candidate?._id as never, + discovered: snapshot.skills[0]!, + commit: snapshot.commit, + now: 3, + }); + + expect(tables.skills[0]).toMatchObject({ + githubCurrentCommit: "b".repeat(40), + githubCurrentContentHash: contentHash, + githubCurrentCandidateId: candidate?._id, + }); + expect(candidate).toMatchObject({ + lifecycleStatus: "promoted", + promotedAt: 3, + }); + + const snapshotA = { ...snapshot, commit: "a".repeat(40) }; + await applyGitHubSkillSourceSyncHandler({ db, scheduler } as never, { + sourceId: "githubSkillSources:current" as never, + repo: "patrick-erichsen/skills", + ownerUserId: "users:patrick" as never, + ownerPublisherId: "publishers:patrick" as never, + githubRepositoryId: "100", + githubOwnerId: "200", + snapshot: snapshotA, + now: 4, + }); + const candidateA = tables.githubSkillCandidates.find( + (row) => row._id === tables.skills[0]?.githubPendingCandidateId, + ); + await upsertGitHubSkillCandidateContentHandler({ db } as never, { + candidateId: candidateA?._id as never, + discovered: snapshotA.skills[0]!, + commit: snapshotA.commit, + now: 5, + }); + expect(tables.skills[0]).toMatchObject({ + githubCurrentCommit: "a".repeat(40), + githubCurrentCandidateId: candidateA?._id, + }); + + await applyGitHubSkillSourceSyncHandler({ db, scheduler } as never, { + sourceId: "githubSkillSources:current" as never, + repo: "patrick-erichsen/skills", + ownerUserId: "users:patrick" as never, + ownerPublisherId: "publishers:patrick" as never, + githubRepositoryId: "100", + githubOwnerId: "200", + snapshot, + now: 6, + }); + expect(tables.githubSkillCandidates).toHaveLength(2); + expect(tables.skills[0]).toMatchObject({ + githubCurrentCommit: "b".repeat(40), + githubCurrentCandidateId: candidate?._id, + }); + + const redirectedSnapshot = { + ...snapshot, + repo: "patrick-erichsen/renamed-skills", + }; + await applyGitHubSkillSourceSyncHandler({ db, scheduler } as never, { + sourceId: "githubSkillSources:current" as never, + repo: "patrick-erichsen/renamed-skills", + ownerUserId: "users:patrick" as never, + ownerPublisherId: "publishers:patrick" as never, + githubRepositoryId: "100", + githubOwnerId: "200", + snapshot: redirectedSnapshot, + now: 7, + }); + const redirectCandidate = tables.githubSkillCandidates.find( + (row) => row._id === tables.skills[0]?.githubPendingCandidateId, + ); + expect(resolveInstallFromTables(tables, "html")).toMatchObject({ + ok: true, + github: { repo: "patrick-erichsen/skills" }, + }); + expect(redirectCandidate).toMatchObject({ + githubRepo: "patrick-erichsen/renamed-skills", + lifecycleStatus: "pending", + }); + await upsertGitHubSkillCandidateContentHandler({ db } as never, { + candidateId: redirectCandidate?._id as never, + discovered: redirectedSnapshot.skills[0]!, + commit: redirectedSnapshot.commit, + now: 8, + }); + expect(resolveInstallFromTables(tables, "html")).toMatchObject({ + ok: true, + github: { repo: "patrick-erichsen/renamed-skills" }, + }); + }); + + it("rejects allowed candidate promotion without the candidate's bound scan identity", async () => { + const { db, tables } = createDb({ + skills: [ + { + _id: "skills:html", + slug: "html", + displayName: "HTML A", + installKind: "github", + githubSourceId: "githubSkillSources:current", + githubPath: "skills/html", + githubCurrentCommit: "a".repeat(40), + githubCurrentContentHash: "a".repeat(64), + githubCurrentStatus: "present", + githubScanStatus: "clean", + githubPendingCandidateId: "githubSkillCandidates:b", + }, + ], + githubSkillCandidates: [ + { + _id: "githubSkillCandidates:b", + skillId: "skills:html", + githubSourceId: "githubSkillSources:current", + githubRepo: "patrick-erichsen/skills", + githubPath: "skills/html", + githubHasSkillCard: false, + githubCommit: "b".repeat(40), + githubContentHash: "b".repeat(64), + displayName: "HTML B", + skillMarkdownPath: "skills/html/SKILL.md", + skillMarkdown: "# HTML B\n", + scanStatus: "clean", + lifecycleStatus: "pending", + verdictSourceScanId: "githubSkillScans:b", + createdAt: 2, + updatedAt: 2, + }, + ], + githubSkillScans: [ + { + _id: "githubSkillScans:b", + skillId: "skills:html", + githubSourceId: "githubSkillSources:current", + contentHash: "b".repeat(64), + commit: "b".repeat(40), + path: "skills/html", + status: "clean", + createdAt: 2, + updatedAt: 2, + }, + ], + }); + + await expect( + applyGitHubSkillVerificationResultHandler({ db } as never, { + skillId: "skills:html" as never, + contentHash: "b".repeat(64), + scanStatus: "clean", + now: 3, + }), + ).resolves.toEqual({ ok: true, skipped: "stale-candidate-verdict" }); + expect(tables.skills[0]).toMatchObject({ + githubCurrentCommit: "a".repeat(40), + githubPendingCandidateId: "githubSkillCandidates:b", + }); + }); + + it("rejects allowed promotion when a legacy candidate has no durable verdict", async () => { + const { db, tables } = createDb({ + skills: [ + { + _id: "skills:html", + slug: "html", + displayName: "HTML A", + installKind: "github", + githubSourceId: "githubSkillSources:current", + githubPath: "skills/html", + githubCurrentCommit: "a".repeat(40), + githubCurrentContentHash: "a".repeat(64), + githubCurrentStatus: "present", + githubScanStatus: "clean", + githubPendingCandidateId: "githubSkillCandidates:b", + }, + ], + githubSkillCandidates: [ + { + _id: "githubSkillCandidates:b", + skillId: "skills:html", + githubSourceId: "githubSkillSources:current", + githubRepo: "patrick-erichsen/skills", + githubPath: "skills/html", + githubHasSkillCard: false, + githubCommit: "b".repeat(40), + githubContentHash: "b".repeat(64), + displayName: "HTML B", + skillMarkdownPath: "skills/html/SKILL.md", + skillMarkdown: "# HTML B\n", + scanStatus: "clean", + lifecycleStatus: "pending", + createdAt: 2, + updatedAt: 2, + }, + ], + }); + + await expect( + applyGitHubSkillVerificationResultHandler({ db } as never, { + skillId: "skills:html" as never, + contentHash: "b".repeat(64), + scanStatus: "clean", + now: 3, + }), + ).resolves.toEqual({ ok: true, skipped: "missing-candidate-verdict" }); + expect(tables.skills[0]).toMatchObject({ + githubCurrentCommit: "a".repeat(40), + githubPendingCandidateId: "githubSkillCandidates:b", + }); + }); + + it("rejects late current-content persistence after a newer pointer wins", async () => { + const staleSnapshot = await buildGitHubSkillSourceSnapshot({ + repo: "patrick-erichsen/skills", + defaultBranch: "main", + commit: "a".repeat(40), + entries: { + "skills/html/SKILL.md": new TextEncoder().encode("# HTML\n"), + }, + }); + const discovered = staleSnapshot.skills[0]!; + const { db, tables } = createDb({ + skills: [ + { + _id: "skills:html", + installKind: "github", + githubSourceId: "githubSkillSources:current", + githubPath: discovered.path, + githubCurrentCommit: "b".repeat(40), + githubCurrentContentHash: discovered.contentHash, + githubCurrentStatus: "present", + }, + ], + githubSkillContents: [ + { + _id: "githubSkillContents:html", + skillId: "skills:html", + githubSourceId: "githubSkillSources:current", + githubPath: discovered.path, + skillMarkdownPath: discovered.skillMarkdownPath, + skillMarkdown: "# HTML newer pointer\n", + githubCommit: "b".repeat(40), + githubContentHash: discovered.contentHash, + fetchedAt: 2, + createdAt: 1, + updatedAt: 2, + }, + ], + }); + + await expect( + upsertGitHubSkillContentHandler({ db } as never, { + skillId: "skills:html" as never, + sourceId: "githubSkillSources:current" as never, + discovered, + commit: staleSnapshot.commit, + now: 3, + }), + ).resolves.toEqual({ ok: true, skipped: "stale-current-pointer" }); + expect(tables.githubSkillContents[0]).toMatchObject({ + skillMarkdown: "# HTML newer pointer\n", + githubCommit: "b".repeat(40), + }); + }); + + it("resolves an archived candidate from its retained repository after a redirect", async () => { + const { db } = createDb({ + githubSkillSources: [ + { + _id: "githubSkillSources:current", + repo: "patrick-erichsen/renamed-skills", + authorizationStatus: "active", + createdAt: 1, + updatedAt: 3, + }, + ], + skills: [ + { + _id: "skills:html", + installKind: "github", + githubSourceId: "githubSkillSources:current", + githubCurrentRepo: "patrick-erichsen/renamed-skills", + githubPath: "skills/html", + githubCurrentCommit: "b".repeat(40), + githubCurrentContentHash: "b".repeat(64), + githubCurrentStatus: "present", + githubScanStatus: "clean", + githubCurrentCandidateId: "githubSkillCandidates:b", + }, + ], + githubSkillCandidates: [ + { + _id: "githubSkillCandidates:a", + skillId: "skills:html", + githubSourceId: "githubSkillSources:current", + githubRepo: "patrick-erichsen/skills", + githubPath: "skills/html", + githubHasSkillCard: false, + githubCommit: "a".repeat(40), + githubContentHash: "a".repeat(64), + displayName: "HTML A", + scanStatus: "clean", + lifecycleStatus: "superseded", + verdictSourceScanId: "githubSkillScans:a", + createdAt: 1, + updatedAt: 2, + }, + { + _id: "githubSkillCandidates:b", + skillId: "skills:html", + githubSourceId: "githubSkillSources:current", + githubRepo: "patrick-erichsen/renamed-skills", + githubPath: "skills/html", + githubHasSkillCard: false, + githubCommit: "b".repeat(40), + githubContentHash: "b".repeat(64), + displayName: "HTML B", + scanStatus: "clean", + lifecycleStatus: "promoted", + verdictSourceScanId: "githubSkillScans:b", + createdAt: 2, + updatedAt: 3, + }, + ], + githubSkillScans: [ + { + _id: "githubSkillScans:a", + skillId: "skills:html", + githubSourceId: "githubSkillSources:current", + contentHash: "a".repeat(64), + commit: "a".repeat(40), + path: "skills/html", + status: "clean", + createdAt: 1, + updatedAt: 1, + }, + { + _id: "githubSkillScans:b", + skillId: "skills:html", + githubSourceId: "githubSkillSources:current", + contentHash: "b".repeat(64), + commit: "b".repeat(40), + path: "skills/html", + status: "clean", + createdAt: 2, + updatedAt: 2, + }, + ], + }); + + await expect( + getArchiveScanBySkillAndContentHashHandler({ db } as never, { + skillId: "skills:html" as never, + commit: "a".repeat(40), + contentHash: "a".repeat(64), + }), + ).resolves.toMatchObject({ + repo: "patrick-erichsen/skills", + path: "skills/html", + commit: "a".repeat(40), + contentHash: "a".repeat(64), + status: "clean", + }); + }); + + it("refuses a legacy current archive without a durable verdict row", async () => { + const { db } = createDb({ + githubSkillSources: [ + { + _id: "githubSkillSources:current", + repo: "patrick-erichsen/skills", + createdAt: 1, + updatedAt: 1, + }, + ], + skills: [ + { + _id: "skills:html", + installKind: "github", + githubSourceId: "githubSkillSources:current", + githubPath: "skills/html", + githubCurrentCommit: "a".repeat(40), + githubCurrentContentHash: "a".repeat(64), + githubCurrentStatus: "present", + githubScanStatus: "clean", + }, + ], + }); + + await expect( + getArchiveScanBySkillAndContentHashHandler({ db } as never, { + skillId: "skills:html" as never, + commit: "a".repeat(40), + contentHash: "a".repeat(64), + }), + ).resolves.toBeNull(); + }); + + it("resolves a deterministic retained archive when redirects share a commit and content hash", async () => { + const commit = "a".repeat(40); + const contentHash = "a".repeat(64); + const { db } = createDb({ + githubSkillSources: [ + { + _id: "githubSkillSources:current", + repo: "patrick-erichsen/current-skills", + createdAt: 1, + updatedAt: 4, + }, + ], + skills: [ + { + _id: "skills:html", + githubCurrentCandidateId: "githubSkillCandidates:current", + }, + ], + githubSkillCandidates: [ + { + _id: "githubSkillCandidates:old", + skillId: "skills:html", + githubSourceId: "githubSkillSources:current", + githubRepo: "patrick-erichsen/old-skills", + githubPath: "skills/html", + githubHasSkillCard: false, + githubCommit: commit, + githubContentHash: contentHash, + displayName: "HTML old", + scanStatus: "clean", + lifecycleStatus: "superseded", + verdictSourceScanId: "githubSkillScans:html", + promotedAt: 1, + createdAt: 1, + updatedAt: 2, + }, + { + _id: "githubSkillCandidates:redirected", + skillId: "skills:html", + githubSourceId: "githubSkillSources:current", + githubRepo: "patrick-erichsen/redirected-skills", + githubPath: "skills/html", + githubHasSkillCard: false, + githubCommit: commit, + githubContentHash: contentHash, + displayName: "HTML redirected", + scanStatus: "clean", + lifecycleStatus: "superseded", + verdictSourceScanId: "githubSkillScans:html", + promotedAt: 3, + createdAt: 2, + updatedAt: 3, + }, + { + _id: "githubSkillCandidates:current", + skillId: "skills:html", + githubSourceId: "githubSkillSources:current", + githubRepo: "patrick-erichsen/current-skills", + githubPath: "skills/html", + githubHasSkillCard: false, + githubCommit: "b".repeat(40), + githubContentHash: "b".repeat(64), + displayName: "HTML current", + scanStatus: "clean", + lifecycleStatus: "promoted", + verdictSourceScanId: "githubSkillScans:current", + promotedAt: 4, + createdAt: 3, + updatedAt: 4, + }, + ], + githubSkillScans: [ + { + _id: "githubSkillScans:html", + skillId: "skills:html", + githubSourceId: "githubSkillSources:current", + contentHash, + commit, + path: "skills/html", + status: "clean", + createdAt: 1, + updatedAt: 1, + }, + ], + }); + + await expect( + getArchiveScanBySkillAndContentHashHandler({ db } as never, { + skillId: "skills:html" as never, + commit, + contentHash, + }), + ).resolves.toMatchObject({ + repo: "patrick-erichsen/redirected-skills", + commit, + contentHash, + }); + }); + + it("keeps a missing skill hidden when it reappears through a repository redirect", async () => { + const snapshot = await buildGitHubSkillSourceSnapshot({ + repo: "patrick-erichsen/renamed-skills", + defaultBranch: "main", + commit: "a".repeat(40), + entries: { + "skills/html/SKILL.md": new TextEncoder().encode("# HTML\n"), + }, + }); + const contentHash = snapshot.skills[0]?.contentHash ?? ""; + const { db, tables } = createDb({ + githubSkillSources: [ + { + _id: "githubSkillSources:current", + repo: "patrick-erichsen/skills", + ownerPublisherId: "publishers:patrick", + githubRepositoryId: "100", + githubOwnerId: "200", + authorizationStatus: "active", + createdAt: 1, + updatedAt: 1, + }, + ], + publishers: [ + { + _id: "publishers:patrick", + kind: "user", + handle: "patrick", + linkedUserId: "users:patrick", + createdAt: 1, + updatedAt: 1, + }, + ], + skills: [ + { + _id: "skills:html", + slug: "html", + displayName: "HTML", + ownerUserId: "users:patrick", + ownerPublisherId: "publishers:patrick", + installKind: "github", + githubSourceId: "githubSkillSources:current", + githubCurrentRepo: "patrick-erichsen/skills", + githubPath: "skills/html", + githubCurrentCommit: snapshot.commit, + githubCurrentContentHash: contentHash, + githubCurrentStatus: "missing", + githubScanStatus: "clean", + githubRemovedAt: 2, + softDeletedAt: 2, + moderationStatus: "hidden", + moderationReason: "github.upstream.removed", + tags: {}, + stats: { downloads: 0, stars: 0, installsCurrent: 0, installsAllTime: 0, versions: 0 }, + createdAt: 1, + updatedAt: 2, + }, + ], + githubSkillScans: [ + { + _id: "githubSkillScans:html", + skillId: "skills:html", + githubSourceId: "githubSkillSources:current", + contentHash, + commit: snapshot.commit, + path: "skills/html", + status: "clean", + createdAt: 1, + updatedAt: 1, + }, + ], + }); + const scheduler = { runAfter: vi.fn(async () => undefined) }; + + await applyGitHubSkillSourceSyncHandler({ db, scheduler } as never, { + sourceId: "githubSkillSources:current" as never, + repo: "patrick-erichsen/renamed-skills", + ownerUserId: "users:patrick" as never, + ownerPublisherId: "publishers:patrick" as never, + githubRepositoryId: "100", + githubOwnerId: "200", + snapshot, + now: 3, + }); + + expect(tables.skills[0]).toMatchObject({ + githubCurrentRepo: "patrick-erichsen/skills", + githubCurrentStatus: "missing", + moderationStatus: "hidden", + }); + expect(tables.githubSkillCandidates).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + githubRepo: "patrick-erichsen/skills", + githubCommit: snapshot.commit, + githubContentHash: contentHash, + lifecycleStatus: "promoted", + verdictSourceScanId: "githubSkillScans:html", + }), + expect.objectContaining({ + githubRepo: "patrick-erichsen/renamed-skills", + lifecycleStatus: "pending", + }), + ]), + ); + }); + + it("rejects stale source observations before they can replace a newer source state", async () => { + const snapshot = await buildGitHubSkillSourceSnapshot({ + repo: "patrick-erichsen/skills", + defaultBranch: "main", + commit: "b".repeat(40), + entries: { + "skills/html/SKILL.md": new TextEncoder().encode("# stale HTML\n"), + }, + }); + const { db, tables } = createDb({ + githubSkillSources: [ + { + _id: "githubSkillSources:current", + repo: "patrick-erichsen/skills", + ownerPublisherId: "publishers:patrick", + githubRepositoryId: "100", + githubOwnerId: "200", + authorizationStatus: "active", + displayManifestCommit: "c".repeat(40), + createdAt: 1, + updatedAt: 20, + }, + ], + }); + + await expect( + applyGitHubSkillSourceSyncHandler({ db } as never, { + sourceId: "githubSkillSources:current" as never, + repo: "patrick-erichsen/skills", + ownerUserId: "users:patrick" as never, + ownerPublisherId: "publishers:patrick" as never, + githubRepositoryId: "100", + githubOwnerId: "200", + expectedSourceUpdatedAt: 10, + snapshot, + now: 30, + }), + ).resolves.toMatchObject({ ok: true, skipped: "stale-source-observation" }); + expect(tables.githubSkillSources[0]).toMatchObject({ + displayManifestCommit: "c".repeat(40), + updatedAt: 20, + }); + expect(tables.githubSkillCandidates ?? []).toEqual([]); + }); + + it("rejects an observation that expected to create a source after another writer created it", async () => { + const snapshot = await buildGitHubSkillSourceSnapshot({ + repo: "patrick-erichsen/skills", + defaultBranch: "main", + commit: "b".repeat(40), + entries: { + "skills/html/SKILL.md": new TextEncoder().encode("# stale HTML\n"), + }, + }); + const { db, tables } = createDb({ + githubSkillSources: [ + { + _id: "githubSkillSources:current", + repo: "patrick-erichsen/skills", + ownerPublisherId: "publishers:patrick", + githubRepositoryId: "100", + githubOwnerId: "200", + authorizationStatus: "active", + displayManifestCommit: "c".repeat(40), + createdAt: 20, + updatedAt: 20, + }, + ], + }); + + await expect( + applyGitHubSkillSourceSyncHandler({ db } as never, { + repo: "patrick-erichsen/skills", + ownerUserId: "users:patrick" as never, + ownerPublisherId: "publishers:patrick" as never, + githubRepositoryId: "100", + githubOwnerId: "200", + expectedSourceUpdatedAt: null, + snapshot, + now: 30, + }), + ).resolves.toMatchObject({ ok: true, skipped: "stale-source-observation" }); + expect(tables.githubSkillSources[0]).toMatchObject({ + displayManifestCommit: "c".repeat(40), + updatedAt: 20, + }); + }); + + it("rejects reassignment of a disconnected source to a different publisher", async () => { + const snapshot = await buildGitHubSkillSourceSnapshot({ + repo: "patrick-erichsen/skills", + defaultBranch: "main", + commit: "b".repeat(40), + entries: { + "skills/html/SKILL.md": new TextEncoder().encode("# HTML\n"), + }, + }); + const { db, tables } = createDb({ + githubSkillSources: [ + { + _id: "githubSkillSources:current", + repo: "patrick-erichsen/skills", + disconnectedOwnerPublisherId: "publishers:patrick", + githubRepositoryId: "100", + githubOwnerId: "200", + authorizationStatus: "revoked", + createdAt: 1, + updatedAt: 2, + }, + ], + }); + + await expect( + applyGitHubSkillSourceSyncHandler({ db } as never, { + sourceId: "githubSkillSources:current" as never, + repo: "patrick-erichsen/skills", + ownerUserId: "users:other" as never, + ownerPublisherId: "publishers:other" as never, + githubRepositoryId: "100", + githubOwnerId: "200", + expectedSourceUpdatedAt: 2, + snapshot, + now: 3, + }), + ).rejects.toThrow(/explicit ownership transfer/i); + expect(tables.githubSkillSources[0]).toMatchObject({ + disconnectedOwnerPublisherId: "publishers:patrick", + authorizationStatus: "revoked", + updatedAt: 2, + }); + }); + + it("rejects a repository redirect that collides with a retained source row", async () => { + const snapshot = await buildGitHubSkillSourceSnapshot({ + repo: "patrick-erichsen/retained-skills", + defaultBranch: "main", + commit: "b".repeat(40), + entries: {}, + }); + const { db, tables } = createDb({ + githubSkillSources: [ + { + _id: "githubSkillSources:current", + repo: "patrick-erichsen/current-skills", + ownerPublisherId: "publishers:patrick", + githubRepositoryId: "100", + githubOwnerId: "200", + authorizationStatus: "active", + createdAt: 1, + updatedAt: 2, + }, + { + _id: "githubSkillSources:retained", + repo: "patrick-erichsen/retained-skills", + disconnectedOwnerPublisherId: "publishers:other", + githubRepositoryId: "300", + githubOwnerId: "400", + authorizationStatus: "revoked", + createdAt: 1, + updatedAt: 2, + }, + ], + }); + + await expect( + applyGitHubSkillSourceSyncHandler({ db } as never, { + sourceId: "githubSkillSources:current" as never, + repo: "patrick-erichsen/retained-skills", + ownerUserId: "users:patrick" as never, + ownerPublisherId: "publishers:patrick" as never, + githubRepositoryId: "100", + githubOwnerId: "200", + expectedSourceUpdatedAt: 2, + snapshot, + now: 3, + }), + ).rejects.toThrow(/retained by another source/i); + expect(tables.githubSkillSources).toHaveLength(2); + expect(tables.githubSkillSources[0]).toMatchObject({ + repo: "patrick-erichsen/current-skills", + updatedAt: 2, + }); + }); + + it("rejects a stale same-hash callback from a superseded source pointer", async () => { + const contentHash = "a".repeat(64); + const { db, tables } = createDb({ + githubSkillSources: [ + { + _id: "githubSkillSources:current", + repo: "patrick-erichsen/skills", + authorizationStatus: "active", + createdAt: 1, + updatedAt: 1, + }, + ], + skills: [ + { + _id: "skills:html", + slug: "html", + displayName: "HTML A", + ownerUserId: "users:patrick", + ownerPublisherId: "publishers:patrick", + installKind: "github", + githubSourceId: "githubSkillSources:current", + githubPath: "skills/html", + githubCurrentCommit: "1".repeat(40), + githubCurrentContentHash: "1".repeat(64), + githubCurrentStatus: "present", + githubScanStatus: "clean", + githubPendingCandidateId: "githubSkillCandidates:return-a", + tags: {}, + stats: { downloads: 0, stars: 0, versions: 0, comments: 0 }, + moderationStatus: "active", + moderationFlags: [], + isSuspicious: false, + createdAt: 1, + updatedAt: 1, + }, + ], + githubSkillCandidates: [ + { + _id: "githubSkillCandidates:return-a", + skillId: "skills:html", + githubSourceId: "githubSkillSources:current", + githubRepo: "patrick-erichsen/skills", + githubPath: "skills/html-renamed", + githubHasSkillCard: false, + githubCommit: "3".repeat(40), + githubContentHash: contentHash, + displayName: "HTML A again", + skillMarkdownPath: "skills/html-renamed/SKILL.md", + skillMarkdown: "# HTML A again\n", + scanStatus: "clean", + lifecycleStatus: "pending", + verdictSourceScanId: "githubSkillScans:return-a", + createdAt: 3, + updatedAt: 3, + }, + ], + githubSkillScans: [ + { + _id: "githubSkillScans:stale-b", + skillId: "skills:html", + githubSourceId: "githubSkillSources:current", + contentHash, + commit: "2".repeat(40), + path: "skills/html-old-pointer", + status: "clean", + createdAt: 2, + updatedAt: 2, + }, + { + _id: "githubSkillScans:return-a", + skillId: "skills:html", + githubSourceId: "githubSkillSources:current", + contentHash, + commit: "3".repeat(40), + path: "skills/html-renamed", + status: "clean", + createdAt: 3, + updatedAt: 3, + }, + ], + }); + + await expect( + applyGitHubSkillVerificationResultHandler({ db } as never, { + skillId: "skills:html" as never, + contentHash, + githubSkillScanId: "githubSkillScans:stale-b" as never, + scanStatus: "clean", + now: 10, + }), + ).resolves.toEqual({ ok: true, skipped: "stale-candidate-verdict" }); + expect(tables.skills[0]).toMatchObject({ + githubCurrentCommit: "1".repeat(40), + githubPendingCandidateId: "githubSkillCandidates:return-a", + }); + + await expect( + applyGitHubSkillVerificationResultHandler({ db } as never, { + skillId: "skills:html" as never, + contentHash, + githubSkillScanId: "githubSkillScans:return-a" as never, + scanStatus: "clean", + now: 11, + }), + ).resolves.toMatchObject({ ok: true, promoted: true }); + expect(tables.skills[0]).toMatchObject({ + githubCurrentCommit: "3".repeat(40), + githubPath: "skills/html-renamed", + githubCurrentCandidateId: "githubSkillCandidates:return-a", + }); + }); + + it("retains a rejected candidate without replacing the allowed GitHub version", async () => { + const { db, tables } = createDb({ + skills: [ + { + _id: "skills:html", + slug: "html", + displayName: "HTML A", + ownerUserId: "users:patrick", + installKind: "github", + githubSourceId: "githubSkillSources:current", + githubPath: "skills/html", + githubCurrentCommit: "a".repeat(40), + githubCurrentContentHash: "a".repeat(64), + githubCurrentStatus: "present", + githubScanStatus: "clean", + githubPendingCandidateId: "githubSkillCandidates:b", + tags: {}, + stats: { downloads: 0, stars: 0, versions: 0, comments: 0 }, + moderationStatus: "active", + moderationFlags: [], + isSuspicious: false, + createdAt: 1, + updatedAt: 1, + }, + ], + githubSkillCandidates: [ + { + _id: "githubSkillCandidates:b", + skillId: "skills:html", + githubSourceId: "githubSkillSources:current", + githubRepo: "patrick-erichsen/skills", + githubPath: "skills/html", + githubHasSkillCard: false, + githubCommit: "b".repeat(40), + githubContentHash: "b".repeat(64), + displayName: "HTML B", + skillMarkdownPath: "skills/html/SKILL.md", + skillMarkdown: "# HTML B\n", + scanStatus: "pending", + lifecycleStatus: "pending", + verdictSourceScanId: "githubSkillScans:b", + createdAt: 2, + updatedAt: 2, + }, + ], + githubSkillScans: [ + { + _id: "githubSkillScans:b", + skillId: "skills:html", + githubSourceId: "githubSkillSources:current", + contentHash: "b".repeat(64), + commit: "b".repeat(40), + path: "skills/html", + status: "malicious", + createdAt: 2, + updatedAt: 3, + }, + ], + }); + + await expect( + applyGitHubSkillVerificationResultHandler({ db } as never, { + skillId: "skills:html" as never, + contentHash: "b".repeat(64), + githubSkillScanId: "githubSkillScans:b" as never, + scanStatus: "malicious", + now: 3, + }), + ).resolves.toEqual({ ok: true, promoted: false }); + + expect(tables.skills[0]).toMatchObject({ + githubCurrentCommit: "a".repeat(40), + githubCurrentContentHash: "a".repeat(64), + githubScanStatus: "clean", + }); + expect(tables.skills[0]).not.toHaveProperty("githubPendingCandidateId"); + expect(tables.githubSkillCandidates[0]).toMatchObject({ + lifecycleStatus: "rejected", + rejectedAt: 3, + scanStatus: "malicious", + }); + }); + + it("keeps a failed candidate addressable so the exact scan can be retried", async () => { + const { db, tables } = createDb({ + skills: [ + { + _id: "skills:html", + slug: "html", + displayName: "HTML A", + ownerUserId: "users:patrick", + installKind: "github", + githubSourceId: "githubSkillSources:current", + githubPath: "skills/html", + githubCurrentCommit: "a".repeat(40), + githubCurrentContentHash: "a".repeat(64), + githubCurrentStatus: "present", + githubScanStatus: "clean", + githubPendingCandidateId: "githubSkillCandidates:b", + tags: {}, + stats: { downloads: 0, stars: 0, versions: 0, comments: 0 }, + moderationStatus: "active", + moderationFlags: [], + isSuspicious: false, + createdAt: 1, + updatedAt: 1, + }, + ], + githubSkillCandidates: [ + { + _id: "githubSkillCandidates:b", + skillId: "skills:html", + githubSourceId: "githubSkillSources:current", + githubRepo: "patrick-erichsen/skills", + githubPath: "skills/html", + githubHasSkillCard: false, + githubCommit: "b".repeat(40), + githubContentHash: "b".repeat(64), + displayName: "HTML B", + skillMarkdownPath: "skills/html/SKILL.md", + skillMarkdown: "# HTML B\n", + scanStatus: "pending", + lifecycleStatus: "pending", + verdictSourceScanId: "githubSkillScans:b", + createdAt: 2, + updatedAt: 2, + }, + ], + githubSkillScans: [ + { + _id: "githubSkillScans:b", + skillId: "skills:html", + githubSourceId: "githubSkillSources:current", + contentHash: "b".repeat(64), + commit: "b".repeat(40), + path: "skills/html", + status: "failed", + createdAt: 2, + updatedAt: 3, + }, + ], + }); + + await applyGitHubSkillVerificationResultHandler({ db } as never, { + skillId: "skills:html" as never, + contentHash: "b".repeat(64), + githubSkillScanId: "githubSkillScans:b" as never, + scanStatus: "failed", + now: 3, + }); + expect(tables.skills[0]).toMatchObject({ + githubCurrentCommit: "a".repeat(40), + githubPendingCandidateId: "githubSkillCandidates:b", + }); + + Object.assign(tables.githubSkillScans[0] ?? {}, { status: "pending" }); + await applyGitHubSkillVerificationResultHandler({ db } as never, { + skillId: "skills:html" as never, + contentHash: "b".repeat(64), + githubSkillScanId: "githubSkillScans:b" as never, + scanStatus: "pending", + now: 4, + }); + expect(tables.githubSkillCandidates[0]).toMatchObject({ + lifecycleStatus: "pending", + scanStatus: "pending", + }); + + Object.assign(tables.githubSkillScans[0] ?? {}, { status: "clean" }); + await applyGitHubSkillVerificationResultHandler({ db } as never, { + skillId: "skills:html" as never, + contentHash: "b".repeat(64), + githubSkillScanId: "githubSkillScans:b" as never, + scanStatus: "clean", + now: 5, + }); + expect(tables.skills[0]).toMatchObject({ + githubCurrentCommit: "b".repeat(40), + githubCurrentCandidateId: "githubSkillCandidates:b", + }); + }); + + it("retains a known-malicious replacement as rejected without making it pending", async () => { + const snapshot = await buildGitHubSkillSourceSnapshot({ + repo: "patrick-erichsen/skills", + defaultBranch: "main", + commit: "b".repeat(40), + entries: { + "skills/html/SKILL.md": new TextEncoder().encode("# HTML B\n"), + }, + }); + const contentHash = snapshot.skills[0]?.contentHash ?? ""; + const { db, tables } = createDb({ + githubSkillSources: [ + { + _id: "githubSkillSources:current", + repo: "patrick-erichsen/skills", + ownerPublisherId: "publishers:patrick", + githubRepositoryId: "100", + githubOwnerId: "200", + authorizationStatus: "active", + createdAt: 1, + updatedAt: 1, + }, + ], + publishers: [ + { + _id: "publishers:patrick", + kind: "user", + handle: "patrick", + displayName: "Patrick", + linkedUserId: "users:patrick", + createdAt: 1, + updatedAt: 1, + }, + ], + skills: [ + { + _id: "skills:html", + slug: "html", + displayName: "HTML A", + ownerUserId: "users:patrick", + ownerPublisherId: "publishers:patrick", + installKind: "github", + githubSourceId: "githubSkillSources:current", + githubCurrentRepo: "patrick-erichsen/skills", + githubPath: "skills/html", + githubCurrentCommit: "a".repeat(40), + githubCurrentContentHash: "a".repeat(64), + githubCurrentStatus: "present", + githubScanStatus: "clean", + tags: {}, + stats: { downloads: 0, stars: 0, versions: 0, comments: 0 }, + moderationStatus: "active", + moderationFlags: [], + isSuspicious: false, + createdAt: 1, + updatedAt: 1, + }, + ], + githubSkillScans: [ + { + _id: "githubSkillScans:b", + skillId: "skills:html", + githubSourceId: "githubSkillSources:current", + contentHash, + commit: "b".repeat(40), + path: "skills/html", + status: "malicious", + createdAt: 2, + updatedAt: 2, + }, + ], + }); + const scheduler = { runAfter: vi.fn(async () => undefined) }; + + await applyGitHubSkillSourceSyncHandler({ db, scheduler } as never, { + sourceId: "githubSkillSources:current" as never, + repo: "patrick-erichsen/skills", + ownerUserId: "users:patrick" as never, + ownerPublisherId: "publishers:patrick" as never, + githubRepositoryId: "100", + githubOwnerId: "200", + snapshot, + now: 3, + }); + + expect(tables.skills[0]).toMatchObject({ + githubCurrentCommit: "a".repeat(40), + githubCurrentContentHash: "a".repeat(64), + }); + expect(tables.skills[0]).not.toHaveProperty("githubPendingCandidateId"); + expect(tables.githubSkillCandidates).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + githubCommit: "b".repeat(40), + scanStatus: "malicious", + lifecycleStatus: "rejected", + rejectedAt: 3, + }), + ]), + ); + expect(scheduler.runAfter).not.toHaveBeenCalled(); + }); + + it("rolls back to a retained candidate with its own allowed verdict", async () => { + const { db, tables } = createDb({ + githubSkillSources: [ + { + _id: "githubSkillSources:current", + repo: "patrick-erichsen/skills", + authorizationStatus: "active", + createdAt: 1, + updatedAt: 1, + }, + ], + skills: [ + { + _id: "skills:html", + slug: "html", + displayName: "HTML B", + ownerUserId: "users:patrick", + ownerPublisherId: "publishers:patrick", + installKind: "github", + githubSourceId: "githubSkillSources:current", + githubPath: "skills/html", + githubCurrentCommit: "b".repeat(40), + githubCurrentContentHash: "b".repeat(64), + githubCurrentStatus: "present", + githubScanStatus: "clean", + githubCurrentCandidateId: "githubSkillCandidates:b", + githubPendingCandidateId: "githubSkillCandidates:c", + tags: {}, + stats: { downloads: 0, stars: 0, versions: 0, comments: 0 }, + moderationStatus: "active", + moderationFlags: [], + isSuspicious: false, + createdAt: 1, + updatedAt: 2, + }, + ], + githubSkillCandidates: [ + { + _id: "githubSkillCandidates:a", + skillId: "skills:html", + githubSourceId: "githubSkillSources:current", + githubRepo: "patrick-erichsen/skills", + githubPath: "skills/html", + githubHasSkillCard: false, + githubCommit: "a".repeat(40), + githubContentHash: "a".repeat(64), + displayName: "HTML A", + skillMarkdownPath: "skills/html/SKILL.md", + skillMarkdown: "# HTML A\n", + scanStatus: "clean", + lifecycleStatus: "superseded", + verdictSourceScanId: "githubSkillScans:a", + promotedAt: 1, + createdAt: 1, + updatedAt: 2, + }, + { + _id: "githubSkillCandidates:b", + skillId: "skills:html", + githubSourceId: "githubSkillSources:current", + githubRepo: "patrick-erichsen/skills", + githubPath: "skills/html", + githubHasSkillCard: false, + githubCommit: "b".repeat(40), + githubContentHash: "b".repeat(64), + displayName: "HTML B", + skillMarkdownPath: "skills/html/SKILL.md", + skillMarkdown: "# HTML B\n", + scanStatus: "clean", + lifecycleStatus: "promoted", + verdictSourceScanId: "githubSkillScans:b", + previousCandidateId: "githubSkillCandidates:a", + promotedAt: 2, + createdAt: 2, + updatedAt: 2, + }, + { + _id: "githubSkillCandidates:c", + skillId: "skills:html", + githubSourceId: "githubSkillSources:current", + githubRepo: "patrick-erichsen/skills", + githubPath: "skills/html", + githubHasSkillCard: false, + githubCommit: "c".repeat(40), + githubContentHash: "c".repeat(64), + displayName: "HTML C", + scanStatus: "pending", + lifecycleStatus: "pending", + previousCandidateId: "githubSkillCandidates:b", + createdAt: 3, + updatedAt: 3, + }, + ], + githubSkillContents: [ + { + _id: "githubSkillContents:html", + skillId: "skills:html", + githubSourceId: "githubSkillSources:current", + githubPath: "skills/html", + skillMarkdownPath: "skills/html/SKILL.md", + skillMarkdown: "# HTML B\n", + githubCommit: "b".repeat(40), + githubContentHash: "b".repeat(64), + fetchedAt: 2, + createdAt: 1, + updatedAt: 2, + }, + ], + githubSkillScans: [ + { + _id: "githubSkillScans:a", + skillId: "skills:html", + githubSourceId: "githubSkillSources:current", + contentHash: "a".repeat(64), + commit: "a".repeat(40), + path: "skills/html", + status: "clean", + createdAt: 1, + updatedAt: 1, + }, + { + _id: "githubSkillScans:b", + skillId: "skills:html", + githubSourceId: "githubSkillSources:current", + contentHash: "b".repeat(64), + commit: "b".repeat(40), + path: "skills/html", + status: "clean", + createdAt: 2, + updatedAt: 2, + }, + ], + }); + + await expect( + rollbackGitHubSkillCandidateHandler({ db } as never, { + skillId: "skills:html" as never, + targetCandidateId: "githubSkillCandidates:a" as never, + confirm: "rollback-github-skill-candidate", + now: 3, + }), + ).resolves.toMatchObject({ ok: true, rolledBack: true }); + + expect(tables.skills[0]).toMatchObject({ + displayName: "HTML A", + githubCurrentCommit: "a".repeat(40), + githubCurrentContentHash: "a".repeat(64), + githubCurrentCandidateId: "githubSkillCandidates:a", + }); + expect(tables.githubSkillCandidates).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + _id: "githubSkillCandidates:a", + lifecycleStatus: "promoted", + }), + expect.objectContaining({ + _id: "githubSkillCandidates:b", + lifecycleStatus: "rolled_back", + rolledBackAt: 3, + }), + expect.objectContaining({ + _id: "githubSkillCandidates:c", + lifecycleStatus: "canceled", + canceledAt: 3, + cancellationReason: "github.rollback", + }), + ]), + ); + expect(tables.githubSkillContents[0]).toMatchObject({ + skillMarkdown: "# HTML A\n", + githubCommit: "a".repeat(40), + githubContentHash: "a".repeat(64), }); }); @@ -2113,7 +3659,11 @@ description: Build HTML artifacts. now: 100, }); - expect(applied.stats).toMatchObject({ changed: 1, inserted: 0, conflicts: 0 }); + expect(applied.stats).toMatchObject({ + changed: 1, + inserted: 0, + conflicts: 0, + }); const pendingSkill = tables.skills[0]; const candidate = tables.githubSkillCandidates[0]; expect(pendingSkill).toMatchObject({ @@ -2140,6 +3690,10 @@ description: Build HTML artifacts. skillMarkdownPath: snapshot.skills[0]?.skillMarkdownPath, skillMarkdown: snapshot.skills[0]?.skillMarkdown, }); + const completedScan = tables.githubSkillScans.find( + (row) => row._id === candidate?.verdictSourceScanId, + ); + Object.assign(completedScan ?? {}, { status: "clean" }); Object.assign(pendingSkill ?? {}, { softDeletedAt: 105, moderationStatus: "hidden", @@ -2149,6 +3703,7 @@ description: Build HTML artifacts. applyGitHubSkillVerificationResultHandler({ db } as never, { skillId: "skills:html" as never, contentHash: snapshot.skills[0]?.contentHash ?? "", + githubSkillScanId: candidate?.verdictSourceScanId as never, scanStatus: "clean", now: 106, }), @@ -2167,6 +3722,7 @@ description: Build HTML artifacts. await applyGitHubSkillVerificationResultHandler({ db } as never, { skillId: "skills:html" as never, contentHash: snapshot.skills[0]?.contentHash ?? "", + githubSkillScanId: candidate?.verdictSourceScanId as never, scanStatus: "clean", now: 110, }); @@ -2188,15 +3744,30 @@ description: Build HTML artifacts. }); expect(tables.skills[0]).not.toHaveProperty("latestVersionId"); expect(tables.skills[0]).not.toHaveProperty("githubPendingCandidateId"); - expect(tables.githubSkillCandidates).toEqual([]); + expect(tables.githubSkillCandidates).toEqual([ + expect.objectContaining({ + _id: candidate?._id, + lifecycleStatus: "promoted", + promotedAt: 110, + }), + ]); expect(tables.skillVersions).toEqual([ - expect.objectContaining({ _id: "skillVersions:html-v1", skillId: "skills:html" }), + expect.objectContaining({ + _id: "skillVersions:html-v1", + skillId: "skills:html", + }), ]); expect(tables.bookmarks).toEqual([ - expect.objectContaining({ _id: "bookmarks:html", skillId: "skills:html" }), + expect.objectContaining({ + _id: "bookmarks:html", + skillId: "skills:html", + }), ]); expect(tables.auditLogs).toEqual([ - expect.objectContaining({ _id: "auditLogs:html", targetId: "skills:html" }), + expect.objectContaining({ + _id: "auditLogs:html", + targetId: "skills:html", + }), ]); expect(tables.skillVersions).toHaveLength(1); }); @@ -2238,7 +3809,13 @@ description: Build HTML artifacts. moderationStatus: "active", moderationVerdict: "clean", tags: {}, - stats: { downloads: 0, stars: 0, installsCurrent: 0, installsAllTime: 0, versions: 0 }, + stats: { + downloads: 0, + stars: 0, + installsCurrent: 0, + installsAllTime: 0, + versions: 0, + }, createdAt: 1, updatedAt: 1, }, @@ -2277,7 +3854,9 @@ description: Build HTML artifacts. "skills/aiq-deploy/skill-card.md": new TextEncoder().encode("# AIQ Card v2\n"), "skills/vision-helper/SKILL.md": new TextEncoder().encode("# Vision Helper\n"), "skills.sh.json": new TextEncoder().encode( - JSON.stringify({ groupings: [{ title: "Agentic AI", skills: ["aiq-deploy"] }] }), + JSON.stringify({ + groupings: [{ title: "Agentic AI", skills: ["aiq-deploy"] }], + }), ), }, }); @@ -2322,7 +3901,13 @@ description: Build HTML artifacts. githubCurrentContentHash: "old-hash", githubScanStatus: "clean", tags: {}, - stats: { downloads: 0, stars: 0, installsCurrent: 0, installsAllTime: 0, versions: 0 }, + stats: { + downloads: 0, + stars: 0, + installsCurrent: 0, + installsAllTime: 0, + versions: 0, + }, createdAt: 1, updatedAt: 1, }, @@ -2333,7 +3918,13 @@ description: Build HTML artifacts. ownerUserId: "users:someone-else", ownerPublisherId: "publishers:someone-else", tags: {}, - stats: { downloads: 0, stars: 0, installsCurrent: 0, installsAllTime: 0, versions: 1 }, + stats: { + downloads: 0, + stars: 0, + installsCurrent: 0, + installsAllTime: 0, + versions: 1, + }, createdAt: 1, updatedAt: 1, }, @@ -2451,7 +4042,13 @@ description: Build HTML artifacts. softDeletedAt: 40, githubScanStatus: "clean", tags: {}, - stats: { downloads: 0, stars: 0, installsCurrent: 0, installsAllTime: 0, versions: 0 }, + stats: { + downloads: 0, + stars: 0, + installsCurrent: 0, + installsAllTime: 0, + versions: 0, + }, createdAt: 1, updatedAt: 60, }, @@ -2513,7 +4110,11 @@ description: Build HTML artifacts. now: 123, }); - expect(result.stats).toMatchObject({ inserted: 1, conflicts: 0, invalid: 0 }); + expect(result.stats).toMatchObject({ + inserted: 1, + conflicts: 0, + invalid: 0, + }); expect(tables.skills).toHaveLength(1); expect(tables.skillVersions ?? []).toEqual([]); expect(tables.githubSkillContents).toEqual([ @@ -2580,7 +4181,13 @@ description: Build HTML artifacts. statsInstallsAllTime: 5, statsSkillsShInstalls: 17, statsGithubStars: 321, - stats: { downloads: 7, stars: 3, installsCurrent: 2, installsAllTime: 5, versions: 0 }, + stats: { + downloads: 7, + stars: 3, + installsCurrent: 2, + installsAllTime: 5, + versions: 0, + }, createdAt: 1, updatedAt: 60, }, @@ -2849,7 +4456,13 @@ description: Build HTML artifacts. githubCurrentStatus: "present", githubScanStatus: "pending", tags: {}, - stats: { downloads: 0, stars: 0, installsCurrent: 0, installsAllTime: 0, versions: 0 }, + stats: { + downloads: 0, + stars: 0, + installsCurrent: 0, + installsAllTime: 0, + versions: 0, + }, createdAt: 1, updatedAt: 1, }, @@ -2926,7 +4539,13 @@ description: Build HTML artifacts. githubCurrentStatus: "present", githubScanStatus: "clean", tags: {}, - stats: { downloads: 0, stars: 0, installsCurrent: 0, installsAllTime: 0, versions: 0 }, + stats: { + downloads: 0, + stars: 0, + installsCurrent: 0, + installsAllTime: 0, + versions: 0, + }, createdAt: 1, updatedAt: 1, }, @@ -3001,7 +4620,13 @@ description: Build HTML artifacts. githubCurrentContentHash: "old-hash", githubScanStatus: "clean", tags: {}, - stats: { downloads: 0, stars: 0, installsCurrent: 0, installsAllTime: 0, versions: 0 }, + stats: { + downloads: 0, + stars: 0, + installsCurrent: 0, + installsAllTime: 0, + versions: 0, + }, createdAt: 1, updatedAt: 1, }, @@ -3213,7 +4838,9 @@ describe("verifyGitHubSkillHandler", () => { }); } if (url.startsWith("https://codeload.github.com/")) { - return new Response(zip, { headers: { "content-length": String(zip.byteLength) } }); + return new Response(zip, { + headers: { "content-length": String(zip.byteLength) }, + }); } return new Response("not found", { status: 404 }); }); @@ -3331,7 +4958,9 @@ describe("verifyGitHubSkillHandler", () => { }); } if (url.startsWith("https://codeload.github.com/")) { - return new Response(zip, { headers: { "content-length": String(zip.byteLength) } }); + return new Response(zip, { + headers: { "content-length": String(zip.byteLength) }, + }); } return new Response("not found", { status: 404 }); }); @@ -3417,7 +5046,9 @@ describe("verifyGitHubSkillHandler", () => { }); } if (url.startsWith("https://codeload.github.com/")) { - return new Response(zip, { headers: { "content-length": String(zip.byteLength) } }); + return new Response(zip, { + headers: { "content-length": String(zip.byteLength) }, + }); } return new Response("not found", { status: 404 }); }); diff --git a/convex/githubSkillSync.ts b/convex/githubSkillSync.ts index 0bda965a..aca61fab 100644 --- a/convex/githubSkillSync.ts +++ b/convex/githubSkillSync.ts @@ -62,7 +62,13 @@ const MAX_SOURCE_SYNC_BATCH_SIZE = 50; type SourceForSync = Pick< Doc<"githubSkillSources">, - "_id" | "repo" | "ownerPublisherId" | "githubRepositoryId" | "githubOwnerId" | "defaultBranch" + | "_id" + | "repo" + | "ownerPublisherId" + | "githubRepositoryId" + | "githubOwnerId" + | "defaultBranch" + | "updatedAt" >; type SourceForSyncPage = { @@ -73,6 +79,7 @@ type SourceForSyncPage = { type SyncOneResult = { ok: true; + skipped?: "stale-source-observation"; repo: string; sourceId?: Id<"githubSkillSources">; commit: string; @@ -260,19 +267,100 @@ const githubSkillScanStatusValidator = v.union( v.literal("failed"), ); +export async function getArchiveScanBySkillAndContentHashHandler( + ctx: QueryCtx, + args: { skillId: Id<"skills">; commit: string; contentHash: string }, +) { + const skill = await ctx.db.get(args.skillId); + if (!skill) return null; + const candidates = await ctx.db + .query("githubSkillCandidates") + .withIndex("by_skill_and_commit_and_content_hash", (q) => + q + .eq("skillId", args.skillId) + .eq("githubCommit", args.commit) + .eq("githubContentHash", args.contentHash), + ) + .collect(); + const eligibleCandidates = candidates.filter( + (candidate) => + (candidate.lifecycleStatus === "promoted" || + candidate.lifecycleStatus === "superseded" || + candidate.lifecycleStatus === "rolled_back") && + (candidate.scanStatus === "clean" || candidate.scanStatus === "suspicious") && + candidate.verdictSourceScanId, + ); + const preferredCandidate = + eligibleCandidates.find((candidate) => candidate._id === skill.githubCurrentCandidateId) ?? + eligibleCandidates + .slice() + .sort( + (a, b) => + (b.promotedAt ?? b.updatedAt) - (a.promotedAt ?? a.updatedAt) || + b._creationTime - a._creationTime || + String(b._id).localeCompare(String(a._id)), + )[0] ?? + null; + if (preferredCandidate?.verdictSourceScanId) { + const [scan, source] = await Promise.all([ + ctx.db.get(preferredCandidate.verdictSourceScanId), + ctx.db.get(preferredCandidate.githubSourceId), + ]); + if ( + scan && + source && + scan.skillId === skill._id && + scan.contentHash === preferredCandidate.githubContentHash && + scan.status === preferredCandidate.scanStatus + ) { + return { + githubSourceId: preferredCandidate.githubSourceId, + repo: preferredCandidate.githubRepo ?? source.repo, + contentHash: preferredCandidate.githubContentHash, + commit: preferredCandidate.githubCommit, + path: preferredCandidate.githubPath, + status: scan.status, + }; + } + } + + if ( + skill.installKind !== "github" || + skill.githubCurrentCommit !== args.commit || + skill.githubCurrentContentHash !== args.contentHash || + !skill.githubSourceId || + !skill.githubPath || + (skill.githubScanStatus !== "clean" && skill.githubScanStatus !== "suspicious") + ) { + return null; + } + const source = await ctx.db.get(skill.githubSourceId); + if (!source) return null; + const scans = await ctx.db + .query("githubSkillScans") + .withIndex("by_skill_and_content_hash", (q) => + q.eq("skillId", skill._id).eq("contentHash", args.contentHash), + ) + .collect(); + const scan = scans.find((candidateScan) => candidateScan.status === skill.githubScanStatus); + if (!scan) return null; + return { + githubSourceId: skill.githubSourceId, + repo: skill.githubCurrentRepo ?? source.repo, + contentHash: skill.githubCurrentContentHash, + commit: skill.githubCurrentCommit, + path: skill.githubPath, + status: scan.status, + }; +} + export const getArchiveScanBySkillAndContentHashInternal = internalQuery({ args: { skillId: v.id("skills"), + commit: v.string(), contentHash: v.string(), }, - handler: async (ctx, args) => { - return await ctx.db - .query("githubSkillScans") - .withIndex("by_skill_and_content_hash", (q) => - q.eq("skillId", args.skillId).eq("contentHash", args.contentHash), - ) - .unique(); - }, + handler: getArchiveScanBySkillAndContentHashHandler, }); export const getSourceByRepoInternal = internalQuery({ @@ -326,16 +414,25 @@ export async function listSourcesForSyncHandler( export const listGitHubSkillContentTargetsInternal = internalQuery({ args: { sourceId: v.id("githubSkillSources") }, handler: async (ctx, args): Promise => { - const [skills, candidates] = await Promise.all([ + const [skills, pendingCandidates, legacyCandidates] = await Promise.all([ ctx.db .query("skills") .withIndex("by_github_source", (q) => q.eq("githubSourceId", args.sourceId)) .collect(), ctx.db .query("githubSkillCandidates") - .withIndex("by_github_source", (q) => q.eq("githubSourceId", args.sourceId)) + .withIndex("by_github_source_and_lifecycle_status", (q) => + q.eq("githubSourceId", args.sourceId).eq("lifecycleStatus", "pending"), + ) + .collect(), + ctx.db + .query("githubSkillCandidates") + .withIndex("by_github_source_and_lifecycle_status", (q) => + q.eq("githubSourceId", args.sourceId).eq("lifecycleStatus", undefined), + ) .collect(), ]); + const candidates = [...pendingCandidates, ...legacyCandidates]; const currentTargets = skills.flatMap((skill) => { if ( skill.installKind !== "github" || @@ -353,12 +450,24 @@ export const listGitHubSkillContentTargetsInternal = internalQuery({ }, ]; }); - const candidateTargets = candidates.map((candidate) => ({ - skillId: candidate.skillId, - githubPath: candidate.githubPath, - githubCurrentContentHash: candidate.githubContentHash, - candidateId: candidate._id, - })); + const pendingCandidateIds = new Set( + skills.flatMap((skill) => + skill.githubPendingCandidateId ? [skill.githubPendingCandidateId] : [], + ), + ); + const candidateTargets = candidates.flatMap((candidate) => + pendingCandidateIds.has(candidate._id) && + (candidate.lifecycleStatus === undefined || candidate.lifecycleStatus === "pending") + ? [ + { + skillId: candidate.skillId, + githubPath: candidate.githubPath, + githubCurrentContentHash: candidate.githubContentHash, + candidateId: candidate._id, + }, + ] + : [], + ); return [...currentTargets, ...candidateTargets]; }, }); @@ -478,6 +587,23 @@ export const recordGitHubSkillSourceSyncAttemptInternal = internalMutation({ handler: recordGitHubSkillSourceSyncAttemptHandler, }); +async function cancelPendingGitHubSkillCandidate( + ctx: MutationCtx, + candidateId: Id<"githubSkillCandidates"> | undefined, + now: number, + reason: string, +) { + if (!candidateId) return; + const candidate = await ctx.db.get(candidateId); + if (!candidate) return; + await ctx.db.patch(candidate._id, { + lifecycleStatus: "canceled", + canceledAt: now, + cancellationReason: reason, + updatedAt: now, + }); +} + export async function revokeGitHubSkillSourceAuthorizationHandler( ctx: MutationCtx, args: { sourceId: Id<"githubSkillSources">; error: string; now?: number }, @@ -487,10 +613,27 @@ export async function revokeGitHubSkillSourceAuthorizationHandler( return { ok: true as const, skipped: "missing-or-legacy-source" as const }; } const now = args.now ?? Date.now(); - const candidates = await ctx.db - .query("githubSkillCandidates") - .withIndex("by_github_source", (q) => q.eq("githubSourceId", source._id)) - .collect(); + const [pendingCandidates, failedCandidates, legacyCandidates] = await Promise.all([ + ctx.db + .query("githubSkillCandidates") + .withIndex("by_github_source_and_lifecycle_status", (q) => + q.eq("githubSourceId", source._id).eq("lifecycleStatus", "pending"), + ) + .collect(), + ctx.db + .query("githubSkillCandidates") + .withIndex("by_github_source_and_lifecycle_status", (q) => + q.eq("githubSourceId", source._id).eq("lifecycleStatus", "failed"), + ) + .collect(), + ctx.db + .query("githubSkillCandidates") + .withIndex("by_github_source_and_lifecycle_status", (q) => + q.eq("githubSourceId", source._id).eq("lifecycleStatus", undefined), + ) + .collect(), + ]); + const candidates = [...pendingCandidates, ...failedCandidates, ...legacyCandidates]; for (const candidate of candidates) { const skill = await ctx.db.get(candidate.skillId); if (skill?.githubPendingCandidateId === candidate._id) { @@ -498,8 +641,13 @@ export async function revokeGitHubSkillSourceAuthorizationHandler( githubPendingCandidateId: undefined, updatedAt: now, }); + await cancelPendingGitHubSkillCandidate( + ctx, + candidate._id, + now, + "github.authorization.revoked", + ); } - await ctx.db.delete(candidate._id); } const skills = await ctx.db @@ -598,7 +746,9 @@ export const getGitHubSkillVerificationTargetInternal = internalQuery({ }, source: { _id: source._id, - repo: source.repo, + repo: exact.candidateId + ? (candidate?.githubRepo ?? source.repo) + : (skill.githubCurrentRepo ?? source.repo), defaultBranch: source.defaultBranch, }, ...(exact.candidateId ? { candidateId: exact.candidateId } : {}), @@ -613,6 +763,7 @@ export type ApplyGitHubSkillSourceSyncArgs = { ownerPublisherId?: Id<"publishers">; githubRepositoryId?: string; githubOwnerId?: string; + expectedSourceUpdatedAt?: number | null; snapshot: GitHubSkillSourceMetadataSnapshot; now?: number; }; @@ -698,7 +849,10 @@ export async function applyGitHubSkillSourceSyncHandler( const previousSkillSnapshot = previousSkill ? ({ ...previousSkill } as Doc<"skills">) : null; await ctx.db.patch(skillPatch.skillId as Id<"skills">, skillPatch.patch); if (previousSkillSnapshot) { - const nextSkillSnapshot = { ...previousSkillSnapshot, ...skillPatch.patch }; + const nextSkillSnapshot = { + ...previousSkillSnapshot, + ...skillPatch.patch, + }; await syncSkillSearchDigestForSkill(ctx, nextSkillSnapshot); await adjustGlobalPublicCountForSkillChange( ctx, @@ -794,7 +948,10 @@ export async function applyGitHubSkillSourceSyncHandler( updatedAt: now, }; await ctx.db.patch(reviveCandidate._id, patch); - const nextSkillSnapshot = { ...previousSkillSnapshot, ...patch } as Doc<"skills">; + const nextSkillSnapshot = { + ...previousSkillSnapshot, + ...patch, + } as Doc<"skills">; const discovered = discoveredBySlug.get(skillInsert.slug); if (discovered) { if (hasGitHubSkillContent(discovered)) { @@ -826,7 +983,11 @@ export async function applyGitHubSkillSourceSyncHandler( const doc = stripUndefined(skillInsert.doc) as Omit, "_id" | "_creationTime">; const skillId = await ctx.db.insert("skills", doc); - const insertedSkill = { ...doc, _id: skillId, _creationTime: now } as Doc<"skills">; + const insertedSkill = { + ...doc, + _id: skillId, + _creationTime: now, + } as Doc<"skills">; await syncSkillSearchDigestForSkill(ctx, insertedSkill); const discovered = discoveredBySlug.get(skillInsert.slug); if (discovered) { @@ -892,12 +1053,55 @@ async function applyGenericGitHubSkillSourceSyncHandler( .query("githubSkillSources") .withIndex("by_repo", (q) => q.eq("repo", args.repo)) .unique())); + if ( + args.expectedSourceUpdatedAt !== undefined && + (args.expectedSourceUpdatedAt === null + ? Boolean(existingSource) + : existingSource?.updatedAt !== args.expectedSourceUpdatedAt) + ) { + return { + ok: true as const, + skipped: "stale-source-observation" as const, + repo: existingSource?.repo ?? args.repo, + sourceId: existingSource?._id, + commit: args.snapshot.commit, + manifestStatus: args.snapshot.manifestStatus, + stats: { + discovered: args.snapshot.skills.length, + inserted: 0, + changed: 0, + unchanged: 0, + removed: 0, + conflicts: 0, + invalid: 0, + revived: 0, + }, + }; + } + const collidingSource = ( + await ctx.db + .query("githubSkillSources") + .withIndex("by_repo", (q) => q.eq("repo", args.repo)) + .take(2) + ).find((source) => source._id !== existingSource?._id); + if (collidingSource) { + throw new ConvexError("GitHub repository name is retained by another source."); + } if ( existingSource?.ownerPublisherId && existingSource.ownerPublisherId !== args.ownerPublisherId ) { throw new ConvexError("GitHub source is already configured for another publisher."); } + if ( + !existingSource?.ownerPublisherId && + existingSource?.disconnectedOwnerPublisherId && + existingSource.disconnectedOwnerPublisherId !== args.ownerPublisherId + ) { + throw new ConvexError( + "Disconnected GitHub source requires an explicit ownership transfer before reassignment.", + ); + } if ( existingSource?.githubRepositoryId && existingSource.githubRepositoryId !== args.githubRepositoryId @@ -920,6 +1124,9 @@ async function applyGenericGitHubSkillSourceSyncHandler( createdAt: args.now, updatedAt: args.now, })); + const sourceUpdatedAt = existingSource + ? Math.max(args.now, existingSource.updatedAt + 1) + : args.now; const publisher = await ctx.db.get(args.ownerPublisherId); if (!publisher || publisher.deletedAt || publisher.deactivatedAt) { throw new ConvexError("GitHub source owner publisher not found."); @@ -927,6 +1134,7 @@ async function applyGenericGitHubSkillSourceSyncHandler( await ctx.db.patch(sourceId, { repo: args.repo, ownerPublisherId: args.ownerPublisherId, + disconnectedOwnerPublisherId: undefined, githubRepositoryId: args.githubRepositoryId, githubOwnerId: args.githubOwnerId, authorizationStatus: "active", @@ -942,17 +1150,28 @@ async function applyGenericGitHubSkillSourceSyncHandler( displayManifestFetchedAt: args.now, displayManifestStatus: args.snapshot.manifestStatus, displayManifest: args.snapshot.manifest, - updatedAt: args.now, + updatedAt: sourceUpdatedAt, }); const sourceSkills = await ctx.db .query("skills") .withIndex("by_github_source", (q) => q.eq("githubSourceId", sourceId)) .collect(); - const sourceCandidates = await ctx.db - .query("githubSkillCandidates") - .withIndex("by_github_source", (q) => q.eq("githubSourceId", sourceId)) - .collect(); + const [pendingSourceCandidates, legacySourceCandidates] = await Promise.all([ + ctx.db + .query("githubSkillCandidates") + .withIndex("by_github_source_and_lifecycle_status", (q) => + q.eq("githubSourceId", sourceId).eq("lifecycleStatus", "pending"), + ) + .collect(), + ctx.db + .query("githubSkillCandidates") + .withIndex("by_github_source_and_lifecycle_status", (q) => + q.eq("githubSourceId", sourceId).eq("lifecycleStatus", undefined), + ) + .collect(), + ]); + const sourceCandidates = [...pendingSourceCandidates, ...legacySourceCandidates]; const sourceSkillByPath = new Map( sourceSkills.flatMap((skill) => (skill.githubPath ? [[skill.githubPath, skill] as const] : [])), ); @@ -1027,6 +1246,7 @@ async function applyGenericGitHubSkillSourceSyncHandler( ownerPublisherId: args.ownerPublisherId, installKind: "github", githubSourceId: sourceId, + githubCurrentRepo: args.repo, githubPath: discovered.path, githubHasSkillCard: Boolean(discovered.skillCardMarkdownPath), githubCurrentCommit: args.snapshot.commit, @@ -1098,13 +1318,47 @@ async function applyGenericGitHubSkillSourceSyncHandler( skill.githubSourceId === sourceId && skill.githubCurrentStatus === "present" && skill.githubCurrentContentHash === discovered.contentHash; + const sameCurrentPointer = + sameCurrentContent && + (skill.githubCurrentRepo ?? existingSource?.repo) === args.repo && + skill.githubPath === discovered.path && + skill.githubCurrentCommit === args.snapshot.commit; + const unchangedAllowedReappearance = + canAutoReviveGitHubSkill(skill) && + (skill.githubCurrentRepo ?? existingSource?.repo) === args.repo && + skill.githubPath === discovered.path && + skill.githubCurrentCommit === args.snapshot.commit && + skill.githubCurrentContentHash === discovered.contentHash && + (skill.githubScanStatus === "clean" || skill.githubScanStatus === "suspicious"); if (skill.softDeletedAt && !canAutoReviveGitHubSkill(skill)) { stats.unchanged += 1; continue; } - if (sameCurrentContent) { + if (unchangedAllowedReappearance) { + const previousSkill = { ...skill }; + const moderation = githubBackedSkillModeration( + skill.githubScanStatus === "suspicious" ? "suspicious" : "clean", + ); + const patch = { + githubCurrentRepo: args.repo, + githubCurrentStatus: "present" as const, + githubCurrentCheckedAt: args.now, + githubRemovedAt: undefined, + softDeletedAt: undefined, + updatedAt: args.now, + ...moderation, + }; + await ctx.db.patch(skill._id, patch); + const nextSkill = { ...previousSkill, ...patch }; + await syncSkillSearchDigestForSkill(ctx, nextSkill); + await adjustGlobalPublicCountForSkillChange(ctx, previousSkill, nextSkill, args.now); + stats.revived += 1; + continue; + } + + if (sameCurrentPointer) { const previousSkill = { ...skill }; const patch = { displayName: discovered.displayName, @@ -1112,6 +1366,7 @@ async function applyGenericGitHubSkillSourceSyncHandler( icon: iconForDiscoveredGitHubSkill(discovered), ownerUserId: args.ownerUserId, ownerPublisherId: args.ownerPublisherId, + githubCurrentRepo: args.repo, githubPath: discovered.path, githubHasSkillCard: Boolean(discovered.skillCardMarkdownPath), githubCurrentCommit: args.snapshot.commit, @@ -1127,22 +1382,30 @@ async function applyGenericGitHubSkillSourceSyncHandler( continue; } - if (hasAllowedGitHubSource || hasAllowedHostedSource) { + if (hasAllowedGitHubSource || hasAllowedHostedSource || canAutoReviveGitHubSkill(skill)) { const candidateId = await upsertGitHubSkillCandidate(ctx, { skill, sourceId, + repo: args.repo, + currentRepo: skill.githubCurrentRepo ?? existingSource?.repo ?? args.repo, discovered, commit: args.snapshot.commit, now: args.now, }); matchedCandidateIds.add(candidateId); - stats.changed += 1; + if (canAutoReviveGitHubSkill(skill)) stats.revived += 1; + else stats.changed += 1; continue; } const previousSkill = { ...skill }; if (skill.githubPendingCandidateId) { - await ctx.db.delete(skill.githubPendingCandidateId); + await cancelPendingGitHubSkillCandidate( + ctx, + skill.githubPendingCandidateId, + args.now, + "github.source.replaced-before-first-verdict", + ); } const moderation = githubBackedSkillModeration("pending"); const patch = { @@ -1153,6 +1416,7 @@ async function applyGenericGitHubSkillSourceSyncHandler( ownerPublisherId: args.ownerPublisherId, installKind: "github" as const, githubSourceId: sourceId, + githubCurrentRepo: args.repo, githubPath: discovered.path, githubHasSkillCard: Boolean(discovered.skillCardMarkdownPath), githubCurrentCommit: args.snapshot.commit, @@ -1185,7 +1449,12 @@ async function applyGenericGitHubSkillSourceSyncHandler( for (const skill of sourceSkills) { if (matchedSkillIds.has(skill._id)) continue; const previousSkill = { ...skill }; - if (skill.githubPendingCandidateId) await ctx.db.delete(skill.githubPendingCandidateId); + await cancelPendingGitHubSkillCandidate( + ctx, + skill.githubPendingCandidateId, + args.now, + "github.upstream.removed", + ); const removedAt = skill.githubRemovedAt ?? args.now; const patch = { githubCurrentStatus: "missing" as const, @@ -1215,14 +1484,19 @@ async function applyGenericGitHubSkillSourceSyncHandler( githubPendingCandidateId: undefined, updatedAt: args.now, }); + await cancelPendingGitHubSkillCandidate( + ctx, + candidate._id, + args.now, + "github.source.observation-superseded", + ); } - await ctx.db.delete(candidate._id); } await ctx.db.patch(sourceId, { lastSyncIssues: issues, lastSyncInvalidSkills: invalidSkills, - updatedAt: args.now, + updatedAt: sourceUpdatedAt, }); return { ok: true, @@ -1241,17 +1515,88 @@ async function upsertGitHubSkillCandidate( args: { skill: Doc<"skills">; sourceId: Id<"githubSkillSources">; + repo: string; + currentRepo: string; discovered: GitHubSkillSourceMetadataSnapshot["skills"][number]; commit: string; now: number; }, ) { - const existing = args.skill.githubPendingCandidateId + const currentCandidateId = await ensureRetainedCurrentGitHubSkillCandidate(ctx, { + skill: args.skill, + repo: args.currentRepo, + now: args.now, + }); + const exactCandidate = await ctx.db + .query("githubSkillCandidates") + .withIndex("by_skill_and_repo_source_commit_path_hash", (q) => + q + .eq("skillId", args.skill._id) + .eq("githubRepo", args.repo) + .eq("githubSourceId", args.sourceId) + .eq("githubCommit", args.commit) + .eq("githubPath", args.discovered.path) + .eq("githubContentHash", args.discovered.contentHash), + ) + .unique(); + if (exactCandidate) { + if ( + exactCandidate.scanStatus === "pending" || + exactCandidate.scanStatus === "clean" || + exactCandidate.scanStatus === "suspicious" + ) { + await ctx.db.patch(exactCandidate._id, { + lifecycleStatus: "pending", + updatedAt: args.now, + }); + await ctx.db.patch(args.skill._id, { + githubPendingCandidateId: exactCandidate._id, + updatedAt: args.now, + }); + const verdictSourceScanId = await scheduleGitHubSkillVerification(ctx, { + skillId: args.skill._id, + contentHash: exactCandidate.githubContentHash, + scanStatus: exactCandidate.scanStatus, + now: args.now, + candidateId: exactCandidate._id, + }); + if (verdictSourceScanId && verdictSourceScanId !== exactCandidate.verdictSourceScanId) { + await ctx.db.patch(exactCandidate._id, { + verdictSourceScanId, + updatedAt: args.now, + }); + } + if ( + verdictSourceScanId && + exactCandidate.skillMarkdown && + exactCandidate.skillMarkdownPath && + (exactCandidate.scanStatus === "clean" || exactCandidate.scanStatus === "suspicious") + ) { + await applyGitHubSkillVerificationResultHandler(ctx, { + skillId: args.skill._id, + contentHash: exactCandidate.githubContentHash, + githubSkillScanId: verdictSourceScanId, + scanStatus: exactCandidate.scanStatus, + now: args.now, + }); + } + } + return exactCandidate._id; + } + const pendingCandidate = args.skill.githubPendingCandidateId ? await ctx.db.get(args.skill.githubPendingCandidateId) - : await ctx.db - .query("githubSkillCandidates") - .withIndex("by_skill", (q) => q.eq("skillId", args.skill._id)) - .unique(); + : null; + if ( + pendingCandidate && + pendingCandidate.githubSourceId === args.sourceId && + pendingCandidate.githubCommit === args.commit && + pendingCandidate.githubPath === args.discovered.path && + pendingCandidate.githubContentHash === args.discovered.contentHash && + (pendingCandidate.lifecycleStatus === undefined || + pendingCandidate.lifecycleStatus === "pending") + ) { + return pendingCandidate._id; + } const reusableScan = await ctx.db .query("githubSkillScans") .withIndex("by_skill_and_content_hash", (q) => @@ -1259,50 +1604,166 @@ async function upsertGitHubSkillCandidate( ) .unique(); const scanStatus = reusableScan?.status ?? "pending"; - const doc = { - skillId: args.skill._id, - githubSourceId: args.sourceId, - githubPath: args.discovered.path, - githubHasSkillCard: Boolean(args.discovered.skillCardMarkdownPath), - githubCommit: args.commit, - githubContentHash: args.discovered.contentHash, - displayName: args.discovered.displayName, - summary: args.discovered.summary, - icon: iconForDiscoveredGitHubSkill(args.discovered), - upstreamVersion: args.discovered.upstreamVersion, - skillMarkdownPath: undefined, - skillMarkdown: undefined, - skillCardMarkdownPath: undefined, - skillCardMarkdown: undefined, - scanStatus, - updatedAt: args.now, - }; - let candidateId: Id<"githubSkillCandidates">; - if (existing) { - candidateId = existing._id; - await ctx.db.patch(existing._id, doc); - } else { - candidateId = await ctx.db.insert("githubSkillCandidates", { - ...stripUndefined(doc), + const lifecycle = + scanStatus === "malicious" + ? { lifecycleStatus: "rejected" as const, rejectedAt: args.now } + : scanStatus === "failed" + ? { lifecycleStatus: "failed" as const, failedAt: args.now } + : { lifecycleStatus: "pending" as const }; + const candidateId = await ctx.db.insert( + "githubSkillCandidates", + stripUndefined({ + skillId: args.skill._id, + githubSourceId: args.sourceId, + githubRepo: args.repo, + githubPath: args.discovered.path, + githubHasSkillCard: Boolean(args.discovered.skillCardMarkdownPath), + githubCommit: args.commit, + githubContentHash: args.discovered.contentHash, + displayName: args.discovered.displayName, + summary: args.discovered.summary, + icon: iconForDiscoveredGitHubSkill(args.discovered), + upstreamVersion: args.discovered.upstreamVersion, + skillMarkdownPath: undefined, + skillMarkdown: undefined, + skillCardMarkdownPath: undefined, + skillCardMarkdown: undefined, + scanStatus, + ...lifecycle, + verdictSourceScanId: reusableScan?._id, + previousCandidateId: currentCandidateId, createdAt: args.now, - } as Omit, "_id" | "_creationTime">); + updatedAt: args.now, + }) as Omit, "_id" | "_creationTime">, + ); + if (pendingCandidate) { + await ctx.db.patch(pendingCandidate._id, { + lifecycleStatus: "superseded", + supersededByCandidateId: candidateId, + supersededAt: args.now, + updatedAt: args.now, + }); } - await ctx.db.patch(args.skill._id, { - githubPendingCandidateId: candidateId, - updatedAt: args.now, - }); + await ctx.db.patch( + args.skill._id, + scanStatus === "malicious" || scanStatus === "failed" + ? { githubPendingCandidateId: undefined, updatedAt: args.now } + : { githubPendingCandidateId: candidateId, updatedAt: args.now }, + ); if (scanStatus === "pending" || scanStatus === "clean" || scanStatus === "suspicious") { - await scheduleGitHubSkillVerification(ctx, { + const verdictSourceScanId = await scheduleGitHubSkillVerification(ctx, { skillId: args.skill._id, contentHash: args.discovered.contentHash, scanStatus, now: args.now, candidateId, }); + if (verdictSourceScanId && verdictSourceScanId !== reusableScan?._id) { + await ctx.db.patch(candidateId, { + verdictSourceScanId, + updatedAt: args.now, + }); + } } return candidateId; } +async function ensureRetainedCurrentGitHubSkillCandidate( + ctx: MutationCtx, + args: { skill: Doc<"skills">; repo: string; now: number }, +): Promise | undefined> { + if (args.skill.githubCurrentCandidateId) return args.skill.githubCurrentCandidateId; + if ( + args.skill.installKind !== "github" || + !args.skill.githubSourceId || + !args.skill.githubPath || + !args.skill.githubCurrentCommit || + !args.skill.githubCurrentContentHash || + (args.skill.githubCurrentStatus !== "present" && + args.skill.githubCurrentStatus !== "missing") || + (args.skill.githubScanStatus !== "clean" && args.skill.githubScanStatus !== "suspicious") + ) { + return undefined; + } + const githubSourceId = args.skill.githubSourceId; + const githubPath = args.skill.githubPath; + const githubCommit = args.skill.githubCurrentCommit; + const githubContentHash = args.skill.githubCurrentContentHash; + + const existing = await ctx.db + .query("githubSkillCandidates") + .withIndex("by_skill_and_repo_source_commit_path_hash", (q) => + q + .eq("skillId", args.skill._id) + .eq("githubRepo", args.repo) + .eq("githubSourceId", githubSourceId) + .eq("githubCommit", githubCommit) + .eq("githubPath", githubPath) + .eq("githubContentHash", githubContentHash), + ) + .unique(); + if (existing) { + await ctx.db.patch(args.skill._id, { + githubCurrentCandidateId: existing._id, + updatedAt: args.now, + }); + return existing._id; + } + + const [scan, content] = await Promise.all([ + ctx.db + .query("githubSkillScans") + .withIndex("by_skill_and_content_hash", (q) => + q.eq("skillId", args.skill._id).eq("contentHash", githubContentHash), + ) + .unique(), + ctx.db + .query("githubSkillContents") + .withIndex("by_skill", (q) => q.eq("skillId", args.skill._id)) + .unique(), + ]); + const allowedScan = scan?.status === args.skill.githubScanStatus ? scan : null; + const exactContent = + content && + content.githubSourceId === githubSourceId && + content.githubPath === githubPath && + content.githubCommit === githubCommit && + content.githubContentHash === githubContentHash + ? content + : null; + const candidateId = await ctx.db.insert( + "githubSkillCandidates", + stripUndefined({ + skillId: args.skill._id, + githubSourceId, + githubRepo: args.repo, + githubPath, + githubHasSkillCard: args.skill.githubHasSkillCard ?? false, + githubCommit, + githubContentHash, + displayName: args.skill.displayName, + summary: args.skill.summary, + icon: args.skill.icon, + upstreamVersion: args.skill.latestVersionSummary?.version, + skillMarkdownPath: exactContent?.skillMarkdownPath, + skillMarkdown: exactContent?.skillMarkdown, + skillCardMarkdownPath: exactContent?.skillCardMarkdownPath, + skillCardMarkdown: exactContent?.skillCardMarkdown, + scanStatus: args.skill.githubScanStatus, + lifecycleStatus: "promoted", + verdictSourceScanId: allowedScan?._id, + promotedAt: args.skill.updatedAt, + createdAt: args.now, + updatedAt: args.now, + }) as Omit, "_id" | "_creationTime">, + ); + await ctx.db.patch(args.skill._id, { + githubCurrentCandidateId: candidateId, + updatedAt: args.now, + }); + return candidateId; +} + function githubSkillSyncConflictIssue( discovered: GitHubSkillSourceMetadataSnapshot["skills"][number], message: string, @@ -1427,6 +1888,18 @@ export async function upsertGitHubSkillContentHandler( now?: number; }, ) { + const skill = await ctx.db.get(args.skillId); + if ( + !skill || + skill.installKind !== "github" || + skill.githubSourceId !== args.sourceId || + skill.githubPath !== args.discovered.path || + skill.githubCurrentCommit !== args.commit || + skill.githubCurrentContentHash !== args.discovered.contentHash || + skill.githubCurrentStatus !== "present" + ) { + return { ok: true as const, skipped: "stale-current-pointer" as const }; + } await upsertGitHubSkillContent(ctx, { skillId: args.skillId, sourceId: args.sourceId, @@ -1481,6 +1954,7 @@ export async function upsertGitHubSkillCandidateContentHandler( return await applyGitHubSkillVerificationResultHandler(ctx, { skillId: candidate.skillId, contentHash: candidate.githubContentHash, + githubSkillScanId: candidate.verdictSourceScanId, scanStatus: candidate.scanStatus, now, }); @@ -1507,7 +1981,7 @@ async function scheduleGitHubSkillVerification( now: number; candidateId?: Id<"githubSkillCandidates">; }, -) { +): Promise | null> { const scan = await ctx.db .query("githubSkillScans") .withIndex("by_skill_and_content_hash", (q) => @@ -1516,7 +1990,7 @@ async function scheduleGitHubSkillVerification( .unique(); if (args.scanStatus !== "pending") { if (scan?.status !== "pending") { - if (scan) return; + if (scan && scan.status === args.scanStatus) return scan._id; await applyGitHubSkillVerificationResultHandler(ctx, { skillId: args.skillId, contentHash: args.contentHash, @@ -1527,18 +2001,19 @@ async function scheduleGitHubSkillVerification( if (scan?.status === "pending" && scan.skillScanRequestId) { const request = await ctx.db.get(scan.skillScanRequestId); const job = request?.securityScanJobId ? await ctx.db.get(request.securityScanJobId) : null; - if (job?.status === "queued" || job?.status === "running") return; - if (request && request.updatedAt > args.now - GITHUB_SKILL_SCAN_ACTION_LEASE_MS) return; + if (job?.status === "queued" || job?.status === "running") return scan._id; + if (request && request.updatedAt > args.now - GITHUB_SKILL_SCAN_ACTION_LEASE_MS) + return scan._id; } if ( scan?.status === "pending" && !scan.skillScanRequestId && scan.updatedAt > args.now - GITHUB_SKILL_SCAN_ACTION_LEASE_MS ) { - return; + return scan._id; } const skill = await ctx.db.get(args.skillId); - if (!skill) return; + if (!skill) return null; const candidate = args.candidateId ? await ctx.db.get(args.candidateId) : null; const target = candidate && @@ -1561,7 +2036,7 @@ async function scheduleGitHubSkillVerification( commit: skill.githubCurrentCommit, } : null; - if (!target) return; + if (!target) return null; const pendingScanInsert = { githubSourceId: target.sourceId, commit: target.commit, @@ -1569,13 +2044,15 @@ async function scheduleGitHubSkillVerification( status: "pending" as const, updatedAt: args.now, }; + let scanId: Id<"githubSkillScans">; if (scan) { + scanId = scan._id; await ctx.db.patch(scan._id, { ...pendingScanInsert, skillScanRequestId: undefined, }); } else { - await ctx.db.insert("githubSkillScans", { + scanId = await ctx.db.insert("githubSkillScans", { skillId: skill._id, contentHash: args.contentHash, ...pendingScanInsert, @@ -1586,6 +2063,7 @@ async function scheduleGitHubSkillVerification( skillId: args.skillId, contentHash: args.contentHash, }); + return scanId; } export const applyGitHubSkillSourceSyncInternal = internalMutation({ @@ -1596,15 +2074,149 @@ export const applyGitHubSkillSourceSyncInternal = internalMutation({ ownerPublisherId: v.optional(v.id("publishers")), githubRepositoryId: v.optional(v.string()), githubOwnerId: v.optional(v.string()), + expectedSourceUpdatedAt: v.optional(v.union(v.number(), v.null())), snapshot: sourceSnapshotValidator, now: v.optional(v.number()), }, handler: applyGitHubSkillSourceSyncHandler, }); +export async function rollbackGitHubSkillCandidateHandler( + ctx: MutationCtx, + args: { + skillId: Id<"skills">; + targetCandidateId: Id<"githubSkillCandidates">; + confirm: string; + now?: number; + }, +) { + if (args.confirm !== "rollback-github-skill-candidate") { + throw new ConvexError("GitHub Skill Sync rollback confirmation required."); + } + const [skill, target] = await Promise.all([ + ctx.db.get(args.skillId), + ctx.db.get(args.targetCandidateId), + ]); + if (!skill || !target || target.skillId !== skill._id) { + throw new ConvexError("GitHub Skill Sync rollback target not found."); + } + if (skill.githubCurrentCandidateId === target._id) { + return { + ok: true as const, + rolledBack: false as const, + reason: "already-current" as const, + }; + } + if ( + (target.scanStatus !== "clean" && target.scanStatus !== "suspicious") || + !target.verdictSourceScanId || + !target.skillMarkdown || + !target.skillMarkdownPath + ) { + throw new ConvexError("GitHub Skill Sync rollback target lacks its own allowed verdict."); + } + const verdictSourceScan = await ctx.db.get(target.verdictSourceScanId); + if ( + !verdictSourceScan || + verdictSourceScan.skillId !== skill._id || + verdictSourceScan.contentHash !== target.githubContentHash || + verdictSourceScan.status !== target.scanStatus + ) { + throw new ConvexError("GitHub Skill Sync rollback verdict no longer matches the target."); + } + const source = await ctx.db.get(target.githubSourceId); + if (!source || source.authorizationStatus === "revoked") { + throw new ConvexError("GitHub Skill Sync rollback source is not authorized."); + } + + const now = args.now ?? Date.now(); + const previousSkill = { ...skill }; + const currentCandidate = skill.githubCurrentCandidateId + ? await ctx.db.get(skill.githubCurrentCandidateId) + : null; + const moderation = githubBackedSkillModeration(target.scanStatus); + await upsertGitHubSkillContent(ctx, { + skillId: skill._id, + sourceId: target.githubSourceId, + discovered: { + slug: skill.slug, + displayName: target.displayName, + summary: target.summary, + upstreamVersion: target.upstreamVersion, + path: target.githubPath, + skillMarkdownPath: target.skillMarkdownPath, + skillMarkdown: target.skillMarkdown, + skillCardMarkdownPath: target.skillCardMarkdownPath, + skillCardMarkdown: target.skillCardMarkdown, + contentHash: target.githubContentHash, + }, + commit: target.githubCommit, + now, + }); + const patch = { + displayName: target.displayName, + summary: target.summary, + icon: target.icon, + installKind: "github" as const, + githubSourceId: target.githubSourceId, + githubCurrentRepo: target.githubRepo ?? source.repo, + githubPath: target.githubPath, + githubHasSkillCard: target.githubHasSkillCard, + githubCurrentCommit: target.githubCommit, + githubCurrentContentHash: target.githubContentHash, + githubCurrentStatus: "present" as const, + githubCurrentCheckedAt: now, + githubScanStatus: target.scanStatus, + githubRemovedAt: undefined, + githubCurrentCandidateId: target._id, + githubPendingCandidateId: undefined, + latestVersionId: undefined, + latestVersionSummary: latestGitHubVersionSummary(target.upstreamVersion, now), + softDeletedAt: undefined, + updatedAt: now, + ...moderation, + }; + if (currentCandidate) { + await ctx.db.patch(currentCandidate._id, { + lifecycleStatus: "rolled_back", + rolledBackAt: now, + updatedAt: now, + }); + } + if (skill.githubPendingCandidateId && skill.githubPendingCandidateId !== target._id) { + await cancelPendingGitHubSkillCandidate( + ctx, + skill.githubPendingCandidateId, + now, + "github.rollback", + ); + } + await ctx.db.patch(target._id, { + lifecycleStatus: "promoted", + promotedAt: now, + updatedAt: now, + }); + await ctx.db.patch(skill._id, patch); + const nextSkill = { ...previousSkill, ...patch }; + await syncSkillSearchDigestForSkill(ctx, nextSkill); + await adjustGlobalPublicCountForSkillChange(ctx, previousSkill, nextSkill, now); + return { ok: true as const, rolledBack: true as const }; +} + +export const rollbackGitHubSkillCandidateInternal = internalMutation({ + args: { + skillId: v.id("skills"), + targetCandidateId: v.id("githubSkillCandidates"), + confirm: v.string(), + now: v.optional(v.number()), + }, + handler: rollbackGitHubSkillCandidateHandler, +}); + export type ApplyGitHubSkillVerificationResultArgs = { skillId: Id<"skills">; contentHash: string; + githubSkillScanId?: Id<"githubSkillScans">; scanStatus: GitHubSkillScanStatus; now?: number; }; @@ -1621,25 +2233,79 @@ export async function applyGitHubSkillVerificationResultHandler( ? await ctx.db.get(skill.githubPendingCandidateId) : null; if (candidate?.githubContentHash === args.contentHash) { + const verdictSourceScan = args.githubSkillScanId + ? await ctx.db.get(args.githubSkillScanId) + : null; + const isAllowedVerdict = args.scanStatus === "clean" || args.scanStatus === "suspicious"; + if (isAllowedVerdict && !args.githubSkillScanId) { + return { + ok: true as const, + skipped: candidate.verdictSourceScanId + ? ("stale-candidate-verdict" as const) + : ("missing-candidate-verdict" as const), + }; + } + if ( + (candidate.verdictSourceScanId && candidate.verdictSourceScanId !== args.githubSkillScanId) || + (args.githubSkillScanId && + (!verdictSourceScan || + verdictSourceScan.skillId !== candidate.skillId || + verdictSourceScan.contentHash !== candidate.githubContentHash || + verdictSourceScan.status !== args.scanStatus)) + ) { + return { ok: true as const, skipped: "stale-candidate-verdict" as const }; + } const now = args.now ?? Date.now(); + const lifecyclePatch = + args.scanStatus === "malicious" + ? { lifecycleStatus: "rejected" as const, rejectedAt: now } + : args.scanStatus === "failed" + ? { lifecycleStatus: "failed" as const, failedAt: now } + : { lifecycleStatus: "pending" as const }; await ctx.db.patch(candidate._id, { scanStatus: args.scanStatus, + ...(args.githubSkillScanId && !candidate.verdictSourceScanId + ? { verdictSourceScanId: args.githubSkillScanId } + : {}), + ...lifecyclePatch, updatedAt: now, }); if (args.scanStatus !== "clean" && args.scanStatus !== "suspicious") { + if (args.scanStatus === "malicious") { + await ctx.db.patch(skill._id, { + githubPendingCandidateId: undefined, + updatedAt: now, + }); + } return { ok: true as const, promoted: false }; } if (!candidate.skillMarkdown || !candidate.skillMarkdownPath) { - return { ok: true as const, skipped: "candidate-content-not-cached" as const }; + return { + ok: true as const, + skipped: "candidate-content-not-cached" as const, + }; } + const isAutomaticReappearance = + canAutoReviveGitHubSkill(skill) && skill.moderationReason === "github.upstream.removed"; if ( - skill.softDeletedAt || - skill.moderationStatus === "hidden" || + (!isAutomaticReappearance && skill.softDeletedAt) || + (!isAutomaticReappearance && skill.moderationStatus === "hidden") || skill.moderationStatus === "removed" ) { - return { ok: true as const, skipped: "skill-no-longer-eligible" as const }; + return { + ok: true as const, + skipped: "skill-no-longer-eligible" as const, + }; } const previousSkill = { ...skill }; + const candidateRepo = + candidate.githubRepo ?? (await ctx.db.get(candidate.githubSourceId))?.repo; + if (!candidateRepo) { + return { + ok: true as const, + skipped: "candidate-source-missing" as const, + }; + } const moderation = githubBackedSkillModeration(args.scanStatus); const patch = { displayName: candidate.displayName, @@ -1647,6 +2313,7 @@ export async function applyGitHubSkillVerificationResultHandler( icon: candidate.icon, installKind: "github" as const, githubSourceId: candidate.githubSourceId, + githubCurrentRepo: candidateRepo, githubPath: candidate.githubPath, githubHasSkillCard: candidate.githubHasSkillCard, githubCurrentCommit: candidate.githubCommit, @@ -1655,6 +2322,7 @@ export async function applyGitHubSkillVerificationResultHandler( githubCurrentCheckedAt: now, githubScanStatus: args.scanStatus, githubRemovedAt: undefined, + githubCurrentCandidateId: candidate._id, githubPendingCandidateId: undefined, latestVersionId: undefined, latestVersionSummary: latestGitHubVersionSummary(candidate.upstreamVersion, now), @@ -1680,11 +2348,27 @@ export async function applyGitHubSkillVerificationResultHandler( commit: candidate.githubCommit, now, }); + const previousCandidate = skill.githubCurrentCandidateId + ? await ctx.db.get(skill.githubCurrentCandidateId) + : null; + if (previousCandidate && previousCandidate._id !== candidate._id) { + await ctx.db.patch(previousCandidate._id, { + lifecycleStatus: "superseded", + supersededByCandidateId: candidate._id, + supersededAt: now, + updatedAt: now, + }); + } await ctx.db.patch(skill._id, patch); const nextSkill = { ...previousSkill, ...patch }; await syncSkillSearchDigestForSkill(ctx, nextSkill); await adjustGlobalPublicCountForSkillChange(ctx, previousSkill, nextSkill, now); - await ctx.db.delete(candidate._id); + await ctx.db.patch(candidate._id, { + lifecycleStatus: "promoted", + previousCandidateId: skill.githubCurrentCandidateId, + promotedAt: now, + updatedAt: now, + }); return { ok: true as const, promoted: true }; } if (skill.installKind !== "github") { @@ -1702,6 +2386,20 @@ export async function applyGitHubSkillVerificationResultHandler( }; } + if (args.githubSkillScanId) { + const scan = await ctx.db.get(args.githubSkillScanId); + if ( + !scan || + scan.skillId !== skill._id || + scan.githubSourceId !== skill.githubSourceId || + scan.path !== skill.githubPath || + scan.commit !== skill.githubCurrentCommit || + scan.contentHash !== skill.githubCurrentContentHash + ) { + return { ok: true as const, skipped: "stale-current-scan" as const }; + } + } + const now = args.now ?? Date.now(); const promote = args.scanStatus === "clean"; const moderation = githubBackedSkillModeration(args.scanStatus); @@ -1723,6 +2421,7 @@ export const applyGitHubSkillVerificationResultInternal = internalMutation({ args: { skillId: v.id("skills"), contentHash: v.string(), + githubSkillScanId: v.optional(v.id("githubSkillScans")), scanStatus: githubSkillScanStatusValidator, now: v.optional(v.number()), }, @@ -1805,10 +2504,16 @@ export async function verifyGitHubSkillHandler( await ctx.runMutation(internal.githubSkillSync.applyGitHubSkillVerificationResultInternal, { skillId: target.skill._id, contentHash: args.contentHash, + githubSkillScanId: prepared.scanId, scanStatus: prepared.scanStatus, }); } - return prepared ?? { ok: true as const, skipped: "scan-request-not-created" as const }; + return ( + prepared ?? { + ok: true as const, + skipped: "scan-request-not-created" as const, + } + ); } let chunkIndex = 0; @@ -1875,6 +2580,7 @@ export async function configurePublicGitHubSkillSourceHandler( ownerPublisherId: args.ownerPublisherId, githubRepositoryId: revalidatedMetadata.repositoryId, githubOwnerId: revalidatedMetadata.ownerId, + expectedSourceUpdatedAt: setup.existingSource?.updatedAt ?? null, snapshot, }); } @@ -1906,6 +2612,7 @@ async function applyFetchedGitHubSkillSourceSnapshot( ownerPublisherId?: Id<"publishers">; githubRepositoryId?: string; githubOwnerId?: string; + expectedSourceUpdatedAt?: number | null; snapshot: GitHubSkillSourceSnapshot; }, ) { @@ -1919,9 +2626,11 @@ async function applyFetchedGitHubSkillSourceSnapshot( ownerPublisherId: args.ownerPublisherId, githubRepositoryId: args.githubRepositoryId, githubOwnerId: args.githubOwnerId, + expectedSourceUpdatedAt: args.expectedSourceUpdatedAt, snapshot: toGitHubSkillSourceMetadataSnapshot(args.snapshot), }, )) as SyncOneResult; + if (result.skipped === "stale-source-observation") return result; await persistGitHubSkillContentsForSnapshot(ctx, result, args.snapshot); return result; } @@ -2049,6 +2758,7 @@ export const syncGitHubSkillSource: ReturnType = action({ ownerPublisherId, githubRepositoryId: revalidatedMetadata.repositoryId, githubOwnerId: revalidatedMetadata.ownerId, + expectedSourceUpdatedAt: source?.updatedAt ?? null, snapshot, }); }, @@ -2066,7 +2776,10 @@ export async function syncGitHubSkillSourcesHandler( MAX_SOURCE_SYNC_BATCH_SIZE, ); const genericEnabled = getRuntimeRolloutCapabilities().githubSkillSync.runtimeEnabled; - logEvent(Events.GitHubSkillSourceSyncStarted, { startedAt, cursor: args.cursor ?? null }); + logEvent(Events.GitHubSkillSourceSyncStarted, { + startedAt, + cursor: args.cursor ?? null, + }); const page = (await ctx.runQuery(internal.githubSkillSync.listSourcesForSyncInternal, { cursor: args.cursor ?? null, batchSize, @@ -2122,6 +2835,7 @@ export async function syncGitHubSkillSourcesHandler( ownerPublisherId: source.ownerPublisherId, githubRepositoryId: revalidatedMetadata.repositoryId, githubOwnerId: revalidatedMetadata.ownerId, + expectedSourceUpdatedAt: source.updatedAt, snapshot, }); results.push(result); diff --git a/convex/lib/installResolver.ts b/convex/lib/installResolver.ts index 629d543d..ec09ac82 100644 --- a/convex/lib/installResolver.ts +++ b/convex/lib/installResolver.ts @@ -13,6 +13,7 @@ export type InstallResolverSkill = { displayName: string; latestVersionSummary?: { version: string } | null; installKind?: "github"; + githubCurrentRepo?: string; githubPath?: string; githubCurrentCommit?: string; githubCurrentContentHash?: string; @@ -106,7 +107,8 @@ export function buildSkillInstallResolution({ if (isSecurityScanStatusBlockedFromPublic(skill.githubScanStatus)) { return block(skill.slug, "github_scan_failed", 403); } - if (!source || !skill.githubPath) { + const repo = skill.githubCurrentRepo ?? source?.repo; + if (!repo || !skill.githubPath) { return block(skill.slug, "github_source_missing", 409); } if ( @@ -128,11 +130,11 @@ export function buildSkillInstallResolution({ slug: skill.slug, installKind: "github", github: { - repo: source.repo, + repo, path: skill.githubPath, commit: skill.githubCurrentCommit, contentHash: skill.githubCurrentContentHash, - sourceUrl: buildGitHubTreeUrl(source.repo, skill.githubCurrentCommit, skill.githubPath), + sourceUrl: buildGitHubTreeUrl(repo, skill.githubCurrentCommit, skill.githubPath), }, }; } diff --git a/convex/lib/retentionPolicy.test.ts b/convex/lib/retentionPolicy.test.ts index 65fbb476..780d6c37 100644 --- a/convex/lib/retentionPolicy.test.ts +++ b/convex/lib/retentionPolicy.test.ts @@ -59,6 +59,15 @@ describe("retention policies", () => { }); }); + it("keeps GitHub Skill Sync candidates and verdicts as durable rollback history", () => { + expect(getRetentionPolicy("githubSkillCandidates")).toMatchObject({ + classification: "permanent", + }); + expect(getRetentionPolicy("githubSkillScans")).toMatchObject({ + classification: "permanent", + }); + }); + it("documents publisher abuse signals as durable review evidence", () => { expect(getRetentionPolicy("publisherAbuseSignals")).toMatchObject({ classification: "permanent", diff --git a/convex/lib/retentionPolicy.ts b/convex/lib/retentionPolicy.ts index e975e61f..21d0e1cf 100644 --- a/convex/lib/retentionPolicy.ts +++ b/convex/lib/retentionPolicy.ts @@ -108,8 +108,10 @@ export const RETENTION_POLICIES = { officialPublishers: permanent("Manual official publisher assignments."), githubSkillSources: permanent("Tracked GitHub source configuration."), githubSkillContents: derived("Cached GitHub source content snapshots.", "githubSkillSources"), - githubSkillCandidates: derived("Pending exact GitHub source candidates.", "githubSkillSources"), - githubSkillScans: derived("Cached GitHub source scan state.", "githubSkillSources"), + githubSkillCandidates: permanent( + "Immutable GitHub source candidate, promotion, rejection, and rollback history.", + ), + githubSkillScans: permanent("Durable exact-content GitHub security verdict history."), skills: permanent("Canonical skill records."), skillSlugAliases: permanent("Historical slug routing aliases."), packages: permanent("Canonical package records."), diff --git a/convex/schema.ts b/convex/schema.ts index 64446a65..bfa4f8af 100644 --- a/convex/schema.ts +++ b/convex/schema.ts @@ -384,6 +384,9 @@ const githubSkillSourceIssueValidator = v.object({ const githubSkillSources = defineTable({ repo: v.string(), ownerPublisherId: v.optional(v.id("publishers")), + // Retains the last owner across disconnects so generic sync cannot silently + // transfer a source by reconnecting its ownerless row. + disconnectedOwnerPublisherId: v.optional(v.id("publishers")), githubRepositoryId: v.optional(v.string()), githubOwnerId: v.optional(v.string()), authorizationStatus: v.optional(v.union(v.literal("active"), v.literal("revoked"))), @@ -503,6 +506,8 @@ const githubSkillCurrentStatusValidator = v.union( const githubSkillCandidates = defineTable({ skillId: v.id("skills"), githubSourceId: v.id("githubSkillSources"), + // Optional only for rows created before version-safe candidate history landed. + githubRepo: v.optional(v.string()), githubPath: v.string(), githubHasSkillCard: v.boolean(), githubCommit: v.string(), @@ -516,12 +521,43 @@ const githubSkillCandidates = defineTable({ skillCardMarkdownPath: v.optional(v.string()), skillCardMarkdown: v.optional(v.string()), scanStatus: githubSkillScanStatusValidator, + lifecycleStatus: v.optional( + v.union( + v.literal("pending"), + v.literal("promoted"), + v.literal("superseded"), + v.literal("rejected"), + v.literal("failed"), + v.literal("canceled"), + v.literal("rolled_back"), + ), + ), + verdictSourceScanId: v.optional(v.id("githubSkillScans")), + previousCandidateId: v.optional(v.id("githubSkillCandidates")), + supersededByCandidateId: v.optional(v.id("githubSkillCandidates")), + promotedAt: v.optional(v.number()), + supersededAt: v.optional(v.number()), + rejectedAt: v.optional(v.number()), + failedAt: v.optional(v.number()), + canceledAt: v.optional(v.number()), + rolledBackAt: v.optional(v.number()), + cancellationReason: v.optional(v.string()), createdAt: v.number(), updatedAt: v.number(), }) .index("by_skill", ["skillId"]) .index("by_skill_and_content_hash", ["skillId", "githubContentHash"]) - .index("by_github_source", ["githubSourceId"]); + .index("by_skill_and_commit_and_content_hash", ["skillId", "githubCommit", "githubContentHash"]) + .index("by_skill_and_repo_source_commit_path_hash", [ + "skillId", + "githubRepo", + "githubSourceId", + "githubCommit", + "githubPath", + "githubContentHash", + ]) + .index("by_github_source", ["githubSourceId"]) + .index("by_github_source_and_lifecycle_status", ["githubSourceId", "lifecycleStatus"]); const githubSkillScans = defineTable({ skillId: v.id("skills"), @@ -842,6 +878,7 @@ const skills = defineTable({ forkOf: forkOfValidator, installKind: v.optional(v.literal("github")), githubSourceId: v.optional(v.id("githubSkillSources")), + githubCurrentRepo: v.optional(v.string()), githubPath: v.optional(v.string()), githubHasSkillCard: v.optional(v.boolean()), githubCurrentCommit: v.optional(v.string()), @@ -850,6 +887,7 @@ const skills = defineTable({ githubCurrentCheckedAt: v.optional(v.number()), githubScanStatus: v.optional(githubSkillScanStatusValidator), githubRemovedAt: v.optional(v.number()), + githubCurrentCandidateId: v.optional(v.id("githubSkillCandidates")), githubPendingCandidateId: v.optional(v.id("githubSkillCandidates")), latestVersionId: v.optional(v.id("skillVersions")), latestVersionSummary: v.optional( diff --git a/convex/securityScan.test.ts b/convex/securityScan.test.ts index 1aebe763..5ab36c17 100644 --- a/convex/securityScan.test.ts +++ b/convex/securityScan.test.ts @@ -1824,6 +1824,9 @@ describe("securityScan", () => { "githubSkillScans:github": { _id: "githubSkillScans:github", skillId: "skills:github", + githubSourceId: "githubSkillSources:github", + commit: "a".repeat(40), + path: "skills/github-skill", contentHash: "content-hash", status: "pending", skillScanRequestId: "skillScanRequests:github", @@ -2815,12 +2818,17 @@ describe("securityScan", () => { "githubSkillScans:github": { _id: "githubSkillScans:github", skillId: "skills:github", + githubSourceId: "githubSkillSources:github", contentHash: "content-hash", + commit: "a".repeat(40), + path: "skills/github-skill", status: "failed", }, "skills:github": { _id: "skills:github", installKind: "github", + githubSourceId: "githubSkillSources:github", + githubPath: "skills/github-skill", githubCurrentStatus: "present", githubCurrentCommit: "a".repeat(40), githubCurrentContentHash: "content-hash", diff --git a/convex/securityScan.ts b/convex/securityScan.ts index de888305..40149a79 100644 --- a/convex/securityScan.ts +++ b/convex/securityScan.ts @@ -2335,6 +2335,7 @@ export const recordGitHubSkillScanResultInternal = internalMutation({ return await applyGitHubSkillVerificationResultHandler(ctx, { skillId: scan.skillId, contentHash: scan.contentHash, + githubSkillScanId: scan._id, scanStatus: args.scanStatus, now, }); @@ -3385,6 +3386,7 @@ export const requeueFailedSecurityScanJobsInternal = internalMutation({ await applyGitHubSkillVerificationResultHandler(ctx, { skillId: scan.skillId, contentHash: scan.contentHash, + githubSkillScanId: scan._id, scanStatus: "pending", now, }); diff --git a/convex/skills.ts b/convex/skills.ts index 2995f9ab..d166b633 100644 --- a/convex/skills.ts +++ b/convex/skills.ts @@ -2675,7 +2675,7 @@ export const getBySlug = query({ const forkOf = await loadPublicSkillReference(ctx, skill.forkOf?.skillId); const canonical = await loadPublicSkillReference(ctx, skill.canonicalSkillId); const githubSource = skill.githubSourceId ? await ctx.db.get(skill.githubSourceId) : null; - const githubSourceRepo = githubSource?.repo; + const githubSourceRepo = skill.githubCurrentRepo ?? githubSource?.repo; const publicSkill = toPublicSkill({ ...skill, badges }); @@ -2844,7 +2844,7 @@ export const getGitHubDownloadTargetInternal = internalQuery({ return { installKind: "github" as const, - repo: source?.repo ?? null, + repo: skill.githubCurrentRepo ?? source?.repo ?? null, path: skill.githubPath ?? null, commit: skill.githubCurrentCommit ?? null, contentHash: skill.githubCurrentContentHash ?? null, @@ -10289,7 +10289,11 @@ export const getGitHubSkillContent = query({ const source = await ctx.db.get(content.githubSourceId); const resultSource = source - ? buildGitHubMarkdownSourceBaseUrl(source.repo, content.githubCommit, content.githubPath) + ? buildGitHubMarkdownSourceBaseUrl( + skill.githubCurrentRepo ?? source.repo, + content.githubCommit, + content.githubPath, + ) : undefined; if (args.kind === "skill-card") { diff --git a/specs/github-backed-skills.md b/specs/github-backed-skills.md index 97915ee3..ec909522 100644 --- a/specs/github-backed-skills.md +++ b/specs/github-backed-skills.md @@ -50,18 +50,21 @@ skills: This table is not an install artifact store. OpenClaw must not install from `githubSkillContents`. -`githubSkillCandidates` stores an exact pending replacement for a canonical -skill while its currently allowed source remains active: +`githubSkillCandidates` stores immutable exact replacement history for a +canonical skill while its currently allowed source remains active: - canonical `skillId` - immutable source repository, path, commit, and folder content hash - bounded display Markdown fetched from that exact commit -- the ClawHub scan state for that exact content +- the ClawHub scan state and durable verdict-source scan identity for that + exact content +- lifecycle state for pending, promoted, superseded, rejected, failed, + canceled, and rolled-back observations Candidates do not create `skillVersions`. A clean or suspicious exact verdict -promotes the candidate onto the existing skill row and then deletes the -candidate. Failed, malicious, stale, removed, or disconnected candidates never -replace the active source. +promotes only the active candidate bound to that verdict onto the existing skill +row. Candidate rows are retained for audit and rollback. Failed, malicious, +stale, removed, or disconnected candidates never replace the active source. ## Dark skills.sh discovery metadata @@ -111,6 +114,7 @@ duplicated alias/adoption row is created for GitHub-backed content. - `installKind: "github"` - `githubSourceId` +- `githubCurrentRepo` - `githubPath` - `githubHasSkillCard` - `githubCurrentCommit` @@ -271,6 +275,17 @@ allowed. Promotion patches the existing canonical skill row in place, preserving its slug, routes, ClawHub metrics, bookmarks, prior hosted versions, and audit relationships. +Repository redirects, commit-only moves, and path-only moves use the same +candidate boundary even when the folder content hash is unchanged. The active +skill keeps its previously allowed repository/path/commit until the replacement +candidate can reuse or receive an allowed verdict. Source mutations carry the +source row version observed before fetching, so a slower stale observation +cannot overwrite a newer synchronized state. Before the first replacement is +created, an older allowed GitHub pointer is materialized into retained candidate +history so it remains available for audit, pinned archive resolution, and +rollback. Late content persistence also rechecks the active pointer before +writing, preventing same-content races from restoring an older commit. + When verification succeeds cleanly: - persist the completed ClawScan, SkillSpector, and static findings on a durable @@ -321,12 +336,13 @@ If the upstream path disappears: The row may remain for audit/history, but users must not silently install an old ClawHub-cached revision after upstream removed or changed it. -Removing a selected repository also removes its pending candidates before -deleting the source, so an in-flight callback cannot promote disconnected -content. Active source-backed skills become missing and hidden. Re-enrollment or -upstream reappearance may revive the same canonical skill row, but the -reappeared exact content returns to pending and must pass scanning before -installation. +Disconnecting a selected repository cancels its pending candidates and revokes +the source, so an in-flight callback cannot promote disconnected content. The +source, candidates, and verdict rows remain as audit history while the publisher +ownership link is cleared. Active source-backed skills become missing and +hidden. Re-enrollment or upstream reappearance may revive the same canonical +skill row; unchanged previously allowed content can reactivate idempotently, +while changed content returns through the candidate gate before installation. ## Install Resolver @@ -354,7 +370,10 @@ returns: ``` OpenClaw downloads the GitHub archive for that commit and extracts only the skill -path. The local lock/origin version is the commit SHA. +path. The local lock/origin version is the commit SHA. Current descriptors and +detail links use the skill's promoted repository pointer, not a newer repository +name still waiting on candidate promotion. Pinned historical archives resolve +repository and path from retained promoted candidate history. Controlled unclaimed skills.sh catalog entries use the repository-qualified reference `skills-sh///`. The colon form