diff --git a/convex/_generated/api.d.ts b/convex/_generated/api.d.ts index 659fca5f..4767d422 100644 --- a/convex/_generated/api.d.ts +++ b/convex/_generated/api.d.ts @@ -65,6 +65,7 @@ import type * as lib_embeddings from "../lib/embeddings.js"; import type * as lib_githubAccount from "../lib/githubAccount.js"; import type * as lib_githubActionsOidc from "../lib/githubActionsOidc.js"; import type * as lib_githubAuth from "../lib/githubAuth.js"; +import type * as lib_githubHandoff from "../lib/githubHandoff.js"; import type * as lib_githubIdentity from "../lib/githubIdentity.js"; import type * as lib_githubImport from "../lib/githubImport.js"; import type * as lib_githubProfileSync from "../lib/githubProfileSync.js"; @@ -216,6 +217,7 @@ declare const fullApi: ApiFromModules<{ "lib/githubAccount": typeof lib_githubAccount; "lib/githubActionsOidc": typeof lib_githubActionsOidc; "lib/githubAuth": typeof lib_githubAuth; + "lib/githubHandoff": typeof lib_githubHandoff; "lib/githubIdentity": typeof lib_githubIdentity; "lib/githubImport": typeof lib_githubImport; "lib/githubProfileSync": typeof lib_githubProfileSync; diff --git a/convex/downloads.test.ts b/convex/downloads.test.ts index 2fe3067b..d0d96182 100644 --- a/convex/downloads.test.ts +++ b/convex/downloads.test.ts @@ -493,4 +493,292 @@ describe("downloads helpers", () => { }), ); }); + + it.each(["clean", "suspicious"] as const)( + "returns a metered public GitHub handoff descriptor for %s scan without scan metadata", + async (scanStatus) => { + const commit = "1".repeat(40); + const runQuery = vi.fn(async (_query: unknown, args: Record) => { + if (isRateLimitArgs(args)) return okRate(); + if ("slug" in args) { + return { + skill: { + _id: "skills:github", + ownerUserId: "users:1", + slug: "aiq-deploy", + tags: {}, + latestVersionId: undefined, + installKind: "github", + githubPath: "skills/aiq-deploy", + githubCurrentCommit: commit, + githubCurrentContentHash: "hash-aiq-deploy", + githubCurrentStatus: "present", + githubScanStatus: scanStatus, + }, + moderationInfo: null, + }; + } + if ("skillId" in args) { + return { + installKind: "github", + repo: "NVIDIA/skills", + path: "skills/aiq-deploy", + commit, + contentHash: "hash-aiq-deploy", + currentStatus: "present", + scanStatus, + removedAt: null, + }; + } + return null; + }); + const runMutation = vi.fn(async (_mutation: unknown, args: Record) => { + if (isRateLimitArgs(args)) return okRate(); + return null; + }); + const runAfter = vi.fn(); + const storageGet = vi.fn(); + + const response = await downloadZipHandler( + { + runQuery, + runMutation, + scheduler: { runAfter }, + storage: { get: storageGet }, + } as unknown as ActionCtx, + new Request("https://example.com/api/v1/download?slug=aiq-deploy", { + headers: { "cf-connecting-ip": "1.2.3.4" }, + }), + ); + + expect(response.status).toBe(200); + expect(response.headers.get("Content-Type")).toBe("application/json"); + expect(storageGet).not.toHaveBeenCalled(); + + const body = await response.json(); + expect(body).toEqual({ + sourceRef: "public-github", + repo: "NVIDIA/skills", + commit, + path: "skills/aiq-deploy", + contentHash: "hash-aiq-deploy", + archiveUrl: `https://api.github.com/repos/NVIDIA/skills/zipball/${commit}`, + }); + expect(body).not.toHaveProperty("scan"); + expect(body).not.toHaveProperty("scanStatus"); + + expect(runAfter).toHaveBeenCalledWith( + expect.any(Number), + expect.anything(), + expect.objectContaining({ + target: { kind: "skill", id: "skills:github" }, + identityKind: "ip", + identityHash: expect.stringMatching(/^[a-f0-9]{64}$/), + }), + ); + }, + ); + + it.each([ + { + name: "pending scan", + skill: { githubCurrentStatus: "present", githubScanStatus: "pending" }, + source: { repo: "NVIDIA/skills" }, + status: 423, + message: "GitHub-backed skill security scan is in progress.", + }, + { + name: "failed scan", + skill: { githubCurrentStatus: "present", githubScanStatus: "failed" }, + source: { repo: "NVIDIA/skills" }, + status: 403, + message: "GitHub-backed skill failed ClawHub security scanning.", + }, + { + name: "malicious scan", + skill: { githubCurrentStatus: "present", githubScanStatus: "malicious" }, + source: { repo: "NVIDIA/skills" }, + status: 403, + message: "GitHub-backed skill failed ClawHub security scanning.", + }, + { + name: "missing upstream path", + skill: { githubCurrentStatus: "missing", githubScanStatus: "clean" }, + source: { repo: "NVIDIA/skills" }, + status: 410, + message: "GitHub-backed skill path is missing upstream.", + }, + { + name: "removed upstream path", + skill: { githubCurrentStatus: "present", githubRemovedAt: 123, githubScanStatus: "clean" }, + source: { repo: "NVIDIA/skills" }, + status: 410, + message: "GitHub-backed skill has been removed upstream.", + }, + { + name: "unknown upstream freshness", + skill: { githubCurrentStatus: "unknown", githubScanStatus: "clean" }, + source: { repo: "NVIDIA/skills" }, + status: 423, + message: "GitHub-backed skill needs an upstream freshness check before download.", + }, + { + name: "incomplete source", + skill: { githubCurrentStatus: "present", githubScanStatus: "clean" }, + source: null, + status: 409, + message: "GitHub-backed skill source metadata is incomplete.", + }, + ])( + "blocks $name GitHub handoffs without scheduling metrics", + async ({ skill, source, status, message }) => { + const runQuery = vi.fn(async (_query: unknown, args: Record) => { + if (isRateLimitArgs(args)) return okRate(); + if ("slug" in args) { + return { + skill: { + _id: "skills:github", + ownerUserId: "users:1", + slug: "aiq-deploy", + tags: {}, + latestVersionId: undefined, + installKind: "github", + githubPath: "skills/aiq-deploy", + githubCurrentCommit: "1".repeat(40), + githubCurrentContentHash: "hash-aiq-deploy", + ...skill, + }, + moderationInfo: null, + }; + } + if ("skillId" in args) { + if (!source) return null; + return { + installKind: "github", + repo: source.repo, + path: "skills/aiq-deploy", + commit: "1".repeat(40), + contentHash: "hash-aiq-deploy", + currentStatus: "present", + scanStatus: "clean", + removedAt: null, + ...skill, + ...(skill.githubCurrentStatus ? { currentStatus: skill.githubCurrentStatus } : {}), + ...(skill.githubScanStatus ? { scanStatus: skill.githubScanStatus } : {}), + ...(skill.githubRemovedAt ? { removedAt: skill.githubRemovedAt } : {}), + }; + } + return null; + }); + const runMutation = vi.fn(async (_mutation: unknown, args: Record) => { + if (isRateLimitArgs(args)) return okRate(); + return null; + }); + const runAfter = vi.fn(); + + const response = await downloadZipHandler( + { + runQuery, + runMutation, + scheduler: { runAfter }, + storage: { get: vi.fn() }, + } as unknown as ActionCtx, + new Request("https://example.com/api/v1/download?slug=aiq-deploy", { + headers: { "cf-connecting-ip": "1.2.3.4" }, + }), + ); + + expect(response.status).toBe(status); + expect(await response.text()).toBe(message); + expect(runAfter).not.toHaveBeenCalled(); + }, + ); + + it.each([ + { + name: "hidden by moderators", + moderationInfo: { + isPendingScan: false, + isMalwareBlocked: false, + isHiddenByMod: true, + isRemoved: false, + }, + status: 403, + message: "This skill is currently unavailable.", + }, + { + name: "removed by moderators", + moderationInfo: { + isPendingScan: false, + isMalwareBlocked: false, + isHiddenByMod: false, + isRemoved: true, + }, + status: 410, + message: "This skill has been removed by a moderator.", + }, + ])( + "blocks $name GitHub handoffs before source descriptor creation", + async ({ moderationInfo, status, message }) => { + const runQuery = vi.fn(async (_query: unknown, args: Record) => { + if (isRateLimitArgs(args)) return okRate(); + if ("slug" in args) { + return { + skill: { + _id: "skills:github", + ownerUserId: "users:1", + slug: "aiq-deploy", + tags: {}, + latestVersionId: undefined, + installKind: "github", + githubPath: "skills/aiq-deploy", + githubCurrentCommit: "1".repeat(40), + githubCurrentContentHash: "hash-aiq-deploy", + githubCurrentStatus: "present", + githubScanStatus: "clean", + }, + moderationInfo, + }; + } + if ("skillId" in args) { + return { + installKind: "github", + repo: "NVIDIA/skills", + path: "skills/aiq-deploy", + commit: "1".repeat(40), + contentHash: "hash-aiq-deploy", + currentStatus: "present", + scanStatus: "clean", + removedAt: null, + }; + } + return null; + }); + const runMutation = vi.fn(async (_mutation: unknown, args: Record) => { + if (isRateLimitArgs(args)) return okRate(); + return null; + }); + const runAfter = vi.fn(); + + const response = await downloadZipHandler( + { + runQuery, + runMutation, + scheduler: { runAfter }, + storage: { get: vi.fn() }, + } as unknown as ActionCtx, + new Request("https://example.com/api/v1/download?slug=aiq-deploy", { + headers: { "cf-connecting-ip": "1.2.3.4" }, + }), + ); + + expect(response.status).toBe(status); + expect(await response.text()).toBe(message); + expect(runQuery).not.toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ skillId: "skills:github" }), + ); + expect(runAfter).not.toHaveBeenCalled(); + }, + ); }); diff --git a/convex/downloads.ts b/convex/downloads.ts index 240c7e79..f8c81f85 100644 --- a/convex/downloads.ts +++ b/convex/downloads.ts @@ -5,18 +5,27 @@ import { httpAction } from "./functions"; import { ambiguousSkillSlugResponse } from "./httpApiV1/shared"; import { getOptionalActiveAuthUserIdFromAction } from "./lib/access"; import { getOptionalApiTokenUserId } from "./lib/apiTokenAuth"; +import { + buildGitHubSkillHandoffDescriptor, + getGitHubHandoffBlock, + isReadyGitHubHandoffTarget, + type GitHubHandoffTarget, +} from "./lib/githubHandoff"; import { corsHeaders, mergeHeaders } from "./lib/httpHeaders"; import { applyRateLimit, getClientIp } from "./lib/httpRateLimit"; -import { getPublicSkillVersionDownloadBlock, isSkillVersionForSkill } from "./lib/skillFileAccess"; +import { + getPublicSkillFileAccessBlock, + getPublicSkillVersionDownloadBlock, + isSkillVersionForSkill, +} from "./lib/skillFileAccess"; import { buildDeterministicZip } from "./lib/skillZip"; const HOUR_MS = 3_600_000; const DOWNLOAD_STAT_JITTER_MS = 60_000; -export async function downloadZipHandler( - ctx: Parameters[0]>[0], - request: Request, -) { +type DownloadCtx = Parameters[0]>[0]; + +export async function downloadZipHandler(ctx: DownloadCtx, request: Request) { const url = new URL(request.url); const slug = url.searchParams.get("slug")?.trim().toLowerCase(); const ownerHandle = @@ -74,6 +83,16 @@ export async function downloadZipHandler( } if (!version || !isSkillVersionForSkill(version, skill._id)) { + if (!versionParam && !tagParam && skill.installKind === "github") { + const moderationBlock = getPublicSkillFileAccessBlock(skillResult.moderationInfo); + if (moderationBlock) { + return new Response(moderationBlock.message, { + status: moderationBlock.status, + headers: mergeHeaders(rate.headers, corsHeaders()), + }); + } + return githubDownloadHandoffResponse(ctx, request, skill._id, rate.headers); + } return new Response("Version not found", { status: 404, headers: mergeHeaders(rate.headers, corsHeaders()), @@ -113,23 +132,7 @@ export async function downloadZipHandler( }); const zipBlob = new Blob([zipArray], { type: "application/zip" }); - try { - const userId = await getOptionalDownloadUserId(ctx, request); - const identity = getDownloadIdentity(request, userId ? String(userId) : null); - if (identity) { - await ctx.scheduler.runAfter( - Math.floor(Math.random() * DOWNLOAD_STAT_JITTER_MS), - internal.downloadMetrics.recordDownloadMetricInternal, - await buildDownloadMetricArgs({ - target: { kind: "skill", id: skill._id }, - identity, - now: Date.now(), - }), - ); - } - } catch { - // Best-effort metric path; do not fail downloads. - } + await scheduleSkillDownloadMetric(ctx, request, skill._id); return new Response(zipBlob, { status: 200, @@ -158,8 +161,69 @@ export function getDownloadIdentityValue(request: Request, userId: string | null return `ip:${ip}`; } +async function githubDownloadHandoffResponse( + ctx: DownloadCtx, + request: Request, + skillId: Id<"skills">, + rateHeaders: HeadersInit, +) { + const target = (await ctx.runQuery(internal.skills.getGitHubDownloadTargetInternal, { + skillId, + })) as GitHubHandoffTarget; + const block = getGitHubHandoffBlock(target); + if (block) { + return new Response(block.message, { + status: block.status, + headers: mergeHeaders(rateHeaders, corsHeaders()), + }); + } + if (!isReadyGitHubHandoffTarget(target)) { + return new Response("GitHub-backed skill source metadata is incomplete.", { + status: 409, + headers: mergeHeaders(rateHeaders, corsHeaders()), + }); + } + + await scheduleSkillDownloadMetric(ctx, request, skillId); + + return Response.json(buildGitHubSkillHandoffDescriptor(target), { + status: 200, + headers: mergeHeaders( + rateHeaders, + { + "Cache-Control": "private, max-age=60", + }, + corsHeaders(), + ), + }); +} + +async function scheduleSkillDownloadMetric( + ctx: DownloadCtx, + request: Request, + skillId: Id<"skills">, +) { + try { + const userId = await getOptionalDownloadUserId(ctx, request); + const identity = getDownloadIdentity(request, userId ? String(userId) : null); + if (identity) { + await ctx.scheduler.runAfter( + Math.floor(Math.random() * DOWNLOAD_STAT_JITTER_MS), + internal.downloadMetrics.recordDownloadMetricInternal, + await buildDownloadMetricArgs({ + target: { kind: "skill", id: skillId }, + identity, + now: Date.now(), + }), + ); + } + } catch { + // Best-effort metric path; do not fail downloads. + } +} + async function getOptionalDownloadUserId( - ctx: Parameters[0]>[0], + ctx: DownloadCtx, request: Request, ): Promise | null> { const apiTokenUserId = await getOptionalApiTokenUserId(ctx, request); diff --git a/convex/httpApiV1.handlers.test.ts b/convex/httpApiV1.handlers.test.ts index 29ebfe66..474823d6 100644 --- a/convex/httpApiV1.handlers.test.ts +++ b/convex/httpApiV1.handlers.test.ts @@ -586,6 +586,128 @@ describe("httpApiV1 handlers", () => { ]); }); + it("skills export includes GitHub-backed skills as public GitHub handoff descriptors", async () => { + vi.mocked(requireApiTokenUser).mockResolvedValue({ + userId: "users:actor", + user: { _id: "users:actor", role: "user" }, + } as never); + vi.mocked(getOptionalApiTokenUser).mockResolvedValue({ + userId: "users:actor", + user: { _id: "users:actor", role: "user" }, + } as never); + const commit = "2".repeat(40); + + const runQuery = vi.fn(async (_query: unknown, args: Record) => { + if ("startDate" in args) { + return { + page: [ + { + skillId: "skills:hosted", + slug: "hosted-demo", + displayName: "Hosted Demo", + latestVersionId: "skillVersions:hosted", + createdAt: 1, + updatedAt: 2, + stats: { downloads: 4 }, + ownerUserId: "users:alice", + ownerHandle: "alice", + ownerDisplayName: "Alice", + }, + { + skillId: "skills:github", + slug: "aiq-deploy", + displayName: "AIQ Deploy", + installKind: "github", + latestVersionId: undefined, + createdAt: 3, + updatedAt: 4, + stats: { downloads: 7 }, + ownerUserId: "users:nvidia", + ownerHandle: "nvidia", + ownerDisplayName: "NVIDIA", + }, + ], + nextCursor: null, + hasMore: false, + }; + } + if (args.versionId === "skillVersions:hosted") { + return { + skillId: "skills:hosted", + version: "1.0.0", + files: [{ storageId: "storage:hosted", path: "SKILL.md" }], + }; + } + if (args.skillId === "skills:github") { + return { + installKind: "github", + repo: "NVIDIA/skills", + path: "skills/aiq-deploy", + commit, + contentHash: "hash-aiq-deploy", + currentStatus: "present", + scanStatus: "suspicious", + removedAt: null, + }; + } + return null; + }); + const storageGet = vi.fn(async () => new Blob(["hosted skill"])); + + const response = await __handlers.exportSkillsV1Handler( + makeCtx({ runQuery, storage: { get: storageGet } }), + new Request("https://example.com/api/v1/skills/export?startDate=1&endDate=5", { + headers: { authorization: "Bearer user-token" }, + }), + ); + + if (response.status !== 200) throw new Error(await response.text()); + expect(response.headers.get("X-Total-Returned")).toBe("2"); + expect(response.headers.get("X-Export-Errors")).toBe("0"); + + const zipEntries = unzipSync(new Uint8Array(await response.arrayBuffer())); + expect(Object.keys(zipEntries).sort()).toEqual([ + "_manifest.json", + "alice/hosted-demo/SKILL.md", + "alice/hosted-demo/_export_skill_meta.json", + "nvidia/aiq-deploy/_export_skill_meta.json", + "nvidia/aiq-deploy/_source_handoff.json", + ]); + expect(zipEntries["_errors.json"]).toBeUndefined(); + + const manifest = JSON.parse(new TextDecoder().decode(zipEntries["_manifest.json"])); + expect(manifest).toEqual([ + expect.objectContaining({ + publisher: "alice", + slug: "hosted-demo", + sourceRef: "public-clawhub", + fileCount: 1, + }), + expect.objectContaining({ + publisher: "nvidia", + slug: "aiq-deploy", + sourceRef: "public-github", + version: null, + fileCount: 0, + }), + ]); + + const handoff = JSON.parse( + new TextDecoder().decode(zipEntries["nvidia/aiq-deploy/_source_handoff.json"]), + ); + expect(handoff).toEqual({ + sourceRef: "public-github", + repo: "NVIDIA/skills", + commit, + path: "skills/aiq-deploy", + contentHash: "hash-aiq-deploy", + archiveUrl: `https://api.github.com/repos/NVIDIA/skills/zipball/${commit}`, + }); + expect(handoff).not.toHaveProperty("scan"); + expect(handoff).not.toHaveProperty("scanStatus"); + expect(storageGet).toHaveBeenCalledTimes(1); + }); + it("skills export skips stale latest versions before reading blobs", async () => { vi.mocked(requireApiTokenUser).mockResolvedValue({ userId: "users:actor", diff --git a/convex/httpApiV1/skillsV1.ts b/convex/httpApiV1/skillsV1.ts index 60427958..6e8edef3 100644 --- a/convex/httpApiV1/skillsV1.ts +++ b/convex/httpApiV1/skillsV1.ts @@ -18,6 +18,13 @@ import { api, internal } from "../_generated/api"; import type { Doc, Id } from "../_generated/dataModel"; import type { ActionCtx } from "../_generated/server"; import { getOptionalApiTokenUserId, requireApiTokenUser } from "../lib/apiTokenAuth"; +import { + buildGitHubSkillHandoffDescriptor, + getGitHubHandoffBlock, + isReadyGitHubHandoffTarget, + type GitHubHandoffTarget, + type ReadyGitHubHandoffTarget, +} from "../lib/githubHandoff"; import { mergeHeaders } from "../lib/httpHeaders"; import { applyRateLimit } from "../lib/httpRateLimit"; import { parseBooleanQueryParam, resolveBooleanQueryParam } from "../lib/httpUtils"; @@ -3447,6 +3454,7 @@ export async function exportSkillsV1Handler(ctx: ActionCtx, request: Request) { slug: string; displayName: string; latestVersionId?: Id<"skillVersions">; + installKind?: "github"; createdAt: number; updatedAt: number; stats?: Record | null; @@ -3507,11 +3515,22 @@ export async function exportSkillsV1Handler(ctx: ActionCtx, request: Request) { }) : Promise.resolve(null), ); + const githubTargets = await chunkedParallel(result.page, 100, (digest) => + isGitHubSourceExportDigest(digest) + ? ctx.runQuery(internal.skills.getGitHubDownloadTargetInternal, { + skillId: digest.skillId, + }) + : Promise.resolve(null), + ); logContext.versionCount = versionDocs.filter(Boolean).length; const exportableVersions: Array | null> = Array.from( { length: result.page.length }, () => null, ); + const exportableGitHubTargets: Array = Array.from( + { length: result.page.length }, + () => null, + ); type BlobTask = { digestIndex: number; fileIndex: number; storageId: Id<"_storage"> }; const blobTasks: BlobTask[] = []; @@ -3522,6 +3541,26 @@ export async function exportSkillsV1Handler(ctx: ActionCtx, request: Request) { const version = versionDocs[i] as Doc<"skillVersions"> | null; if (!version) { + if (isGitHubSourceExportDigest(digest)) { + const target = githubTargets[i] as GitHubHandoffTarget; + const block = getGitHubHandoffBlock(target); + if (block) { + exportErrors.push({ + slug: digest.slug, + error: block.message, + }); + continue; + } + if (!isReadyGitHubHandoffTarget(target)) { + exportErrors.push({ + slug: digest.slug, + error: "GitHub-backed skill source metadata is incomplete.", + }); + continue; + } + exportableGitHubTargets[i] = target; + continue; + } exportErrors.push({ slug: digest.slug, error: `version not found (latestVersionId: ${digest.latestVersionId ?? "null"})`, @@ -3601,7 +3640,6 @@ export async function exportSkillsV1Handler(ctx: ActionCtx, request: Request) { version?: string; files?: Array<{ storageId: Id<"_storage">; path: string }>; } | null; - if (!version?.files) continue; if (!validateSlug(digest.slug)) continue; const publisherSegment = getExportPublisherSegment(digest); @@ -3613,6 +3651,38 @@ export async function exportSkillsV1Handler(ctx: ActionCtx, request: Request) { continue; } const exportRoot = `${publisherSegment}/${digest.slug}`; + const githubTarget = exportableGitHubTargets[i]; + if (githubTarget) { + const descriptor = buildGitHubSkillHandoffDescriptor(githubTarget); + zipEntries.push({ + path: `${exportRoot}/_source_handoff.json`, + bytes: new TextEncoder().encode(JSON.stringify(descriptor, null, 2)), + }); + + const skillMeta = buildExportSkillMeta(digest, { + sourceRef: "public-github", + version: null, + }); + zipEntries.push({ + path: `${exportRoot}/_export_skill_meta.json`, + bytes: new TextEncoder().encode(JSON.stringify(skillMeta, null, 2)), + }); + + manifest.push({ + publisher: publisherSegment, + slug: digest.slug, + sourceRef: "public-github", + version: null, + displayName: digest.displayName, + createdAt: digest.createdAt, + updatedAt: digest.updatedAt, + stats: (digest.stats as Record) ?? null, + fileCount: 0, + }); + continue; + } + + if (!version?.files) continue; const digestBlobs = blobsByDigest.get(i); if (!digestBlobs) continue; @@ -3650,18 +3720,10 @@ export async function exportSkillsV1Handler(ctx: ActionCtx, request: Request) { fileCount++; } - const skillMeta = { - slug: digest.slug, - displayName: digest.displayName, + const skillMeta = buildExportSkillMeta(digest, { + sourceRef: "public-clawhub", version: version.version ?? null, - createdAt: digest.createdAt, - updatedAt: digest.updatedAt, - stats: digest.stats ?? null, - owner: { - handle: digest.ownerHandle ?? null, - displayName: digest.ownerDisplayName ?? null, - }, - }; + }); zipEntries.push({ path: `${exportRoot}/_export_skill_meta.json`, bytes: new TextEncoder().encode(JSON.stringify(skillMeta, null, 2)), @@ -3670,6 +3732,7 @@ export async function exportSkillsV1Handler(ctx: ActionCtx, request: Request) { manifest.push({ publisher: publisherSegment, slug: digest.slug, + sourceRef: "public-clawhub", version: version.version ?? null, displayName: digest.displayName, createdAt: digest.createdAt, @@ -3720,3 +3783,37 @@ function getExportPublisherSegment(digest: { const fallback = String(digest.ownerUserId).replace(/[^a-zA-Z0-9._-]/g, "-"); return validateSlug(fallback) ? fallback : null; } + +function isGitHubSourceExportDigest(digest: { + installKind?: "github"; + latestVersionId?: Id<"skillVersions">; +}) { + return digest.installKind === "github" && !digest.latestVersionId; +} + +function buildExportSkillMeta( + digest: { + slug: string; + displayName: string; + createdAt: number; + updatedAt: number; + stats?: Record | null; + ownerHandle?: string | null; + ownerDisplayName?: string | null; + }, + source: { sourceRef: "public-clawhub" | "public-github"; version: string | null }, +) { + return { + slug: digest.slug, + displayName: digest.displayName, + sourceRef: source.sourceRef, + version: source.version, + createdAt: digest.createdAt, + updatedAt: digest.updatedAt, + stats: digest.stats ?? null, + owner: { + handle: digest.ownerHandle ?? null, + displayName: digest.ownerDisplayName ?? null, + }, + }; +} diff --git a/convex/lib/githubHandoff.ts b/convex/lib/githubHandoff.ts new file mode 100644 index 00000000..a1fa7c62 --- /dev/null +++ b/convex/lib/githubHandoff.ts @@ -0,0 +1,121 @@ +export type GitHubHandoffBlock = { status: 403 | 409 | 410 | 423; message: string }; + +export type GitHubHandoffTarget = { + installKind: "github"; + repo: string | null; + path: string | null; + commit: string | null; + contentHash: string | null; + currentStatus: "present" | "missing" | "unknown" | null; + scanStatus: "clean" | "suspicious" | "malicious" | "pending" | "failed" | null; + removedAt: number | null; +} | null; + +export type ReadyGitHubHandoffTarget = { + installKind: "github"; + repo: string; + path: string; + commit: string; + contentHash: string; + currentStatus: "present"; + scanStatus: "clean" | "suspicious"; + removedAt: number | null; +}; + +type GitHubHandoffScanStatus = NonNullable["scanStatus"]; + +export function getGitHubHandoffBlock(target: GitHubHandoffTarget): GitHubHandoffBlock | null { + if (!target || target.installKind !== "github") { + return { + status: 409, + message: "GitHub-backed skill source metadata is incomplete.", + }; + } + if (target.removedAt) { + return { + status: 410, + message: "GitHub-backed skill has been removed upstream.", + }; + } + if (target.currentStatus === "missing") { + return { + status: 410, + message: "GitHub-backed skill path is missing upstream.", + }; + } + if (target.scanStatus === "failed" || target.scanStatus === "malicious") { + return { + status: 403, + message: "GitHub-backed skill failed ClawHub security scanning.", + }; + } + if (!target.repo || !target.path) { + return { + status: 409, + message: "GitHub-backed skill source metadata is incomplete.", + }; + } + if ( + target.currentStatus !== "present" || + !target.commit || + !target.contentHash || + !isValidGitHubRepo(target.repo) + ) { + return { + status: 423, + message: "GitHub-backed skill needs an upstream freshness check before download.", + }; + } + if (!isSuccessfulGitHubHandoffScanStatus(target.scanStatus)) { + return { + status: 423, + message: "GitHub-backed skill security scan is in progress.", + }; + } + return null; +} + +export function isReadyGitHubHandoffTarget( + target: GitHubHandoffTarget, +): target is ReadyGitHubHandoffTarget { + return Boolean( + target && + target.installKind === "github" && + target.repo && + target.path && + target.commit && + target.contentHash && + target.currentStatus === "present" && + isSuccessfulGitHubHandoffScanStatus(target.scanStatus) && + isValidGitHubRepo(target.repo), + ); +} + +export function buildGitHubSkillHandoffDescriptor(target: ReadyGitHubHandoffTarget) { + return { + sourceRef: "public-github" as const, + repo: target.repo, + commit: target.commit, + path: target.path, + contentHash: target.contentHash, + archiveUrl: buildGitHubZipballUrl(target.repo, target.commit), + }; +} + +export function isSuccessfulGitHubHandoffScanStatus(scanStatus: GitHubHandoffScanStatus) { + // Suspicious is review-worthy but not a hard public download block; pending, + // failed, and malicious states still block handoff above. + return scanStatus === "clean" || scanStatus === "suspicious"; +} + +function buildGitHubZipballUrl(repo: string, commit: string) { + const [owner, name] = repo.split("/"); + return `https://api.github.com/repos/${encodeURIComponent(owner)}/${encodeURIComponent( + name, + )}/zipball/${encodeURIComponent(commit)}`; +} + +function isValidGitHubRepo(repo: string) { + const [owner, name, ...rest] = repo.split("/"); + return Boolean(owner && name && rest.length === 0); +} diff --git a/convex/lib/skillZip.ts b/convex/lib/skillZip.ts index 20eaff3b..87093689 100644 --- a/convex/lib/skillZip.ts +++ b/convex/lib/skillZip.ts @@ -81,6 +81,7 @@ export function buildDeterministicPackageZip(entries: ZipEntry[]) { export interface MergedExportManifestEntry { publisher: string; slug: string; + sourceRef?: "public-clawhub" | "public-github"; version: string | null; displayName: string; createdAt: number; diff --git a/convex/skills.exportList.test.ts b/convex/skills.exportList.test.ts index bad20639..6815e3df 100644 --- a/convex/skills.exportList.test.ts +++ b/convex/skills.exportList.test.ts @@ -66,6 +66,34 @@ describe("skills.listByDateRange export list", () => { getPageMock.mockResolvedValue({ page: [ digest({ slug: "exportable" }), + digest({ + slug: "github-clean", + latestVersionId: undefined, + installKind: "github", + githubCurrentStatus: "present", + githubScanStatus: "clean", + }), + digest({ + slug: "github-suspicious", + latestVersionId: undefined, + installKind: "github", + githubCurrentStatus: "present", + githubScanStatus: "suspicious", + }), + digest({ + slug: "github-pending", + latestVersionId: undefined, + installKind: "github", + githubCurrentStatus: "present", + githubScanStatus: "pending", + }), + digest({ + slug: "github-missing", + latestVersionId: undefined, + installKind: "github", + githubCurrentStatus: "missing", + githubScanStatus: "clean", + }), digest({ slug: "missing-version", latestVersionId: undefined }), digest({ slug: "hidden", moderationStatus: "hidden" }), digest({ slug: "malicious", moderationFlags: ["blocked.malware"] }), @@ -77,7 +105,11 @@ describe("skills.listByDateRange export list", () => { const result = await listByDateRangeHandler({ db: {} }, { startDate: 1, endDate: 5 }); - expect(result.page.map((item) => item.slug)).toEqual(["exportable"]); + expect(result.page.map((item) => item.slug)).toEqual([ + "exportable", + "github-clean", + "github-suspicious", + ]); expect(getPageMock).toHaveBeenCalledWith( expect.anything(), expect.objectContaining({ diff --git a/convex/skills.ts b/convex/skills.ts index a4fd937b..4244b17d 100644 --- a/convex/skills.ts +++ b/convex/skills.ts @@ -2744,6 +2744,26 @@ export const getBySlug = query({ }, }); +export const getGitHubDownloadTargetInternal = internalQuery({ + args: { skillId: v.id("skills") }, + handler: async (ctx, args) => { + const skill = await ctx.db.get(args.skillId); + if (!skill || skill.installKind !== "github") return null; + const source = skill.githubSourceId ? await ctx.db.get(skill.githubSourceId) : null; + + return { + installKind: "github" as const, + repo: source?.repo ?? null, + path: skill.githubPath ?? null, + commit: skill.githubCurrentCommit ?? null, + contentHash: skill.githubCurrentContentHash ?? null, + currentStatus: skill.githubCurrentStatus ?? null, + scanStatus: skill.githubScanStatus ?? null, + removedAt: skill.githubRemovedAt ?? null, + }; + }, +}); + export const getVerifyTargetBySlugInternal = internalQuery({ args: { slug: v.string(), ownerHandle: v.optional(v.string()) }, handler: async (ctx, args) => { @@ -12765,10 +12785,22 @@ export const listByDateRange = internalQuery({ function isExportableSkillDigest( skill: Pick< Doc<"skillSearchDigest">, - "latestVersionId" | "softDeletedAt" | "moderationStatus" | "moderationFlags" + | "latestVersionId" + | "installKind" + | "githubCurrentStatus" + | "githubScanStatus" + | "softDeletedAt" + | "moderationStatus" + | "moderationFlags" >, ) { - return Boolean(skill.latestVersionId) && isPublicSkillDoc(skill); + if (!isPublicSkillDoc(skill)) return false; + if (skill.latestVersionId) return true; + return ( + skill.installKind === "github" && + skill.githubCurrentStatus === "present" && + (skill.githubScanStatus === "clean" || skill.githubScanStatus === "suspicious") + ); } export const __test = { diff --git a/docs/api.md b/docs/api.md index c6840795..0760e820 100644 --- a/docs/api.md +++ b/docs/api.md @@ -98,6 +98,13 @@ Public read: - `GET /api/v1/skills/{slug}/file?path=&version=&tag=` - `GET /api/v1/resolve?slug=&hash=` - `GET /api/v1/download?slug=&version=&tag=` + - Hosted skills return deterministic ZIP bytes. + - Current GitHub-backed skills with a `clean` or `suspicious` scan return a + JSON `public-github` handoff descriptor instead of ClawHub bytes. +- `GET /api/v1/skills/export?startDate=&endDate=&limit=&cursor=` + - Hosted skills are exported as stored files. + - Current GitHub-backed skills with a `clean` or `suspicious` scan are exported + as `public-github` handoff descriptors. - `GET /api/v1/packages?limit=&cursor=&sort=` - `sort`: `updated` (default), `recommended`, `downloads`, legacy alias `installs` - Invalid `sort` values return `400` @@ -124,6 +131,7 @@ Auth required: - `POST /api/v1/skills/{slug}/transfer/accept` - `POST /api/v1/skills/{slug}/transfer/reject` - `POST /api/v1/skills/{slug}/transfer/cancel` +- `GET /api/v1/skills/export?startDate=&endDate=&limit=&cursor=` - `GET /api/v1/plugins/export?startDate=&endDate=&limit=&cursor=&family=` - `GET /api/v1/transfers/incoming` - `GET /api/v1/transfers/outgoing` diff --git a/docs/http-api.md b/docs/http-api.md index b9f1dc37..1a6e6f5f 100644 --- a/docs/http-api.md +++ b/docs/http-api.md @@ -604,6 +604,43 @@ Legacy v1 filter aliases remain accepted on read endpoints: Legacy aliases are not accepted as stored or author-declared category values. +### `GET /api/v1/skills/export` + +Bulk export of latest public skills for offline analysis. + +Auth: + +- API token required. + +Query params: + +- `startDate` (required): Unix milliseconds lower bound for skill `updatedAt`. +- `endDate` (required): Unix milliseconds upper bound for skill `updatedAt`. +- `limit` (optional): integer (1-250), default `250`. +- `cursor` (optional): pagination cursor from the previous response. + +Response: + +- Body: ZIP archive. +- Each exported skill is rooted at `{publisher}/{slug}/`. +- Hosted skills include the latest stored version files and are listed in + `_manifest.json` with `sourceRef: "public-clawhub"`. +- Current GitHub-backed skills with a `clean` or `suspicious` scan include + `_source_handoff.json` with `sourceRef: "public-github"`, repo, commit, path, + content hash, and archive URL. They do not include ClawHub-hosted source files. +- Each skill includes `_export_skill_meta.json`. +- `_manifest.json` is always included at the ZIP root. +- `_errors.json` is included when individual skills or files could not be + exported. + +Headers: + +- `X-Next-Cursor` +- `X-Has-More` +- `X-Total-Returned` +- `X-Date-Range` +- `X-Export-Errors` + ### `GET /api/v1/plugins/export` Bulk export of latest public plugin releases for offline analysis. @@ -1228,7 +1265,9 @@ Response: ### `GET /api/v1/download` -Downloads a zip of a skill version. +Downloads a hosted skill version ZIP, or returns a GitHub source handoff for a +current GitHub-backed skill with a `clean` or `suspicious` scan and no hosted +version. Query params: @@ -1240,7 +1279,11 @@ Notes: - If neither `version` nor `tag` is provided, the latest version is used. - Soft-deleted versions return `410`. -- Download stats are counted as unique identities per hour (`userId` when API token is valid, otherwise IP). +- GitHub-backed skill handoffs do not proxy or mirror bytes. The JSON response + includes `sourceRef: "public-github"`, `repo`, `commit`, `path`, `contentHash`, + and `archiveUrl`; scan/current state is a gate and is not included as success + payload metadata. +- Download stats are counted as unique identities per UTC day (`userId` when API token is valid, otherwise IP). ## Auth endpoints (Bearer token) diff --git a/public/api/v1/openapi.json b/public/api/v1/openapi.json index 58088d3f..519e394d 100644 --- a/public/api/v1/openapi.json +++ b/public/api/v1/openapi.json @@ -27,6 +27,42 @@ } } }, + "GitHubSkillDownloadHandoff": { + "type": "object", + "additionalProperties": false, + "required": [ + "sourceRef", + "repo", + "commit", + "path", + "contentHash", + "archiveUrl" + ], + "properties": { + "sourceRef": { + "type": "string", + "enum": [ + "public-github" + ] + }, + "repo": { + "type": "string" + }, + "commit": { + "type": "string" + }, + "path": { + "type": "string" + }, + "contentHash": { + "type": "string" + }, + "archiveUrl": { + "type": "string", + "format": "uri" + } + } + }, "Owner": { "type": [ "object", @@ -1619,7 +1655,7 @@ }, "/api/v1/download": { "get": { - "summary": "Download zip", + "summary": "Download skill ZIP or GitHub handoff", "parameters": [ { "name": "slug", @@ -1648,13 +1684,18 @@ ], "responses": { "200": { - "description": "Zip", + "description": "Hosted ZIP bytes or a GitHub-backed source handoff descriptor.", "content": { "application/zip": { "schema": { "type": "string", "format": "binary" } + }, + "application/json": { + "schema": { + "$ref": "#/components/schemas/GitHubSkillDownloadHandoff" + } } } } diff --git a/specs/github-backed-skills.md b/specs/github-backed-skills.md index b0bf8d31..db54962d 100644 --- a/specs/github-backed-skills.md +++ b/specs/github-backed-skills.md @@ -235,6 +235,20 @@ bypass failed, suspicious, malicious, missing, or removed upstream states. ClawHub must not create hosted `skillVersions` or ClawHub download artifacts for GitHub-backed skills. +`GET /api/v1/download` may still be used as a metered source handoff for current +GitHub-backed skills whose scan verdict is `clean` or `suspicious`. In that case +ClawHub returns only stored fetch coordinates (`sourceRef: "public-github"`, +repo, commit, path, content hash, and an archive URL) after checking current +upstream and scan state. It must not fetch GitHub, expand archives, proxy bytes, +create `skillVersions`, or include detailed scan metadata in the successful +payload. + +`GET /api/v1/skills/export` follows the same no-mirror contract. Hosted skills +continue to export stored version files with `sourceRef: "public-clawhub"`. +Current GitHub-backed skills whose scan verdict is `clean` or `suspicious` are +included as `sourceRef: "public-github"` entries with `_source_handoff.json` +coordinates, not ClawHub-hosted source files. + This avoids two NVIDIA concerns: - Signature drift: any byte-level transformation in a mirror can invalidate diff --git a/src/__tests__/openapi-contract.test.ts b/src/__tests__/openapi-contract.test.ts index b36b7a19..6a96ae35 100644 --- a/src/__tests__/openapi-contract.test.ts +++ b/src/__tests__/openapi-contract.test.ts @@ -17,6 +17,13 @@ function sortValuesForPath(paths: unknown, path: string) { return property(property(sortParameter, "schema"), "enum"); } +function responseContentForPath(paths: unknown, path: string, status: string) { + const routePath = property(paths, path); + const getOperation = property(routePath, "get"); + const responses = property(getOperation, "responses"); + return property(property(responses, status), "content"); +} + describe("OpenAPI contract", () => { it("documents accepted skills sort aliases", async () => { const specPath = new URL("../../public/api/v1/openapi.json", import.meta.url); @@ -50,4 +57,29 @@ describe("OpenAPI contract", () => { expect(sortValuesForPath(paths, "/api/v1/code-plugins")).toEqual(sortValues); expect(sortValuesForPath(paths, "/api/v1/bundle-plugins")).toEqual(sortValues); }); + + it("documents skill downloads as either hosted ZIP bytes or GitHub handoff JSON", async () => { + const specPath = new URL("../../public/api/v1/openapi.json", import.meta.url); + const spec: unknown = JSON.parse(await readFile(specPath, "utf8")); + const paths = property(spec, "paths"); + const schemas = property(property(spec, "components"), "schemas"); + + const responseContent = responseContentForPath(paths, "/api/v1/download", "200"); + expect(property(responseContent, "application/zip")).toBeTruthy(); + expect(property(property(responseContent, "application/json"), "schema")).toEqual({ + $ref: "#/components/schemas/GitHubSkillDownloadHandoff", + }); + + const handoffSchema = property(schemas, "GitHubSkillDownloadHandoff"); + expect(property(handoffSchema, "required")).toEqual([ + "sourceRef", + "repo", + "commit", + "path", + "contentHash", + "archiveUrl", + ]); + expect(property(property(handoffSchema, "properties"), "scan")).toBeUndefined(); + expect(property(property(handoffSchema, "properties"), "scanStatus")).toBeUndefined(); + }); });