feat: add github-backed skill handoffs to downloads and export

This commit is contained in:
Patrick Erichsen
2026-06-23 17:08:46 -07:00
committed by GitHub
parent 35cc191d48
commit 63ca5c2a2e
14 changed files with 939 additions and 42 deletions
+2
View File
@@ -65,6 +65,7 @@ import type * as lib_embeddings from "../lib/embeddings.js";
import type * as lib_githubAccount from "../lib/githubAccount.js";
import type * as lib_githubActionsOidc from "../lib/githubActionsOidc.js";
import type * as lib_githubAuth from "../lib/githubAuth.js";
import type * as lib_githubHandoff from "../lib/githubHandoff.js";
import type * as lib_githubIdentity from "../lib/githubIdentity.js";
import type * as lib_githubImport from "../lib/githubImport.js";
import type * as lib_githubProfileSync from "../lib/githubProfileSync.js";
@@ -216,6 +217,7 @@ declare const fullApi: ApiFromModules<{
"lib/githubAccount": typeof lib_githubAccount;
"lib/githubActionsOidc": typeof lib_githubActionsOidc;
"lib/githubAuth": typeof lib_githubAuth;
"lib/githubHandoff": typeof lib_githubHandoff;
"lib/githubIdentity": typeof lib_githubIdentity;
"lib/githubImport": typeof lib_githubImport;
"lib/githubProfileSync": typeof lib_githubProfileSync;
+288
View File
@@ -493,4 +493,292 @@ describe("downloads helpers", () => {
}),
);
});
it.each(["clean", "suspicious"] as const)(
"returns a metered public GitHub handoff descriptor for %s scan without scan metadata",
async (scanStatus) => {
const commit = "1".repeat(40);
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
if (isRateLimitArgs(args)) return okRate();
if ("slug" in args) {
return {
skill: {
_id: "skills:github",
ownerUserId: "users:1",
slug: "aiq-deploy",
tags: {},
latestVersionId: undefined,
installKind: "github",
githubPath: "skills/aiq-deploy",
githubCurrentCommit: commit,
githubCurrentContentHash: "hash-aiq-deploy",
githubCurrentStatus: "present",
githubScanStatus: scanStatus,
},
moderationInfo: null,
};
}
if ("skillId" in args) {
return {
installKind: "github",
repo: "NVIDIA/skills",
path: "skills/aiq-deploy",
commit,
contentHash: "hash-aiq-deploy",
currentStatus: "present",
scanStatus,
removedAt: null,
};
}
return null;
});
const runMutation = vi.fn(async (_mutation: unknown, args: Record<string, unknown>) => {
if (isRateLimitArgs(args)) return okRate();
return null;
});
const runAfter = vi.fn();
const storageGet = vi.fn();
const response = await downloadZipHandler(
{
runQuery,
runMutation,
scheduler: { runAfter },
storage: { get: storageGet },
} as unknown as ActionCtx,
new Request("https://example.com/api/v1/download?slug=aiq-deploy", {
headers: { "cf-connecting-ip": "1.2.3.4" },
}),
);
expect(response.status).toBe(200);
expect(response.headers.get("Content-Type")).toBe("application/json");
expect(storageGet).not.toHaveBeenCalled();
const body = await response.json();
expect(body).toEqual({
sourceRef: "public-github",
repo: "NVIDIA/skills",
commit,
path: "skills/aiq-deploy",
contentHash: "hash-aiq-deploy",
archiveUrl: `https://api.github.com/repos/NVIDIA/skills/zipball/${commit}`,
});
expect(body).not.toHaveProperty("scan");
expect(body).not.toHaveProperty("scanStatus");
expect(runAfter).toHaveBeenCalledWith(
expect.any(Number),
expect.anything(),
expect.objectContaining({
target: { kind: "skill", id: "skills:github" },
identityKind: "ip",
identityHash: expect.stringMatching(/^[a-f0-9]{64}$/),
}),
);
},
);
it.each([
{
name: "pending scan",
skill: { githubCurrentStatus: "present", githubScanStatus: "pending" },
source: { repo: "NVIDIA/skills" },
status: 423,
message: "GitHub-backed skill security scan is in progress.",
},
{
name: "failed scan",
skill: { githubCurrentStatus: "present", githubScanStatus: "failed" },
source: { repo: "NVIDIA/skills" },
status: 403,
message: "GitHub-backed skill failed ClawHub security scanning.",
},
{
name: "malicious scan",
skill: { githubCurrentStatus: "present", githubScanStatus: "malicious" },
source: { repo: "NVIDIA/skills" },
status: 403,
message: "GitHub-backed skill failed ClawHub security scanning.",
},
{
name: "missing upstream path",
skill: { githubCurrentStatus: "missing", githubScanStatus: "clean" },
source: { repo: "NVIDIA/skills" },
status: 410,
message: "GitHub-backed skill path is missing upstream.",
},
{
name: "removed upstream path",
skill: { githubCurrentStatus: "present", githubRemovedAt: 123, githubScanStatus: "clean" },
source: { repo: "NVIDIA/skills" },
status: 410,
message: "GitHub-backed skill has been removed upstream.",
},
{
name: "unknown upstream freshness",
skill: { githubCurrentStatus: "unknown", githubScanStatus: "clean" },
source: { repo: "NVIDIA/skills" },
status: 423,
message: "GitHub-backed skill needs an upstream freshness check before download.",
},
{
name: "incomplete source",
skill: { githubCurrentStatus: "present", githubScanStatus: "clean" },
source: null,
status: 409,
message: "GitHub-backed skill source metadata is incomplete.",
},
])(
"blocks $name GitHub handoffs without scheduling metrics",
async ({ skill, source, status, message }) => {
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
if (isRateLimitArgs(args)) return okRate();
if ("slug" in args) {
return {
skill: {
_id: "skills:github",
ownerUserId: "users:1",
slug: "aiq-deploy",
tags: {},
latestVersionId: undefined,
installKind: "github",
githubPath: "skills/aiq-deploy",
githubCurrentCommit: "1".repeat(40),
githubCurrentContentHash: "hash-aiq-deploy",
...skill,
},
moderationInfo: null,
};
}
if ("skillId" in args) {
if (!source) return null;
return {
installKind: "github",
repo: source.repo,
path: "skills/aiq-deploy",
commit: "1".repeat(40),
contentHash: "hash-aiq-deploy",
currentStatus: "present",
scanStatus: "clean",
removedAt: null,
...skill,
...(skill.githubCurrentStatus ? { currentStatus: skill.githubCurrentStatus } : {}),
...(skill.githubScanStatus ? { scanStatus: skill.githubScanStatus } : {}),
...(skill.githubRemovedAt ? { removedAt: skill.githubRemovedAt } : {}),
};
}
return null;
});
const runMutation = vi.fn(async (_mutation: unknown, args: Record<string, unknown>) => {
if (isRateLimitArgs(args)) return okRate();
return null;
});
const runAfter = vi.fn();
const response = await downloadZipHandler(
{
runQuery,
runMutation,
scheduler: { runAfter },
storage: { get: vi.fn() },
} as unknown as ActionCtx,
new Request("https://example.com/api/v1/download?slug=aiq-deploy", {
headers: { "cf-connecting-ip": "1.2.3.4" },
}),
);
expect(response.status).toBe(status);
expect(await response.text()).toBe(message);
expect(runAfter).not.toHaveBeenCalled();
},
);
it.each([
{
name: "hidden by moderators",
moderationInfo: {
isPendingScan: false,
isMalwareBlocked: false,
isHiddenByMod: true,
isRemoved: false,
},
status: 403,
message: "This skill is currently unavailable.",
},
{
name: "removed by moderators",
moderationInfo: {
isPendingScan: false,
isMalwareBlocked: false,
isHiddenByMod: false,
isRemoved: true,
},
status: 410,
message: "This skill has been removed by a moderator.",
},
])(
"blocks $name GitHub handoffs before source descriptor creation",
async ({ moderationInfo, status, message }) => {
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
if (isRateLimitArgs(args)) return okRate();
if ("slug" in args) {
return {
skill: {
_id: "skills:github",
ownerUserId: "users:1",
slug: "aiq-deploy",
tags: {},
latestVersionId: undefined,
installKind: "github",
githubPath: "skills/aiq-deploy",
githubCurrentCommit: "1".repeat(40),
githubCurrentContentHash: "hash-aiq-deploy",
githubCurrentStatus: "present",
githubScanStatus: "clean",
},
moderationInfo,
};
}
if ("skillId" in args) {
return {
installKind: "github",
repo: "NVIDIA/skills",
path: "skills/aiq-deploy",
commit: "1".repeat(40),
contentHash: "hash-aiq-deploy",
currentStatus: "present",
scanStatus: "clean",
removedAt: null,
};
}
return null;
});
const runMutation = vi.fn(async (_mutation: unknown, args: Record<string, unknown>) => {
if (isRateLimitArgs(args)) return okRate();
return null;
});
const runAfter = vi.fn();
const response = await downloadZipHandler(
{
runQuery,
runMutation,
scheduler: { runAfter },
storage: { get: vi.fn() },
} as unknown as ActionCtx,
new Request("https://example.com/api/v1/download?slug=aiq-deploy", {
headers: { "cf-connecting-ip": "1.2.3.4" },
}),
);
expect(response.status).toBe(status);
expect(await response.text()).toBe(message);
expect(runQuery).not.toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({ skillId: "skills:github" }),
);
expect(runAfter).not.toHaveBeenCalled();
},
);
});
+87 -23
View File
@@ -5,18 +5,27 @@ import { httpAction } from "./functions";
import { ambiguousSkillSlugResponse } from "./httpApiV1/shared";
import { getOptionalActiveAuthUserIdFromAction } from "./lib/access";
import { getOptionalApiTokenUserId } from "./lib/apiTokenAuth";
import {
buildGitHubSkillHandoffDescriptor,
getGitHubHandoffBlock,
isReadyGitHubHandoffTarget,
type GitHubHandoffTarget,
} from "./lib/githubHandoff";
import { corsHeaders, mergeHeaders } from "./lib/httpHeaders";
import { applyRateLimit, getClientIp } from "./lib/httpRateLimit";
import { getPublicSkillVersionDownloadBlock, isSkillVersionForSkill } from "./lib/skillFileAccess";
import {
getPublicSkillFileAccessBlock,
getPublicSkillVersionDownloadBlock,
isSkillVersionForSkill,
} from "./lib/skillFileAccess";
import { buildDeterministicZip } from "./lib/skillZip";
const HOUR_MS = 3_600_000;
const DOWNLOAD_STAT_JITTER_MS = 60_000;
export async function downloadZipHandler(
ctx: Parameters<Parameters<typeof httpAction>[0]>[0],
request: Request,
) {
type DownloadCtx = Parameters<Parameters<typeof httpAction>[0]>[0];
export async function downloadZipHandler(ctx: DownloadCtx, request: Request) {
const url = new URL(request.url);
const slug = url.searchParams.get("slug")?.trim().toLowerCase();
const ownerHandle =
@@ -74,6 +83,16 @@ export async function downloadZipHandler(
}
if (!version || !isSkillVersionForSkill(version, skill._id)) {
if (!versionParam && !tagParam && skill.installKind === "github") {
const moderationBlock = getPublicSkillFileAccessBlock(skillResult.moderationInfo);
if (moderationBlock) {
return new Response(moderationBlock.message, {
status: moderationBlock.status,
headers: mergeHeaders(rate.headers, corsHeaders()),
});
}
return githubDownloadHandoffResponse(ctx, request, skill._id, rate.headers);
}
return new Response("Version not found", {
status: 404,
headers: mergeHeaders(rate.headers, corsHeaders()),
@@ -113,23 +132,7 @@ export async function downloadZipHandler(
});
const zipBlob = new Blob([zipArray], { type: "application/zip" });
try {
const userId = await getOptionalDownloadUserId(ctx, request);
const identity = getDownloadIdentity(request, userId ? String(userId) : null);
if (identity) {
await ctx.scheduler.runAfter(
Math.floor(Math.random() * DOWNLOAD_STAT_JITTER_MS),
internal.downloadMetrics.recordDownloadMetricInternal,
await buildDownloadMetricArgs({
target: { kind: "skill", id: skill._id },
identity,
now: Date.now(),
}),
);
}
} catch {
// Best-effort metric path; do not fail downloads.
}
await scheduleSkillDownloadMetric(ctx, request, skill._id);
return new Response(zipBlob, {
status: 200,
@@ -158,8 +161,69 @@ export function getDownloadIdentityValue(request: Request, userId: string | null
return `ip:${ip}`;
}
async function githubDownloadHandoffResponse(
ctx: DownloadCtx,
request: Request,
skillId: Id<"skills">,
rateHeaders: HeadersInit,
) {
const target = (await ctx.runQuery(internal.skills.getGitHubDownloadTargetInternal, {
skillId,
})) as GitHubHandoffTarget;
const block = getGitHubHandoffBlock(target);
if (block) {
return new Response(block.message, {
status: block.status,
headers: mergeHeaders(rateHeaders, corsHeaders()),
});
}
if (!isReadyGitHubHandoffTarget(target)) {
return new Response("GitHub-backed skill source metadata is incomplete.", {
status: 409,
headers: mergeHeaders(rateHeaders, corsHeaders()),
});
}
await scheduleSkillDownloadMetric(ctx, request, skillId);
return Response.json(buildGitHubSkillHandoffDescriptor(target), {
status: 200,
headers: mergeHeaders(
rateHeaders,
{
"Cache-Control": "private, max-age=60",
},
corsHeaders(),
),
});
}
async function scheduleSkillDownloadMetric(
ctx: DownloadCtx,
request: Request,
skillId: Id<"skills">,
) {
try {
const userId = await getOptionalDownloadUserId(ctx, request);
const identity = getDownloadIdentity(request, userId ? String(userId) : null);
if (identity) {
await ctx.scheduler.runAfter(
Math.floor(Math.random() * DOWNLOAD_STAT_JITTER_MS),
internal.downloadMetrics.recordDownloadMetricInternal,
await buildDownloadMetricArgs({
target: { kind: "skill", id: skillId },
identity,
now: Date.now(),
}),
);
}
} catch {
// Best-effort metric path; do not fail downloads.
}
}
async function getOptionalDownloadUserId(
ctx: Parameters<Parameters<typeof httpAction>[0]>[0],
ctx: DownloadCtx,
request: Request,
): Promise<Id<"users"> | null> {
const apiTokenUserId = await getOptionalApiTokenUserId(ctx, request);
+122
View File
@@ -586,6 +586,128 @@ describe("httpApiV1 handlers", () => {
]);
});
it("skills export includes GitHub-backed skills as public GitHub handoff descriptors", async () => {
vi.mocked(requireApiTokenUser).mockResolvedValue({
userId: "users:actor",
user: { _id: "users:actor", role: "user" },
} as never);
vi.mocked(getOptionalApiTokenUser).mockResolvedValue({
userId: "users:actor",
user: { _id: "users:actor", role: "user" },
} as never);
const commit = "2".repeat(40);
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
if ("startDate" in args) {
return {
page: [
{
skillId: "skills:hosted",
slug: "hosted-demo",
displayName: "Hosted Demo",
latestVersionId: "skillVersions:hosted",
createdAt: 1,
updatedAt: 2,
stats: { downloads: 4 },
ownerUserId: "users:alice",
ownerHandle: "alice",
ownerDisplayName: "Alice",
},
{
skillId: "skills:github",
slug: "aiq-deploy",
displayName: "AIQ Deploy",
installKind: "github",
latestVersionId: undefined,
createdAt: 3,
updatedAt: 4,
stats: { downloads: 7 },
ownerUserId: "users:nvidia",
ownerHandle: "nvidia",
ownerDisplayName: "NVIDIA",
},
],
nextCursor: null,
hasMore: false,
};
}
if (args.versionId === "skillVersions:hosted") {
return {
skillId: "skills:hosted",
version: "1.0.0",
files: [{ storageId: "storage:hosted", path: "SKILL.md" }],
};
}
if (args.skillId === "skills:github") {
return {
installKind: "github",
repo: "NVIDIA/skills",
path: "skills/aiq-deploy",
commit,
contentHash: "hash-aiq-deploy",
currentStatus: "present",
scanStatus: "suspicious",
removedAt: null,
};
}
return null;
});
const storageGet = vi.fn(async () => new Blob(["hosted skill"]));
const response = await __handlers.exportSkillsV1Handler(
makeCtx({ runQuery, storage: { get: storageGet } }),
new Request("https://example.com/api/v1/skills/export?startDate=1&endDate=5", {
headers: { authorization: "Bearer user-token" },
}),
);
if (response.status !== 200) throw new Error(await response.text());
expect(response.headers.get("X-Total-Returned")).toBe("2");
expect(response.headers.get("X-Export-Errors")).toBe("0");
const zipEntries = unzipSync(new Uint8Array(await response.arrayBuffer()));
expect(Object.keys(zipEntries).sort()).toEqual([
"_manifest.json",
"alice/hosted-demo/SKILL.md",
"alice/hosted-demo/_export_skill_meta.json",
"nvidia/aiq-deploy/_export_skill_meta.json",
"nvidia/aiq-deploy/_source_handoff.json",
]);
expect(zipEntries["_errors.json"]).toBeUndefined();
const manifest = JSON.parse(new TextDecoder().decode(zipEntries["_manifest.json"]));
expect(manifest).toEqual([
expect.objectContaining({
publisher: "alice",
slug: "hosted-demo",
sourceRef: "public-clawhub",
fileCount: 1,
}),
expect.objectContaining({
publisher: "nvidia",
slug: "aiq-deploy",
sourceRef: "public-github",
version: null,
fileCount: 0,
}),
]);
const handoff = JSON.parse(
new TextDecoder().decode(zipEntries["nvidia/aiq-deploy/_source_handoff.json"]),
);
expect(handoff).toEqual({
sourceRef: "public-github",
repo: "NVIDIA/skills",
commit,
path: "skills/aiq-deploy",
contentHash: "hash-aiq-deploy",
archiveUrl: `https://api.github.com/repos/NVIDIA/skills/zipball/${commit}`,
});
expect(handoff).not.toHaveProperty("scan");
expect(handoff).not.toHaveProperty("scanStatus");
expect(storageGet).toHaveBeenCalledTimes(1);
});
it("skills export skips stale latest versions before reading blobs", async () => {
vi.mocked(requireApiTokenUser).mockResolvedValue({
userId: "users:actor",
+109 -12
View File
@@ -18,6 +18,13 @@ import { api, internal } from "../_generated/api";
import type { Doc, Id } from "../_generated/dataModel";
import type { ActionCtx } from "../_generated/server";
import { getOptionalApiTokenUserId, requireApiTokenUser } from "../lib/apiTokenAuth";
import {
buildGitHubSkillHandoffDescriptor,
getGitHubHandoffBlock,
isReadyGitHubHandoffTarget,
type GitHubHandoffTarget,
type ReadyGitHubHandoffTarget,
} from "../lib/githubHandoff";
import { mergeHeaders } from "../lib/httpHeaders";
import { applyRateLimit } from "../lib/httpRateLimit";
import { parseBooleanQueryParam, resolveBooleanQueryParam } from "../lib/httpUtils";
@@ -3447,6 +3454,7 @@ export async function exportSkillsV1Handler(ctx: ActionCtx, request: Request) {
slug: string;
displayName: string;
latestVersionId?: Id<"skillVersions">;
installKind?: "github";
createdAt: number;
updatedAt: number;
stats?: Record<string, unknown> | null;
@@ -3507,11 +3515,22 @@ export async function exportSkillsV1Handler(ctx: ActionCtx, request: Request) {
})
: Promise.resolve(null),
);
const githubTargets = await chunkedParallel(result.page, 100, (digest) =>
isGitHubSourceExportDigest(digest)
? ctx.runQuery(internal.skills.getGitHubDownloadTargetInternal, {
skillId: digest.skillId,
})
: Promise.resolve(null),
);
logContext.versionCount = versionDocs.filter(Boolean).length;
const exportableVersions: Array<Doc<"skillVersions"> | null> = Array.from(
{ length: result.page.length },
() => null,
);
const exportableGitHubTargets: Array<ReadyGitHubHandoffTarget | null> = Array.from(
{ length: result.page.length },
() => null,
);
type BlobTask = { digestIndex: number; fileIndex: number; storageId: Id<"_storage"> };
const blobTasks: BlobTask[] = [];
@@ -3522,6 +3541,26 @@ export async function exportSkillsV1Handler(ctx: ActionCtx, request: Request) {
const version = versionDocs[i] as Doc<"skillVersions"> | null;
if (!version) {
if (isGitHubSourceExportDigest(digest)) {
const target = githubTargets[i] as GitHubHandoffTarget;
const block = getGitHubHandoffBlock(target);
if (block) {
exportErrors.push({
slug: digest.slug,
error: block.message,
});
continue;
}
if (!isReadyGitHubHandoffTarget(target)) {
exportErrors.push({
slug: digest.slug,
error: "GitHub-backed skill source metadata is incomplete.",
});
continue;
}
exportableGitHubTargets[i] = target;
continue;
}
exportErrors.push({
slug: digest.slug,
error: `version not found (latestVersionId: ${digest.latestVersionId ?? "null"})`,
@@ -3601,7 +3640,6 @@ export async function exportSkillsV1Handler(ctx: ActionCtx, request: Request) {
version?: string;
files?: Array<{ storageId: Id<"_storage">; path: string }>;
} | null;
if (!version?.files) continue;
if (!validateSlug(digest.slug)) continue;
const publisherSegment = getExportPublisherSegment(digest);
@@ -3613,6 +3651,38 @@ export async function exportSkillsV1Handler(ctx: ActionCtx, request: Request) {
continue;
}
const exportRoot = `${publisherSegment}/${digest.slug}`;
const githubTarget = exportableGitHubTargets[i];
if (githubTarget) {
const descriptor = buildGitHubSkillHandoffDescriptor(githubTarget);
zipEntries.push({
path: `${exportRoot}/_source_handoff.json`,
bytes: new TextEncoder().encode(JSON.stringify(descriptor, null, 2)),
});
const skillMeta = buildExportSkillMeta(digest, {
sourceRef: "public-github",
version: null,
});
zipEntries.push({
path: `${exportRoot}/_export_skill_meta.json`,
bytes: new TextEncoder().encode(JSON.stringify(skillMeta, null, 2)),
});
manifest.push({
publisher: publisherSegment,
slug: digest.slug,
sourceRef: "public-github",
version: null,
displayName: digest.displayName,
createdAt: digest.createdAt,
updatedAt: digest.updatedAt,
stats: (digest.stats as Record<string, unknown>) ?? null,
fileCount: 0,
});
continue;
}
if (!version?.files) continue;
const digestBlobs = blobsByDigest.get(i);
if (!digestBlobs) continue;
@@ -3650,18 +3720,10 @@ export async function exportSkillsV1Handler(ctx: ActionCtx, request: Request) {
fileCount++;
}
const skillMeta = {
slug: digest.slug,
displayName: digest.displayName,
const skillMeta = buildExportSkillMeta(digest, {
sourceRef: "public-clawhub",
version: version.version ?? null,
createdAt: digest.createdAt,
updatedAt: digest.updatedAt,
stats: digest.stats ?? null,
owner: {
handle: digest.ownerHandle ?? null,
displayName: digest.ownerDisplayName ?? null,
},
};
});
zipEntries.push({
path: `${exportRoot}/_export_skill_meta.json`,
bytes: new TextEncoder().encode(JSON.stringify(skillMeta, null, 2)),
@@ -3670,6 +3732,7 @@ export async function exportSkillsV1Handler(ctx: ActionCtx, request: Request) {
manifest.push({
publisher: publisherSegment,
slug: digest.slug,
sourceRef: "public-clawhub",
version: version.version ?? null,
displayName: digest.displayName,
createdAt: digest.createdAt,
@@ -3720,3 +3783,37 @@ function getExportPublisherSegment(digest: {
const fallback = String(digest.ownerUserId).replace(/[^a-zA-Z0-9._-]/g, "-");
return validateSlug(fallback) ? fallback : null;
}
function isGitHubSourceExportDigest(digest: {
installKind?: "github";
latestVersionId?: Id<"skillVersions">;
}) {
return digest.installKind === "github" && !digest.latestVersionId;
}
function buildExportSkillMeta(
digest: {
slug: string;
displayName: string;
createdAt: number;
updatedAt: number;
stats?: Record<string, unknown> | null;
ownerHandle?: string | null;
ownerDisplayName?: string | null;
},
source: { sourceRef: "public-clawhub" | "public-github"; version: string | null },
) {
return {
slug: digest.slug,
displayName: digest.displayName,
sourceRef: source.sourceRef,
version: source.version,
createdAt: digest.createdAt,
updatedAt: digest.updatedAt,
stats: digest.stats ?? null,
owner: {
handle: digest.ownerHandle ?? null,
displayName: digest.ownerDisplayName ?? null,
},
};
}
+121
View File
@@ -0,0 +1,121 @@
export type GitHubHandoffBlock = { status: 403 | 409 | 410 | 423; message: string };
export type GitHubHandoffTarget = {
installKind: "github";
repo: string | null;
path: string | null;
commit: string | null;
contentHash: string | null;
currentStatus: "present" | "missing" | "unknown" | null;
scanStatus: "clean" | "suspicious" | "malicious" | "pending" | "failed" | null;
removedAt: number | null;
} | null;
export type ReadyGitHubHandoffTarget = {
installKind: "github";
repo: string;
path: string;
commit: string;
contentHash: string;
currentStatus: "present";
scanStatus: "clean" | "suspicious";
removedAt: number | null;
};
type GitHubHandoffScanStatus = NonNullable<GitHubHandoffTarget>["scanStatus"];
export function getGitHubHandoffBlock(target: GitHubHandoffTarget): GitHubHandoffBlock | null {
if (!target || target.installKind !== "github") {
return {
status: 409,
message: "GitHub-backed skill source metadata is incomplete.",
};
}
if (target.removedAt) {
return {
status: 410,
message: "GitHub-backed skill has been removed upstream.",
};
}
if (target.currentStatus === "missing") {
return {
status: 410,
message: "GitHub-backed skill path is missing upstream.",
};
}
if (target.scanStatus === "failed" || target.scanStatus === "malicious") {
return {
status: 403,
message: "GitHub-backed skill failed ClawHub security scanning.",
};
}
if (!target.repo || !target.path) {
return {
status: 409,
message: "GitHub-backed skill source metadata is incomplete.",
};
}
if (
target.currentStatus !== "present" ||
!target.commit ||
!target.contentHash ||
!isValidGitHubRepo(target.repo)
) {
return {
status: 423,
message: "GitHub-backed skill needs an upstream freshness check before download.",
};
}
if (!isSuccessfulGitHubHandoffScanStatus(target.scanStatus)) {
return {
status: 423,
message: "GitHub-backed skill security scan is in progress.",
};
}
return null;
}
export function isReadyGitHubHandoffTarget(
target: GitHubHandoffTarget,
): target is ReadyGitHubHandoffTarget {
return Boolean(
target &&
target.installKind === "github" &&
target.repo &&
target.path &&
target.commit &&
target.contentHash &&
target.currentStatus === "present" &&
isSuccessfulGitHubHandoffScanStatus(target.scanStatus) &&
isValidGitHubRepo(target.repo),
);
}
export function buildGitHubSkillHandoffDescriptor(target: ReadyGitHubHandoffTarget) {
return {
sourceRef: "public-github" as const,
repo: target.repo,
commit: target.commit,
path: target.path,
contentHash: target.contentHash,
archiveUrl: buildGitHubZipballUrl(target.repo, target.commit),
};
}
export function isSuccessfulGitHubHandoffScanStatus(scanStatus: GitHubHandoffScanStatus) {
// Suspicious is review-worthy but not a hard public download block; pending,
// failed, and malicious states still block handoff above.
return scanStatus === "clean" || scanStatus === "suspicious";
}
function buildGitHubZipballUrl(repo: string, commit: string) {
const [owner, name] = repo.split("/");
return `https://api.github.com/repos/${encodeURIComponent(owner)}/${encodeURIComponent(
name,
)}/zipball/${encodeURIComponent(commit)}`;
}
function isValidGitHubRepo(repo: string) {
const [owner, name, ...rest] = repo.split("/");
return Boolean(owner && name && rest.length === 0);
}
+1
View File
@@ -81,6 +81,7 @@ export function buildDeterministicPackageZip(entries: ZipEntry[]) {
export interface MergedExportManifestEntry {
publisher: string;
slug: string;
sourceRef?: "public-clawhub" | "public-github";
version: string | null;
displayName: string;
createdAt: number;
+33 -1
View File
@@ -66,6 +66,34 @@ describe("skills.listByDateRange export list", () => {
getPageMock.mockResolvedValue({
page: [
digest({ slug: "exportable" }),
digest({
slug: "github-clean",
latestVersionId: undefined,
installKind: "github",
githubCurrentStatus: "present",
githubScanStatus: "clean",
}),
digest({
slug: "github-suspicious",
latestVersionId: undefined,
installKind: "github",
githubCurrentStatus: "present",
githubScanStatus: "suspicious",
}),
digest({
slug: "github-pending",
latestVersionId: undefined,
installKind: "github",
githubCurrentStatus: "present",
githubScanStatus: "pending",
}),
digest({
slug: "github-missing",
latestVersionId: undefined,
installKind: "github",
githubCurrentStatus: "missing",
githubScanStatus: "clean",
}),
digest({ slug: "missing-version", latestVersionId: undefined }),
digest({ slug: "hidden", moderationStatus: "hidden" }),
digest({ slug: "malicious", moderationFlags: ["blocked.malware"] }),
@@ -77,7 +105,11 @@ describe("skills.listByDateRange export list", () => {
const result = await listByDateRangeHandler({ db: {} }, { startDate: 1, endDate: 5 });
expect(result.page.map((item) => item.slug)).toEqual(["exportable"]);
expect(result.page.map((item) => item.slug)).toEqual([
"exportable",
"github-clean",
"github-suspicious",
]);
expect(getPageMock).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
+34 -2
View File
@@ -2744,6 +2744,26 @@ export const getBySlug = query({
},
});
export const getGitHubDownloadTargetInternal = internalQuery({
args: { skillId: v.id("skills") },
handler: async (ctx, args) => {
const skill = await ctx.db.get(args.skillId);
if (!skill || skill.installKind !== "github") return null;
const source = skill.githubSourceId ? await ctx.db.get(skill.githubSourceId) : null;
return {
installKind: "github" as const,
repo: source?.repo ?? null,
path: skill.githubPath ?? null,
commit: skill.githubCurrentCommit ?? null,
contentHash: skill.githubCurrentContentHash ?? null,
currentStatus: skill.githubCurrentStatus ?? null,
scanStatus: skill.githubScanStatus ?? null,
removedAt: skill.githubRemovedAt ?? null,
};
},
});
export const getVerifyTargetBySlugInternal = internalQuery({
args: { slug: v.string(), ownerHandle: v.optional(v.string()) },
handler: async (ctx, args) => {
@@ -12765,10 +12785,22 @@ export const listByDateRange = internalQuery({
function isExportableSkillDigest(
skill: Pick<
Doc<"skillSearchDigest">,
"latestVersionId" | "softDeletedAt" | "moderationStatus" | "moderationFlags"
| "latestVersionId"
| "installKind"
| "githubCurrentStatus"
| "githubScanStatus"
| "softDeletedAt"
| "moderationStatus"
| "moderationFlags"
>,
) {
return Boolean(skill.latestVersionId) && isPublicSkillDoc(skill);
if (!isPublicSkillDoc(skill)) return false;
if (skill.latestVersionId) return true;
return (
skill.installKind === "github" &&
skill.githubCurrentStatus === "present" &&
(skill.githubScanStatus === "clean" || skill.githubScanStatus === "suspicious")
);
}
export const __test = {
+8
View File
@@ -98,6 +98,13 @@ Public read:
- `GET /api/v1/skills/{slug}/file?path=&version=&tag=`
- `GET /api/v1/resolve?slug=&hash=`
- `GET /api/v1/download?slug=&version=&tag=`
- Hosted skills return deterministic ZIP bytes.
- Current GitHub-backed skills with a `clean` or `suspicious` scan return a
JSON `public-github` handoff descriptor instead of ClawHub bytes.
- `GET /api/v1/skills/export?startDate=&endDate=&limit=&cursor=`
- Hosted skills are exported as stored files.
- Current GitHub-backed skills with a `clean` or `suspicious` scan are exported
as `public-github` handoff descriptors.
- `GET /api/v1/packages?limit=&cursor=&sort=`
- `sort`: `updated` (default), `recommended`, `downloads`, legacy alias `installs`
- Invalid `sort` values return `400`
@@ -124,6 +131,7 @@ Auth required:
- `POST /api/v1/skills/{slug}/transfer/accept`
- `POST /api/v1/skills/{slug}/transfer/reject`
- `POST /api/v1/skills/{slug}/transfer/cancel`
- `GET /api/v1/skills/export?startDate=&endDate=&limit=&cursor=`
- `GET /api/v1/plugins/export?startDate=&endDate=&limit=&cursor=&family=`
- `GET /api/v1/transfers/incoming`
- `GET /api/v1/transfers/outgoing`
+45 -2
View File
@@ -604,6 +604,43 @@ Legacy v1 filter aliases remain accepted on read endpoints:
Legacy aliases are not accepted as stored or author-declared category values.
### `GET /api/v1/skills/export`
Bulk export of latest public skills for offline analysis.
Auth:
- API token required.
Query params:
- `startDate` (required): Unix milliseconds lower bound for skill `updatedAt`.
- `endDate` (required): Unix milliseconds upper bound for skill `updatedAt`.
- `limit` (optional): integer (1-250), default `250`.
- `cursor` (optional): pagination cursor from the previous response.
Response:
- Body: ZIP archive.
- Each exported skill is rooted at `{publisher}/{slug}/`.
- Hosted skills include the latest stored version files and are listed in
`_manifest.json` with `sourceRef: "public-clawhub"`.
- Current GitHub-backed skills with a `clean` or `suspicious` scan include
`_source_handoff.json` with `sourceRef: "public-github"`, repo, commit, path,
content hash, and archive URL. They do not include ClawHub-hosted source files.
- Each skill includes `_export_skill_meta.json`.
- `_manifest.json` is always included at the ZIP root.
- `_errors.json` is included when individual skills or files could not be
exported.
Headers:
- `X-Next-Cursor`
- `X-Has-More`
- `X-Total-Returned`
- `X-Date-Range`
- `X-Export-Errors`
### `GET /api/v1/plugins/export`
Bulk export of latest public plugin releases for offline analysis.
@@ -1228,7 +1265,9 @@ Response:
### `GET /api/v1/download`
Downloads a zip of a skill version.
Downloads a hosted skill version ZIP, or returns a GitHub source handoff for a
current GitHub-backed skill with a `clean` or `suspicious` scan and no hosted
version.
Query params:
@@ -1240,7 +1279,11 @@ Notes:
- If neither `version` nor `tag` is provided, the latest version is used.
- Soft-deleted versions return `410`.
- Download stats are counted as unique identities per hour (`userId` when API token is valid, otherwise IP).
- GitHub-backed skill handoffs do not proxy or mirror bytes. The JSON response
includes `sourceRef: "public-github"`, `repo`, `commit`, `path`, `contentHash`,
and `archiveUrl`; scan/current state is a gate and is not included as success
payload metadata.
- Download stats are counted as unique identities per UTC day (`userId` when API token is valid, otherwise IP).
## Auth endpoints (Bearer token)
+43 -2
View File
@@ -27,6 +27,42 @@
}
}
},
"GitHubSkillDownloadHandoff": {
"type": "object",
"additionalProperties": false,
"required": [
"sourceRef",
"repo",
"commit",
"path",
"contentHash",
"archiveUrl"
],
"properties": {
"sourceRef": {
"type": "string",
"enum": [
"public-github"
]
},
"repo": {
"type": "string"
},
"commit": {
"type": "string"
},
"path": {
"type": "string"
},
"contentHash": {
"type": "string"
},
"archiveUrl": {
"type": "string",
"format": "uri"
}
}
},
"Owner": {
"type": [
"object",
@@ -1619,7 +1655,7 @@
},
"/api/v1/download": {
"get": {
"summary": "Download zip",
"summary": "Download skill ZIP or GitHub handoff",
"parameters": [
{
"name": "slug",
@@ -1648,13 +1684,18 @@
],
"responses": {
"200": {
"description": "Zip",
"description": "Hosted ZIP bytes or a GitHub-backed source handoff descriptor.",
"content": {
"application/zip": {
"schema": {
"type": "string",
"format": "binary"
}
},
"application/json": {
"schema": {
"$ref": "#/components/schemas/GitHubSkillDownloadHandoff"
}
}
}
}
+14
View File
@@ -235,6 +235,20 @@ bypass failed, suspicious, malicious, missing, or removed upstream states.
ClawHub must not create hosted `skillVersions` or ClawHub download artifacts for
GitHub-backed skills.
`GET /api/v1/download` may still be used as a metered source handoff for current
GitHub-backed skills whose scan verdict is `clean` or `suspicious`. In that case
ClawHub returns only stored fetch coordinates (`sourceRef: "public-github"`,
repo, commit, path, content hash, and an archive URL) after checking current
upstream and scan state. It must not fetch GitHub, expand archives, proxy bytes,
create `skillVersions`, or include detailed scan metadata in the successful
payload.
`GET /api/v1/skills/export` follows the same no-mirror contract. Hosted skills
continue to export stored version files with `sourceRef: "public-clawhub"`.
Current GitHub-backed skills whose scan verdict is `clean` or `suspicious` are
included as `sourceRef: "public-github"` entries with `_source_handoff.json`
coordinates, not ClawHub-hosted source files.
This avoids two NVIDIA concerns:
- Signature drift: any byte-level transformation in a mirror can invalidate
+32
View File
@@ -17,6 +17,13 @@ function sortValuesForPath(paths: unknown, path: string) {
return property(property(sortParameter, "schema"), "enum");
}
function responseContentForPath(paths: unknown, path: string, status: string) {
const routePath = property(paths, path);
const getOperation = property(routePath, "get");
const responses = property(getOperation, "responses");
return property(property(responses, status), "content");
}
describe("OpenAPI contract", () => {
it("documents accepted skills sort aliases", async () => {
const specPath = new URL("../../public/api/v1/openapi.json", import.meta.url);
@@ -50,4 +57,29 @@ describe("OpenAPI contract", () => {
expect(sortValuesForPath(paths, "/api/v1/code-plugins")).toEqual(sortValues);
expect(sortValuesForPath(paths, "/api/v1/bundle-plugins")).toEqual(sortValues);
});
it("documents skill downloads as either hosted ZIP bytes or GitHub handoff JSON", async () => {
const specPath = new URL("../../public/api/v1/openapi.json", import.meta.url);
const spec: unknown = JSON.parse(await readFile(specPath, "utf8"));
const paths = property(spec, "paths");
const schemas = property(property(spec, "components"), "schemas");
const responseContent = responseContentForPath(paths, "/api/v1/download", "200");
expect(property(responseContent, "application/zip")).toBeTruthy();
expect(property(property(responseContent, "application/json"), "schema")).toEqual({
$ref: "#/components/schemas/GitHubSkillDownloadHandoff",
});
const handoffSchema = property(schemas, "GitHubSkillDownloadHandoff");
expect(property(handoffSchema, "required")).toEqual([
"sourceRef",
"repo",
"commit",
"path",
"contentHash",
"archiveUrl",
]);
expect(property(property(handoffSchema, "properties"), "scan")).toBeUndefined();
expect(property(property(handoffSchema, "properties"), "scanStatus")).toBeUndefined();
});
});