diff --git a/bun.lock b/bun.lock index d246c932..e5789c15 100644 --- a/bun.lock +++ b/bun.lock @@ -41,6 +41,7 @@ "fflate": "0.8.3", "h3": "2.0.1-rc.25", "ignore": "7.0.6", + "jose": "6.2.3", "lucide-react": "1.28.0", "mermaid": "^11.16.1", "mime": "4.1.0", diff --git a/convex/downloads.test.ts b/convex/downloads.test.ts index 43de37dc..9bcb2a33 100644 --- a/convex/downloads.test.ts +++ b/convex/downloads.test.ts @@ -1,7 +1,20 @@ import type { RateLimitArgs, RateLimitReturns } from "@convex-dev/rate-limiter"; +import { unzipSync } from "fflate"; +import { exportJWK, exportPKCS8, generateKeyPair } from "jose"; import { afterEach, describe, expect, it, vi } from "vitest"; import type { ActionCtx } from "./_generated/server"; -import { __test, downloadZipHandler } from "./downloads"; +import { __test, downloadZipHandler, recordArchiveDownloadMetricHandler } from "./downloads"; +import { + ARCHIVE_MANIFEST_AUDIENCE, + ARCHIVE_MANIFEST_CONTENT_TYPE, + ARCHIVE_MANIFEST_JWS_TYPE, + ARCHIVE_METRIC_AUDIENCE, + ARCHIVE_METRIC_JWS_TYPE, + type ArchiveMetricPayload, + signArchivePayload, + type SkillArchiveManifest, + verifyArchivePayloadWithLocalJwks, +} from "./lib/archiveManifest"; function isRateLimitArgs(args: unknown): args is RateLimitArgs { if (!args || typeof args !== "object") return false; @@ -35,8 +48,30 @@ function stubZipResponse() { vi.stubGlobal("Response", MockResponse as unknown as typeof Response); } +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise((resolvePromise) => { + resolve = resolvePromise; + }); + return { promise, resolve }; +} + +function streamingBlob(text: string) { + const bytes = new TextEncoder().encode(text); + return { + stream: () => + new ReadableStream({ + start(controller) { + controller.enqueue(bytes); + controller.close(); + }, + }), + } as Blob; +} + describe("downloads helpers", () => { afterEach(() => { + vi.restoreAllMocks(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); }); @@ -79,7 +114,6 @@ describe("downloads helpers", () => { it("schedules zip download stats outside the response path", async () => { vi.stubEnv("TRUST_FORWARDED_IPS", "true"); - stubZipResponse(); const runQuery = vi.fn(async (_query: unknown, args: Record) => { if ("slug" in args) { @@ -111,23 +145,28 @@ describe("downloads helpers", () => { return { mutation, args }; }); const runAfter = vi.fn(); - const storageGet = vi.fn().mockResolvedValue(new Blob(["hello"], { type: "text/markdown" })); + const storageGet = vi.fn().mockResolvedValue(streamingBlob("hello")); const response = await downloadZipHandler( { runQuery, runMutation, scheduler: { runAfter }, - storage: { get: storageGet }, + storage: { + get: storageGet, + getMetadata: vi.fn().mockResolvedValue({}), + }, } as unknown as ActionCtx, - new Request("https://example.com/api/v1/download?slug=demo", { + new Request("https://preview-branch-123.convex.site/api/v1/download?slug=demo", { headers: { "cf-connecting-ip": "1.2.3.4" }, }), ); expect(response.status).toBe(200); expect(response.headers.get("Content-Type")).toBe("application/zip"); + const archive = new Uint8Array(await response.arrayBuffer()); expect(storageGet).toHaveBeenCalledWith("_storage:1"); + expect(new TextDecoder().decode(unzipSync(archive)["SKILL.md"])).toBe("hello"); const recordCalls = runAfter.mock.calls.filter(([, , args]) => { if (!args || typeof args !== "object") return false; @@ -152,6 +191,373 @@ describe("downloads helpers", () => { }); }); + it("returns a bounded archive manifest to the Nitro streaming owner", async () => { + vi.stubEnv("CLAWHUB_PREVIEW", "1"); + vi.stubEnv("TRUST_FORWARDED_IPS", "true"); + vi.spyOn(Date, "now").mockReturnValue(10_000); + const keyPair = await generateKeyPair("RS256", { extractable: true }); + const privateKey = await exportPKCS8(keyPair.privateKey); + const publicKey = await exportJWK(keyPair.publicKey); + const jwks = JSON.stringify({ keys: [{ use: "sig", ...publicKey }] }); + vi.stubEnv("JWT_PRIVATE_KEY", privateKey); + vi.stubEnv("JWKS", jwks); + const runQuery = vi.fn(async (_query: unknown, args: Record) => { + if ("slug" in args) { + return { + skill: { + _id: "skills:1", + ownerUserId: "users:1", + slug: "demo", + tags: {}, + latestVersionId: "skillVersions:1", + }, + moderationInfo: null, + }; + } + if ("versionId" in args) { + return { + _id: "skillVersions:1", + skillId: "skills:1", + version: "1.0.0+build", + createdAt: 3, + files: [ + { path: "SKILL.md", storageId: "_storage:1" }, + { path: "missing.txt", storageId: "_storage:missing" }, + ], + softDeletedAt: undefined, + }; + } + return null; + }); + const runMutation = vi.fn(async (_mutation: unknown, args: Record) => { + if (isRateLimitArgs(args)) return okRate(); + return null; + }); + const runAfter = vi.fn(); + const storageGet = vi.fn(); + const storageGetUrl = vi.fn(async (storageId: string) => + storageId === "_storage:1" + ? "https://preview-branch-123.convex.cloud/api/storage/storage-1" + : null, + ); + + const response = await downloadZipHandler( + { + runQuery, + runMutation, + scheduler: { runAfter }, + storage: { + get: storageGet, + getUrl: storageGetUrl, + getMetadata: vi.fn(), + }, + } as unknown as ActionCtx, + new Request("https://preview-branch-123.convex.site/api/v1/download?slug=demo", { + headers: { + "cf-connecting-ip": "1.2.3.4", + "x-clawhub-archive-manifest": "v1", + "x-clawhub-vercel-oidc-token": "vercel-oidc", + }, + }), + { verifyArchiveRequester: vi.fn(async () => undefined) }, + ); + + expect(response.status).toBe(200); + expect(response.headers.get("Content-Type")).toBe(ARCHIVE_MANIFEST_CONTENT_TYPE); + expect(response.headers.get("Cache-Control")).toBe("private, no-store"); + const manifest = (await verifyArchivePayloadWithLocalJwks( + await response.text(), + ARCHIVE_MANIFEST_JWS_TYPE, + jwks, + )) as SkillArchiveManifest; + expect(manifest).toEqual({ + schema: "clawhub.skill-archive-manifest.v1", + issuer: "https://preview-branch-123.convex.site", + audience: ARCHIVE_MANIFEST_AUDIENCE, + issuedAt: 10_000, + expiresAt: 40_000, + filename: "demo-1.0.0+build.zip", + meta: { + ownerId: "users:1", + slug: "demo", + version: "1.0.0+build", + publishedAt: 3, + }, + entries: [ + { + path: "SKILL.md", + url: "https://preview-branch-123.convex.cloud/api/storage/storage-1", + }, + ], + metricToken: expect.any(String), + }); + const metricPayload = (await verifyArchivePayloadWithLocalJwks( + manifest.metricToken!, + ARCHIVE_METRIC_JWS_TYPE, + jwks, + )) as ArchiveMetricPayload; + expect(metricPayload).toMatchObject({ + schema: "clawhub.archive-download-metric.v1", + issuer: "https://preview-branch-123.convex.site", + audience: ARCHIVE_METRIC_AUDIENCE, + issuedAt: 10_000, + expiresAt: 40_000, + metric: { + target: { kind: "skill", id: "skills:1" }, + identityKind: "ip", + identityHash: expect.any(String), + dayStart: 0, + occurredAt: 10_000, + }, + }); + expect(storageGet).not.toHaveBeenCalled(); + expect(storageGetUrl).toHaveBeenCalledTimes(2); + expect(runAfter).not.toHaveBeenCalled(); + }); + + it("rejects a direct manifest request without the Nitro Vercel identity", async () => { + vi.stubEnv("CLAWHUB_PREVIEW", "1"); + const runQuery = vi.fn(); + const runMutation = vi.fn(async (_mutation: unknown, args: Record) => { + if (isRateLimitArgs(args)) return okRate(); + return null; + }); + + const verifyArchiveRequester = vi.fn(async () => { + throw new Error("invalid Vercel identity"); + }); + const response = await downloadZipHandler( + { runQuery, runMutation } as unknown as ActionCtx, + new Request("https://preview-branch-123.convex.site/api/v1/download?slug=demo", { + headers: { + "x-clawhub-archive-manifest": "v1", + "x-clawhub-vercel-oidc-token": "client-forgery", + }, + }), + { verifyArchiveRequester }, + ); + + expect(response.status).toBe(401); + expect(response.headers.get("Cache-Control")).toBe("no-store"); + expect(verifyArchiveRequester).toHaveBeenCalledWith("client-forgery", "preview"); + expect(runQuery).not.toHaveBeenCalled(); + }); + + it("records only a valid, unexpired archive metric capability", async () => { + const keyPair = await generateKeyPair("RS256", { extractable: true }); + const privateKey = await exportPKCS8(keyPair.privateKey); + const publicKey = await exportJWK(keyPair.publicKey); + const jwks = JSON.stringify({ keys: [{ use: "sig", ...publicKey }] }); + vi.stubEnv("JWKS", jwks); + vi.spyOn(Date, "now").mockReturnValue(2_000); + const payload: ArchiveMetricPayload = { + schema: "clawhub.archive-download-metric.v1", + issuer: "https://example.com", + audience: ARCHIVE_METRIC_AUDIENCE, + issuedAt: 1_000, + expiresAt: 31_000, + metric: { + target: { kind: "skill", id: "skills:1" }, + identityKind: "ip", + identityHash: "identity-hash", + dayStart: 0, + occurredAt: 1_000, + }, + }; + const token = await signArchivePayload(payload, ARCHIVE_METRIC_JWS_TYPE, privateKey); + const runAfter = vi.fn(); + const ctx = { scheduler: { runAfter } } as unknown as ActionCtx; + + const response = await recordArchiveDownloadMetricHandler( + ctx, + new Request("https://example.com/api/internal/archive-download-metric", { + method: "POST", + body: token, + }), + ); + expect(response.status).toBe(204); + expect(runAfter).toHaveBeenCalledWith(expect.any(Number), expect.anything(), payload.metric); + + const [header, body, signature] = token.split("."); + const modifiedBody = `${body!.slice(0, -1)}${body!.endsWith("A") ? "B" : "A"}`; + const modifiedToken = `${header}.${modifiedBody}.${signature}`; + const modifiedResponse = await recordArchiveDownloadMetricHandler( + ctx, + new Request("https://example.com/api/internal/archive-download-metric", { + method: "POST", + body: modifiedToken, + }), + ); + expect(modifiedResponse.status).toBe(401); + + const expiredToken = await signArchivePayload( + { ...payload, expiresAt: 1_500 }, + ARCHIVE_METRIC_JWS_TYPE, + privateKey, + ); + const expiredResponse = await recordArchiveDownloadMetricHandler( + ctx, + new Request("https://example.com/api/internal/archive-download-metric", { + method: "POST", + body: expiredToken, + }), + ); + expect(expiredResponse.status).toBe(401); + expect(runAfter).toHaveBeenCalledTimes(1); + }); + + it("streams stored file chunks, stays deterministic, and skips a Blob that vanishes", async () => { + const firstChunk = new Uint8Array(64 * 1024).fill(0x61); + const secondChunk = new TextEncoder().encode("streamed body\n"); + const releaseSecondChunk = deferred(); + const arrayBuffer = vi.fn(() => Promise.reject(new Error("whole Blob read"))); + const stream = vi.fn( + () => + new ReadableStream({ + start(controller) { + controller.enqueue(firstChunk); + }, + async pull(controller) { + await releaseSecondChunk.promise; + controller.enqueue(secondChunk); + controller.close(); + }, + }), + ); + const storageGetMetadata = vi.fn().mockResolvedValue({}); + const storageGet = vi.fn(async (storageId: string) => { + if (storageId === "_storage:skill") { + return { arrayBuffer, stream } as unknown as Blob; + } + if (storageId === "_storage:notes") { + return { + stream: () => + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("supporting notes\n")); + controller.close(); + }, + }), + } as Blob; + } + return null; + }); + const runQuery = vi.fn(async (_query: unknown, args: Record) => { + if ("slug" in args) { + return { + skill: { + _id: "skills:1", + ownerUserId: "users:1", + slug: "demo", + tags: {}, + latestVersionId: "skillVersions:1", + }, + moderationInfo: null, + }; + } + if ("versionId" in args) { + return { + _id: "skillVersions:1", + skillId: "skills:1", + version: "1.0.0", + createdAt: 3, + files: [ + { path: "a.txt", storageId: "_storage:skill" }, + { path: "b.txt", storageId: "_storage:notes" }, + { path: "missing.txt", storageId: "_storage:missing" }, + ], + softDeletedAt: undefined, + }; + } + return null; + }); + const runMutation = vi.fn(async (_mutation: unknown, args: Record) => { + if (isRateLimitArgs(args)) return okRate(); + return null; + }); + + const response = await Promise.race([ + downloadZipHandler( + { + runQuery, + runMutation, + scheduler: { runAfter: vi.fn() }, + storage: { get: storageGet, getMetadata: storageGetMetadata }, + } as unknown as ActionCtx, + new Request("https://example.com/api/v1/download?slug=demo"), + ), + new Promise((_, reject) => { + setTimeout( + () => reject(new Error("download handler read archive bodies before responding")), + 1_000, + ); + }), + ]); + + expect(response.status).toBe(200); + expect(storageGetMetadata).not.toHaveBeenCalled(); + expect(storageGet).not.toHaveBeenCalled(); + + const reader = response.body!.getReader(); + const firstArchiveChunk = await reader.read(); + expect(firstArchiveChunk.done).toBe(false); + expect(stream).toHaveBeenCalledTimes(1); + expect(arrayBuffer).not.toHaveBeenCalled(); + releaseSecondChunk.resolve(); + + const archiveChunks = [firstArchiveChunk.value!]; + for (;;) { + const chunk = await reader.read(); + if (chunk.done) break; + archiveChunks.push(chunk.value); + } + const responseBytes = Uint8Array.from(archiveChunks.flatMap((chunk) => [...chunk])); + const unzipped = unzipSync(responseBytes); + expect(Object.keys(unzipped).sort()).toEqual(["_meta.json", "a.txt", "b.txt"]); + expect(unzipped["a.txt"]).toEqual(Uint8Array.from([...firstChunk, ...secondChunk])); + expect(new TextDecoder().decode(unzipped["b.txt"])).toBe("supporting notes\n"); + expect(storageGet).toHaveBeenCalledWith("_storage:missing"); + + const repeatResponse = await downloadZipHandler( + { + runQuery, + runMutation, + scheduler: { runAfter: vi.fn() }, + storage: { + get: vi.fn(async (storageId: string) => { + if (storageId === "_storage:skill") { + return { + stream: () => + new ReadableStream({ + start(controller) { + controller.enqueue(firstChunk); + controller.enqueue(secondChunk); + controller.close(); + }, + }), + } as Blob; + } + if (storageId === "_storage:notes") { + return { + stream: () => + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("supporting notes\n")); + controller.close(); + }, + }), + } as Blob; + } + return null; + }), + getMetadata: storageGetMetadata, + }, + } as unknown as ActionCtx, + new Request("https://example.com/api/v1/download?slug=demo"), + ); + expect(new Uint8Array(await repeatResponse.arrayBuffer())).toEqual(responseBytes); + }); + it("returns 410 for an explicitly requested revoked version", async () => { const runQuery = vi.fn(async (_query: unknown, args: Record) => { if ("slug" in args) { @@ -195,7 +601,7 @@ describe("downloads helpers", () => { runQuery, runMutation, scheduler: { runAfter }, - storage: { get: storageGet }, + storage: { get: storageGet, getMetadata: vi.fn().mockResolvedValue({}) }, } as unknown as ActionCtx, new Request("https://example.com/api/v1/download?slug=demo&version=1.0.0"), ); @@ -241,7 +647,10 @@ describe("downloads helpers", () => { runQuery, runMutation, scheduler: { runAfter: vi.fn() }, - storage: { get: vi.fn().mockResolvedValue(new Blob(["hello"])) }, + storage: { + get: vi.fn().mockResolvedValue(streamingBlob("hello")), + getMetadata: vi.fn().mockResolvedValue({}), + }, } as unknown as ActionCtx, new Request("https://example.com/api/v1/download?slug=demo&ownerHandle=clawkit"), ); @@ -302,7 +711,7 @@ describe("downloads helpers", () => { runQuery, runMutation, scheduler: { runAfter: vi.fn() }, - storage: { get: storageGet }, + storage: { get: storageGet, getMetadata: vi.fn().mockResolvedValue({}) }, } as unknown as ActionCtx, new Request("https://example.com/api/v1/download?slug=demo&tag=old", { headers: { "cf-connecting-ip": "1.2.3.4" }, @@ -403,7 +812,7 @@ describe("downloads helpers", () => { runQuery, runMutation, scheduler: { runAfter: vi.fn() }, - storage: { get: storageGet }, + storage: { get: storageGet, getMetadata: vi.fn().mockResolvedValue({}) }, } as unknown as ActionCtx, new Request("https://example.com/api/v1/download?slug=demo&version=1.0.0", { headers: { "cf-connecting-ip": "1.2.3.4" }, @@ -453,17 +862,21 @@ describe("downloads helpers", () => { }); const runMutation = vi.fn(async (_mutation: unknown, args: Record) => { if (isRateLimitArgs(args)) return okRate(); + if (Object.keys(args).length === 0) return "https://upload.example"; return { tokenTouched: "tokenId" in args }; }); const runAfter = vi.fn(); - const storageGet = vi.fn().mockResolvedValue(new Blob(["hello"], { type: "text/markdown" })); + const storageGet = vi.fn().mockResolvedValue(streamingBlob("hello")); const response = await downloadZipHandler( { runQuery, runMutation, scheduler: { runAfter }, - storage: { get: storageGet }, + storage: { + get: storageGet, + getMetadata: vi.fn().mockResolvedValue({}), + }, } as unknown as ActionCtx, new Request("https://example.com/api/v1/download?slug=demo", { headers: { @@ -516,17 +929,21 @@ describe("downloads helpers", () => { }); const runMutation = vi.fn(async (_mutation: unknown, args: Record) => { if (isRateLimitArgs(args)) return okRate(); + if (Object.keys(args).length === 0) return "https://upload.example"; return { mutationRecorded: true }; }); const runAfter = vi.fn(); - const storageGet = vi.fn().mockResolvedValue(new Blob(["hello"], { type: "text/markdown" })); + const storageGet = vi.fn().mockResolvedValue(streamingBlob("hello")); const response = await downloadZipHandler( { runQuery, runMutation, scheduler: { runAfter }, - storage: { get: storageGet }, + storage: { + get: storageGet, + getMetadata: vi.fn().mockResolvedValue({}), + }, } as unknown as ActionCtx, new Request("https://example.com/api/v1/download?slug=demo", { headers: { "cf-connecting-ip": "1.2.3.4" }, diff --git a/convex/downloads.ts b/convex/downloads.ts index 7288e7a0..52cb71a1 100644 --- a/convex/downloads.ts +++ b/convex/downloads.ts @@ -5,6 +5,24 @@ import { httpAction } from "./functions"; import { ambiguousSkillSlugResponse } from "./httpApiV1/shared"; import { getOptionalActiveAuthUserIdFromAction } from "./lib/access"; import { getOptionalApiTokenUserId } from "./lib/apiTokenAuth"; +import { + ARCHIVE_MANIFEST_AUDIENCE, + ARCHIVE_MANIFEST_CONTENT_TYPE, + ARCHIVE_MANIFEST_JWS_TYPE, + ARCHIVE_METRIC_AUDIENCE, + ARCHIVE_METRIC_JWS_TYPE, + type ArchiveMetricArgs, + type ArchiveMetricPayload, + signArchivePayload, + type SkillArchiveManifest, + verifyArchivePayloadWithLocalJwks, +} from "./lib/archiveManifest"; +import { + ARCHIVE_REQUEST_IDENTITY_HEADER, + expectedVercelEnvironmentForConvexSite, + type ClawHubVercelEnvironment, + verifyClawHubVercelOidcToken, +} from "./lib/clawhubVercelOidc"; import { buildGitHubSkillHandoffDescriptor, getGitHubHandoffBlock, @@ -18,14 +36,35 @@ import { getPublicSkillVersionDownloadBlock, isSkillVersionForSkill, } from "./lib/skillFileAccess"; -import { buildDeterministicZip } from "./lib/skillZip"; +import { buildDeterministicZipStream } from "./lib/skillZip"; const HOUR_MS = 3_600_000; const DOWNLOAD_STAT_JITTER_MS = 60_000; +const ARCHIVE_MANIFEST_REQUEST_HEADER = "x-clawhub-archive-manifest"; +const ARCHIVE_MANIFEST_TTL_MS = 30_000; +const ARCHIVE_MANIFEST_CLOCK_SKEW_MS = 5_000; +const MAX_ARCHIVE_MANIFEST_FILES = 8_192; +const MAX_ARCHIVE_MANIFEST_BYTES = 4 * 1024 * 1024; +const MAX_ARCHIVE_METRIC_TOKEN_BYTES = 16 * 1024; type DownloadCtx = Parameters[0]>[0]; -export async function downloadZipHandler(ctx: DownloadCtx, request: Request) { +type DownloadDependencies = { + verifyArchiveRequester: ( + token: string, + expectedEnvironment: ClawHubVercelEnvironment, + ) => Promise; +}; + +const DEFAULT_DOWNLOAD_DEPENDENCIES: DownloadDependencies = { + verifyArchiveRequester: verifyClawHubVercelOidcToken, +}; + +export async function downloadZipHandler( + ctx: DownloadCtx, + request: Request, + dependencies: DownloadDependencies = DEFAULT_DOWNLOAD_DEPENDENCIES, +) { const url = new URL(request.url); const slug = url.searchParams.get("slug")?.trim().toLowerCase(); const ownerHandle = @@ -42,6 +81,20 @@ export async function downloadZipHandler(ctx: DownloadCtx, request: Request) { }); } + const manifestRequested = request.headers.get(ARCHIVE_MANIFEST_REQUEST_HEADER) === "v1"; + if (manifestRequested) { + const token = request.headers.get(ARCHIVE_REQUEST_IDENTITY_HEADER)?.trim(); + const expectedEnvironment = expectedVercelEnvironmentForConvexSite(request.url); + if (!token || !expectedEnvironment) { + return unauthorizedArchiveManifestResponse(); + } + try { + await dependencies.verifyArchiveRequester(token, expectedEnvironment); + } catch { + return unauthorizedArchiveManifestResponse(); + } + } + const rate = await applyRateLimit(ctx, request, "download"); if (!rate.ok) return rate.response; @@ -117,24 +170,77 @@ export async function downloadZipHandler(ctx: DownloadCtx, request: Request) { }); } - const entries: Array<{ path: string; bytes: Uint8Array }> = []; - for (const file of version.files) { - const blob = await ctx.storage.get(file.storageId); - if (!blob) continue; - const buffer = new Uint8Array(await blob.arrayBuffer()); - entries.push({ path: file.path, bytes: buffer }); - } - const zipArray = buildDeterministicZip(entries, { + const meta = { ownerId: String(skill.ownerUserId), slug: skill.slug, version: version.version, publishedAt: version.createdAt, - }); - const zipBlob = new Blob([zipArray], { type: "application/zip" }); + }; + + if (manifestRequested) { + if (version.files.length > MAX_ARCHIVE_MANIFEST_FILES) { + return new Response("Skill archive contains too many files", { + status: 413, + headers: mergeHeaders(rate.headers, corsHeaders()), + }); + } + const entries: Array<{ path: string; url: string }> = []; + for (const file of version.files) { + const fileUrl = await ctx.storage.getUrl(file.storageId); + if (fileUrl) entries.push({ path: file.path, url: fileUrl }); + } + const issuedAt = Date.now(); + const expiresAt = issuedAt + ARCHIVE_MANIFEST_TTL_MS; + const issuer = url.origin; + const metricToken = await buildArchiveDownloadMetricToken( + ctx, + request, + skill._id, + issuer, + issuedAt, + expiresAt, + ); + const manifest: SkillArchiveManifest = { + schema: "clawhub.skill-archive-manifest.v1", + issuer, + audience: ARCHIVE_MANIFEST_AUDIENCE, + issuedAt, + expiresAt, + filename: `${slug}-${version.version}.zip`, + meta, + entries, + ...(metricToken ? { metricToken } : {}), + }; + const signedManifest = await signArchivePayload(manifest, ARCHIVE_MANIFEST_JWS_TYPE); + if (new TextEncoder().encode(signedManifest).byteLength > MAX_ARCHIVE_MANIFEST_BYTES) { + return new Response("Skill archive manifest is too large", { + status: 413, + headers: mergeHeaders(rate.headers, corsHeaders()), + }); + } + + return new Response(signedManifest, { + status: 200, + headers: mergeHeaders( + rate.headers, + { + "Content-Type": ARCHIVE_MANIFEST_CONTENT_TYPE, + "Cache-Control": "private, no-store", + }, + corsHeaders(), + ), + }); + } + + const entries = version.files.map((file) => ({ + path: file.path, + openStream: async () => (await ctx.storage.get(file.storageId))?.stream() ?? null, + })); + const zipStream = buildDeterministicZipStream(entries, meta); await scheduleSkillDownloadMetric(ctx, request, skill._id); - return new Response(zipBlob, { + return new Response(zipStream, { status: 200, headers: mergeHeaders( rate.headers, @@ -150,6 +256,56 @@ export async function downloadZipHandler(ctx: DownloadCtx, request: Request) { export const downloadZip = httpAction(downloadZipHandler); +function unauthorizedArchiveManifestResponse() { + return new Response("Unauthorized archive manifest request", { + status: 401, + headers: { "Cache-Control": "no-store" }, + }); +} + +export async function recordArchiveDownloadMetricHandler(ctx: DownloadCtx, request: Request) { + const token = await readBoundedRequestText(request, MAX_ARCHIVE_METRIC_TOKEN_BYTES); + if (!token) { + return new Response("Invalid archive metric capability", { + status: 400, + headers: { "Cache-Control": "no-store" }, + }); + } + + let value: unknown; + try { + value = await verifyArchivePayloadWithLocalJwks(token, ARCHIVE_METRIC_JWS_TYPE); + } catch { + return new Response("Invalid archive metric capability", { + status: 401, + headers: { "Cache-Control": "no-store" }, + }); + } + const payload = parseArchiveMetricPayload(value, new URL(request.url).origin, Date.now()); + if (!payload) { + return new Response("Invalid archive metric capability", { + status: 401, + headers: { "Cache-Control": "no-store" }, + }); + } + + try { + await ctx.scheduler.runAfter( + Math.floor(Math.random() * DOWNLOAD_STAT_JITTER_MS), + internal.downloadMetrics.recordDownloadMetricInternal, + { + ...payload.metric, + target: { kind: "skill", id: payload.metric.target.id as Id<"skills"> }, + }, + ); + } catch { + // Metrics remain best-effort and must not affect an archive already being streamed. + } + return new Response(null, { status: 204, headers: { "Cache-Control": "no-store" } }); +} + +export const recordArchiveDownloadMetric = httpAction(recordArchiveDownloadMetricHandler); + export function getHourStart(timestamp: number) { return Math.floor(timestamp / HOUR_MS) * HOUR_MS; } @@ -222,6 +378,98 @@ export async function scheduleSkillDownloadMetric( } } +async function buildArchiveDownloadMetricToken( + ctx: DownloadCtx, + request: Request, + skillId: Id<"skills">, + issuer: string, + issuedAt: number, + expiresAt: number, +) { + try { + const userId = await getOptionalDownloadUserId(ctx, request); + const identity = getDownloadIdentity(request, userId ? String(userId) : null); + if (!identity) return undefined; + const metricArgs = await buildDownloadMetricArgs({ + target: { kind: "skill", id: skillId }, + identity, + now: issuedAt, + }); + const metric: ArchiveMetricArgs = { + ...metricArgs, + target: { kind: "skill", id: String(skillId) }, + }; + const payload: ArchiveMetricPayload = { + schema: "clawhub.archive-download-metric.v1", + issuer, + audience: ARCHIVE_METRIC_AUDIENCE, + issuedAt, + expiresAt, + metric, + }; + return await signArchivePayload(payload, ARCHIVE_METRIC_JWS_TYPE); + } catch { + return undefined; + } +} + +function parseArchiveMetricPayload( + value: unknown, + expectedIssuer: string, + now: number, +): ArchiveMetricPayload | null { + if (!value || typeof value !== "object") return null; + const payload = value as Partial; + if (payload.schema !== "clawhub.archive-download-metric.v1") return null; + if (payload.issuer !== expectedIssuer || payload.audience !== ARCHIVE_METRIC_AUDIENCE) + return null; + if (!Number.isFinite(payload.issuedAt) || !Number.isFinite(payload.expiresAt)) return null; + const issuedAt = payload.issuedAt as number; + const expiresAt = payload.expiresAt as number; + if (issuedAt > now + ARCHIVE_MANIFEST_CLOCK_SKEW_MS || expiresAt <= now) return null; + if (expiresAt <= issuedAt || expiresAt - issuedAt > ARCHIVE_MANIFEST_TTL_MS) return null; + if (!payload.metric || typeof payload.metric !== "object") return null; + const metric = payload.metric as Partial; + if (metric.target?.kind !== "skill" || typeof metric.target.id !== "string") return null; + if (metric.identityKind !== "user" && metric.identityKind !== "ip") return null; + if (typeof metric.identityHash !== "string" || metric.identityHash.length === 0) return null; + if (!Number.isFinite(metric.dayStart) || !Number.isFinite(metric.occurredAt)) return null; + return payload as ArchiveMetricPayload; +} + +async function readBoundedRequestText(request: Request, maxBytes: number) { + const contentLength = request.headers.get("content-length"); + if (contentLength) { + const declaredBytes = Number.parseInt(contentLength, 10); + if (Number.isFinite(declaredBytes) && declaredBytes > maxBytes) return null; + } + const reader = request.body?.getReader(); + if (!reader) return null; + const chunks: Uint8Array[] = []; + let totalBytes = 0; + try { + for (;;) { + const chunk = await reader.read(); + if (chunk.done) break; + totalBytes += chunk.value.byteLength; + if (totalBytes > maxBytes) { + await reader.cancel().catch(() => undefined); + return null; + } + chunks.push(chunk.value); + } + } finally { + reader.releaseLock(); + } + const bytes = new Uint8Array(totalBytes); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + return new TextDecoder().decode(bytes); +} + async function getOptionalDownloadUserId( ctx: DownloadCtx, request: Request, diff --git a/convex/http.ts b/convex/http.ts index 33926d7d..84255a6b 100644 --- a/convex/http.ts +++ b/convex/http.ts @@ -2,7 +2,7 @@ import { ApiRoutes, LegacyApiRoutes } from "clawhub-schema"; import { httpRouter } from "convex/server"; import { agentSkillsHttp } from "./agentSkillsHttp"; import { auth } from "./auth"; -import { downloadZip } from "./downloads"; +import { downloadZip, recordArchiveDownloadMetric } from "./downloads"; import { cliPublishHttp, cliDeviceCodeHttp, @@ -100,6 +100,12 @@ http.route({ handler: downloadZip, }); +http.route({ + path: "/api/internal/archive-download-metric", + method: "POST", + handler: recordArchiveDownloadMetric, +}); + http.route({ path: ApiRoutes.search, method: "GET", diff --git a/convex/lib/archiveManifest.ts b/convex/lib/archiveManifest.ts new file mode 100644 index 00000000..29ab9c08 --- /dev/null +++ b/convex/lib/archiveManifest.ts @@ -0,0 +1,70 @@ +import { CompactSign, compactVerify, createLocalJWKSet, importPKCS8 } from "jose"; + +export const ARCHIVE_MANIFEST_CONTENT_TYPE = "application/vnd.clawhub.skill-archive-manifest+jws"; +export const ARCHIVE_MANIFEST_AUDIENCE = "clawhub.nitro-skill-archive"; +export const ARCHIVE_MANIFEST_JWS_TYPE = "clawhub-skill-archive+jws"; +export const ARCHIVE_METRIC_AUDIENCE = "clawhub.convex-download-metric"; +export const ARCHIVE_METRIC_JWS_TYPE = "clawhub-download-metric+jws"; + +export type ArchiveMetricArgs = { + target: { kind: "skill"; id: string }; + identityKind: "user" | "ip"; + identityHash: string; + dayStart: number; + occurredAt?: number; +}; + +export type SkillArchiveManifest = { + schema: "clawhub.skill-archive-manifest.v1"; + issuer: string; + audience: typeof ARCHIVE_MANIFEST_AUDIENCE; + issuedAt: number; + expiresAt: number; + filename: string; + meta: { + ownerId: string; + slug: string; + version: string; + publishedAt: number; + }; + entries: Array<{ path: string; url: string }>; + metricToken?: string; +}; + +export type ArchiveMetricPayload = { + schema: "clawhub.archive-download-metric.v1"; + issuer: string; + audience: typeof ARCHIVE_METRIC_AUDIENCE; + issuedAt: number; + expiresAt: number; + metric: ArchiveMetricArgs; +}; + +export async function signArchivePayload( + payload: SkillArchiveManifest | ArchiveMetricPayload, + type: typeof ARCHIVE_MANIFEST_JWS_TYPE | typeof ARCHIVE_METRIC_JWS_TYPE, + privateKeyPem = process.env.JWT_PRIVATE_KEY, +) { + if (!privateKeyPem) throw new Error("JWT_PRIVATE_KEY is required to sign archive capabilities"); + const privateKey = await importPKCS8(privateKeyPem, "RS256"); + const bytes = Uint8Array.from(new TextEncoder().encode(JSON.stringify(payload))); + return await new CompactSign(bytes) + .setProtectedHeader({ alg: "RS256", typ: type }) + .sign(privateKey); +} + +export async function verifyArchivePayloadWithLocalJwks( + token: string, + expectedType: typeof ARCHIVE_MANIFEST_JWS_TYPE | typeof ARCHIVE_METRIC_JWS_TYPE, + jwksJson = process.env.JWKS, +): Promise { + if (!jwksJson) throw new Error("JWKS is required to verify archive capabilities"); + const jwks = JSON.parse(jwksJson) as Parameters[0]; + const verified = await compactVerify(token, createLocalJWKSet(jwks), { + algorithms: ["RS256"], + }); + if (verified.protectedHeader.typ !== expectedType) { + throw new Error("Unexpected archive capability type"); + } + return JSON.parse(new TextDecoder().decode(verified.payload)) as unknown; +} diff --git a/convex/lib/clawhubVercelOidc.test.ts b/convex/lib/clawhubVercelOidc.test.ts new file mode 100644 index 00000000..708cb01d --- /dev/null +++ b/convex/lib/clawhubVercelOidc.test.ts @@ -0,0 +1,78 @@ +/* @vitest-environment node */ + +import { createLocalJWKSet, exportJWK, generateKeyPair, SignJWT } from "jose"; +import { describe, expect, it } from "vitest"; +import { + CLAWHUB_VERCEL_OWNER_ID, + CLAWHUB_VERCEL_PROJECT, + CLAWHUB_VERCEL_PROJECT_ID, + CLAWHUB_VERCEL_TEAM, + expectedVercelEnvironmentForConvexSite, + verifyClawHubVercelOidcToken, +} from "./clawhubVercelOidc"; + +describe("ClawHub Vercel OIDC", () => { + it("binds each Convex site class to its Vercel environment", () => { + expect( + expectedVercelEnvironmentForConvexSite( + "https://migrated-production.convex.site/api/v1/download", + { CLAWHUB_ENV: "production" }, + ), + ).toBe("production"); + expect( + expectedVercelEnvironmentForConvexSite( + "https://academic-chihuahua-392.convex.site/api/v1/download", + { CLAWHUB_ENV: "test" }, + ), + ).toBe("preview"); + expect( + expectedVercelEnvironmentForConvexSite( + "https://preview-branch-123.convex.site/api/v1/download", + { CLAWHUB_PREVIEW: "1" }, + ), + ).toBe("preview"); + expect(expectedVercelEnvironmentForConvexSite("http://127.0.0.1:3211/api/v1/download")).toBe( + "development", + ); + expect( + expectedVercelEnvironmentForConvexSite("https://attacker.example/api/v1/download", { + CLAWHUB_ENV: "production", + }), + ).toBeNull(); + expect( + expectedVercelEnvironmentForConvexSite( + "https://unclassified.convex.site/api/v1/download", + {}, + ), + ).toBeNull(); + }); + + it("accepts only the ClawHub project identity for the expected environment", async () => { + const keyPair = await generateKeyPair("RS256", { extractable: true }); + const publicKey = await exportJWK(keyPair.publicKey); + const jwks = createLocalJWKSet({ keys: [{ use: "sig", ...publicKey }] }); + const token = await new SignJWT({ + owner_id: CLAWHUB_VERCEL_OWNER_ID, + project_id: CLAWHUB_VERCEL_PROJECT_ID, + environment: "preview", + }) + .setProtectedHeader({ alg: "RS256" }) + .setIssuer(`https://oidc.vercel.com/${CLAWHUB_VERCEL_TEAM}`) + .setAudience(`https://vercel.com/${CLAWHUB_VERCEL_TEAM}`) + .setSubject( + `owner:${CLAWHUB_VERCEL_TEAM}:project:${CLAWHUB_VERCEL_PROJECT}:environment:preview`, + ) + .setIssuedAt() + .setExpirationTime("5m") + .sign(keyPair.privateKey); + + await expect(verifyClawHubVercelOidcToken(token, "preview", jwks)).resolves.toMatchObject({ + owner_id: CLAWHUB_VERCEL_OWNER_ID, + project_id: CLAWHUB_VERCEL_PROJECT_ID, + environment: "preview", + }); + await expect(verifyClawHubVercelOidcToken(token, "production", jwks)).rejects.toThrow( + "Invalid ClawHub Vercel identity", + ); + }); +}); diff --git a/convex/lib/clawhubVercelOidc.ts b/convex/lib/clawhubVercelOidc.ts new file mode 100644 index 00000000..e906ebe0 --- /dev/null +++ b/convex/lib/clawhubVercelOidc.ts @@ -0,0 +1,66 @@ +import { createRemoteJWKSet, jwtVerify, type JWTVerifyGetKey } from "jose"; + +export const CLAWHUB_VERCEL_OWNER_ID = "team_pLdjXbfy0XvPRiNmAygTjTSH"; +export const CLAWHUB_VERCEL_PROJECT_ID = "prj_UVAJPNPYrBwTEkPJwkpEySsge8Mc"; +export const CLAWHUB_VERCEL_TEAM = "openclaw-foundation"; +export const CLAWHUB_VERCEL_PROJECT = "clawhub"; +export const ARCHIVE_REQUEST_IDENTITY_HEADER = "x-clawhub-vercel-oidc-token"; + +const VERCEL_OIDC_ISSUER = `https://oidc.vercel.com/${CLAWHUB_VERCEL_TEAM}`; +const VERCEL_OIDC_AUDIENCE = `https://vercel.com/${CLAWHUB_VERCEL_TEAM}`; +const VERCEL_OIDC_JWKS = createRemoteJWKSet(new URL(`${VERCEL_OIDC_ISSUER}/.well-known/jwks`)); + +type ClawHubArchiveRuntimeEnvironment = { + CLAWHUB_ENV?: string; + CLAWHUB_PREVIEW?: string; +}; + +export type ClawHubVercelEnvironment = "development" | "preview" | "production"; + +export function expectedVercelEnvironmentForConvexSite( + requestUrl: string, + env: ClawHubArchiveRuntimeEnvironment = process.env, +): ClawHubVercelEnvironment | null { + let url: URL; + try { + url = new URL(requestUrl); + } catch { + return null; + } + if (url.protocol === "http:" && ["localhost", "127.0.0.1", "[::1]"].includes(url.hostname)) { + return "development"; + } + if (url.protocol !== "https:" || !url.hostname.endsWith(".convex.site")) return null; + + const runtimeEnvironment = env.CLAWHUB_ENV?.trim(); + if (env.CLAWHUB_PREVIEW === "1") { + return runtimeEnvironment && runtimeEnvironment !== "preview" ? null : "preview"; + } + if (runtimeEnvironment === "production") return "production"; + // ClawHub Test is an app-level label on a Vercel preview-target deployment. + if (runtimeEnvironment === "test") return "preview"; + return null; +} + +export async function verifyClawHubVercelOidcToken( + token: string, + expectedEnvironment: ClawHubVercelEnvironment, + jwks: JWTVerifyGetKey = VERCEL_OIDC_JWKS, +) { + const verified = await jwtVerify(token, jwks, { + algorithms: ["RS256"], + issuer: VERCEL_OIDC_ISSUER, + audience: VERCEL_OIDC_AUDIENCE, + }); + const payload = verified.payload; + if ( + payload.owner_id !== CLAWHUB_VERCEL_OWNER_ID || + payload.project_id !== CLAWHUB_VERCEL_PROJECT_ID || + payload.environment !== expectedEnvironment || + payload.sub !== + `owner:${CLAWHUB_VERCEL_TEAM}:project:${CLAWHUB_VERCEL_PROJECT}:environment:${expectedEnvironment}` + ) { + throw new Error("Invalid ClawHub Vercel identity"); + } + return payload; +} diff --git a/convex/lib/skillZip.ts b/convex/lib/skillZip.ts index c4b06bcd..51b991a4 100644 --- a/convex/lib/skillZip.ts +++ b/convex/lib/skillZip.ts @@ -1,11 +1,16 @@ import { findClawPackagePathHierarchyCollision, isSafeClawPackagePath } from "clawhub-schema"; -import { zipSync } from "fflate"; +import { Zip, ZipDeflate, zipSync } from "fflate"; type ZipEntry = { path: string; bytes: Uint8Array; }; +export type AsyncZipEntry = { + path: string; + openStream: () => Promise | null>; +}; + export type SkillZipMeta = { ownerId: string; slug: string; @@ -16,6 +21,9 @@ export type SkillZipMeta = { type ZipInput = Record; const FIXED_ZIP_DATE = new Date(1980, 0, 1, 0, 0, 0); +// Storage response chunk boundaries vary with transport backpressure; normalize +// them so identical files still produce byte-for-byte identical archives. +const ZIP_INPUT_CHUNK_BYTES = 64 * 1024; // ==================== Zip Slip Protection ==================== @@ -68,6 +76,154 @@ export function buildDeterministicZip(entries: ZipEntry[], meta?: SkillZipMeta) return Uint8Array.from(zipSync(zipData, { level: 6 })); } +export function buildDeterministicZipStream(entries: AsyncZipEntry[], meta?: SkillZipMeta) { + const orderedEntries = orderZipEntries(entries, meta); + const output: Uint8Array[] = []; + let entryIndex = 0; + let current: + | { + reader: ReadableStreamDefaultReader; + zipEntry: ZipDeflate; + inputBuffer: Uint8Array; + inputBufferLength: number; + sourceChunk?: Uint8Array; + sourceOffset: number; + sourceDone: boolean; + } + | undefined; + let archiveEnded = false; + let archiveDone = false; + let archiveError: unknown; + + const archive = new Zip((error, chunk, final) => { + if (error) archiveError = error; + if (chunk?.length) output.push(chunk); + if (final) archiveDone = true; + }); + + const advance = async () => { + if (archiveError) throw archiveError; + + if (current) { + while (current.inputBufferLength < ZIP_INPUT_CHUNK_BYTES && !current.sourceDone) { + if (!current.sourceChunk || current.sourceOffset === current.sourceChunk.byteLength) { + const next = await current.reader.read(); + if (next.done) { + current.sourceDone = true; + break; + } + current.sourceChunk = next.value; + current.sourceOffset = 0; + if (next.value.byteLength === 0) continue; + } + + const sourceBytesRemaining = current.sourceChunk.byteLength - current.sourceOffset; + const outputBytesRemaining = ZIP_INPUT_CHUNK_BYTES - current.inputBufferLength; + const bytesToCopy = Math.min(sourceBytesRemaining, outputBytesRemaining); + current.inputBuffer.set( + current.sourceChunk.subarray(current.sourceOffset, current.sourceOffset + bytesToCopy), + current.inputBufferLength, + ); + current.sourceOffset += bytesToCopy; + current.inputBufferLength += bytesToCopy; + } + + if (current.inputBufferLength > 0) { + current.zipEntry.push( + current.inputBuffer.subarray(0, current.inputBufferLength), + current.sourceDone, + ); + current.inputBuffer = new Uint8Array(ZIP_INPUT_CHUNK_BYTES); + current.inputBufferLength = 0; + } else if (current.sourceDone) { + current.zipEntry.push(new Uint8Array(0), true); + } + if (current.sourceDone) { + current.reader.releaseLock(); + current = undefined; + } + if (archiveError) throw archiveError; + return; + } + + while (entryIndex < orderedEntries.length) { + const entry = orderedEntries[entryIndex++]; + const stream = await entry.openStream(); + // A storage reference can become stale after the version document was read. + // Do not commit a ZIP header until the Blob is known to still exist. + if (!stream) continue; + + const zipEntry = new ZipDeflate(entry.path, { level: 6 }); + zipEntry.mtime = FIXED_ZIP_DATE; + archive.add(zipEntry); + current = { + reader: stream.getReader(), + zipEntry, + inputBuffer: new Uint8Array(ZIP_INPUT_CHUNK_BYTES), + inputBufferLength: 0, + sourceOffset: 0, + sourceDone: false, + }; + return; + } + + if (!archiveEnded) { + archiveEnded = true; + archive.end(); + if (archiveError) throw archiveError; + } + }; + + return new ReadableStream( + { + async pull(controller) { + try { + for (;;) { + if (output.length > 0 || archiveDone) break; + await advance(); + } + const chunk = output.shift(); + if (chunk) controller.enqueue(chunk); + else controller.close(); + } catch (error) { + archive.terminate(); + await current?.reader.cancel(error); + controller.error(error); + } + }, + async cancel(reason) { + archive.terminate(); + await current?.reader.cancel(reason); + }, + }, + { highWaterMark: 0 }, + ); +} + +function orderZipEntries(entries: AsyncZipEntry[], meta?: SkillZipMeta) { + const sorted = [...entries].sort((a, b) => a.path.localeCompare(b.path)); + const byPath = new Map(sorted.map((entry) => [entry.path, entry])); + const zipDataOrder: Record = {}; + for (const entry of sorted) zipDataOrder[entry.path] = true; + + if (meta) { + const metaBytes = new TextEncoder().encode(JSON.stringify(buildSkillMeta(meta), null, 2)); + byPath.set("_meta.json", { + path: "_meta.json", + openStream: async () => + new ReadableStream({ + start(controller) { + controller.enqueue(metaBytes); + controller.close(); + }, + }), + }); + zipDataOrder["_meta.json"] = true; + } + + return Object.keys(zipDataOrder).map((path) => byPath.get(path)!); +} + export function buildDeterministicPackageZip(entries: ZipEntry[]) { const unsafeEntry = entries.find((entry) => !isSafeClawPackagePath(entry.path)); if (unsafeEntry) { diff --git a/docs/http-api.md b/docs/http-api.md index 12c46596..e352f085 100644 --- a/docs/http-api.md +++ b/docs/http-api.md @@ -1315,6 +1315,9 @@ Notes: - If neither `version` nor `tag` is provided, the latest version is used. - Soft-deleted versions return `410`. +- Hosted skill versions return a streamed deterministic ZIP with + `Content-Disposition: attachment; filename="-.zip"`. ClawHub + applies moderation, rate limiting, and download metering before streaming. - GitHub-backed skill handoffs do not proxy or mirror bytes. The JSON response includes `sourceRef: "public-github"`, `repo`, `commit`, `path`, `contentHash`, and `archiveUrl`; scan/current state is a gate and is not included as success diff --git a/package.json b/package.json index 04a66228..47a8ab59 100644 --- a/package.json +++ b/package.json @@ -129,6 +129,7 @@ "fflate": "0.8.3", "h3": "2.0.1-rc.25", "ignore": "7.0.6", + "jose": "6.2.3", "lucide-react": "1.28.0", "mermaid": "^11.16.1", "mime": "4.1.0", diff --git a/public/api/v1/openapi.json b/public/api/v1/openapi.json index c0a148a1..e9562082 100644 --- a/public/api/v1/openapi.json +++ b/public/api/v1/openapi.json @@ -1682,25 +1682,25 @@ } } ], - "responses": { - "200": { - "description": "Hosted ZIP bytes or a GitHub-backed source handoff descriptor.", - "content": { - "application/zip": { - "schema": { - "type": "string", - "format": "binary" - } - }, - "application/json": { - "schema": { - "$ref": "#/components/schemas/GitHubSkillDownloadHandoff" - } + "responses": { + "200": { + "description": "Hosted ZIP bytes or a GitHub-backed source handoff descriptor.", + "content": { + "application/zip": { + "schema": { + "type": "string", + "format": "binary" + } + }, + "application/json": { + "schema": { + "$ref": "#/components/schemas/GitHubSkillDownloadHandoff" } } } } } + } }, "/api/v1/packages": { "get": { diff --git a/server/convexProxy.test.ts b/server/convexProxy.test.ts index 8bdb37ae..848e2d5e 100644 --- a/server/convexProxy.test.ts +++ b/server/convexProxy.test.ts @@ -1,16 +1,37 @@ /* @vitest-environment node */ +import { unzipSync } from "fflate"; import { mockEvent } from "h3"; +import { createLocalJWKSet, exportJWK, exportPKCS8, generateKeyPair } from "jose"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { + ARCHIVE_MANIFEST_AUDIENCE, + ARCHIVE_MANIFEST_CONTENT_TYPE, + ARCHIVE_MANIFEST_JWS_TYPE, + signArchivePayload, + type SkillArchiveManifest, +} from "../convex/lib/archiveManifest"; import { buildConvexProxyTarget, isConvexProxyMethodAllowed, proxyConvexRequest, + resolveConvexStorageOrigin, resolveConvexProxyEnv, + verifySignedArchiveManifest, } from "./convexProxy"; +const TEST_ARCHIVE_DEPENDENCIES = { + getArchiveRequestToken: async () => "verified-vercel-oidc", + verifyArchiveManifest: async (token: string) => ({ + ...(JSON.parse(token) as Record), + issuer: "https://preview-branch-123.convex.site", + audience: ARCHIVE_MANIFEST_AUDIENCE, + }), +}; + describe("Convex HTTP proxy", () => { afterEach(() => { + vi.restoreAllMocks(); vi.unstubAllGlobals(); }); @@ -69,6 +90,22 @@ describe("Convex HTTP proxy", () => { }); }); + it("accepts storage only from the Convex deployment paired to the selected site", () => { + expect( + resolveConvexStorageOrigin("https://preview-branch-123.convex.site/api/v1/download", { + VITE_CONVEX_SITE_URL: "https://preview-branch-123.convex.site", + VITE_CONVEX_URL: "https://preview-branch-123.convex.cloud", + CONVEX_URL: "https://wry-manatee-359.convex.cloud", + }), + ).toBe("https://preview-branch-123.convex.cloud"); + expect( + resolveConvexStorageOrigin("https://preview-branch-123.convex.site/api/v1/download", { + VITE_CONVEX_SITE_URL: "https://preview-branch-123.convex.site", + VITE_CONVEX_URL: "https://wry-manatee-359.convex.cloud", + }), + ).toBeNull(); + }); + it("proxies reads and exposes the non-secret preview deployment name for proof", async () => { const fetchMock = vi.fn(async () => { return new Response(JSON.stringify({ ok: true }), { @@ -92,6 +129,489 @@ describe("Convex HTTP proxy", () => { expect(response.headers.get("X-ClawHub-Preview-Backend")).toBe("preview-branch-123"); }); + it("streams hosted downloads from a Convex manifest with the final attachment filename", async () => { + const storedBody = new TextEncoder().encode("# streamed skill\n"); + const fetchMock = vi.fn(async (input: string | URL | Request, _init?: RequestInit) => { + const url = input.toString(); + if (url.startsWith("https://preview-branch-123.convex.site/api/v1/download")) { + return Response.json( + { + schema: "clawhub.skill-archive-manifest.v1", + issuedAt: 1_000, + expiresAt: 31_000, + filename: "demo-1.0.0+build.zip", + meta: { + ownerId: "users:1", + slug: "demo", + version: "1.0.0+build", + publishedAt: 3, + }, + entries: [ + { + path: "SKILL.md", + url: "https://preview-branch-123.convex.cloud/api/storage/storage-1", + }, + { + path: "stale.txt", + url: "https://preview-branch-123.convex.cloud/api/storage/storage-missing", + }, + ], + metricToken: "signed-metric-capability", + }, + { + headers: { + "cache-control": "private, no-store", + "content-digest": "sha-256=:manifest-digest:", + "content-encoding": "gzip", + "content-type": ARCHIVE_MANIFEST_CONTENT_TYPE, + etag: '"manifest-etag"', + "x-ratelimit-remaining": "49", + }, + }, + ); + } + if (url === "https://preview-branch-123.convex.cloud/api/storage/storage-1") { + return new Response(storedBody, { status: 200 }); + } + if (url === "https://preview-branch-123.convex.cloud/api/storage/storage-missing") { + return new Response("missing", { status: 404 }); + } + if (url === "https://preview-branch-123.convex.site/api/internal/archive-download-metric") { + return new Response(null, { status: 204 }); + } + throw new Error(`Unexpected fetch: ${url}`); + }); + vi.stubGlobal("fetch", fetchMock); + vi.spyOn(Date, "now").mockReturnValue(2_000); + const event = mockEvent("https://preview.example/api/v1/download?slug=demo", { + headers: { "x-clawhub-vercel-oidc-token": "client-forgery" }, + }); + + const response = await proxyConvexRequest( + event, + { + VERCEL_ENV: "preview", + VITE_CONVEX_SITE_URL: "https://preview-branch-123.convex.site", + VITE_CONVEX_URL: "https://preview-branch-123.convex.cloud", + }, + TEST_ARCHIVE_DEPENDENCIES, + ); + + expect(response.status).toBe(200); + expect(response.headers.get("Content-Type")).toBe("application/zip"); + expect(response.headers.get("Content-Disposition")).toBe( + 'attachment; filename="demo-1.0.0+build.zip"', + ); + expect(response.headers.get("Cache-Control")).toBe("private, no-store"); + expect(response.headers.get("Content-Digest")).toBeNull(); + expect(response.headers.get("Content-Encoding")).toBeNull(); + expect(response.headers.get("ETag")).toBeNull(); + expect(response.headers.get("X-RateLimit-Remaining")).toBe("49"); + expect(response.headers.get("X-ClawHub-Preview-Backend")).toBe("preview-branch-123"); + const archive = unzipSync(new Uint8Array(await response.arrayBuffer())); + expect(archive["SKILL.md"]).toEqual(storedBody); + expect(Object.keys(archive).sort()).toEqual(["SKILL.md", "_meta.json"]); + expect(fetchMock.mock.calls[0]?.[0]).toBe( + "https://preview-branch-123.convex.site/api/v1/download?slug=demo", + ); + expect( + new Headers(fetchMock.mock.calls[0]?.[1]?.headers).get("x-clawhub-archive-manifest"), + ).toBe("v1"); + expect( + new Headers(fetchMock.mock.calls[0]?.[1]?.headers).get("x-clawhub-vercel-oidc-token"), + ).toBe("verified-vercel-oidc"); + const metricCall = fetchMock.mock.calls.find( + ([input]) => + input.toString() === + "https://preview-branch-123.convex.site/api/internal/archive-download-metric", + ); + expect(metricCall?.[1]).toMatchObject({ + method: "POST", + body: "signed-metric-capability", + }); + const fetchedUrls = fetchMock.mock.calls.map(([input]) => input.toString()); + expect( + fetchedUrls.indexOf( + "https://preview-branch-123.convex.site/api/internal/archive-download-metric", + ), + ).toBeGreaterThan( + fetchedUrls.indexOf("https://preview-branch-123.convex.cloud/api/storage/storage-1"), + ); + }); + + it("does not record a download when every source Blob has vanished", async () => { + const fetchMock = vi.fn(async (input: string | URL | Request) => { + const url = input.toString(); + if (url.startsWith("https://preview-branch-123.convex.site/api/v1/download")) { + return Response.json( + { + schema: "clawhub.skill-archive-manifest.v1", + issuedAt: 1_000, + expiresAt: 31_000, + filename: "demo-1.0.0.zip", + meta: { + ownerId: "users:1", + slug: "demo", + version: "1.0.0", + publishedAt: 3, + }, + entries: [ + { + path: "stale.txt", + url: "https://preview-branch-123.convex.cloud/api/storage/storage-missing", + }, + ], + metricToken: "signed-metric-capability", + }, + { headers: { "content-type": ARCHIVE_MANIFEST_CONTENT_TYPE } }, + ); + } + if (url === "https://preview-branch-123.convex.cloud/api/storage/storage-missing") { + return new Response("missing", { status: 404 }); + } + throw new Error(`Unexpected fetch: ${url}`); + }); + vi.stubGlobal("fetch", fetchMock); + vi.spyOn(Date, "now").mockReturnValue(2_000); + + const response = await proxyConvexRequest( + mockEvent("https://preview.example/api/v1/download?slug=demo"), + { + VERCEL_ENV: "preview", + VITE_CONVEX_SITE_URL: "https://preview-branch-123.convex.site", + VITE_CONVEX_URL: "https://preview-branch-123.convex.cloud", + }, + TEST_ARCHIVE_DEPENDENCIES, + ); + const archive = unzipSync(new Uint8Array(await response.arrayBuffer())); + + expect(Object.keys(archive)).toEqual(["_meta.json"]); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + + it("rejects an unsigned archive manifest from the paired Convex origin", async () => { + const fetchMock = vi.fn(async (input: string | URL | Request) => { + const url = input.toString(); + if (url.startsWith("https://preview-branch-123.convex.site/api/v1/download")) { + return Response.json( + { + schema: "clawhub.skill-archive-manifest.v1", + issuedAt: 1_000, + expiresAt: 31_000, + filename: "demo-1.0.0.zip", + meta: { + ownerId: "users:1", + slug: "demo", + version: "1.0.0", + publishedAt: 3, + }, + entries: [ + { + path: "SKILL.md", + url: "https://preview-branch-123.convex.cloud/api/storage/storage-1", + }, + ], + }, + { headers: { "content-type": "application/vnd.clawhub.skill-archive-manifest+json" } }, + ); + } + if (url === "https://preview-branch-123.convex.cloud/api/storage/storage-1") { + return new Response("should not be fetched"); + } + throw new Error(`Unexpected fetch: ${url}`); + }); + vi.stubGlobal("fetch", fetchMock); + vi.spyOn(Date, "now").mockReturnValue(2_000); + + const response = await proxyConvexRequest( + mockEvent("https://preview.example/api/v1/download?slug=demo"), + { + VERCEL_ENV: "preview", + VITE_CONVEX_SITE_URL: "https://preview-branch-123.convex.site", + VITE_CONVEX_URL: "https://preview-branch-123.convex.cloud", + }, + TEST_ARCHIVE_DEPENDENCIES, + ); + + expect(response.status).toBe(502); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it("verifies the paired Convex signature and rejects a modified manifest", async () => { + const keyPair = await generateKeyPair("RS256", { extractable: true }); + const privateKey = await exportPKCS8(keyPair.privateKey); + const publicKey = await exportJWK(keyPair.publicKey); + const manifest: SkillArchiveManifest = { + schema: "clawhub.skill-archive-manifest.v1", + issuer: "https://preview-branch-123.convex.site", + audience: ARCHIVE_MANIFEST_AUDIENCE, + issuedAt: 1_000, + expiresAt: 31_000, + filename: "demo-1.0.0.zip", + meta: { + ownerId: "users:1", + slug: "demo", + version: "1.0.0", + publishedAt: 3, + }, + entries: [ + { + path: "SKILL.md", + url: "https://preview-branch-123.convex.cloud/api/storage/storage-1", + }, + ], + }; + const token = await signArchivePayload(manifest, ARCHIVE_MANIFEST_JWS_TYPE, privateKey); + const localJwks = createLocalJWKSet({ keys: [{ use: "sig", ...publicKey }] }); + + await expect( + verifySignedArchiveManifest( + token, + "https://preview-branch-123.convex.site/api/v1/download", + localJwks, + ), + ).resolves.toEqual(manifest); + const [header, payload, signature] = token.split("."); + const modifiedPayload = `${payload!.slice(0, -1)}${payload!.endsWith("A") ? "B" : "A"}`; + await expect( + verifySignedArchiveManifest( + `${header}.${modifiedPayload}.${signature}`, + "https://preview-branch-123.convex.site/api/v1/download", + localJwks, + ), + ).rejects.toThrow(); + }); + + it("produces identical archive bytes when storage streams use different chunk boundaries", async () => { + // Cross many normalization boundaries without making full-suite coverage + // instrumentation dominate the repository's 15-second test timeout. + const storedBody = new Uint8Array(512 * 1024); + let randomState = 0x3451cafe; + for (let index = 0; index < storedBody.length; index += 1) { + randomState ^= randomState << 13; + randomState ^= randomState >>> 17; + randomState ^= randomState << 5; + storedBody[index] = randomState; + } + + let storageRequest = 0; + const fetchMock = vi.fn(async (input: string | URL | Request) => { + const url = input.toString(); + if (url.startsWith("https://preview-branch-123.convex.site/api/v1/download")) { + return Response.json( + { + schema: "clawhub.skill-archive-manifest.v1", + issuedAt: 1_000, + expiresAt: 31_000, + filename: "demo-1.0.0.zip", + meta: { + ownerId: "users:1", + slug: "demo", + version: "1.0.0", + publishedAt: 3, + }, + entries: [ + { + path: "SKILL.md", + url: "https://preview-branch-123.convex.cloud/api/storage/storage-1", + }, + ], + }, + { headers: { "content-type": ARCHIVE_MANIFEST_CONTENT_TYPE } }, + ); + } + if (url === "https://preview-branch-123.convex.cloud/api/storage/storage-1") { + const chunkSize = storageRequest++ === 0 ? 16_381 : 65_521; + let offset = 0; + return new Response( + new ReadableStream({ + pull(controller) { + if (offset >= storedBody.length) { + controller.close(); + return; + } + const nextOffset = Math.min(offset + chunkSize, storedBody.length); + controller.enqueue(storedBody.slice(offset, nextOffset)); + offset = nextOffset; + }, + }), + ); + } + throw new Error(`Unexpected fetch: ${url}`); + }); + vi.stubGlobal("fetch", fetchMock); + vi.spyOn(Date, "now").mockReturnValue(2_000); + const env = { + VERCEL_ENV: "preview", + VITE_CONVEX_SITE_URL: "https://preview-branch-123.convex.site", + VITE_CONVEX_URL: "https://preview-branch-123.convex.cloud", + }; + + const firstResponse = await proxyConvexRequest( + mockEvent("https://preview.example/api/v1/download?slug=demo"), + env, + TEST_ARCHIVE_DEPENDENCIES, + ); + const firstArchive = new Uint8Array(await firstResponse.arrayBuffer()); + const secondResponse = await proxyConvexRequest( + mockEvent("https://preview.example/api/v1/download?slug=demo"), + env, + TEST_ARCHIVE_DEPENDENCIES, + ); + const secondArchive = new Uint8Array(await secondResponse.arrayBuffer()); + + expect(secondArchive.byteLength).toBe(firstArchive.byteLength); + expect(secondArchive.every((byte, index) => byte === firstArchive[index])).toBe(true); + expect(unzipSync(secondArchive)["SKILL.md"]).toEqual(storedBody); + }); + + it("rejects archive manifests that point outside the paired Convex storage origin", async () => { + const fetchMock = vi.fn(async (input: string | URL | Request) => { + const url = input.toString(); + if (url.startsWith("https://preview-branch-123.convex.site/api/v1/download")) { + return Response.json( + { + schema: "clawhub.skill-archive-manifest.v1", + issuedAt: 1_000, + expiresAt: 31_000, + filename: "demo-1.0.0.zip", + meta: { + ownerId: "users:1", + slug: "demo", + version: "1.0.0", + publishedAt: 3, + }, + entries: [{ path: "SKILL.md", url: "https://attacker.example/private" }], + }, + { headers: { "content-type": ARCHIVE_MANIFEST_CONTENT_TYPE } }, + ); + } + throw new Error(`Security boundary crossed: ${url}`); + }); + vi.stubGlobal("fetch", fetchMock); + vi.spyOn(Date, "now").mockReturnValue(2_000); + const event = mockEvent("https://preview.example/api/v1/download?slug=demo"); + + const response = await proxyConvexRequest( + event, + { + VERCEL_ENV: "preview", + VITE_CONVEX_SITE_URL: "https://preview-branch-123.convex.site", + VITE_CONVEX_URL: "https://preview-branch-123.convex.cloud", + }, + TEST_ARCHIVE_DEPENDENCIES, + ); + + expect(response.status).toBe(502); + expect(await response.text()).toBe("Invalid or expired archive manifest"); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it("rejects an expired archive manifest before fetching any stored file", async () => { + const fetchMock = vi.fn(async () => + Response.json( + { + schema: "clawhub.skill-archive-manifest.v1", + issuedAt: 1_000, + expiresAt: 2_000, + filename: "demo-1.0.0.zip", + meta: { + ownerId: "users:1", + slug: "demo", + version: "1.0.0", + publishedAt: 3, + }, + entries: [ + { + path: "SKILL.md", + url: "https://preview-branch-123.convex.cloud/api/storage/storage-1", + }, + ], + }, + { headers: { "content-type": ARCHIVE_MANIFEST_CONTENT_TYPE } }, + ), + ); + vi.stubGlobal("fetch", fetchMock); + vi.spyOn(Date, "now").mockReturnValue(2_001); + + const response = await proxyConvexRequest( + mockEvent("https://preview.example/api/v1/download?slug=demo"), + { + VERCEL_ENV: "preview", + VITE_CONVEX_SITE_URL: "https://preview-branch-123.convex.site", + VITE_CONVEX_URL: "https://preview-branch-123.convex.cloud", + }, + TEST_ARCHIVE_DEPENDENCIES, + ); + + expect(response.status).toBe(502); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it("does not accept a client-supplied manifest replay", async () => { + const fetchMock = vi.fn( + async (_input: string | URL | Request, _init?: RequestInit) => + new Response("Skill not found", { status: 404 }), + ); + vi.stubGlobal("fetch", fetchMock); + const event = mockEvent("https://preview.example/api/v1/download?slug=demo&manifest=replayed", { + headers: { "x-clawhub-archive-manifest": "replayed-v0" }, + }); + + const response = await proxyConvexRequest( + event, + { + VERCEL_ENV: "preview", + VITE_CONVEX_SITE_URL: "https://preview-branch-123.convex.site", + VITE_CONVEX_URL: "https://preview-branch-123.convex.cloud", + }, + TEST_ARCHIVE_DEPENDENCIES, + ); + + expect(response.status).toBe(404); + expect(fetchMock.mock.calls[0]?.[0]).toBe( + "https://preview-branch-123.convex.site/api/v1/download?slug=demo&manifest=replayed", + ); + expect( + new Headers(fetchMock.mock.calls[0]?.[1]?.headers).get("x-clawhub-archive-manifest"), + ).toBe("v1"); + }); + + it("rejects an oversized manifest body", async () => { + const bodyRead = vi.fn(); + const fetchMock = vi.fn( + async () => + new Response( + new ReadableStream({ + pull(controller) { + bodyRead(); + controller.enqueue(new Uint8Array(1024 * 1024)); + }, + }), + { + headers: { + "content-type": ARCHIVE_MANIFEST_CONTENT_TYPE, + }, + }, + ), + ); + vi.stubGlobal("fetch", fetchMock); + + const response = await proxyConvexRequest( + mockEvent("https://preview.example/api/v1/download?slug=demo"), + { + VERCEL_ENV: "preview", + VITE_CONVEX_SITE_URL: "https://preview-branch-123.convex.site", + VITE_CONVEX_URL: "https://preview-branch-123.convex.cloud", + }, + TEST_ARCHIVE_DEPENDENCIES, + ); + + expect(response.status).toBe(502); + expect(bodyRead.mock.calls.length).toBeGreaterThanOrEqual(5); + expect(bodyRead.mock.calls.length).toBeLessThanOrEqual(6); + }); + it("exposes the permanent Test backend name for deployment proof", async () => { vi.stubGlobal( "fetch", diff --git a/server/convexProxy.ts b/server/convexProxy.ts index 833f4165..f184e78f 100644 --- a/server/convexProxy.ts +++ b/server/convexProxy.ts @@ -1,6 +1,44 @@ +import { getVercelOidcToken } from "@vercel/oidc"; import { defineEventHandler, getRequestURL, proxyRequest, type H3Event } from "h3"; +import { compactVerify, type CompactVerifyGetKey, createRemoteJWKSet } from "jose"; +import { + ARCHIVE_MANIFEST_AUDIENCE, + ARCHIVE_MANIFEST_CONTENT_TYPE, + ARCHIVE_MANIFEST_JWS_TYPE, + type SkillArchiveManifest, +} from "../convex/lib/archiveManifest"; +import { + ARCHIVE_REQUEST_IDENTITY_HEADER, + CLAWHUB_VERCEL_PROJECT, + CLAWHUB_VERCEL_TEAM, +} from "../convex/lib/clawhubVercelOidc"; +import { + buildDeterministicZipStream, + type SkillZipMeta, + validateFilePath, + validateSlug, +} from "../convex/lib/skillZip"; import { convexDeploymentName, resolveConvexSiteUrl } from "../src/lib/convexDeploymentUrl"; +const ARCHIVE_MANIFEST_REQUEST_HEADER = "x-clawhub-archive-manifest"; +const ARCHIVE_MANIFEST_MAX_AGE_MS = 60_000; +const ARCHIVE_MANIFEST_CLOCK_SKEW_MS = 5_000; +const MAX_ARCHIVE_MANIFEST_ENTRIES = 8_192; +const MAX_ARCHIVE_ENTRY_URL_LENGTH = 4_096; +const MAX_ARCHIVE_MANIFEST_BYTES = 4 * 1024 * 1024; +const ARCHIVE_FILENAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._+-]{0,499}\.zip$/; +const ARCHIVE_REPRESENTATION_HEADERS = [ + "accept-ranges", + "content-digest", + "content-encoding", + "content-length", + "content-md5", + "content-range", + "digest", + "etag", + "last-modified", +] as const; + type ProxyEnv = { CONVEX_URL?: string; VERCEL_ENV?: string; @@ -10,6 +48,18 @@ type ProxyEnv = { VITE_CONVEX_URL?: string; }; +type ProxyDependencies = { + getArchiveRequestToken: () => Promise; + verifyArchiveManifest: (token: string, target: string) => Promise; +}; + +const DEFAULT_PROXY_DEPENDENCIES: ProxyDependencies = { + getArchiveRequestToken: () => + getVercelOidcToken({ team: CLAWHUB_VERCEL_TEAM, project: CLAWHUB_VERCEL_PROJECT }), + verifyArchiveManifest: verifySignedArchiveManifest, +}; +const archiveJwksByOrigin = new Map>(); + const BUNDLED_PROXY_ENV: ProxyEnv = { VITE_CLAWHUB_DEPLOY_ENV: import.meta.env.VITE_CLAWHUB_DEPLOY_ENV, VITE_CONVEX_SITE_URL: import.meta.env.VITE_CONVEX_SITE_URL, @@ -62,6 +112,7 @@ export function buildConvexProxyTarget(pathAndQuery: string, env: ProxyEnv) { export async function proxyConvexRequest( event: H3Event, env: ProxyEnv = resolveConvexProxyEnv(process.env), + dependencies: ProxyDependencies = DEFAULT_PROXY_DEPENDENCIES, ): Promise { if (!isConvexProxyMethodAllowed(event.req.method, env)) { return new Response("Disposable previews are read-only.", { @@ -76,7 +127,30 @@ export async function proxyConvexRequest( const requestUrl = getRequestURL(event); const target = buildConvexProxyTarget(`${requestUrl.pathname}${requestUrl.search}`, env); - const proxied = await proxyRequest(event, target); + const isArchiveRequest = isSkillDownloadPath(new URL(target).pathname); + let archiveRequestToken: string | undefined; + if (isArchiveRequest) { + try { + archiveRequestToken = await dependencies.getArchiveRequestToken(); + } catch { + return new Response("Archive streaming identity unavailable", { + status: 503, + headers: { "Cache-Control": "no-store" }, + }); + } + } + const proxied = await proxyRequest(event, target, { + ...(isArchiveRequest + ? { + fetchOptions: { + headers: { + [ARCHIVE_MANIFEST_REQUEST_HEADER]: "v1", + [ARCHIVE_REQUEST_IDENTITY_HEADER]: archiveRequestToken!, + }, + }, + } + : {}), + }); // H3's HTTPResponse is not guaranteed to share Nitro's bundled class identity. // Normalize it before crossing that boundary or Nitro can stringify the wrapper. const response = new Response(proxied.body, { @@ -84,6 +158,22 @@ export async function proxyConvexRequest( statusText: proxied.statusText, headers: proxied.headers, }); + const contentType = response.headers.get("content-type")?.split(";", 1)[0]?.trim(); + if (contentType === ARCHIVE_MANIFEST_CONTENT_TYPE) { + return streamSkillArchive( + response, + env, + target, + event.req.signal, + dependencies.verifyArchiveManifest, + ); + } + if ( + isSkillDownloadPath(new URL(target).pathname) && + contentType?.startsWith("application/vnd.clawhub.skill-archive-manifest") + ) { + return new Response("Invalid or expired archive manifest", { status: 502 }); + } if (isPreviewFrontend(env) || isTestFrontend(env)) { const deployment = convexDeploymentName(target); if (deployment) { @@ -96,4 +186,231 @@ export async function proxyConvexRequest( return response; } +function isSkillDownloadPath(pathname: string) { + return pathname === "/api/v1/download" || pathname === "/api/download"; +} + +async function streamSkillArchive( + manifestResponse: Response, + env: ProxyEnv, + target: string, + signal: AbortSignal, + verifyArchiveManifest: ProxyDependencies["verifyArchiveManifest"], +) { + let value: unknown; + try { + const token = await readBoundedArchiveManifest(manifestResponse); + if (!token) return new Response("Invalid or expired archive manifest", { status: 502 }); + value = await verifyArchiveManifest(token, target); + } catch { + return new Response("Invalid or expired archive manifest", { status: 502 }); + } + const expectedStorageOrigin = resolveConvexStorageOrigin(target, env); + const manifest = parseSkillArchiveManifest( + value, + new URL(target).origin, + expectedStorageOrigin, + Date.now(), + ); + if (!manifest) { + return new Response("Invalid or expired archive manifest", { status: 502 }); + } + + let metricRecorded = false; + const recordMetric = async () => { + if (metricRecorded || !manifest.metricToken) return; + metricRecorded = true; + try { + await fetch(new URL("/api/internal/archive-download-metric", target), { + method: "POST", + headers: { "content-type": "application/jose" }, + body: manifest.metricToken, + }); + } catch { + // Download metrics remain best-effort and never interrupt archive bytes. + } + }; + + const stream = buildDeterministicZipStream( + manifest.entries.map((entry) => ({ + path: entry.path, + openStream: async () => { + const response = await fetch(entry.url, { redirect: "error", signal }); + if (response.status === 404) return null; + if (!response.ok || !response.body) { + throw new Error(`Failed to fetch archive entry: ${response.status}`); + } + await recordMetric(); + return response.body; + }, + })), + manifest.meta, + ); + const headers = new Headers(manifestResponse.headers); + for (const name of ARCHIVE_REPRESENTATION_HEADERS) headers.delete(name); + headers.set("content-type", "application/zip"); + headers.set("content-disposition", `attachment; filename="${manifest.filename}"`); + const response = new Response(stream, { status: 200, headers }); + if (isPreviewFrontend(env) || isTestFrontend(env)) { + const deployment = convexDeploymentName(target); + if (deployment) { + response.headers.set( + isTestFrontend(env) ? "X-ClawHub-Test-Backend" : "X-ClawHub-Preview-Backend", + deployment, + ); + } + } + return response; +} + +async function readBoundedArchiveManifest(response: Response) { + const contentLength = response.headers.get("content-length"); + if (contentLength) { + const declaredBytes = Number.parseInt(contentLength, 10); + if (Number.isFinite(declaredBytes) && declaredBytes > MAX_ARCHIVE_MANIFEST_BYTES) return null; + } + const reader = response.body?.getReader(); + if (!reader) return null; + const chunks: Uint8Array[] = []; + let totalBytes = 0; + try { + for (;;) { + const chunk = await reader.read(); + if (chunk.done) break; + totalBytes += chunk.value.byteLength; + if (totalBytes > MAX_ARCHIVE_MANIFEST_BYTES) { + await reader.cancel().catch(() => undefined); + return null; + } + chunks.push(chunk.value); + } + } finally { + reader.releaseLock(); + } + const bytes = new Uint8Array(totalBytes); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + return new TextDecoder().decode(bytes); +} + +function parseSkillArchiveManifest( + value: unknown, + expectedIssuer: string, + expectedStorageOrigin: string | null, + now: number, +): SkillArchiveManifest | null { + if (!value || typeof value !== "object") return null; + const manifest = value as Partial; + if (manifest.schema !== "clawhub.skill-archive-manifest.v1") return null; + if ( + manifest.issuer !== expectedIssuer || + manifest.audience !== ARCHIVE_MANIFEST_AUDIENCE || + !expectedStorageOrigin + ) { + return null; + } + if (!Number.isFinite(manifest.issuedAt) || !Number.isFinite(manifest.expiresAt)) return null; + const issuedAt = manifest.issuedAt as number; + const expiresAt = manifest.expiresAt as number; + if (issuedAt > now + ARCHIVE_MANIFEST_CLOCK_SKEW_MS || expiresAt <= now) return null; + if (expiresAt <= issuedAt || expiresAt - issuedAt > ARCHIVE_MANIFEST_MAX_AGE_MS) return null; + if (typeof manifest.filename !== "string" || !ARCHIVE_FILENAME_PATTERN.test(manifest.filename)) { + return null; + } + if (!isSkillZipMeta(manifest.meta)) return null; + if (!Array.isArray(manifest.entries) || manifest.entries.length > MAX_ARCHIVE_MANIFEST_ENTRIES) { + return null; + } + if ( + manifest.metricToken !== undefined && + (typeof manifest.metricToken !== "string" || manifest.metricToken.length > 16 * 1024) + ) { + return null; + } + + const seenPaths = new Set(); + for (const entry of manifest.entries) { + if (!entry || typeof entry !== "object") return null; + if (typeof entry.path !== "string" || !validateFilePath(entry.path)) return null; + if (seenPaths.has(entry.path)) return null; + seenPaths.add(entry.path); + if (typeof entry.url !== "string" || entry.url.length > MAX_ARCHIVE_ENTRY_URL_LENGTH) { + return null; + } + let url: URL; + try { + url = new URL(entry.url); + } catch { + return null; + } + if (url.username || url.password || url.origin !== expectedStorageOrigin) return null; + if (!url.pathname.startsWith("/api/storage/")) return null; + } + return manifest as SkillArchiveManifest; +} + +function isSkillZipMeta(value: unknown): value is SkillZipMeta { + if (!value || typeof value !== "object") return false; + const meta = value as Partial; + return ( + typeof meta.ownerId === "string" && + typeof meta.slug === "string" && + validateSlug(meta.slug) && + typeof meta.version === "string" && + meta.version.length > 0 && + meta.version.length <= 200 && + typeof meta.publishedAt === "number" && + Number.isFinite(meta.publishedAt) + ); +} + +export async function verifySignedArchiveManifest( + token: string, + target: string, + jwksOverride?: CompactVerifyGetKey, +) { + const targetOrigin = new URL(target).origin; + let jwks = jwksOverride ?? archiveJwksByOrigin.get(targetOrigin); + if (!jwks) { + const remoteJwks = createRemoteJWKSet(new URL("/.well-known/jwks.json", targetOrigin)); + archiveJwksByOrigin.set(targetOrigin, remoteJwks); + jwks = remoteJwks; + } + const verified = await compactVerify(token, jwks, { algorithms: ["RS256"] }); + if (verified.protectedHeader.typ !== ARCHIVE_MANIFEST_JWS_TYPE) { + throw new Error("Unexpected archive manifest signature type"); + } + return JSON.parse(new TextDecoder().decode(verified.payload)) as unknown; +} + +export function resolveConvexStorageOrigin(target: string, env: ProxyEnv) { + let targetOrigin: string; + let selectedSiteOrigin: string; + let cloudUrl: URL; + try { + targetOrigin = new URL(target).origin; + selectedSiteOrigin = resolveConvexSiteUrl(env); + cloudUrl = new URL(env.VITE_CONVEX_URL ?? env.CONVEX_URL ?? ""); + } catch { + return null; + } + if (targetOrigin !== selectedSiteOrigin) return null; + + const isLocalCloud = + cloudUrl.protocol === "http:" && + ["localhost", "127.0.0.1", "[::1]"].includes(cloudUrl.hostname); + if (isLocalCloud) return cloudUrl.origin; + if (cloudUrl.protocol !== "https:" || !cloudUrl.hostname.endsWith(".convex.cloud")) { + return null; + } + + const targetDeployment = convexDeploymentName(target); + const cloudDeployment = cloudUrl.hostname.slice(0, -".convex.cloud".length); + if (targetDeployment && targetDeployment !== cloudDeployment) return null; + return cloudUrl.origin; +} + export default defineEventHandler((event) => proxyConvexRequest(event)); diff --git a/specs/spec.md b/specs/spec.md index 7aa1bcd1..abde80d4 100644 --- a/specs/spec.md +++ b/specs/spec.md @@ -163,7 +163,37 @@ Local fixture data lives in `convex/devSeed.ts` and `fixtures/public-corpus/`. ## Download API - JSON API for skill metadata + versions. -- Download endpoint returns zip of a version (HTTP action). +- Convex remains the download control plane: it resolves the version, applies + moderation and rate limits, preserves auth-derived metering, and returns the + Nitro API owner a bounded, no-store manifest of Convex File Storage URLs. +- Nitro streams those source files into the deterministic ZIP with backpressure + and owns the public response headers. Archive bytes must not pass through a + Convex HTTP action because those responses are capped at 20 MiB. +- The manifest is an internal server-to-server capability, not a client token. + Nitro overwrites the internal request headers and authenticates to Convex with + its Vercel OIDC identity; Convex verifies the Vercel signature plus the exact + ClawHub team, project, subject, audience, and target environment before + returning any storage URL. The permanent ClawHub Test frontend is a Vercel + preview-target deployment, so its OIDC environment and subject use `preview`; + the app-level `test` label is not an OIDC trust claim. Convex derives this + expected target from its explicit runtime environment markers, not a fixed + deployment hostname, and fails closed when a remote runtime is unclassified. + Nitro never accepts a + client-supplied manifest and + accepts Convex's response only as a short-lived RS256 JWS signed by the + selected Convex deployment's existing auth key and verified from that + deployment's JWKS endpoint. The signed issuer, audience, type, and time bounds + must match the request, and source URLs are allowed only on the single + build-paired Convex deployment's `/api/storage/` surface. Convex storage URLs + are reusable bearer URLs, so they must never appear in the public response. + Nitro preserves control-plane headers such as rate-limit state but discards + manifest representation metadata before emitting the generated ZIP headers. +- Download metering is also a signed, short-lived capability. It contains only + the existing pre-hashed identity and metric arguments, stays inside the + Convex-to-Nitro boundary, and is returned to Convex only after Nitro opens the + first live source Blob. An archive whose source Blobs are all stale must not + count. Capability replay remains harmless because the existing + target/identity/day metric mutation is idempotent. - Soft-delete versions; downloads remain for non-deleted versions only. ## UI (SPA) @@ -187,5 +217,6 @@ Local fixture data lives in `convex/devSeed.ts` and `fixtures/public-corpus/`. ## Open questions (carry forward) - Embeddings provider key + rate limits. -- Zip generation memory limits (optimize with streaming if needed). +- ZIP generation must remain backpressured; never buffer a whole stored entry + or completed archive in either Convex or Nitro. - GitHub App repo sync (phase 2). diff --git a/src/components/SkillDetailTabs.test.tsx b/src/components/SkillDetailTabs.test.tsx index 66fb216b..60bdfe0c 100644 --- a/src/components/SkillDetailTabs.test.tsx +++ b/src/components/SkillDetailTabs.test.tsx @@ -215,7 +215,7 @@ describe("SkillDetailTabs README links", () => { ); const href = screen.getByRole("link", { name: "Download version v1.0.0" }).getAttribute("href"); - const url = new URL(href ?? ""); + const url = new URL(href ?? "", "https://clawhub.ai"); expect(url.pathname).toBe("/api/v1/download"); expect(url.searchParams.get("slug")).toBe("api-gateway"); expect(url.searchParams.get("ownerHandle")).toBe("clawkit"); diff --git a/src/components/SkillVersionsPanel.tsx b/src/components/SkillVersionsPanel.tsx index b2eecb83..1603af1f 100644 --- a/src/components/SkillVersionsPanel.tsx +++ b/src/components/SkillVersionsPanel.tsx @@ -6,7 +6,6 @@ import { api } from "../../convex/_generated/api"; import type { Doc } from "../../convex/_generated/dataModel"; import type { Id } from "../../convex/_generated/dataModel"; import { getUserFacingConvexError } from "../lib/convexError"; -import { getRuntimeEnv } from "../lib/runtimeEnv"; import { Badge } from "./ui/badge"; import { Button } from "./ui/button"; import { VersionChangelog } from "./VersionChangelog"; @@ -39,7 +38,6 @@ function mergeSkillVersions(...groups: Doc<"skillVersions">[][]) { } function buildVersionDownloadHref( - convexSiteUrl: string, skillSlug: string, ownerHandle: string | null | undefined, version: string, @@ -48,7 +46,7 @@ function buildVersionDownloadHref( const normalizedOwner = ownerHandle?.trim().replace(/^@+/, ""); if (normalizedOwner) params.set("ownerHandle", normalizedOwner); params.set("version", version); - return `${convexSiteUrl}/api/v1/download?${params.toString()}`; + return `/api/v1/download?${params.toString()}`; } export function SkillVersionsPanel({ @@ -62,7 +60,6 @@ export function SkillVersionsPanel({ ownerHandle, suppressedMessage, }: SkillVersionsPanelProps) { - const convexSiteUrl = getRuntimeEnv("VITE_CONVEX_SITE_URL") ?? "https://clawhub.ai"; const deleteOwnedVersion = useMutation(api.skills.deleteOwnedVersion); const restoreOwnedVersion = useMutation(api.skills.restoreOwnedVersion); const { @@ -222,12 +219,8 @@ export function SkillVersionsPanel({ <> {!nixPlugin && isAvailable ? (