mirror of
https://github.com/openclaw/clawhub.git
synced 2026-08-14 00:47:57 +00:00
fix: stream legacy skill downloads (#3451)
* fix: stream legacy skill downloads * fix: stream zip entries in bounded chunks * fix: stream large archives through api owner * fix: normalize streamed archive chunks * fix: authenticate archive streaming handoff * fix: authenticate archive manifest requests * test: bound archive determinism fixture * fix: align archive oidc trust with vercel targets * fix: harden archive runtime boundaries * test: isolate archive proxy credentials * test: exercise streamed manifest size cap
This commit is contained in:
Patrick Erichsen
@@ -41,6 +41,7 @@
|
||||
"fflate": "0.8.3",
|
||||
"h3": "2.0.1-rc.25",
|
||||
"ignore": "7.0.6",
|
||||
"jose": "6.2.3",
|
||||
"lucide-react": "1.28.0",
|
||||
"mermaid": "^11.16.1",
|
||||
"mime": "4.1.0",
|
||||
|
||||
+430
-13
@@ -1,7 +1,20 @@
|
||||
import type { RateLimitArgs, RateLimitReturns } from "@convex-dev/rate-limiter";
|
||||
import { unzipSync } from "fflate";
|
||||
import { exportJWK, exportPKCS8, generateKeyPair } from "jose";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { ActionCtx } from "./_generated/server";
|
||||
import { __test, downloadZipHandler } from "./downloads";
|
||||
import { __test, downloadZipHandler, recordArchiveDownloadMetricHandler } from "./downloads";
|
||||
import {
|
||||
ARCHIVE_MANIFEST_AUDIENCE,
|
||||
ARCHIVE_MANIFEST_CONTENT_TYPE,
|
||||
ARCHIVE_MANIFEST_JWS_TYPE,
|
||||
ARCHIVE_METRIC_AUDIENCE,
|
||||
ARCHIVE_METRIC_JWS_TYPE,
|
||||
type ArchiveMetricPayload,
|
||||
signArchivePayload,
|
||||
type SkillArchiveManifest,
|
||||
verifyArchivePayloadWithLocalJwks,
|
||||
} from "./lib/archiveManifest";
|
||||
|
||||
function isRateLimitArgs(args: unknown): args is RateLimitArgs {
|
||||
if (!args || typeof args !== "object") return false;
|
||||
@@ -35,8 +48,30 @@ function stubZipResponse() {
|
||||
vi.stubGlobal("Response", MockResponse as unknown as typeof Response);
|
||||
}
|
||||
|
||||
function deferred<T>() {
|
||||
let resolve!: (value: T) => void;
|
||||
const promise = new Promise<T>((resolvePromise) => {
|
||||
resolve = resolvePromise;
|
||||
});
|
||||
return { promise, resolve };
|
||||
}
|
||||
|
||||
function streamingBlob(text: string) {
|
||||
const bytes = new TextEncoder().encode(text);
|
||||
return {
|
||||
stream: () =>
|
||||
new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(bytes);
|
||||
controller.close();
|
||||
},
|
||||
}),
|
||||
} as Blob;
|
||||
}
|
||||
|
||||
describe("downloads helpers", () => {
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllEnvs();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
@@ -79,7 +114,6 @@ describe("downloads helpers", () => {
|
||||
|
||||
it("schedules zip download stats outside the response path", async () => {
|
||||
vi.stubEnv("TRUST_FORWARDED_IPS", "true");
|
||||
stubZipResponse();
|
||||
|
||||
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
|
||||
if ("slug" in args) {
|
||||
@@ -111,23 +145,28 @@ describe("downloads helpers", () => {
|
||||
return { mutation, args };
|
||||
});
|
||||
const runAfter = vi.fn();
|
||||
const storageGet = vi.fn().mockResolvedValue(new Blob(["hello"], { type: "text/markdown" }));
|
||||
const storageGet = vi.fn().mockResolvedValue(streamingBlob("hello"));
|
||||
|
||||
const response = await downloadZipHandler(
|
||||
{
|
||||
runQuery,
|
||||
runMutation,
|
||||
scheduler: { runAfter },
|
||||
storage: { get: storageGet },
|
||||
storage: {
|
||||
get: storageGet,
|
||||
getMetadata: vi.fn().mockResolvedValue({}),
|
||||
},
|
||||
} as unknown as ActionCtx,
|
||||
new Request("https://example.com/api/v1/download?slug=demo", {
|
||||
new Request("https://preview-branch-123.convex.site/api/v1/download?slug=demo", {
|
||||
headers: { "cf-connecting-ip": "1.2.3.4" },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get("Content-Type")).toBe("application/zip");
|
||||
const archive = new Uint8Array(await response.arrayBuffer());
|
||||
expect(storageGet).toHaveBeenCalledWith("_storage:1");
|
||||
expect(new TextDecoder().decode(unzipSync(archive)["SKILL.md"])).toBe("hello");
|
||||
|
||||
const recordCalls = runAfter.mock.calls.filter(([, , args]) => {
|
||||
if (!args || typeof args !== "object") return false;
|
||||
@@ -152,6 +191,373 @@ describe("downloads helpers", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("returns a bounded archive manifest to the Nitro streaming owner", async () => {
|
||||
vi.stubEnv("CLAWHUB_PREVIEW", "1");
|
||||
vi.stubEnv("TRUST_FORWARDED_IPS", "true");
|
||||
vi.spyOn(Date, "now").mockReturnValue(10_000);
|
||||
const keyPair = await generateKeyPair("RS256", { extractable: true });
|
||||
const privateKey = await exportPKCS8(keyPair.privateKey);
|
||||
const publicKey = await exportJWK(keyPair.publicKey);
|
||||
const jwks = JSON.stringify({ keys: [{ use: "sig", ...publicKey }] });
|
||||
vi.stubEnv("JWT_PRIVATE_KEY", privateKey);
|
||||
vi.stubEnv("JWKS", jwks);
|
||||
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
|
||||
if ("slug" in args) {
|
||||
return {
|
||||
skill: {
|
||||
_id: "skills:1",
|
||||
ownerUserId: "users:1",
|
||||
slug: "demo",
|
||||
tags: {},
|
||||
latestVersionId: "skillVersions:1",
|
||||
},
|
||||
moderationInfo: null,
|
||||
};
|
||||
}
|
||||
if ("versionId" in args) {
|
||||
return {
|
||||
_id: "skillVersions:1",
|
||||
skillId: "skills:1",
|
||||
version: "1.0.0+build",
|
||||
createdAt: 3,
|
||||
files: [
|
||||
{ path: "SKILL.md", storageId: "_storage:1" },
|
||||
{ path: "missing.txt", storageId: "_storage:missing" },
|
||||
],
|
||||
softDeletedAt: undefined,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
});
|
||||
const runMutation = vi.fn(async (_mutation: unknown, args: Record<string, unknown>) => {
|
||||
if (isRateLimitArgs(args)) return okRate();
|
||||
return null;
|
||||
});
|
||||
const runAfter = vi.fn();
|
||||
const storageGet = vi.fn();
|
||||
const storageGetUrl = vi.fn(async (storageId: string) =>
|
||||
storageId === "_storage:1"
|
||||
? "https://preview-branch-123.convex.cloud/api/storage/storage-1"
|
||||
: null,
|
||||
);
|
||||
|
||||
const response = await downloadZipHandler(
|
||||
{
|
||||
runQuery,
|
||||
runMutation,
|
||||
scheduler: { runAfter },
|
||||
storage: {
|
||||
get: storageGet,
|
||||
getUrl: storageGetUrl,
|
||||
getMetadata: vi.fn(),
|
||||
},
|
||||
} as unknown as ActionCtx,
|
||||
new Request("https://preview-branch-123.convex.site/api/v1/download?slug=demo", {
|
||||
headers: {
|
||||
"cf-connecting-ip": "1.2.3.4",
|
||||
"x-clawhub-archive-manifest": "v1",
|
||||
"x-clawhub-vercel-oidc-token": "vercel-oidc",
|
||||
},
|
||||
}),
|
||||
{ verifyArchiveRequester: vi.fn(async () => undefined) },
|
||||
);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get("Content-Type")).toBe(ARCHIVE_MANIFEST_CONTENT_TYPE);
|
||||
expect(response.headers.get("Cache-Control")).toBe("private, no-store");
|
||||
const manifest = (await verifyArchivePayloadWithLocalJwks(
|
||||
await response.text(),
|
||||
ARCHIVE_MANIFEST_JWS_TYPE,
|
||||
jwks,
|
||||
)) as SkillArchiveManifest;
|
||||
expect(manifest).toEqual({
|
||||
schema: "clawhub.skill-archive-manifest.v1",
|
||||
issuer: "https://preview-branch-123.convex.site",
|
||||
audience: ARCHIVE_MANIFEST_AUDIENCE,
|
||||
issuedAt: 10_000,
|
||||
expiresAt: 40_000,
|
||||
filename: "demo-1.0.0+build.zip",
|
||||
meta: {
|
||||
ownerId: "users:1",
|
||||
slug: "demo",
|
||||
version: "1.0.0+build",
|
||||
publishedAt: 3,
|
||||
},
|
||||
entries: [
|
||||
{
|
||||
path: "SKILL.md",
|
||||
url: "https://preview-branch-123.convex.cloud/api/storage/storage-1",
|
||||
},
|
||||
],
|
||||
metricToken: expect.any(String),
|
||||
});
|
||||
const metricPayload = (await verifyArchivePayloadWithLocalJwks(
|
||||
manifest.metricToken!,
|
||||
ARCHIVE_METRIC_JWS_TYPE,
|
||||
jwks,
|
||||
)) as ArchiveMetricPayload;
|
||||
expect(metricPayload).toMatchObject({
|
||||
schema: "clawhub.archive-download-metric.v1",
|
||||
issuer: "https://preview-branch-123.convex.site",
|
||||
audience: ARCHIVE_METRIC_AUDIENCE,
|
||||
issuedAt: 10_000,
|
||||
expiresAt: 40_000,
|
||||
metric: {
|
||||
target: { kind: "skill", id: "skills:1" },
|
||||
identityKind: "ip",
|
||||
identityHash: expect.any(String),
|
||||
dayStart: 0,
|
||||
occurredAt: 10_000,
|
||||
},
|
||||
});
|
||||
expect(storageGet).not.toHaveBeenCalled();
|
||||
expect(storageGetUrl).toHaveBeenCalledTimes(2);
|
||||
expect(runAfter).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects a direct manifest request without the Nitro Vercel identity", async () => {
|
||||
vi.stubEnv("CLAWHUB_PREVIEW", "1");
|
||||
const runQuery = vi.fn();
|
||||
const runMutation = vi.fn(async (_mutation: unknown, args: Record<string, unknown>) => {
|
||||
if (isRateLimitArgs(args)) return okRate();
|
||||
return null;
|
||||
});
|
||||
|
||||
const verifyArchiveRequester = vi.fn(async () => {
|
||||
throw new Error("invalid Vercel identity");
|
||||
});
|
||||
const response = await downloadZipHandler(
|
||||
{ runQuery, runMutation } as unknown as ActionCtx,
|
||||
new Request("https://preview-branch-123.convex.site/api/v1/download?slug=demo", {
|
||||
headers: {
|
||||
"x-clawhub-archive-manifest": "v1",
|
||||
"x-clawhub-vercel-oidc-token": "client-forgery",
|
||||
},
|
||||
}),
|
||||
{ verifyArchiveRequester },
|
||||
);
|
||||
|
||||
expect(response.status).toBe(401);
|
||||
expect(response.headers.get("Cache-Control")).toBe("no-store");
|
||||
expect(verifyArchiveRequester).toHaveBeenCalledWith("client-forgery", "preview");
|
||||
expect(runQuery).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("records only a valid, unexpired archive metric capability", async () => {
|
||||
const keyPair = await generateKeyPair("RS256", { extractable: true });
|
||||
const privateKey = await exportPKCS8(keyPair.privateKey);
|
||||
const publicKey = await exportJWK(keyPair.publicKey);
|
||||
const jwks = JSON.stringify({ keys: [{ use: "sig", ...publicKey }] });
|
||||
vi.stubEnv("JWKS", jwks);
|
||||
vi.spyOn(Date, "now").mockReturnValue(2_000);
|
||||
const payload: ArchiveMetricPayload = {
|
||||
schema: "clawhub.archive-download-metric.v1",
|
||||
issuer: "https://example.com",
|
||||
audience: ARCHIVE_METRIC_AUDIENCE,
|
||||
issuedAt: 1_000,
|
||||
expiresAt: 31_000,
|
||||
metric: {
|
||||
target: { kind: "skill", id: "skills:1" },
|
||||
identityKind: "ip",
|
||||
identityHash: "identity-hash",
|
||||
dayStart: 0,
|
||||
occurredAt: 1_000,
|
||||
},
|
||||
};
|
||||
const token = await signArchivePayload(payload, ARCHIVE_METRIC_JWS_TYPE, privateKey);
|
||||
const runAfter = vi.fn();
|
||||
const ctx = { scheduler: { runAfter } } as unknown as ActionCtx;
|
||||
|
||||
const response = await recordArchiveDownloadMetricHandler(
|
||||
ctx,
|
||||
new Request("https://example.com/api/internal/archive-download-metric", {
|
||||
method: "POST",
|
||||
body: token,
|
||||
}),
|
||||
);
|
||||
expect(response.status).toBe(204);
|
||||
expect(runAfter).toHaveBeenCalledWith(expect.any(Number), expect.anything(), payload.metric);
|
||||
|
||||
const [header, body, signature] = token.split(".");
|
||||
const modifiedBody = `${body!.slice(0, -1)}${body!.endsWith("A") ? "B" : "A"}`;
|
||||
const modifiedToken = `${header}.${modifiedBody}.${signature}`;
|
||||
const modifiedResponse = await recordArchiveDownloadMetricHandler(
|
||||
ctx,
|
||||
new Request("https://example.com/api/internal/archive-download-metric", {
|
||||
method: "POST",
|
||||
body: modifiedToken,
|
||||
}),
|
||||
);
|
||||
expect(modifiedResponse.status).toBe(401);
|
||||
|
||||
const expiredToken = await signArchivePayload(
|
||||
{ ...payload, expiresAt: 1_500 },
|
||||
ARCHIVE_METRIC_JWS_TYPE,
|
||||
privateKey,
|
||||
);
|
||||
const expiredResponse = await recordArchiveDownloadMetricHandler(
|
||||
ctx,
|
||||
new Request("https://example.com/api/internal/archive-download-metric", {
|
||||
method: "POST",
|
||||
body: expiredToken,
|
||||
}),
|
||||
);
|
||||
expect(expiredResponse.status).toBe(401);
|
||||
expect(runAfter).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("streams stored file chunks, stays deterministic, and skips a Blob that vanishes", async () => {
|
||||
const firstChunk = new Uint8Array(64 * 1024).fill(0x61);
|
||||
const secondChunk = new TextEncoder().encode("streamed body\n");
|
||||
const releaseSecondChunk = deferred<void>();
|
||||
const arrayBuffer = vi.fn(() => Promise.reject(new Error("whole Blob read")));
|
||||
const stream = vi.fn(
|
||||
() =>
|
||||
new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(firstChunk);
|
||||
},
|
||||
async pull(controller) {
|
||||
await releaseSecondChunk.promise;
|
||||
controller.enqueue(secondChunk);
|
||||
controller.close();
|
||||
},
|
||||
}),
|
||||
);
|
||||
const storageGetMetadata = vi.fn().mockResolvedValue({});
|
||||
const storageGet = vi.fn(async (storageId: string) => {
|
||||
if (storageId === "_storage:skill") {
|
||||
return { arrayBuffer, stream } as unknown as Blob;
|
||||
}
|
||||
if (storageId === "_storage:notes") {
|
||||
return {
|
||||
stream: () =>
|
||||
new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(new TextEncoder().encode("supporting notes\n"));
|
||||
controller.close();
|
||||
},
|
||||
}),
|
||||
} as Blob;
|
||||
}
|
||||
return null;
|
||||
});
|
||||
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
|
||||
if ("slug" in args) {
|
||||
return {
|
||||
skill: {
|
||||
_id: "skills:1",
|
||||
ownerUserId: "users:1",
|
||||
slug: "demo",
|
||||
tags: {},
|
||||
latestVersionId: "skillVersions:1",
|
||||
},
|
||||
moderationInfo: null,
|
||||
};
|
||||
}
|
||||
if ("versionId" in args) {
|
||||
return {
|
||||
_id: "skillVersions:1",
|
||||
skillId: "skills:1",
|
||||
version: "1.0.0",
|
||||
createdAt: 3,
|
||||
files: [
|
||||
{ path: "a.txt", storageId: "_storage:skill" },
|
||||
{ path: "b.txt", storageId: "_storage:notes" },
|
||||
{ path: "missing.txt", storageId: "_storage:missing" },
|
||||
],
|
||||
softDeletedAt: undefined,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
});
|
||||
const runMutation = vi.fn(async (_mutation: unknown, args: Record<string, unknown>) => {
|
||||
if (isRateLimitArgs(args)) return okRate();
|
||||
return null;
|
||||
});
|
||||
|
||||
const response = await Promise.race([
|
||||
downloadZipHandler(
|
||||
{
|
||||
runQuery,
|
||||
runMutation,
|
||||
scheduler: { runAfter: vi.fn() },
|
||||
storage: { get: storageGet, getMetadata: storageGetMetadata },
|
||||
} as unknown as ActionCtx,
|
||||
new Request("https://example.com/api/v1/download?slug=demo"),
|
||||
),
|
||||
new Promise<never>((_, reject) => {
|
||||
setTimeout(
|
||||
() => reject(new Error("download handler read archive bodies before responding")),
|
||||
1_000,
|
||||
);
|
||||
}),
|
||||
]);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(storageGetMetadata).not.toHaveBeenCalled();
|
||||
expect(storageGet).not.toHaveBeenCalled();
|
||||
|
||||
const reader = response.body!.getReader();
|
||||
const firstArchiveChunk = await reader.read();
|
||||
expect(firstArchiveChunk.done).toBe(false);
|
||||
expect(stream).toHaveBeenCalledTimes(1);
|
||||
expect(arrayBuffer).not.toHaveBeenCalled();
|
||||
releaseSecondChunk.resolve();
|
||||
|
||||
const archiveChunks = [firstArchiveChunk.value!];
|
||||
for (;;) {
|
||||
const chunk = await reader.read();
|
||||
if (chunk.done) break;
|
||||
archiveChunks.push(chunk.value);
|
||||
}
|
||||
const responseBytes = Uint8Array.from(archiveChunks.flatMap((chunk) => [...chunk]));
|
||||
const unzipped = unzipSync(responseBytes);
|
||||
expect(Object.keys(unzipped).sort()).toEqual(["_meta.json", "a.txt", "b.txt"]);
|
||||
expect(unzipped["a.txt"]).toEqual(Uint8Array.from([...firstChunk, ...secondChunk]));
|
||||
expect(new TextDecoder().decode(unzipped["b.txt"])).toBe("supporting notes\n");
|
||||
expect(storageGet).toHaveBeenCalledWith("_storage:missing");
|
||||
|
||||
const repeatResponse = await downloadZipHandler(
|
||||
{
|
||||
runQuery,
|
||||
runMutation,
|
||||
scheduler: { runAfter: vi.fn() },
|
||||
storage: {
|
||||
get: vi.fn(async (storageId: string) => {
|
||||
if (storageId === "_storage:skill") {
|
||||
return {
|
||||
stream: () =>
|
||||
new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(firstChunk);
|
||||
controller.enqueue(secondChunk);
|
||||
controller.close();
|
||||
},
|
||||
}),
|
||||
} as Blob;
|
||||
}
|
||||
if (storageId === "_storage:notes") {
|
||||
return {
|
||||
stream: () =>
|
||||
new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(new TextEncoder().encode("supporting notes\n"));
|
||||
controller.close();
|
||||
},
|
||||
}),
|
||||
} as Blob;
|
||||
}
|
||||
return null;
|
||||
}),
|
||||
getMetadata: storageGetMetadata,
|
||||
},
|
||||
} as unknown as ActionCtx,
|
||||
new Request("https://example.com/api/v1/download?slug=demo"),
|
||||
);
|
||||
expect(new Uint8Array(await repeatResponse.arrayBuffer())).toEqual(responseBytes);
|
||||
});
|
||||
|
||||
it("returns 410 for an explicitly requested revoked version", async () => {
|
||||
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
|
||||
if ("slug" in args) {
|
||||
@@ -195,7 +601,7 @@ describe("downloads helpers", () => {
|
||||
runQuery,
|
||||
runMutation,
|
||||
scheduler: { runAfter },
|
||||
storage: { get: storageGet },
|
||||
storage: { get: storageGet, getMetadata: vi.fn().mockResolvedValue({}) },
|
||||
} as unknown as ActionCtx,
|
||||
new Request("https://example.com/api/v1/download?slug=demo&version=1.0.0"),
|
||||
);
|
||||
@@ -241,7 +647,10 @@ describe("downloads helpers", () => {
|
||||
runQuery,
|
||||
runMutation,
|
||||
scheduler: { runAfter: vi.fn() },
|
||||
storage: { get: vi.fn().mockResolvedValue(new Blob(["hello"])) },
|
||||
storage: {
|
||||
get: vi.fn().mockResolvedValue(streamingBlob("hello")),
|
||||
getMetadata: vi.fn().mockResolvedValue({}),
|
||||
},
|
||||
} as unknown as ActionCtx,
|
||||
new Request("https://example.com/api/v1/download?slug=demo&ownerHandle=clawkit"),
|
||||
);
|
||||
@@ -302,7 +711,7 @@ describe("downloads helpers", () => {
|
||||
runQuery,
|
||||
runMutation,
|
||||
scheduler: { runAfter: vi.fn() },
|
||||
storage: { get: storageGet },
|
||||
storage: { get: storageGet, getMetadata: vi.fn().mockResolvedValue({}) },
|
||||
} as unknown as ActionCtx,
|
||||
new Request("https://example.com/api/v1/download?slug=demo&tag=old", {
|
||||
headers: { "cf-connecting-ip": "1.2.3.4" },
|
||||
@@ -403,7 +812,7 @@ describe("downloads helpers", () => {
|
||||
runQuery,
|
||||
runMutation,
|
||||
scheduler: { runAfter: vi.fn() },
|
||||
storage: { get: storageGet },
|
||||
storage: { get: storageGet, getMetadata: vi.fn().mockResolvedValue({}) },
|
||||
} as unknown as ActionCtx,
|
||||
new Request("https://example.com/api/v1/download?slug=demo&version=1.0.0", {
|
||||
headers: { "cf-connecting-ip": "1.2.3.4" },
|
||||
@@ -453,17 +862,21 @@ describe("downloads helpers", () => {
|
||||
});
|
||||
const runMutation = vi.fn(async (_mutation: unknown, args: Record<string, unknown>) => {
|
||||
if (isRateLimitArgs(args)) return okRate();
|
||||
if (Object.keys(args).length === 0) return "https://upload.example";
|
||||
return { tokenTouched: "tokenId" in args };
|
||||
});
|
||||
const runAfter = vi.fn();
|
||||
const storageGet = vi.fn().mockResolvedValue(new Blob(["hello"], { type: "text/markdown" }));
|
||||
const storageGet = vi.fn().mockResolvedValue(streamingBlob("hello"));
|
||||
|
||||
const response = await downloadZipHandler(
|
||||
{
|
||||
runQuery,
|
||||
runMutation,
|
||||
scheduler: { runAfter },
|
||||
storage: { get: storageGet },
|
||||
storage: {
|
||||
get: storageGet,
|
||||
getMetadata: vi.fn().mockResolvedValue({}),
|
||||
},
|
||||
} as unknown as ActionCtx,
|
||||
new Request("https://example.com/api/v1/download?slug=demo", {
|
||||
headers: {
|
||||
@@ -516,17 +929,21 @@ describe("downloads helpers", () => {
|
||||
});
|
||||
const runMutation = vi.fn(async (_mutation: unknown, args: Record<string, unknown>) => {
|
||||
if (isRateLimitArgs(args)) return okRate();
|
||||
if (Object.keys(args).length === 0) return "https://upload.example";
|
||||
return { mutationRecorded: true };
|
||||
});
|
||||
const runAfter = vi.fn();
|
||||
const storageGet = vi.fn().mockResolvedValue(new Blob(["hello"], { type: "text/markdown" }));
|
||||
const storageGet = vi.fn().mockResolvedValue(streamingBlob("hello"));
|
||||
|
||||
const response = await downloadZipHandler(
|
||||
{
|
||||
runQuery,
|
||||
runMutation,
|
||||
scheduler: { runAfter },
|
||||
storage: { get: storageGet },
|
||||
storage: {
|
||||
get: storageGet,
|
||||
getMetadata: vi.fn().mockResolvedValue({}),
|
||||
},
|
||||
} as unknown as ActionCtx,
|
||||
new Request("https://example.com/api/v1/download?slug=demo", {
|
||||
headers: { "cf-connecting-ip": "1.2.3.4" },
|
||||
|
||||
+261
-13
@@ -5,6 +5,24 @@ import { httpAction } from "./functions";
|
||||
import { ambiguousSkillSlugResponse } from "./httpApiV1/shared";
|
||||
import { getOptionalActiveAuthUserIdFromAction } from "./lib/access";
|
||||
import { getOptionalApiTokenUserId } from "./lib/apiTokenAuth";
|
||||
import {
|
||||
ARCHIVE_MANIFEST_AUDIENCE,
|
||||
ARCHIVE_MANIFEST_CONTENT_TYPE,
|
||||
ARCHIVE_MANIFEST_JWS_TYPE,
|
||||
ARCHIVE_METRIC_AUDIENCE,
|
||||
ARCHIVE_METRIC_JWS_TYPE,
|
||||
type ArchiveMetricArgs,
|
||||
type ArchiveMetricPayload,
|
||||
signArchivePayload,
|
||||
type SkillArchiveManifest,
|
||||
verifyArchivePayloadWithLocalJwks,
|
||||
} from "./lib/archiveManifest";
|
||||
import {
|
||||
ARCHIVE_REQUEST_IDENTITY_HEADER,
|
||||
expectedVercelEnvironmentForConvexSite,
|
||||
type ClawHubVercelEnvironment,
|
||||
verifyClawHubVercelOidcToken,
|
||||
} from "./lib/clawhubVercelOidc";
|
||||
import {
|
||||
buildGitHubSkillHandoffDescriptor,
|
||||
getGitHubHandoffBlock,
|
||||
@@ -18,14 +36,35 @@ import {
|
||||
getPublicSkillVersionDownloadBlock,
|
||||
isSkillVersionForSkill,
|
||||
} from "./lib/skillFileAccess";
|
||||
import { buildDeterministicZip } from "./lib/skillZip";
|
||||
import { buildDeterministicZipStream } from "./lib/skillZip";
|
||||
|
||||
const HOUR_MS = 3_600_000;
|
||||
const DOWNLOAD_STAT_JITTER_MS = 60_000;
|
||||
const ARCHIVE_MANIFEST_REQUEST_HEADER = "x-clawhub-archive-manifest";
|
||||
const ARCHIVE_MANIFEST_TTL_MS = 30_000;
|
||||
const ARCHIVE_MANIFEST_CLOCK_SKEW_MS = 5_000;
|
||||
const MAX_ARCHIVE_MANIFEST_FILES = 8_192;
|
||||
const MAX_ARCHIVE_MANIFEST_BYTES = 4 * 1024 * 1024;
|
||||
const MAX_ARCHIVE_METRIC_TOKEN_BYTES = 16 * 1024;
|
||||
|
||||
type DownloadCtx = Parameters<Parameters<typeof httpAction>[0]>[0];
|
||||
|
||||
export async function downloadZipHandler(ctx: DownloadCtx, request: Request) {
|
||||
type DownloadDependencies = {
|
||||
verifyArchiveRequester: (
|
||||
token: string,
|
||||
expectedEnvironment: ClawHubVercelEnvironment,
|
||||
) => Promise<unknown>;
|
||||
};
|
||||
|
||||
const DEFAULT_DOWNLOAD_DEPENDENCIES: DownloadDependencies = {
|
||||
verifyArchiveRequester: verifyClawHubVercelOidcToken,
|
||||
};
|
||||
|
||||
export async function downloadZipHandler(
|
||||
ctx: DownloadCtx,
|
||||
request: Request,
|
||||
dependencies: DownloadDependencies = DEFAULT_DOWNLOAD_DEPENDENCIES,
|
||||
) {
|
||||
const url = new URL(request.url);
|
||||
const slug = url.searchParams.get("slug")?.trim().toLowerCase();
|
||||
const ownerHandle =
|
||||
@@ -42,6 +81,20 @@ export async function downloadZipHandler(ctx: DownloadCtx, request: Request) {
|
||||
});
|
||||
}
|
||||
|
||||
const manifestRequested = request.headers.get(ARCHIVE_MANIFEST_REQUEST_HEADER) === "v1";
|
||||
if (manifestRequested) {
|
||||
const token = request.headers.get(ARCHIVE_REQUEST_IDENTITY_HEADER)?.trim();
|
||||
const expectedEnvironment = expectedVercelEnvironmentForConvexSite(request.url);
|
||||
if (!token || !expectedEnvironment) {
|
||||
return unauthorizedArchiveManifestResponse();
|
||||
}
|
||||
try {
|
||||
await dependencies.verifyArchiveRequester(token, expectedEnvironment);
|
||||
} catch {
|
||||
return unauthorizedArchiveManifestResponse();
|
||||
}
|
||||
}
|
||||
|
||||
const rate = await applyRateLimit(ctx, request, "download");
|
||||
if (!rate.ok) return rate.response;
|
||||
|
||||
@@ -117,24 +170,77 @@ export async function downloadZipHandler(ctx: DownloadCtx, request: Request) {
|
||||
});
|
||||
}
|
||||
|
||||
const entries: Array<{ path: string; bytes: Uint8Array }> = [];
|
||||
for (const file of version.files) {
|
||||
const blob = await ctx.storage.get(file.storageId);
|
||||
if (!blob) continue;
|
||||
const buffer = new Uint8Array(await blob.arrayBuffer());
|
||||
entries.push({ path: file.path, bytes: buffer });
|
||||
}
|
||||
const zipArray = buildDeterministicZip(entries, {
|
||||
const meta = {
|
||||
ownerId: String(skill.ownerUserId),
|
||||
slug: skill.slug,
|
||||
version: version.version,
|
||||
publishedAt: version.createdAt,
|
||||
});
|
||||
const zipBlob = new Blob([zipArray], { type: "application/zip" });
|
||||
};
|
||||
|
||||
if (manifestRequested) {
|
||||
if (version.files.length > MAX_ARCHIVE_MANIFEST_FILES) {
|
||||
return new Response("Skill archive contains too many files", {
|
||||
status: 413,
|
||||
headers: mergeHeaders(rate.headers, corsHeaders()),
|
||||
});
|
||||
}
|
||||
const entries: Array<{ path: string; url: string }> = [];
|
||||
for (const file of version.files) {
|
||||
const fileUrl = await ctx.storage.getUrl(file.storageId);
|
||||
if (fileUrl) entries.push({ path: file.path, url: fileUrl });
|
||||
}
|
||||
const issuedAt = Date.now();
|
||||
const expiresAt = issuedAt + ARCHIVE_MANIFEST_TTL_MS;
|
||||
const issuer = url.origin;
|
||||
const metricToken = await buildArchiveDownloadMetricToken(
|
||||
ctx,
|
||||
request,
|
||||
skill._id,
|
||||
issuer,
|
||||
issuedAt,
|
||||
expiresAt,
|
||||
);
|
||||
const manifest: SkillArchiveManifest = {
|
||||
schema: "clawhub.skill-archive-manifest.v1",
|
||||
issuer,
|
||||
audience: ARCHIVE_MANIFEST_AUDIENCE,
|
||||
issuedAt,
|
||||
expiresAt,
|
||||
filename: `${slug}-${version.version}.zip`,
|
||||
meta,
|
||||
entries,
|
||||
...(metricToken ? { metricToken } : {}),
|
||||
};
|
||||
const signedManifest = await signArchivePayload(manifest, ARCHIVE_MANIFEST_JWS_TYPE);
|
||||
if (new TextEncoder().encode(signedManifest).byteLength > MAX_ARCHIVE_MANIFEST_BYTES) {
|
||||
return new Response("Skill archive manifest is too large", {
|
||||
status: 413,
|
||||
headers: mergeHeaders(rate.headers, corsHeaders()),
|
||||
});
|
||||
}
|
||||
|
||||
return new Response(signedManifest, {
|
||||
status: 200,
|
||||
headers: mergeHeaders(
|
||||
rate.headers,
|
||||
{
|
||||
"Content-Type": ARCHIVE_MANIFEST_CONTENT_TYPE,
|
||||
"Cache-Control": "private, no-store",
|
||||
},
|
||||
corsHeaders(),
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
const entries = version.files.map((file) => ({
|
||||
path: file.path,
|
||||
openStream: async () => (await ctx.storage.get(file.storageId))?.stream() ?? null,
|
||||
}));
|
||||
const zipStream = buildDeterministicZipStream(entries, meta);
|
||||
|
||||
await scheduleSkillDownloadMetric(ctx, request, skill._id);
|
||||
|
||||
return new Response(zipBlob, {
|
||||
return new Response(zipStream, {
|
||||
status: 200,
|
||||
headers: mergeHeaders(
|
||||
rate.headers,
|
||||
@@ -150,6 +256,56 @@ export async function downloadZipHandler(ctx: DownloadCtx, request: Request) {
|
||||
|
||||
export const downloadZip = httpAction(downloadZipHandler);
|
||||
|
||||
function unauthorizedArchiveManifestResponse() {
|
||||
return new Response("Unauthorized archive manifest request", {
|
||||
status: 401,
|
||||
headers: { "Cache-Control": "no-store" },
|
||||
});
|
||||
}
|
||||
|
||||
export async function recordArchiveDownloadMetricHandler(ctx: DownloadCtx, request: Request) {
|
||||
const token = await readBoundedRequestText(request, MAX_ARCHIVE_METRIC_TOKEN_BYTES);
|
||||
if (!token) {
|
||||
return new Response("Invalid archive metric capability", {
|
||||
status: 400,
|
||||
headers: { "Cache-Control": "no-store" },
|
||||
});
|
||||
}
|
||||
|
||||
let value: unknown;
|
||||
try {
|
||||
value = await verifyArchivePayloadWithLocalJwks(token, ARCHIVE_METRIC_JWS_TYPE);
|
||||
} catch {
|
||||
return new Response("Invalid archive metric capability", {
|
||||
status: 401,
|
||||
headers: { "Cache-Control": "no-store" },
|
||||
});
|
||||
}
|
||||
const payload = parseArchiveMetricPayload(value, new URL(request.url).origin, Date.now());
|
||||
if (!payload) {
|
||||
return new Response("Invalid archive metric capability", {
|
||||
status: 401,
|
||||
headers: { "Cache-Control": "no-store" },
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
await ctx.scheduler.runAfter(
|
||||
Math.floor(Math.random() * DOWNLOAD_STAT_JITTER_MS),
|
||||
internal.downloadMetrics.recordDownloadMetricInternal,
|
||||
{
|
||||
...payload.metric,
|
||||
target: { kind: "skill", id: payload.metric.target.id as Id<"skills"> },
|
||||
},
|
||||
);
|
||||
} catch {
|
||||
// Metrics remain best-effort and must not affect an archive already being streamed.
|
||||
}
|
||||
return new Response(null, { status: 204, headers: { "Cache-Control": "no-store" } });
|
||||
}
|
||||
|
||||
export const recordArchiveDownloadMetric = httpAction(recordArchiveDownloadMetricHandler);
|
||||
|
||||
export function getHourStart(timestamp: number) {
|
||||
return Math.floor(timestamp / HOUR_MS) * HOUR_MS;
|
||||
}
|
||||
@@ -222,6 +378,98 @@ export async function scheduleSkillDownloadMetric(
|
||||
}
|
||||
}
|
||||
|
||||
async function buildArchiveDownloadMetricToken(
|
||||
ctx: DownloadCtx,
|
||||
request: Request,
|
||||
skillId: Id<"skills">,
|
||||
issuer: string,
|
||||
issuedAt: number,
|
||||
expiresAt: number,
|
||||
) {
|
||||
try {
|
||||
const userId = await getOptionalDownloadUserId(ctx, request);
|
||||
const identity = getDownloadIdentity(request, userId ? String(userId) : null);
|
||||
if (!identity) return undefined;
|
||||
const metricArgs = await buildDownloadMetricArgs({
|
||||
target: { kind: "skill", id: skillId },
|
||||
identity,
|
||||
now: issuedAt,
|
||||
});
|
||||
const metric: ArchiveMetricArgs = {
|
||||
...metricArgs,
|
||||
target: { kind: "skill", id: String(skillId) },
|
||||
};
|
||||
const payload: ArchiveMetricPayload = {
|
||||
schema: "clawhub.archive-download-metric.v1",
|
||||
issuer,
|
||||
audience: ARCHIVE_METRIC_AUDIENCE,
|
||||
issuedAt,
|
||||
expiresAt,
|
||||
metric,
|
||||
};
|
||||
return await signArchivePayload(payload, ARCHIVE_METRIC_JWS_TYPE);
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function parseArchiveMetricPayload(
|
||||
value: unknown,
|
||||
expectedIssuer: string,
|
||||
now: number,
|
||||
): ArchiveMetricPayload | null {
|
||||
if (!value || typeof value !== "object") return null;
|
||||
const payload = value as Partial<ArchiveMetricPayload>;
|
||||
if (payload.schema !== "clawhub.archive-download-metric.v1") return null;
|
||||
if (payload.issuer !== expectedIssuer || payload.audience !== ARCHIVE_METRIC_AUDIENCE)
|
||||
return null;
|
||||
if (!Number.isFinite(payload.issuedAt) || !Number.isFinite(payload.expiresAt)) return null;
|
||||
const issuedAt = payload.issuedAt as number;
|
||||
const expiresAt = payload.expiresAt as number;
|
||||
if (issuedAt > now + ARCHIVE_MANIFEST_CLOCK_SKEW_MS || expiresAt <= now) return null;
|
||||
if (expiresAt <= issuedAt || expiresAt - issuedAt > ARCHIVE_MANIFEST_TTL_MS) return null;
|
||||
if (!payload.metric || typeof payload.metric !== "object") return null;
|
||||
const metric = payload.metric as Partial<ArchiveMetricArgs>;
|
||||
if (metric.target?.kind !== "skill" || typeof metric.target.id !== "string") return null;
|
||||
if (metric.identityKind !== "user" && metric.identityKind !== "ip") return null;
|
||||
if (typeof metric.identityHash !== "string" || metric.identityHash.length === 0) return null;
|
||||
if (!Number.isFinite(metric.dayStart) || !Number.isFinite(metric.occurredAt)) return null;
|
||||
return payload as ArchiveMetricPayload;
|
||||
}
|
||||
|
||||
async function readBoundedRequestText(request: Request, maxBytes: number) {
|
||||
const contentLength = request.headers.get("content-length");
|
||||
if (contentLength) {
|
||||
const declaredBytes = Number.parseInt(contentLength, 10);
|
||||
if (Number.isFinite(declaredBytes) && declaredBytes > maxBytes) return null;
|
||||
}
|
||||
const reader = request.body?.getReader();
|
||||
if (!reader) return null;
|
||||
const chunks: Uint8Array[] = [];
|
||||
let totalBytes = 0;
|
||||
try {
|
||||
for (;;) {
|
||||
const chunk = await reader.read();
|
||||
if (chunk.done) break;
|
||||
totalBytes += chunk.value.byteLength;
|
||||
if (totalBytes > maxBytes) {
|
||||
await reader.cancel().catch(() => undefined);
|
||||
return null;
|
||||
}
|
||||
chunks.push(chunk.value);
|
||||
}
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
const bytes = new Uint8Array(totalBytes);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
bytes.set(chunk, offset);
|
||||
offset += chunk.byteLength;
|
||||
}
|
||||
return new TextDecoder().decode(bytes);
|
||||
}
|
||||
|
||||
async function getOptionalDownloadUserId(
|
||||
ctx: DownloadCtx,
|
||||
request: Request,
|
||||
|
||||
+7
-1
@@ -2,7 +2,7 @@ import { ApiRoutes, LegacyApiRoutes } from "clawhub-schema";
|
||||
import { httpRouter } from "convex/server";
|
||||
import { agentSkillsHttp } from "./agentSkillsHttp";
|
||||
import { auth } from "./auth";
|
||||
import { downloadZip } from "./downloads";
|
||||
import { downloadZip, recordArchiveDownloadMetric } from "./downloads";
|
||||
import {
|
||||
cliPublishHttp,
|
||||
cliDeviceCodeHttp,
|
||||
@@ -100,6 +100,12 @@ http.route({
|
||||
handler: downloadZip,
|
||||
});
|
||||
|
||||
http.route({
|
||||
path: "/api/internal/archive-download-metric",
|
||||
method: "POST",
|
||||
handler: recordArchiveDownloadMetric,
|
||||
});
|
||||
|
||||
http.route({
|
||||
path: ApiRoutes.search,
|
||||
method: "GET",
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
import { CompactSign, compactVerify, createLocalJWKSet, importPKCS8 } from "jose";
|
||||
|
||||
export const ARCHIVE_MANIFEST_CONTENT_TYPE = "application/vnd.clawhub.skill-archive-manifest+jws";
|
||||
export const ARCHIVE_MANIFEST_AUDIENCE = "clawhub.nitro-skill-archive";
|
||||
export const ARCHIVE_MANIFEST_JWS_TYPE = "clawhub-skill-archive+jws";
|
||||
export const ARCHIVE_METRIC_AUDIENCE = "clawhub.convex-download-metric";
|
||||
export const ARCHIVE_METRIC_JWS_TYPE = "clawhub-download-metric+jws";
|
||||
|
||||
export type ArchiveMetricArgs = {
|
||||
target: { kind: "skill"; id: string };
|
||||
identityKind: "user" | "ip";
|
||||
identityHash: string;
|
||||
dayStart: number;
|
||||
occurredAt?: number;
|
||||
};
|
||||
|
||||
export type SkillArchiveManifest = {
|
||||
schema: "clawhub.skill-archive-manifest.v1";
|
||||
issuer: string;
|
||||
audience: typeof ARCHIVE_MANIFEST_AUDIENCE;
|
||||
issuedAt: number;
|
||||
expiresAt: number;
|
||||
filename: string;
|
||||
meta: {
|
||||
ownerId: string;
|
||||
slug: string;
|
||||
version: string;
|
||||
publishedAt: number;
|
||||
};
|
||||
entries: Array<{ path: string; url: string }>;
|
||||
metricToken?: string;
|
||||
};
|
||||
|
||||
export type ArchiveMetricPayload = {
|
||||
schema: "clawhub.archive-download-metric.v1";
|
||||
issuer: string;
|
||||
audience: typeof ARCHIVE_METRIC_AUDIENCE;
|
||||
issuedAt: number;
|
||||
expiresAt: number;
|
||||
metric: ArchiveMetricArgs;
|
||||
};
|
||||
|
||||
export async function signArchivePayload(
|
||||
payload: SkillArchiveManifest | ArchiveMetricPayload,
|
||||
type: typeof ARCHIVE_MANIFEST_JWS_TYPE | typeof ARCHIVE_METRIC_JWS_TYPE,
|
||||
privateKeyPem = process.env.JWT_PRIVATE_KEY,
|
||||
) {
|
||||
if (!privateKeyPem) throw new Error("JWT_PRIVATE_KEY is required to sign archive capabilities");
|
||||
const privateKey = await importPKCS8(privateKeyPem, "RS256");
|
||||
const bytes = Uint8Array.from(new TextEncoder().encode(JSON.stringify(payload)));
|
||||
return await new CompactSign(bytes)
|
||||
.setProtectedHeader({ alg: "RS256", typ: type })
|
||||
.sign(privateKey);
|
||||
}
|
||||
|
||||
export async function verifyArchivePayloadWithLocalJwks(
|
||||
token: string,
|
||||
expectedType: typeof ARCHIVE_MANIFEST_JWS_TYPE | typeof ARCHIVE_METRIC_JWS_TYPE,
|
||||
jwksJson = process.env.JWKS,
|
||||
): Promise<unknown> {
|
||||
if (!jwksJson) throw new Error("JWKS is required to verify archive capabilities");
|
||||
const jwks = JSON.parse(jwksJson) as Parameters<typeof createLocalJWKSet>[0];
|
||||
const verified = await compactVerify(token, createLocalJWKSet(jwks), {
|
||||
algorithms: ["RS256"],
|
||||
});
|
||||
if (verified.protectedHeader.typ !== expectedType) {
|
||||
throw new Error("Unexpected archive capability type");
|
||||
}
|
||||
return JSON.parse(new TextDecoder().decode(verified.payload)) as unknown;
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
/* @vitest-environment node */
|
||||
|
||||
import { createLocalJWKSet, exportJWK, generateKeyPair, SignJWT } from "jose";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
CLAWHUB_VERCEL_OWNER_ID,
|
||||
CLAWHUB_VERCEL_PROJECT,
|
||||
CLAWHUB_VERCEL_PROJECT_ID,
|
||||
CLAWHUB_VERCEL_TEAM,
|
||||
expectedVercelEnvironmentForConvexSite,
|
||||
verifyClawHubVercelOidcToken,
|
||||
} from "./clawhubVercelOidc";
|
||||
|
||||
describe("ClawHub Vercel OIDC", () => {
|
||||
it("binds each Convex site class to its Vercel environment", () => {
|
||||
expect(
|
||||
expectedVercelEnvironmentForConvexSite(
|
||||
"https://migrated-production.convex.site/api/v1/download",
|
||||
{ CLAWHUB_ENV: "production" },
|
||||
),
|
||||
).toBe("production");
|
||||
expect(
|
||||
expectedVercelEnvironmentForConvexSite(
|
||||
"https://academic-chihuahua-392.convex.site/api/v1/download",
|
||||
{ CLAWHUB_ENV: "test" },
|
||||
),
|
||||
).toBe("preview");
|
||||
expect(
|
||||
expectedVercelEnvironmentForConvexSite(
|
||||
"https://preview-branch-123.convex.site/api/v1/download",
|
||||
{ CLAWHUB_PREVIEW: "1" },
|
||||
),
|
||||
).toBe("preview");
|
||||
expect(expectedVercelEnvironmentForConvexSite("http://127.0.0.1:3211/api/v1/download")).toBe(
|
||||
"development",
|
||||
);
|
||||
expect(
|
||||
expectedVercelEnvironmentForConvexSite("https://attacker.example/api/v1/download", {
|
||||
CLAWHUB_ENV: "production",
|
||||
}),
|
||||
).toBeNull();
|
||||
expect(
|
||||
expectedVercelEnvironmentForConvexSite(
|
||||
"https://unclassified.convex.site/api/v1/download",
|
||||
{},
|
||||
),
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it("accepts only the ClawHub project identity for the expected environment", async () => {
|
||||
const keyPair = await generateKeyPair("RS256", { extractable: true });
|
||||
const publicKey = await exportJWK(keyPair.publicKey);
|
||||
const jwks = createLocalJWKSet({ keys: [{ use: "sig", ...publicKey }] });
|
||||
const token = await new SignJWT({
|
||||
owner_id: CLAWHUB_VERCEL_OWNER_ID,
|
||||
project_id: CLAWHUB_VERCEL_PROJECT_ID,
|
||||
environment: "preview",
|
||||
})
|
||||
.setProtectedHeader({ alg: "RS256" })
|
||||
.setIssuer(`https://oidc.vercel.com/${CLAWHUB_VERCEL_TEAM}`)
|
||||
.setAudience(`https://vercel.com/${CLAWHUB_VERCEL_TEAM}`)
|
||||
.setSubject(
|
||||
`owner:${CLAWHUB_VERCEL_TEAM}:project:${CLAWHUB_VERCEL_PROJECT}:environment:preview`,
|
||||
)
|
||||
.setIssuedAt()
|
||||
.setExpirationTime("5m")
|
||||
.sign(keyPair.privateKey);
|
||||
|
||||
await expect(verifyClawHubVercelOidcToken(token, "preview", jwks)).resolves.toMatchObject({
|
||||
owner_id: CLAWHUB_VERCEL_OWNER_ID,
|
||||
project_id: CLAWHUB_VERCEL_PROJECT_ID,
|
||||
environment: "preview",
|
||||
});
|
||||
await expect(verifyClawHubVercelOidcToken(token, "production", jwks)).rejects.toThrow(
|
||||
"Invalid ClawHub Vercel identity",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,66 @@
|
||||
import { createRemoteJWKSet, jwtVerify, type JWTVerifyGetKey } from "jose";
|
||||
|
||||
export const CLAWHUB_VERCEL_OWNER_ID = "team_pLdjXbfy0XvPRiNmAygTjTSH";
|
||||
export const CLAWHUB_VERCEL_PROJECT_ID = "prj_UVAJPNPYrBwTEkPJwkpEySsge8Mc";
|
||||
export const CLAWHUB_VERCEL_TEAM = "openclaw-foundation";
|
||||
export const CLAWHUB_VERCEL_PROJECT = "clawhub";
|
||||
export const ARCHIVE_REQUEST_IDENTITY_HEADER = "x-clawhub-vercel-oidc-token";
|
||||
|
||||
const VERCEL_OIDC_ISSUER = `https://oidc.vercel.com/${CLAWHUB_VERCEL_TEAM}`;
|
||||
const VERCEL_OIDC_AUDIENCE = `https://vercel.com/${CLAWHUB_VERCEL_TEAM}`;
|
||||
const VERCEL_OIDC_JWKS = createRemoteJWKSet(new URL(`${VERCEL_OIDC_ISSUER}/.well-known/jwks`));
|
||||
|
||||
type ClawHubArchiveRuntimeEnvironment = {
|
||||
CLAWHUB_ENV?: string;
|
||||
CLAWHUB_PREVIEW?: string;
|
||||
};
|
||||
|
||||
export type ClawHubVercelEnvironment = "development" | "preview" | "production";
|
||||
|
||||
export function expectedVercelEnvironmentForConvexSite(
|
||||
requestUrl: string,
|
||||
env: ClawHubArchiveRuntimeEnvironment = process.env,
|
||||
): ClawHubVercelEnvironment | null {
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(requestUrl);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (url.protocol === "http:" && ["localhost", "127.0.0.1", "[::1]"].includes(url.hostname)) {
|
||||
return "development";
|
||||
}
|
||||
if (url.protocol !== "https:" || !url.hostname.endsWith(".convex.site")) return null;
|
||||
|
||||
const runtimeEnvironment = env.CLAWHUB_ENV?.trim();
|
||||
if (env.CLAWHUB_PREVIEW === "1") {
|
||||
return runtimeEnvironment && runtimeEnvironment !== "preview" ? null : "preview";
|
||||
}
|
||||
if (runtimeEnvironment === "production") return "production";
|
||||
// ClawHub Test is an app-level label on a Vercel preview-target deployment.
|
||||
if (runtimeEnvironment === "test") return "preview";
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function verifyClawHubVercelOidcToken(
|
||||
token: string,
|
||||
expectedEnvironment: ClawHubVercelEnvironment,
|
||||
jwks: JWTVerifyGetKey = VERCEL_OIDC_JWKS,
|
||||
) {
|
||||
const verified = await jwtVerify(token, jwks, {
|
||||
algorithms: ["RS256"],
|
||||
issuer: VERCEL_OIDC_ISSUER,
|
||||
audience: VERCEL_OIDC_AUDIENCE,
|
||||
});
|
||||
const payload = verified.payload;
|
||||
if (
|
||||
payload.owner_id !== CLAWHUB_VERCEL_OWNER_ID ||
|
||||
payload.project_id !== CLAWHUB_VERCEL_PROJECT_ID ||
|
||||
payload.environment !== expectedEnvironment ||
|
||||
payload.sub !==
|
||||
`owner:${CLAWHUB_VERCEL_TEAM}:project:${CLAWHUB_VERCEL_PROJECT}:environment:${expectedEnvironment}`
|
||||
) {
|
||||
throw new Error("Invalid ClawHub Vercel identity");
|
||||
}
|
||||
return payload;
|
||||
}
|
||||
+157
-1
@@ -1,11 +1,16 @@
|
||||
import { findClawPackagePathHierarchyCollision, isSafeClawPackagePath } from "clawhub-schema";
|
||||
import { zipSync } from "fflate";
|
||||
import { Zip, ZipDeflate, zipSync } from "fflate";
|
||||
|
||||
type ZipEntry = {
|
||||
path: string;
|
||||
bytes: Uint8Array;
|
||||
};
|
||||
|
||||
export type AsyncZipEntry = {
|
||||
path: string;
|
||||
openStream: () => Promise<ReadableStream<Uint8Array> | null>;
|
||||
};
|
||||
|
||||
export type SkillZipMeta = {
|
||||
ownerId: string;
|
||||
slug: string;
|
||||
@@ -16,6 +21,9 @@ export type SkillZipMeta = {
|
||||
type ZipInput = Record<string, Uint8Array | [Uint8Array, { mtime?: Date }]>;
|
||||
|
||||
const FIXED_ZIP_DATE = new Date(1980, 0, 1, 0, 0, 0);
|
||||
// Storage response chunk boundaries vary with transport backpressure; normalize
|
||||
// them so identical files still produce byte-for-byte identical archives.
|
||||
const ZIP_INPUT_CHUNK_BYTES = 64 * 1024;
|
||||
|
||||
// ==================== Zip Slip Protection ====================
|
||||
|
||||
@@ -68,6 +76,154 @@ export function buildDeterministicZip(entries: ZipEntry[], meta?: SkillZipMeta)
|
||||
return Uint8Array.from(zipSync(zipData, { level: 6 }));
|
||||
}
|
||||
|
||||
export function buildDeterministicZipStream(entries: AsyncZipEntry[], meta?: SkillZipMeta) {
|
||||
const orderedEntries = orderZipEntries(entries, meta);
|
||||
const output: Uint8Array[] = [];
|
||||
let entryIndex = 0;
|
||||
let current:
|
||||
| {
|
||||
reader: ReadableStreamDefaultReader<Uint8Array>;
|
||||
zipEntry: ZipDeflate;
|
||||
inputBuffer: Uint8Array;
|
||||
inputBufferLength: number;
|
||||
sourceChunk?: Uint8Array;
|
||||
sourceOffset: number;
|
||||
sourceDone: boolean;
|
||||
}
|
||||
| undefined;
|
||||
let archiveEnded = false;
|
||||
let archiveDone = false;
|
||||
let archiveError: unknown;
|
||||
|
||||
const archive = new Zip((error, chunk, final) => {
|
||||
if (error) archiveError = error;
|
||||
if (chunk?.length) output.push(chunk);
|
||||
if (final) archiveDone = true;
|
||||
});
|
||||
|
||||
const advance = async () => {
|
||||
if (archiveError) throw archiveError;
|
||||
|
||||
if (current) {
|
||||
while (current.inputBufferLength < ZIP_INPUT_CHUNK_BYTES && !current.sourceDone) {
|
||||
if (!current.sourceChunk || current.sourceOffset === current.sourceChunk.byteLength) {
|
||||
const next = await current.reader.read();
|
||||
if (next.done) {
|
||||
current.sourceDone = true;
|
||||
break;
|
||||
}
|
||||
current.sourceChunk = next.value;
|
||||
current.sourceOffset = 0;
|
||||
if (next.value.byteLength === 0) continue;
|
||||
}
|
||||
|
||||
const sourceBytesRemaining = current.sourceChunk.byteLength - current.sourceOffset;
|
||||
const outputBytesRemaining = ZIP_INPUT_CHUNK_BYTES - current.inputBufferLength;
|
||||
const bytesToCopy = Math.min(sourceBytesRemaining, outputBytesRemaining);
|
||||
current.inputBuffer.set(
|
||||
current.sourceChunk.subarray(current.sourceOffset, current.sourceOffset + bytesToCopy),
|
||||
current.inputBufferLength,
|
||||
);
|
||||
current.sourceOffset += bytesToCopy;
|
||||
current.inputBufferLength += bytesToCopy;
|
||||
}
|
||||
|
||||
if (current.inputBufferLength > 0) {
|
||||
current.zipEntry.push(
|
||||
current.inputBuffer.subarray(0, current.inputBufferLength),
|
||||
current.sourceDone,
|
||||
);
|
||||
current.inputBuffer = new Uint8Array(ZIP_INPUT_CHUNK_BYTES);
|
||||
current.inputBufferLength = 0;
|
||||
} else if (current.sourceDone) {
|
||||
current.zipEntry.push(new Uint8Array(0), true);
|
||||
}
|
||||
if (current.sourceDone) {
|
||||
current.reader.releaseLock();
|
||||
current = undefined;
|
||||
}
|
||||
if (archiveError) throw archiveError;
|
||||
return;
|
||||
}
|
||||
|
||||
while (entryIndex < orderedEntries.length) {
|
||||
const entry = orderedEntries[entryIndex++];
|
||||
const stream = await entry.openStream();
|
||||
// A storage reference can become stale after the version document was read.
|
||||
// Do not commit a ZIP header until the Blob is known to still exist.
|
||||
if (!stream) continue;
|
||||
|
||||
const zipEntry = new ZipDeflate(entry.path, { level: 6 });
|
||||
zipEntry.mtime = FIXED_ZIP_DATE;
|
||||
archive.add(zipEntry);
|
||||
current = {
|
||||
reader: stream.getReader(),
|
||||
zipEntry,
|
||||
inputBuffer: new Uint8Array(ZIP_INPUT_CHUNK_BYTES),
|
||||
inputBufferLength: 0,
|
||||
sourceOffset: 0,
|
||||
sourceDone: false,
|
||||
};
|
||||
return;
|
||||
}
|
||||
|
||||
if (!archiveEnded) {
|
||||
archiveEnded = true;
|
||||
archive.end();
|
||||
if (archiveError) throw archiveError;
|
||||
}
|
||||
};
|
||||
|
||||
return new ReadableStream<Uint8Array>(
|
||||
{
|
||||
async pull(controller) {
|
||||
try {
|
||||
for (;;) {
|
||||
if (output.length > 0 || archiveDone) break;
|
||||
await advance();
|
||||
}
|
||||
const chunk = output.shift();
|
||||
if (chunk) controller.enqueue(chunk);
|
||||
else controller.close();
|
||||
} catch (error) {
|
||||
archive.terminate();
|
||||
await current?.reader.cancel(error);
|
||||
controller.error(error);
|
||||
}
|
||||
},
|
||||
async cancel(reason) {
|
||||
archive.terminate();
|
||||
await current?.reader.cancel(reason);
|
||||
},
|
||||
},
|
||||
{ highWaterMark: 0 },
|
||||
);
|
||||
}
|
||||
|
||||
function orderZipEntries(entries: AsyncZipEntry[], meta?: SkillZipMeta) {
|
||||
const sorted = [...entries].sort((a, b) => a.path.localeCompare(b.path));
|
||||
const byPath = new Map(sorted.map((entry) => [entry.path, entry]));
|
||||
const zipDataOrder: Record<string, true> = {};
|
||||
for (const entry of sorted) zipDataOrder[entry.path] = true;
|
||||
|
||||
if (meta) {
|
||||
const metaBytes = new TextEncoder().encode(JSON.stringify(buildSkillMeta(meta), null, 2));
|
||||
byPath.set("_meta.json", {
|
||||
path: "_meta.json",
|
||||
openStream: async () =>
|
||||
new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(metaBytes);
|
||||
controller.close();
|
||||
},
|
||||
}),
|
||||
});
|
||||
zipDataOrder["_meta.json"] = true;
|
||||
}
|
||||
|
||||
return Object.keys(zipDataOrder).map((path) => byPath.get(path)!);
|
||||
}
|
||||
|
||||
export function buildDeterministicPackageZip(entries: ZipEntry[]) {
|
||||
const unsafeEntry = entries.find((entry) => !isSafeClawPackagePath(entry.path));
|
||||
if (unsafeEntry) {
|
||||
|
||||
@@ -1315,6 +1315,9 @@ Notes:
|
||||
|
||||
- If neither `version` nor `tag` is provided, the latest version is used.
|
||||
- Soft-deleted versions return `410`.
|
||||
- Hosted skill versions return a streamed deterministic ZIP with
|
||||
`Content-Disposition: attachment; filename="<slug>-<version>.zip"`. ClawHub
|
||||
applies moderation, rate limiting, and download metering before streaming.
|
||||
- GitHub-backed skill handoffs do not proxy or mirror bytes. The JSON response
|
||||
includes `sourceRef: "public-github"`, `repo`, `commit`, `path`, `contentHash`,
|
||||
and `archiveUrl`; scan/current state is a gate and is not included as success
|
||||
|
||||
@@ -129,6 +129,7 @@
|
||||
"fflate": "0.8.3",
|
||||
"h3": "2.0.1-rc.25",
|
||||
"ignore": "7.0.6",
|
||||
"jose": "6.2.3",
|
||||
"lucide-react": "1.28.0",
|
||||
"mermaid": "^11.16.1",
|
||||
"mime": "4.1.0",
|
||||
|
||||
+14
-14
@@ -1682,25 +1682,25 @@
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Hosted ZIP bytes or a GitHub-backed source handoff descriptor.",
|
||||
"content": {
|
||||
"application/zip": {
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"format": "binary"
|
||||
}
|
||||
},
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/GitHubSkillDownloadHandoff"
|
||||
}
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Hosted ZIP bytes or a GitHub-backed source handoff descriptor.",
|
||||
"content": {
|
||||
"application/zip": {
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"format": "binary"
|
||||
}
|
||||
},
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/GitHubSkillDownloadHandoff"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/packages": {
|
||||
"get": {
|
||||
|
||||
@@ -1,16 +1,37 @@
|
||||
/* @vitest-environment node */
|
||||
|
||||
import { unzipSync } from "fflate";
|
||||
import { mockEvent } from "h3";
|
||||
import { createLocalJWKSet, exportJWK, exportPKCS8, generateKeyPair } from "jose";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
ARCHIVE_MANIFEST_AUDIENCE,
|
||||
ARCHIVE_MANIFEST_CONTENT_TYPE,
|
||||
ARCHIVE_MANIFEST_JWS_TYPE,
|
||||
signArchivePayload,
|
||||
type SkillArchiveManifest,
|
||||
} from "../convex/lib/archiveManifest";
|
||||
import {
|
||||
buildConvexProxyTarget,
|
||||
isConvexProxyMethodAllowed,
|
||||
proxyConvexRequest,
|
||||
resolveConvexStorageOrigin,
|
||||
resolveConvexProxyEnv,
|
||||
verifySignedArchiveManifest,
|
||||
} from "./convexProxy";
|
||||
|
||||
const TEST_ARCHIVE_DEPENDENCIES = {
|
||||
getArchiveRequestToken: async () => "verified-vercel-oidc",
|
||||
verifyArchiveManifest: async (token: string) => ({
|
||||
...(JSON.parse(token) as Record<string, unknown>),
|
||||
issuer: "https://preview-branch-123.convex.site",
|
||||
audience: ARCHIVE_MANIFEST_AUDIENCE,
|
||||
}),
|
||||
};
|
||||
|
||||
describe("Convex HTTP proxy", () => {
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
@@ -69,6 +90,22 @@ describe("Convex HTTP proxy", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("accepts storage only from the Convex deployment paired to the selected site", () => {
|
||||
expect(
|
||||
resolveConvexStorageOrigin("https://preview-branch-123.convex.site/api/v1/download", {
|
||||
VITE_CONVEX_SITE_URL: "https://preview-branch-123.convex.site",
|
||||
VITE_CONVEX_URL: "https://preview-branch-123.convex.cloud",
|
||||
CONVEX_URL: "https://wry-manatee-359.convex.cloud",
|
||||
}),
|
||||
).toBe("https://preview-branch-123.convex.cloud");
|
||||
expect(
|
||||
resolveConvexStorageOrigin("https://preview-branch-123.convex.site/api/v1/download", {
|
||||
VITE_CONVEX_SITE_URL: "https://preview-branch-123.convex.site",
|
||||
VITE_CONVEX_URL: "https://wry-manatee-359.convex.cloud",
|
||||
}),
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it("proxies reads and exposes the non-secret preview deployment name for proof", async () => {
|
||||
const fetchMock = vi.fn(async () => {
|
||||
return new Response(JSON.stringify({ ok: true }), {
|
||||
@@ -92,6 +129,489 @@ describe("Convex HTTP proxy", () => {
|
||||
expect(response.headers.get("X-ClawHub-Preview-Backend")).toBe("preview-branch-123");
|
||||
});
|
||||
|
||||
it("streams hosted downloads from a Convex manifest with the final attachment filename", async () => {
|
||||
const storedBody = new TextEncoder().encode("# streamed skill\n");
|
||||
const fetchMock = vi.fn(async (input: string | URL | Request, _init?: RequestInit) => {
|
||||
const url = input.toString();
|
||||
if (url.startsWith("https://preview-branch-123.convex.site/api/v1/download")) {
|
||||
return Response.json(
|
||||
{
|
||||
schema: "clawhub.skill-archive-manifest.v1",
|
||||
issuedAt: 1_000,
|
||||
expiresAt: 31_000,
|
||||
filename: "demo-1.0.0+build.zip",
|
||||
meta: {
|
||||
ownerId: "users:1",
|
||||
slug: "demo",
|
||||
version: "1.0.0+build",
|
||||
publishedAt: 3,
|
||||
},
|
||||
entries: [
|
||||
{
|
||||
path: "SKILL.md",
|
||||
url: "https://preview-branch-123.convex.cloud/api/storage/storage-1",
|
||||
},
|
||||
{
|
||||
path: "stale.txt",
|
||||
url: "https://preview-branch-123.convex.cloud/api/storage/storage-missing",
|
||||
},
|
||||
],
|
||||
metricToken: "signed-metric-capability",
|
||||
},
|
||||
{
|
||||
headers: {
|
||||
"cache-control": "private, no-store",
|
||||
"content-digest": "sha-256=:manifest-digest:",
|
||||
"content-encoding": "gzip",
|
||||
"content-type": ARCHIVE_MANIFEST_CONTENT_TYPE,
|
||||
etag: '"manifest-etag"',
|
||||
"x-ratelimit-remaining": "49",
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
if (url === "https://preview-branch-123.convex.cloud/api/storage/storage-1") {
|
||||
return new Response(storedBody, { status: 200 });
|
||||
}
|
||||
if (url === "https://preview-branch-123.convex.cloud/api/storage/storage-missing") {
|
||||
return new Response("missing", { status: 404 });
|
||||
}
|
||||
if (url === "https://preview-branch-123.convex.site/api/internal/archive-download-metric") {
|
||||
return new Response(null, { status: 204 });
|
||||
}
|
||||
throw new Error(`Unexpected fetch: ${url}`);
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
vi.spyOn(Date, "now").mockReturnValue(2_000);
|
||||
const event = mockEvent("https://preview.example/api/v1/download?slug=demo", {
|
||||
headers: { "x-clawhub-vercel-oidc-token": "client-forgery" },
|
||||
});
|
||||
|
||||
const response = await proxyConvexRequest(
|
||||
event,
|
||||
{
|
||||
VERCEL_ENV: "preview",
|
||||
VITE_CONVEX_SITE_URL: "https://preview-branch-123.convex.site",
|
||||
VITE_CONVEX_URL: "https://preview-branch-123.convex.cloud",
|
||||
},
|
||||
TEST_ARCHIVE_DEPENDENCIES,
|
||||
);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get("Content-Type")).toBe("application/zip");
|
||||
expect(response.headers.get("Content-Disposition")).toBe(
|
||||
'attachment; filename="demo-1.0.0+build.zip"',
|
||||
);
|
||||
expect(response.headers.get("Cache-Control")).toBe("private, no-store");
|
||||
expect(response.headers.get("Content-Digest")).toBeNull();
|
||||
expect(response.headers.get("Content-Encoding")).toBeNull();
|
||||
expect(response.headers.get("ETag")).toBeNull();
|
||||
expect(response.headers.get("X-RateLimit-Remaining")).toBe("49");
|
||||
expect(response.headers.get("X-ClawHub-Preview-Backend")).toBe("preview-branch-123");
|
||||
const archive = unzipSync(new Uint8Array(await response.arrayBuffer()));
|
||||
expect(archive["SKILL.md"]).toEqual(storedBody);
|
||||
expect(Object.keys(archive).sort()).toEqual(["SKILL.md", "_meta.json"]);
|
||||
expect(fetchMock.mock.calls[0]?.[0]).toBe(
|
||||
"https://preview-branch-123.convex.site/api/v1/download?slug=demo",
|
||||
);
|
||||
expect(
|
||||
new Headers(fetchMock.mock.calls[0]?.[1]?.headers).get("x-clawhub-archive-manifest"),
|
||||
).toBe("v1");
|
||||
expect(
|
||||
new Headers(fetchMock.mock.calls[0]?.[1]?.headers).get("x-clawhub-vercel-oidc-token"),
|
||||
).toBe("verified-vercel-oidc");
|
||||
const metricCall = fetchMock.mock.calls.find(
|
||||
([input]) =>
|
||||
input.toString() ===
|
||||
"https://preview-branch-123.convex.site/api/internal/archive-download-metric",
|
||||
);
|
||||
expect(metricCall?.[1]).toMatchObject({
|
||||
method: "POST",
|
||||
body: "signed-metric-capability",
|
||||
});
|
||||
const fetchedUrls = fetchMock.mock.calls.map(([input]) => input.toString());
|
||||
expect(
|
||||
fetchedUrls.indexOf(
|
||||
"https://preview-branch-123.convex.site/api/internal/archive-download-metric",
|
||||
),
|
||||
).toBeGreaterThan(
|
||||
fetchedUrls.indexOf("https://preview-branch-123.convex.cloud/api/storage/storage-1"),
|
||||
);
|
||||
});
|
||||
|
||||
it("does not record a download when every source Blob has vanished", async () => {
|
||||
const fetchMock = vi.fn(async (input: string | URL | Request) => {
|
||||
const url = input.toString();
|
||||
if (url.startsWith("https://preview-branch-123.convex.site/api/v1/download")) {
|
||||
return Response.json(
|
||||
{
|
||||
schema: "clawhub.skill-archive-manifest.v1",
|
||||
issuedAt: 1_000,
|
||||
expiresAt: 31_000,
|
||||
filename: "demo-1.0.0.zip",
|
||||
meta: {
|
||||
ownerId: "users:1",
|
||||
slug: "demo",
|
||||
version: "1.0.0",
|
||||
publishedAt: 3,
|
||||
},
|
||||
entries: [
|
||||
{
|
||||
path: "stale.txt",
|
||||
url: "https://preview-branch-123.convex.cloud/api/storage/storage-missing",
|
||||
},
|
||||
],
|
||||
metricToken: "signed-metric-capability",
|
||||
},
|
||||
{ headers: { "content-type": ARCHIVE_MANIFEST_CONTENT_TYPE } },
|
||||
);
|
||||
}
|
||||
if (url === "https://preview-branch-123.convex.cloud/api/storage/storage-missing") {
|
||||
return new Response("missing", { status: 404 });
|
||||
}
|
||||
throw new Error(`Unexpected fetch: ${url}`);
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
vi.spyOn(Date, "now").mockReturnValue(2_000);
|
||||
|
||||
const response = await proxyConvexRequest(
|
||||
mockEvent("https://preview.example/api/v1/download?slug=demo"),
|
||||
{
|
||||
VERCEL_ENV: "preview",
|
||||
VITE_CONVEX_SITE_URL: "https://preview-branch-123.convex.site",
|
||||
VITE_CONVEX_URL: "https://preview-branch-123.convex.cloud",
|
||||
},
|
||||
TEST_ARCHIVE_DEPENDENCIES,
|
||||
);
|
||||
const archive = unzipSync(new Uint8Array(await response.arrayBuffer()));
|
||||
|
||||
expect(Object.keys(archive)).toEqual(["_meta.json"]);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("rejects an unsigned archive manifest from the paired Convex origin", async () => {
|
||||
const fetchMock = vi.fn(async (input: string | URL | Request) => {
|
||||
const url = input.toString();
|
||||
if (url.startsWith("https://preview-branch-123.convex.site/api/v1/download")) {
|
||||
return Response.json(
|
||||
{
|
||||
schema: "clawhub.skill-archive-manifest.v1",
|
||||
issuedAt: 1_000,
|
||||
expiresAt: 31_000,
|
||||
filename: "demo-1.0.0.zip",
|
||||
meta: {
|
||||
ownerId: "users:1",
|
||||
slug: "demo",
|
||||
version: "1.0.0",
|
||||
publishedAt: 3,
|
||||
},
|
||||
entries: [
|
||||
{
|
||||
path: "SKILL.md",
|
||||
url: "https://preview-branch-123.convex.cloud/api/storage/storage-1",
|
||||
},
|
||||
],
|
||||
},
|
||||
{ headers: { "content-type": "application/vnd.clawhub.skill-archive-manifest+json" } },
|
||||
);
|
||||
}
|
||||
if (url === "https://preview-branch-123.convex.cloud/api/storage/storage-1") {
|
||||
return new Response("should not be fetched");
|
||||
}
|
||||
throw new Error(`Unexpected fetch: ${url}`);
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
vi.spyOn(Date, "now").mockReturnValue(2_000);
|
||||
|
||||
const response = await proxyConvexRequest(
|
||||
mockEvent("https://preview.example/api/v1/download?slug=demo"),
|
||||
{
|
||||
VERCEL_ENV: "preview",
|
||||
VITE_CONVEX_SITE_URL: "https://preview-branch-123.convex.site",
|
||||
VITE_CONVEX_URL: "https://preview-branch-123.convex.cloud",
|
||||
},
|
||||
TEST_ARCHIVE_DEPENDENCIES,
|
||||
);
|
||||
|
||||
expect(response.status).toBe(502);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("verifies the paired Convex signature and rejects a modified manifest", async () => {
|
||||
const keyPair = await generateKeyPair("RS256", { extractable: true });
|
||||
const privateKey = await exportPKCS8(keyPair.privateKey);
|
||||
const publicKey = await exportJWK(keyPair.publicKey);
|
||||
const manifest: SkillArchiveManifest = {
|
||||
schema: "clawhub.skill-archive-manifest.v1",
|
||||
issuer: "https://preview-branch-123.convex.site",
|
||||
audience: ARCHIVE_MANIFEST_AUDIENCE,
|
||||
issuedAt: 1_000,
|
||||
expiresAt: 31_000,
|
||||
filename: "demo-1.0.0.zip",
|
||||
meta: {
|
||||
ownerId: "users:1",
|
||||
slug: "demo",
|
||||
version: "1.0.0",
|
||||
publishedAt: 3,
|
||||
},
|
||||
entries: [
|
||||
{
|
||||
path: "SKILL.md",
|
||||
url: "https://preview-branch-123.convex.cloud/api/storage/storage-1",
|
||||
},
|
||||
],
|
||||
};
|
||||
const token = await signArchivePayload(manifest, ARCHIVE_MANIFEST_JWS_TYPE, privateKey);
|
||||
const localJwks = createLocalJWKSet({ keys: [{ use: "sig", ...publicKey }] });
|
||||
|
||||
await expect(
|
||||
verifySignedArchiveManifest(
|
||||
token,
|
||||
"https://preview-branch-123.convex.site/api/v1/download",
|
||||
localJwks,
|
||||
),
|
||||
).resolves.toEqual(manifest);
|
||||
const [header, payload, signature] = token.split(".");
|
||||
const modifiedPayload = `${payload!.slice(0, -1)}${payload!.endsWith("A") ? "B" : "A"}`;
|
||||
await expect(
|
||||
verifySignedArchiveManifest(
|
||||
`${header}.${modifiedPayload}.${signature}`,
|
||||
"https://preview-branch-123.convex.site/api/v1/download",
|
||||
localJwks,
|
||||
),
|
||||
).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("produces identical archive bytes when storage streams use different chunk boundaries", async () => {
|
||||
// Cross many normalization boundaries without making full-suite coverage
|
||||
// instrumentation dominate the repository's 15-second test timeout.
|
||||
const storedBody = new Uint8Array(512 * 1024);
|
||||
let randomState = 0x3451cafe;
|
||||
for (let index = 0; index < storedBody.length; index += 1) {
|
||||
randomState ^= randomState << 13;
|
||||
randomState ^= randomState >>> 17;
|
||||
randomState ^= randomState << 5;
|
||||
storedBody[index] = randomState;
|
||||
}
|
||||
|
||||
let storageRequest = 0;
|
||||
const fetchMock = vi.fn(async (input: string | URL | Request) => {
|
||||
const url = input.toString();
|
||||
if (url.startsWith("https://preview-branch-123.convex.site/api/v1/download")) {
|
||||
return Response.json(
|
||||
{
|
||||
schema: "clawhub.skill-archive-manifest.v1",
|
||||
issuedAt: 1_000,
|
||||
expiresAt: 31_000,
|
||||
filename: "demo-1.0.0.zip",
|
||||
meta: {
|
||||
ownerId: "users:1",
|
||||
slug: "demo",
|
||||
version: "1.0.0",
|
||||
publishedAt: 3,
|
||||
},
|
||||
entries: [
|
||||
{
|
||||
path: "SKILL.md",
|
||||
url: "https://preview-branch-123.convex.cloud/api/storage/storage-1",
|
||||
},
|
||||
],
|
||||
},
|
||||
{ headers: { "content-type": ARCHIVE_MANIFEST_CONTENT_TYPE } },
|
||||
);
|
||||
}
|
||||
if (url === "https://preview-branch-123.convex.cloud/api/storage/storage-1") {
|
||||
const chunkSize = storageRequest++ === 0 ? 16_381 : 65_521;
|
||||
let offset = 0;
|
||||
return new Response(
|
||||
new ReadableStream<Uint8Array>({
|
||||
pull(controller) {
|
||||
if (offset >= storedBody.length) {
|
||||
controller.close();
|
||||
return;
|
||||
}
|
||||
const nextOffset = Math.min(offset + chunkSize, storedBody.length);
|
||||
controller.enqueue(storedBody.slice(offset, nextOffset));
|
||||
offset = nextOffset;
|
||||
},
|
||||
}),
|
||||
);
|
||||
}
|
||||
throw new Error(`Unexpected fetch: ${url}`);
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
vi.spyOn(Date, "now").mockReturnValue(2_000);
|
||||
const env = {
|
||||
VERCEL_ENV: "preview",
|
||||
VITE_CONVEX_SITE_URL: "https://preview-branch-123.convex.site",
|
||||
VITE_CONVEX_URL: "https://preview-branch-123.convex.cloud",
|
||||
};
|
||||
|
||||
const firstResponse = await proxyConvexRequest(
|
||||
mockEvent("https://preview.example/api/v1/download?slug=demo"),
|
||||
env,
|
||||
TEST_ARCHIVE_DEPENDENCIES,
|
||||
);
|
||||
const firstArchive = new Uint8Array(await firstResponse.arrayBuffer());
|
||||
const secondResponse = await proxyConvexRequest(
|
||||
mockEvent("https://preview.example/api/v1/download?slug=demo"),
|
||||
env,
|
||||
TEST_ARCHIVE_DEPENDENCIES,
|
||||
);
|
||||
const secondArchive = new Uint8Array(await secondResponse.arrayBuffer());
|
||||
|
||||
expect(secondArchive.byteLength).toBe(firstArchive.byteLength);
|
||||
expect(secondArchive.every((byte, index) => byte === firstArchive[index])).toBe(true);
|
||||
expect(unzipSync(secondArchive)["SKILL.md"]).toEqual(storedBody);
|
||||
});
|
||||
|
||||
it("rejects archive manifests that point outside the paired Convex storage origin", async () => {
|
||||
const fetchMock = vi.fn(async (input: string | URL | Request) => {
|
||||
const url = input.toString();
|
||||
if (url.startsWith("https://preview-branch-123.convex.site/api/v1/download")) {
|
||||
return Response.json(
|
||||
{
|
||||
schema: "clawhub.skill-archive-manifest.v1",
|
||||
issuedAt: 1_000,
|
||||
expiresAt: 31_000,
|
||||
filename: "demo-1.0.0.zip",
|
||||
meta: {
|
||||
ownerId: "users:1",
|
||||
slug: "demo",
|
||||
version: "1.0.0",
|
||||
publishedAt: 3,
|
||||
},
|
||||
entries: [{ path: "SKILL.md", url: "https://attacker.example/private" }],
|
||||
},
|
||||
{ headers: { "content-type": ARCHIVE_MANIFEST_CONTENT_TYPE } },
|
||||
);
|
||||
}
|
||||
throw new Error(`Security boundary crossed: ${url}`);
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
vi.spyOn(Date, "now").mockReturnValue(2_000);
|
||||
const event = mockEvent("https://preview.example/api/v1/download?slug=demo");
|
||||
|
||||
const response = await proxyConvexRequest(
|
||||
event,
|
||||
{
|
||||
VERCEL_ENV: "preview",
|
||||
VITE_CONVEX_SITE_URL: "https://preview-branch-123.convex.site",
|
||||
VITE_CONVEX_URL: "https://preview-branch-123.convex.cloud",
|
||||
},
|
||||
TEST_ARCHIVE_DEPENDENCIES,
|
||||
);
|
||||
|
||||
expect(response.status).toBe(502);
|
||||
expect(await response.text()).toBe("Invalid or expired archive manifest");
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("rejects an expired archive manifest before fetching any stored file", async () => {
|
||||
const fetchMock = vi.fn(async () =>
|
||||
Response.json(
|
||||
{
|
||||
schema: "clawhub.skill-archive-manifest.v1",
|
||||
issuedAt: 1_000,
|
||||
expiresAt: 2_000,
|
||||
filename: "demo-1.0.0.zip",
|
||||
meta: {
|
||||
ownerId: "users:1",
|
||||
slug: "demo",
|
||||
version: "1.0.0",
|
||||
publishedAt: 3,
|
||||
},
|
||||
entries: [
|
||||
{
|
||||
path: "SKILL.md",
|
||||
url: "https://preview-branch-123.convex.cloud/api/storage/storage-1",
|
||||
},
|
||||
],
|
||||
},
|
||||
{ headers: { "content-type": ARCHIVE_MANIFEST_CONTENT_TYPE } },
|
||||
),
|
||||
);
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
vi.spyOn(Date, "now").mockReturnValue(2_001);
|
||||
|
||||
const response = await proxyConvexRequest(
|
||||
mockEvent("https://preview.example/api/v1/download?slug=demo"),
|
||||
{
|
||||
VERCEL_ENV: "preview",
|
||||
VITE_CONVEX_SITE_URL: "https://preview-branch-123.convex.site",
|
||||
VITE_CONVEX_URL: "https://preview-branch-123.convex.cloud",
|
||||
},
|
||||
TEST_ARCHIVE_DEPENDENCIES,
|
||||
);
|
||||
|
||||
expect(response.status).toBe(502);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not accept a client-supplied manifest replay", async () => {
|
||||
const fetchMock = vi.fn(
|
||||
async (_input: string | URL | Request, _init?: RequestInit) =>
|
||||
new Response("Skill not found", { status: 404 }),
|
||||
);
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
const event = mockEvent("https://preview.example/api/v1/download?slug=demo&manifest=replayed", {
|
||||
headers: { "x-clawhub-archive-manifest": "replayed-v0" },
|
||||
});
|
||||
|
||||
const response = await proxyConvexRequest(
|
||||
event,
|
||||
{
|
||||
VERCEL_ENV: "preview",
|
||||
VITE_CONVEX_SITE_URL: "https://preview-branch-123.convex.site",
|
||||
VITE_CONVEX_URL: "https://preview-branch-123.convex.cloud",
|
||||
},
|
||||
TEST_ARCHIVE_DEPENDENCIES,
|
||||
);
|
||||
|
||||
expect(response.status).toBe(404);
|
||||
expect(fetchMock.mock.calls[0]?.[0]).toBe(
|
||||
"https://preview-branch-123.convex.site/api/v1/download?slug=demo&manifest=replayed",
|
||||
);
|
||||
expect(
|
||||
new Headers(fetchMock.mock.calls[0]?.[1]?.headers).get("x-clawhub-archive-manifest"),
|
||||
).toBe("v1");
|
||||
});
|
||||
|
||||
it("rejects an oversized manifest body", async () => {
|
||||
const bodyRead = vi.fn();
|
||||
const fetchMock = vi.fn(
|
||||
async () =>
|
||||
new Response(
|
||||
new ReadableStream({
|
||||
pull(controller) {
|
||||
bodyRead();
|
||||
controller.enqueue(new Uint8Array(1024 * 1024));
|
||||
},
|
||||
}),
|
||||
{
|
||||
headers: {
|
||||
"content-type": ARCHIVE_MANIFEST_CONTENT_TYPE,
|
||||
},
|
||||
},
|
||||
),
|
||||
);
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const response = await proxyConvexRequest(
|
||||
mockEvent("https://preview.example/api/v1/download?slug=demo"),
|
||||
{
|
||||
VERCEL_ENV: "preview",
|
||||
VITE_CONVEX_SITE_URL: "https://preview-branch-123.convex.site",
|
||||
VITE_CONVEX_URL: "https://preview-branch-123.convex.cloud",
|
||||
},
|
||||
TEST_ARCHIVE_DEPENDENCIES,
|
||||
);
|
||||
|
||||
expect(response.status).toBe(502);
|
||||
expect(bodyRead.mock.calls.length).toBeGreaterThanOrEqual(5);
|
||||
expect(bodyRead.mock.calls.length).toBeLessThanOrEqual(6);
|
||||
});
|
||||
|
||||
it("exposes the permanent Test backend name for deployment proof", async () => {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
|
||||
+318
-1
@@ -1,6 +1,44 @@
|
||||
import { getVercelOidcToken } from "@vercel/oidc";
|
||||
import { defineEventHandler, getRequestURL, proxyRequest, type H3Event } from "h3";
|
||||
import { compactVerify, type CompactVerifyGetKey, createRemoteJWKSet } from "jose";
|
||||
import {
|
||||
ARCHIVE_MANIFEST_AUDIENCE,
|
||||
ARCHIVE_MANIFEST_CONTENT_TYPE,
|
||||
ARCHIVE_MANIFEST_JWS_TYPE,
|
||||
type SkillArchiveManifest,
|
||||
} from "../convex/lib/archiveManifest";
|
||||
import {
|
||||
ARCHIVE_REQUEST_IDENTITY_HEADER,
|
||||
CLAWHUB_VERCEL_PROJECT,
|
||||
CLAWHUB_VERCEL_TEAM,
|
||||
} from "../convex/lib/clawhubVercelOidc";
|
||||
import {
|
||||
buildDeterministicZipStream,
|
||||
type SkillZipMeta,
|
||||
validateFilePath,
|
||||
validateSlug,
|
||||
} from "../convex/lib/skillZip";
|
||||
import { convexDeploymentName, resolveConvexSiteUrl } from "../src/lib/convexDeploymentUrl";
|
||||
|
||||
const ARCHIVE_MANIFEST_REQUEST_HEADER = "x-clawhub-archive-manifest";
|
||||
const ARCHIVE_MANIFEST_MAX_AGE_MS = 60_000;
|
||||
const ARCHIVE_MANIFEST_CLOCK_SKEW_MS = 5_000;
|
||||
const MAX_ARCHIVE_MANIFEST_ENTRIES = 8_192;
|
||||
const MAX_ARCHIVE_ENTRY_URL_LENGTH = 4_096;
|
||||
const MAX_ARCHIVE_MANIFEST_BYTES = 4 * 1024 * 1024;
|
||||
const ARCHIVE_FILENAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._+-]{0,499}\.zip$/;
|
||||
const ARCHIVE_REPRESENTATION_HEADERS = [
|
||||
"accept-ranges",
|
||||
"content-digest",
|
||||
"content-encoding",
|
||||
"content-length",
|
||||
"content-md5",
|
||||
"content-range",
|
||||
"digest",
|
||||
"etag",
|
||||
"last-modified",
|
||||
] as const;
|
||||
|
||||
type ProxyEnv = {
|
||||
CONVEX_URL?: string;
|
||||
VERCEL_ENV?: string;
|
||||
@@ -10,6 +48,18 @@ type ProxyEnv = {
|
||||
VITE_CONVEX_URL?: string;
|
||||
};
|
||||
|
||||
type ProxyDependencies = {
|
||||
getArchiveRequestToken: () => Promise<string>;
|
||||
verifyArchiveManifest: (token: string, target: string) => Promise<unknown>;
|
||||
};
|
||||
|
||||
const DEFAULT_PROXY_DEPENDENCIES: ProxyDependencies = {
|
||||
getArchiveRequestToken: () =>
|
||||
getVercelOidcToken({ team: CLAWHUB_VERCEL_TEAM, project: CLAWHUB_VERCEL_PROJECT }),
|
||||
verifyArchiveManifest: verifySignedArchiveManifest,
|
||||
};
|
||||
const archiveJwksByOrigin = new Map<string, ReturnType<typeof createRemoteJWKSet>>();
|
||||
|
||||
const BUNDLED_PROXY_ENV: ProxyEnv = {
|
||||
VITE_CLAWHUB_DEPLOY_ENV: import.meta.env.VITE_CLAWHUB_DEPLOY_ENV,
|
||||
VITE_CONVEX_SITE_URL: import.meta.env.VITE_CONVEX_SITE_URL,
|
||||
@@ -62,6 +112,7 @@ export function buildConvexProxyTarget(pathAndQuery: string, env: ProxyEnv) {
|
||||
export async function proxyConvexRequest(
|
||||
event: H3Event,
|
||||
env: ProxyEnv = resolveConvexProxyEnv(process.env),
|
||||
dependencies: ProxyDependencies = DEFAULT_PROXY_DEPENDENCIES,
|
||||
): Promise<Response> {
|
||||
if (!isConvexProxyMethodAllowed(event.req.method, env)) {
|
||||
return new Response("Disposable previews are read-only.", {
|
||||
@@ -76,7 +127,30 @@ export async function proxyConvexRequest(
|
||||
|
||||
const requestUrl = getRequestURL(event);
|
||||
const target = buildConvexProxyTarget(`${requestUrl.pathname}${requestUrl.search}`, env);
|
||||
const proxied = await proxyRequest(event, target);
|
||||
const isArchiveRequest = isSkillDownloadPath(new URL(target).pathname);
|
||||
let archiveRequestToken: string | undefined;
|
||||
if (isArchiveRequest) {
|
||||
try {
|
||||
archiveRequestToken = await dependencies.getArchiveRequestToken();
|
||||
} catch {
|
||||
return new Response("Archive streaming identity unavailable", {
|
||||
status: 503,
|
||||
headers: { "Cache-Control": "no-store" },
|
||||
});
|
||||
}
|
||||
}
|
||||
const proxied = await proxyRequest(event, target, {
|
||||
...(isArchiveRequest
|
||||
? {
|
||||
fetchOptions: {
|
||||
headers: {
|
||||
[ARCHIVE_MANIFEST_REQUEST_HEADER]: "v1",
|
||||
[ARCHIVE_REQUEST_IDENTITY_HEADER]: archiveRequestToken!,
|
||||
},
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
// H3's HTTPResponse is not guaranteed to share Nitro's bundled class identity.
|
||||
// Normalize it before crossing that boundary or Nitro can stringify the wrapper.
|
||||
const response = new Response(proxied.body, {
|
||||
@@ -84,6 +158,22 @@ export async function proxyConvexRequest(
|
||||
statusText: proxied.statusText,
|
||||
headers: proxied.headers,
|
||||
});
|
||||
const contentType = response.headers.get("content-type")?.split(";", 1)[0]?.trim();
|
||||
if (contentType === ARCHIVE_MANIFEST_CONTENT_TYPE) {
|
||||
return streamSkillArchive(
|
||||
response,
|
||||
env,
|
||||
target,
|
||||
event.req.signal,
|
||||
dependencies.verifyArchiveManifest,
|
||||
);
|
||||
}
|
||||
if (
|
||||
isSkillDownloadPath(new URL(target).pathname) &&
|
||||
contentType?.startsWith("application/vnd.clawhub.skill-archive-manifest")
|
||||
) {
|
||||
return new Response("Invalid or expired archive manifest", { status: 502 });
|
||||
}
|
||||
if (isPreviewFrontend(env) || isTestFrontend(env)) {
|
||||
const deployment = convexDeploymentName(target);
|
||||
if (deployment) {
|
||||
@@ -96,4 +186,231 @@ export async function proxyConvexRequest(
|
||||
return response;
|
||||
}
|
||||
|
||||
function isSkillDownloadPath(pathname: string) {
|
||||
return pathname === "/api/v1/download" || pathname === "/api/download";
|
||||
}
|
||||
|
||||
async function streamSkillArchive(
|
||||
manifestResponse: Response,
|
||||
env: ProxyEnv,
|
||||
target: string,
|
||||
signal: AbortSignal,
|
||||
verifyArchiveManifest: ProxyDependencies["verifyArchiveManifest"],
|
||||
) {
|
||||
let value: unknown;
|
||||
try {
|
||||
const token = await readBoundedArchiveManifest(manifestResponse);
|
||||
if (!token) return new Response("Invalid or expired archive manifest", { status: 502 });
|
||||
value = await verifyArchiveManifest(token, target);
|
||||
} catch {
|
||||
return new Response("Invalid or expired archive manifest", { status: 502 });
|
||||
}
|
||||
const expectedStorageOrigin = resolveConvexStorageOrigin(target, env);
|
||||
const manifest = parseSkillArchiveManifest(
|
||||
value,
|
||||
new URL(target).origin,
|
||||
expectedStorageOrigin,
|
||||
Date.now(),
|
||||
);
|
||||
if (!manifest) {
|
||||
return new Response("Invalid or expired archive manifest", { status: 502 });
|
||||
}
|
||||
|
||||
let metricRecorded = false;
|
||||
const recordMetric = async () => {
|
||||
if (metricRecorded || !manifest.metricToken) return;
|
||||
metricRecorded = true;
|
||||
try {
|
||||
await fetch(new URL("/api/internal/archive-download-metric", target), {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/jose" },
|
||||
body: manifest.metricToken,
|
||||
});
|
||||
} catch {
|
||||
// Download metrics remain best-effort and never interrupt archive bytes.
|
||||
}
|
||||
};
|
||||
|
||||
const stream = buildDeterministicZipStream(
|
||||
manifest.entries.map((entry) => ({
|
||||
path: entry.path,
|
||||
openStream: async () => {
|
||||
const response = await fetch(entry.url, { redirect: "error", signal });
|
||||
if (response.status === 404) return null;
|
||||
if (!response.ok || !response.body) {
|
||||
throw new Error(`Failed to fetch archive entry: ${response.status}`);
|
||||
}
|
||||
await recordMetric();
|
||||
return response.body;
|
||||
},
|
||||
})),
|
||||
manifest.meta,
|
||||
);
|
||||
const headers = new Headers(manifestResponse.headers);
|
||||
for (const name of ARCHIVE_REPRESENTATION_HEADERS) headers.delete(name);
|
||||
headers.set("content-type", "application/zip");
|
||||
headers.set("content-disposition", `attachment; filename="${manifest.filename}"`);
|
||||
const response = new Response(stream, { status: 200, headers });
|
||||
if (isPreviewFrontend(env) || isTestFrontend(env)) {
|
||||
const deployment = convexDeploymentName(target);
|
||||
if (deployment) {
|
||||
response.headers.set(
|
||||
isTestFrontend(env) ? "X-ClawHub-Test-Backend" : "X-ClawHub-Preview-Backend",
|
||||
deployment,
|
||||
);
|
||||
}
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
async function readBoundedArchiveManifest(response: Response) {
|
||||
const contentLength = response.headers.get("content-length");
|
||||
if (contentLength) {
|
||||
const declaredBytes = Number.parseInt(contentLength, 10);
|
||||
if (Number.isFinite(declaredBytes) && declaredBytes > MAX_ARCHIVE_MANIFEST_BYTES) return null;
|
||||
}
|
||||
const reader = response.body?.getReader();
|
||||
if (!reader) return null;
|
||||
const chunks: Uint8Array[] = [];
|
||||
let totalBytes = 0;
|
||||
try {
|
||||
for (;;) {
|
||||
const chunk = await reader.read();
|
||||
if (chunk.done) break;
|
||||
totalBytes += chunk.value.byteLength;
|
||||
if (totalBytes > MAX_ARCHIVE_MANIFEST_BYTES) {
|
||||
await reader.cancel().catch(() => undefined);
|
||||
return null;
|
||||
}
|
||||
chunks.push(chunk.value);
|
||||
}
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
const bytes = new Uint8Array(totalBytes);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
bytes.set(chunk, offset);
|
||||
offset += chunk.byteLength;
|
||||
}
|
||||
return new TextDecoder().decode(bytes);
|
||||
}
|
||||
|
||||
function parseSkillArchiveManifest(
|
||||
value: unknown,
|
||||
expectedIssuer: string,
|
||||
expectedStorageOrigin: string | null,
|
||||
now: number,
|
||||
): SkillArchiveManifest | null {
|
||||
if (!value || typeof value !== "object") return null;
|
||||
const manifest = value as Partial<SkillArchiveManifest>;
|
||||
if (manifest.schema !== "clawhub.skill-archive-manifest.v1") return null;
|
||||
if (
|
||||
manifest.issuer !== expectedIssuer ||
|
||||
manifest.audience !== ARCHIVE_MANIFEST_AUDIENCE ||
|
||||
!expectedStorageOrigin
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
if (!Number.isFinite(manifest.issuedAt) || !Number.isFinite(manifest.expiresAt)) return null;
|
||||
const issuedAt = manifest.issuedAt as number;
|
||||
const expiresAt = manifest.expiresAt as number;
|
||||
if (issuedAt > now + ARCHIVE_MANIFEST_CLOCK_SKEW_MS || expiresAt <= now) return null;
|
||||
if (expiresAt <= issuedAt || expiresAt - issuedAt > ARCHIVE_MANIFEST_MAX_AGE_MS) return null;
|
||||
if (typeof manifest.filename !== "string" || !ARCHIVE_FILENAME_PATTERN.test(manifest.filename)) {
|
||||
return null;
|
||||
}
|
||||
if (!isSkillZipMeta(manifest.meta)) return null;
|
||||
if (!Array.isArray(manifest.entries) || manifest.entries.length > MAX_ARCHIVE_MANIFEST_ENTRIES) {
|
||||
return null;
|
||||
}
|
||||
if (
|
||||
manifest.metricToken !== undefined &&
|
||||
(typeof manifest.metricToken !== "string" || manifest.metricToken.length > 16 * 1024)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const seenPaths = new Set<string>();
|
||||
for (const entry of manifest.entries) {
|
||||
if (!entry || typeof entry !== "object") return null;
|
||||
if (typeof entry.path !== "string" || !validateFilePath(entry.path)) return null;
|
||||
if (seenPaths.has(entry.path)) return null;
|
||||
seenPaths.add(entry.path);
|
||||
if (typeof entry.url !== "string" || entry.url.length > MAX_ARCHIVE_ENTRY_URL_LENGTH) {
|
||||
return null;
|
||||
}
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(entry.url);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (url.username || url.password || url.origin !== expectedStorageOrigin) return null;
|
||||
if (!url.pathname.startsWith("/api/storage/")) return null;
|
||||
}
|
||||
return manifest as SkillArchiveManifest;
|
||||
}
|
||||
|
||||
function isSkillZipMeta(value: unknown): value is SkillZipMeta {
|
||||
if (!value || typeof value !== "object") return false;
|
||||
const meta = value as Partial<SkillZipMeta>;
|
||||
return (
|
||||
typeof meta.ownerId === "string" &&
|
||||
typeof meta.slug === "string" &&
|
||||
validateSlug(meta.slug) &&
|
||||
typeof meta.version === "string" &&
|
||||
meta.version.length > 0 &&
|
||||
meta.version.length <= 200 &&
|
||||
typeof meta.publishedAt === "number" &&
|
||||
Number.isFinite(meta.publishedAt)
|
||||
);
|
||||
}
|
||||
|
||||
export async function verifySignedArchiveManifest(
|
||||
token: string,
|
||||
target: string,
|
||||
jwksOverride?: CompactVerifyGetKey,
|
||||
) {
|
||||
const targetOrigin = new URL(target).origin;
|
||||
let jwks = jwksOverride ?? archiveJwksByOrigin.get(targetOrigin);
|
||||
if (!jwks) {
|
||||
const remoteJwks = createRemoteJWKSet(new URL("/.well-known/jwks.json", targetOrigin));
|
||||
archiveJwksByOrigin.set(targetOrigin, remoteJwks);
|
||||
jwks = remoteJwks;
|
||||
}
|
||||
const verified = await compactVerify(token, jwks, { algorithms: ["RS256"] });
|
||||
if (verified.protectedHeader.typ !== ARCHIVE_MANIFEST_JWS_TYPE) {
|
||||
throw new Error("Unexpected archive manifest signature type");
|
||||
}
|
||||
return JSON.parse(new TextDecoder().decode(verified.payload)) as unknown;
|
||||
}
|
||||
|
||||
export function resolveConvexStorageOrigin(target: string, env: ProxyEnv) {
|
||||
let targetOrigin: string;
|
||||
let selectedSiteOrigin: string;
|
||||
let cloudUrl: URL;
|
||||
try {
|
||||
targetOrigin = new URL(target).origin;
|
||||
selectedSiteOrigin = resolveConvexSiteUrl(env);
|
||||
cloudUrl = new URL(env.VITE_CONVEX_URL ?? env.CONVEX_URL ?? "");
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (targetOrigin !== selectedSiteOrigin) return null;
|
||||
|
||||
const isLocalCloud =
|
||||
cloudUrl.protocol === "http:" &&
|
||||
["localhost", "127.0.0.1", "[::1]"].includes(cloudUrl.hostname);
|
||||
if (isLocalCloud) return cloudUrl.origin;
|
||||
if (cloudUrl.protocol !== "https:" || !cloudUrl.hostname.endsWith(".convex.cloud")) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const targetDeployment = convexDeploymentName(target);
|
||||
const cloudDeployment = cloudUrl.hostname.slice(0, -".convex.cloud".length);
|
||||
if (targetDeployment && targetDeployment !== cloudDeployment) return null;
|
||||
return cloudUrl.origin;
|
||||
}
|
||||
|
||||
export default defineEventHandler((event) => proxyConvexRequest(event));
|
||||
|
||||
+33
-2
@@ -163,7 +163,37 @@ Local fixture data lives in `convex/devSeed.ts` and `fixtures/public-corpus/`.
|
||||
## Download API
|
||||
|
||||
- JSON API for skill metadata + versions.
|
||||
- Download endpoint returns zip of a version (HTTP action).
|
||||
- Convex remains the download control plane: it resolves the version, applies
|
||||
moderation and rate limits, preserves auth-derived metering, and returns the
|
||||
Nitro API owner a bounded, no-store manifest of Convex File Storage URLs.
|
||||
- Nitro streams those source files into the deterministic ZIP with backpressure
|
||||
and owns the public response headers. Archive bytes must not pass through a
|
||||
Convex HTTP action because those responses are capped at 20 MiB.
|
||||
- The manifest is an internal server-to-server capability, not a client token.
|
||||
Nitro overwrites the internal request headers and authenticates to Convex with
|
||||
its Vercel OIDC identity; Convex verifies the Vercel signature plus the exact
|
||||
ClawHub team, project, subject, audience, and target environment before
|
||||
returning any storage URL. The permanent ClawHub Test frontend is a Vercel
|
||||
preview-target deployment, so its OIDC environment and subject use `preview`;
|
||||
the app-level `test` label is not an OIDC trust claim. Convex derives this
|
||||
expected target from its explicit runtime environment markers, not a fixed
|
||||
deployment hostname, and fails closed when a remote runtime is unclassified.
|
||||
Nitro never accepts a
|
||||
client-supplied manifest and
|
||||
accepts Convex's response only as a short-lived RS256 JWS signed by the
|
||||
selected Convex deployment's existing auth key and verified from that
|
||||
deployment's JWKS endpoint. The signed issuer, audience, type, and time bounds
|
||||
must match the request, and source URLs are allowed only on the single
|
||||
build-paired Convex deployment's `/api/storage/` surface. Convex storage URLs
|
||||
are reusable bearer URLs, so they must never appear in the public response.
|
||||
Nitro preserves control-plane headers such as rate-limit state but discards
|
||||
manifest representation metadata before emitting the generated ZIP headers.
|
||||
- Download metering is also a signed, short-lived capability. It contains only
|
||||
the existing pre-hashed identity and metric arguments, stays inside the
|
||||
Convex-to-Nitro boundary, and is returned to Convex only after Nitro opens the
|
||||
first live source Blob. An archive whose source Blobs are all stale must not
|
||||
count. Capability replay remains harmless because the existing
|
||||
target/identity/day metric mutation is idempotent.
|
||||
- Soft-delete versions; downloads remain for non-deleted versions only.
|
||||
|
||||
## UI (SPA)
|
||||
@@ -187,5 +217,6 @@ Local fixture data lives in `convex/devSeed.ts` and `fixtures/public-corpus/`.
|
||||
## Open questions (carry forward)
|
||||
|
||||
- Embeddings provider key + rate limits.
|
||||
- Zip generation memory limits (optimize with streaming if needed).
|
||||
- ZIP generation must remain backpressured; never buffer a whole stored entry
|
||||
or completed archive in either Convex or Nitro.
|
||||
- GitHub App repo sync (phase 2).
|
||||
|
||||
@@ -215,7 +215,7 @@ describe("SkillDetailTabs README links", () => {
|
||||
);
|
||||
|
||||
const href = screen.getByRole("link", { name: "Download version v1.0.0" }).getAttribute("href");
|
||||
const url = new URL(href ?? "");
|
||||
const url = new URL(href ?? "", "https://clawhub.ai");
|
||||
expect(url.pathname).toBe("/api/v1/download");
|
||||
expect(url.searchParams.get("slug")).toBe("api-gateway");
|
||||
expect(url.searchParams.get("ownerHandle")).toBe("clawkit");
|
||||
|
||||
@@ -6,7 +6,6 @@ import { api } from "../../convex/_generated/api";
|
||||
import type { Doc } from "../../convex/_generated/dataModel";
|
||||
import type { Id } from "../../convex/_generated/dataModel";
|
||||
import { getUserFacingConvexError } from "../lib/convexError";
|
||||
import { getRuntimeEnv } from "../lib/runtimeEnv";
|
||||
import { Badge } from "./ui/badge";
|
||||
import { Button } from "./ui/button";
|
||||
import { VersionChangelog } from "./VersionChangelog";
|
||||
@@ -39,7 +38,6 @@ function mergeSkillVersions(...groups: Doc<"skillVersions">[][]) {
|
||||
}
|
||||
|
||||
function buildVersionDownloadHref(
|
||||
convexSiteUrl: string,
|
||||
skillSlug: string,
|
||||
ownerHandle: string | null | undefined,
|
||||
version: string,
|
||||
@@ -48,7 +46,7 @@ function buildVersionDownloadHref(
|
||||
const normalizedOwner = ownerHandle?.trim().replace(/^@+/, "");
|
||||
if (normalizedOwner) params.set("ownerHandle", normalizedOwner);
|
||||
params.set("version", version);
|
||||
return `${convexSiteUrl}/api/v1/download?${params.toString()}`;
|
||||
return `/api/v1/download?${params.toString()}`;
|
||||
}
|
||||
|
||||
export function SkillVersionsPanel({
|
||||
@@ -62,7 +60,6 @@ export function SkillVersionsPanel({
|
||||
ownerHandle,
|
||||
suppressedMessage,
|
||||
}: SkillVersionsPanelProps) {
|
||||
const convexSiteUrl = getRuntimeEnv("VITE_CONVEX_SITE_URL") ?? "https://clawhub.ai";
|
||||
const deleteOwnedVersion = useMutation(api.skills.deleteOwnedVersion);
|
||||
const restoreOwnedVersion = useMutation(api.skills.restoreOwnedVersion);
|
||||
const {
|
||||
@@ -222,12 +219,8 @@ export function SkillVersionsPanel({
|
||||
<>
|
||||
{!nixPlugin && isAvailable ? (
|
||||
<a
|
||||
href={buildVersionDownloadHref(
|
||||
convexSiteUrl,
|
||||
skillSlug,
|
||||
ownerHandle,
|
||||
version.version,
|
||||
)}
|
||||
href={buildVersionDownloadHref(skillSlug, ownerHandle, version.version)}
|
||||
download={`${skillSlug}-${version.version}.zip`}
|
||||
className="skill-version-release-download skill-version-release-download-labeled"
|
||||
aria-label={`Download version v${version.version}`}
|
||||
>
|
||||
|
||||
Reference in New Issue
Block a user