mirror of
https://github.com/openclaw/clawhub.git
synced 2026-08-14 00:47:57 +00:00
fix: require multipart package publishes
Require multipart package publishes so public clients cannot provide trusted file or artifact metadata. Preserve released CLI multipart field names (`files` and `clawpack`), derive stored file and npm-pack artifact metadata from uploaded bytes, and keep large ClawPack tarballs working through ticketed staged uploads. Tests: - bunx vitest run convex/uploads.test.ts --reporter=dot - bunx vitest run convex/httpApi.handlers.test.ts convex/httpApiV1.handlers.test.ts --testNamePattern "package publish|multipart package publish|multipart ClawPack|staged ClawPack|browser session auth|cliUploadUrl|upload tickets" --reporter=dot - bun run --cwd packages/clawhub test:src -- src/cli/commands/packages.test.ts --testNamePattern "ClawPack tarballs over|stages ClawPack|publishes a ClawPack|cleans generated ClawPack" - bun run format:check - bun run lint - bunx tsc -p packages/schema/tsconfig.json --noEmit --pretty false - bunx tsc -p packages/clawhub/tsconfig.json --noEmit --pretty false - bunx tsc --noEmit --pretty false - bun run ci:unit - git diff --check origin/main
This commit is contained in:
@@ -4,13 +4,15 @@ import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
vi.mock("./lib/apiTokenAuth", () => ({
|
||||
getOptionalApiTokenUser: vi.fn(),
|
||||
requireApiTokenUser: vi.fn(),
|
||||
requirePackagePublishAuth: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("./skills", () => ({
|
||||
publishVersionForUser: vi.fn(),
|
||||
}));
|
||||
|
||||
const { getOptionalApiTokenUser, requireApiTokenUser } = await import("./lib/apiTokenAuth");
|
||||
const { getOptionalApiTokenUser, requireApiTokenUser, requirePackagePublishAuth } =
|
||||
await import("./lib/apiTokenAuth");
|
||||
const { publishVersionForUser } = await import("./skills");
|
||||
const { __handlers } = await import("./httpApi");
|
||||
const { hashSkillFiles } = await import("./lib/skills");
|
||||
@@ -23,6 +25,7 @@ describe("httpApi handlers", () => {
|
||||
afterEach(() => {
|
||||
vi.mocked(getOptionalApiTokenUser).mockReset();
|
||||
vi.mocked(requireApiTokenUser).mockReset();
|
||||
vi.mocked(requirePackagePublishAuth).mockReset();
|
||||
vi.mocked(publishVersionForUser).mockReset();
|
||||
});
|
||||
|
||||
@@ -444,18 +447,51 @@ describe("httpApi handlers", () => {
|
||||
});
|
||||
|
||||
it("cliUploadUrlHttp returns uploadUrl", async () => {
|
||||
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: "user1" } as never);
|
||||
const runMutation = vi.fn().mockResolvedValue("https://upload.local");
|
||||
vi.mocked(requirePackagePublishAuth).mockResolvedValueOnce({
|
||||
kind: "user",
|
||||
userId: "user1",
|
||||
} as never);
|
||||
const runMutation = vi.fn().mockResolvedValue({
|
||||
uploadUrl: "https://upload.local",
|
||||
uploadTicket: "packagePublishUploadTickets:1",
|
||||
});
|
||||
const response = await __handlers.cliUploadUrlHandler(
|
||||
makeCtx({ runMutation }),
|
||||
new Request("https://x/api/cli/upload-url", { method: "POST" }),
|
||||
);
|
||||
expect(response.status).toBe(200);
|
||||
expect(await response.json()).toEqual({ uploadUrl: "https://upload.local" });
|
||||
expect(await response.json()).toEqual({
|
||||
uploadUrl: "https://upload.local",
|
||||
uploadTicket: "packagePublishUploadTickets:1",
|
||||
});
|
||||
});
|
||||
|
||||
it("cliUploadUrlHttp accepts package publish tokens", async () => {
|
||||
vi.mocked(requirePackagePublishAuth).mockResolvedValueOnce({
|
||||
kind: "github-actions",
|
||||
publishToken: { _id: "packagePublishTokens:1" },
|
||||
} as never);
|
||||
const runMutation = vi.fn().mockResolvedValue({
|
||||
uploadUrl: "https://upload.local/package",
|
||||
uploadTicket: "packagePublishUploadTickets:2",
|
||||
});
|
||||
const response = await __handlers.cliUploadUrlHandler(
|
||||
makeCtx({ runMutation }),
|
||||
new Request("https://x/api/cli/upload-url", { method: "POST" }),
|
||||
);
|
||||
expect(response.status).toBe(200);
|
||||
expect(await response.json()).toEqual({
|
||||
uploadUrl: "https://upload.local/package",
|
||||
uploadTicket: "packagePublishUploadTickets:2",
|
||||
});
|
||||
expect(runMutation).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({ publishTokenId: "packagePublishTokens:1" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("cliUploadUrlHttp returns 401 when unauthorized", async () => {
|
||||
vi.mocked(requireApiTokenUser).mockRejectedValueOnce(new Error("Unauthorized"));
|
||||
vi.mocked(requirePackagePublishAuth).mockRejectedValueOnce(new Error("Unauthorized"));
|
||||
const response = await __handlers.cliUploadUrlHandler(
|
||||
makeCtx({}),
|
||||
new Request("https://x/api/cli/upload-url", { method: "POST" }),
|
||||
|
||||
+11
-6
@@ -10,7 +10,7 @@ import { api, internal } from "./_generated/api";
|
||||
import type { Id } from "./_generated/dataModel";
|
||||
import type { ActionCtx } from "./_generated/server";
|
||||
import { httpAction } from "./functions";
|
||||
import { requireApiTokenUser } from "./lib/apiTokenAuth";
|
||||
import { requireApiTokenUser, requirePackagePublishAuth } from "./lib/apiTokenAuth";
|
||||
import { corsHeaders, mergeHeaders } from "./lib/httpHeaders";
|
||||
import { applyRateLimit } from "./lib/httpRateLimit";
|
||||
import { parseBooleanQueryParam, resolveBooleanQueryParam } from "./lib/httpUtils";
|
||||
@@ -148,11 +148,16 @@ export const cliWhoamiHttp = httpAction(cliWhoamiHandler);
|
||||
|
||||
async function cliUploadUrlHandler(ctx: ActionCtx, request: Request) {
|
||||
try {
|
||||
const { userId } = await requireApiTokenUser(ctx, request);
|
||||
const uploadUrl = await ctx.runMutation(internal.uploads.generateUploadUrlForUserInternal, {
|
||||
userId,
|
||||
});
|
||||
return json({ uploadUrl });
|
||||
const auth = await requirePackagePublishAuth(ctx, request);
|
||||
const upload =
|
||||
auth.kind === "user"
|
||||
? await ctx.runMutation(internal.uploads.createPackagePublishUploadForUserInternal, {
|
||||
userId: auth.userId,
|
||||
})
|
||||
: await ctx.runMutation(internal.uploads.createPackagePublishUploadForTokenInternal, {
|
||||
publishTokenId: auth.publishToken._id,
|
||||
});
|
||||
return json(upload);
|
||||
} catch (error) {
|
||||
return text(formatAuthFailure(error), 401);
|
||||
}
|
||||
|
||||
@@ -99,8 +99,8 @@ function writeTarString(target: Uint8Array, offset: number, width: number, value
|
||||
target.set(encoded.subarray(0, width), offset);
|
||||
}
|
||||
|
||||
function tarFile(path: string, content: string) {
|
||||
const bytes = new TextEncoder().encode(content);
|
||||
function tarFile(path: string, content: string | Uint8Array) {
|
||||
const bytes = typeof content === "string" ? new TextEncoder().encode(content) : content;
|
||||
const header = new Uint8Array(TAR_BLOCK_SIZE);
|
||||
writeTarString(header, 0, 100, path);
|
||||
writeTarString(header, 100, 8, tarOctal(0o644, 8));
|
||||
@@ -123,7 +123,7 @@ function tarFile(path: string, content: string) {
|
||||
return [header, body];
|
||||
}
|
||||
|
||||
function npmPackFixture(files: Record<string, string>) {
|
||||
function npmPackFixture(files: Record<string, string | Uint8Array>) {
|
||||
const parts: Uint8Array[] = [];
|
||||
for (const [path, content] of Object.entries(files)) {
|
||||
parts.push(...tarFile(path, content));
|
||||
@@ -145,13 +145,35 @@ function bytesToArrayBuffer(bytes: Uint8Array): ArrayBuffer {
|
||||
return copy.buffer;
|
||||
}
|
||||
|
||||
function packagePublishMetadata(overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
name: "demo-plugin",
|
||||
family: "bundle-plugin",
|
||||
version: "1.0.0",
|
||||
changelog: "init",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function packagePublishForm(payload: Record<string, unknown>) {
|
||||
const form = new FormData();
|
||||
form.set("payload", JSON.stringify(payload));
|
||||
return form;
|
||||
}
|
||||
|
||||
function makeCtx(partial: Record<string, unknown>) {
|
||||
const rateLimitStatus =
|
||||
typeof partial.rateLimitStatus === "function"
|
||||
? (partial.rateLimitStatus as (args: RateLimitArgs) => unknown)
|
||||
: null;
|
||||
const partialRunQuery =
|
||||
typeof partial.runQuery === "function"
|
||||
? (partial.runQuery as (query: unknown, args: Record<string, unknown>) => unknown)
|
||||
: null;
|
||||
const runQuery = vi.fn(async (query: unknown, args: Record<string, unknown>) => {
|
||||
if (isRateLimitArgs(args)) return { ...okRate(), limit: args.limit };
|
||||
if (isRateLimitArgs(args)) {
|
||||
return rateLimitStatus?.(args) ?? { ...okRate(), limit: args.limit };
|
||||
}
|
||||
return partialRunQuery ? await partialRunQuery(query, args) : null;
|
||||
});
|
||||
const runMutation =
|
||||
@@ -9495,37 +9517,26 @@ describe("httpApiV1 handlers", () => {
|
||||
const runAction = vi
|
||||
.fn()
|
||||
.mockResolvedValue({ ok: true, packageId: "pkg:1", releaseId: "rel:1" });
|
||||
const form = packagePublishForm(
|
||||
packagePublishMetadata({
|
||||
ownerHandle: "openclaw",
|
||||
bundle: { hostTargets: ["desktop"] },
|
||||
}),
|
||||
);
|
||||
form.append("files", new File(["{}"], "openclaw.plugin.json", { type: "application/json" }));
|
||||
|
||||
const response = await __handlers.publishPackageV1Handler(
|
||||
makeCtx({ runAction, runMutation }),
|
||||
makeCtx({
|
||||
runAction,
|
||||
runMutation,
|
||||
storage: {
|
||||
store: vi.fn(async (entry: File) => `storage:${entry.name}`),
|
||||
},
|
||||
}),
|
||||
new Request("https://example.com/api/v1/packages", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: "Bearer clh_test",
|
||||
"content-type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
name: "demo-plugin",
|
||||
ownerHandle: "openclaw",
|
||||
family: "bundle-plugin",
|
||||
version: "1.0.0",
|
||||
changelog: "init",
|
||||
bundle: { hostTargets: ["desktop"] },
|
||||
files: [
|
||||
{
|
||||
path: "openclaw.plugin.json",
|
||||
size: 2,
|
||||
storageId: "storage:1",
|
||||
sha256: "a".repeat(64),
|
||||
},
|
||||
{
|
||||
path: ".codex-plugin/plugin.json",
|
||||
size: 2,
|
||||
storageId: "storage:1",
|
||||
sha256: "a".repeat(64),
|
||||
},
|
||||
],
|
||||
}),
|
||||
headers: { Authorization: "Bearer clh_test" },
|
||||
body: form,
|
||||
}),
|
||||
);
|
||||
|
||||
@@ -9563,6 +9574,53 @@ describe("httpApiV1 handlers", () => {
|
||||
'Documents read from or written to the "publishers" table changed while this mutation was being run and on every subsequent retry.',
|
||||
),
|
||||
);
|
||||
const form = packagePublishForm(
|
||||
packagePublishMetadata({
|
||||
ownerHandle: "openclaw",
|
||||
bundle: { hostTargets: ["desktop"] },
|
||||
}),
|
||||
);
|
||||
const pack = npmPackFixture({
|
||||
"package/package.json": JSON.stringify({ name: "demo-plugin", version: "1.0.0" }),
|
||||
"package/openclaw.plugin.json": JSON.stringify({ id: "demo.plugin" }),
|
||||
"package/dist/index.js": "export const demo = true;\n",
|
||||
});
|
||||
form.append(
|
||||
"clawpack",
|
||||
new File([bytesToArrayBuffer(pack)], "demo-plugin-1.0.0.tgz", {
|
||||
type: "application/octet-stream",
|
||||
}),
|
||||
);
|
||||
|
||||
const response = await __handlers.publishPackageV1Handler(
|
||||
makeCtx({
|
||||
runAction,
|
||||
runMutation,
|
||||
storage: { store: vi.fn(async (_entry: Blob) => "storage:1") },
|
||||
}),
|
||||
new Request("https://example.com/api/v1/packages", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: "Bearer clh_test",
|
||||
},
|
||||
body: form,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(503);
|
||||
expect(response.headers.get("Retry-After")).toBe("1");
|
||||
await expect(response.text()).resolves.toContain("Transient ClawHub write contention");
|
||||
});
|
||||
|
||||
it("package publish rejects JSON request bodies before publish actions run", async () => {
|
||||
vi.mocked(getOptionalApiTokenUserId).mockResolvedValue("users:1" as never);
|
||||
vi.mocked(requirePackagePublishAuth).mockResolvedValue({
|
||||
kind: "user",
|
||||
userId: "users:1",
|
||||
user: { _id: "users:1", handle: "p" },
|
||||
} as never);
|
||||
const runMutation = vi.fn().mockResolvedValue(okRate());
|
||||
const runAction = vi.fn();
|
||||
|
||||
const response = await __handlers.publishPackageV1Handler(
|
||||
makeCtx({ runAction, runMutation }),
|
||||
@@ -9572,34 +9630,39 @@ describe("httpApiV1 handlers", () => {
|
||||
Authorization: "Bearer clh_test",
|
||||
"content-type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
name: "demo-plugin",
|
||||
ownerHandle: "openclaw",
|
||||
family: "bundle-plugin",
|
||||
version: "1.0.0",
|
||||
changelog: "init",
|
||||
bundle: { hostTargets: ["desktop"] },
|
||||
files: [
|
||||
{
|
||||
path: "openclaw.plugin.json",
|
||||
size: 2,
|
||||
storageId: "storage:1",
|
||||
sha256: "a".repeat(64),
|
||||
},
|
||||
{
|
||||
path: ".codex-plugin/plugin.json",
|
||||
size: 2,
|
||||
storageId: "storage:1",
|
||||
sha256: "a".repeat(64),
|
||||
},
|
||||
],
|
||||
}),
|
||||
body: JSON.stringify(packagePublishMetadata()),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(503);
|
||||
expect(response.headers.get("Retry-After")).toBe("1");
|
||||
await expect(response.text()).resolves.toContain("Transient ClawHub write contention");
|
||||
expect(response.status).toBe(415);
|
||||
expect(await response.text()).toBe("Package publish requires multipart/form-data");
|
||||
expect(runAction).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("package publish rejects browser session auth when token auth is not an API token", async () => {
|
||||
vi.mocked(getAuthUserId).mockResolvedValue("users:session" as never);
|
||||
vi.mocked(requirePackagePublishAuth).mockRejectedValue(new Error("Unauthorized"));
|
||||
const runMutation = vi.fn().mockResolvedValue(okRate());
|
||||
const runAction = vi.fn();
|
||||
const form = packagePublishForm(packagePublishMetadata());
|
||||
form.append("files", new File(["{}"], "openclaw.plugin.json", { type: "application/json" }));
|
||||
|
||||
const response = await __handlers.publishPackageV1Handler(
|
||||
makeCtx({
|
||||
runAction,
|
||||
runMutation,
|
||||
storage: { store: vi.fn(async () => "storage:plugin") },
|
||||
}),
|
||||
new Request("https://example.com/api/v1/packages", {
|
||||
method: "POST",
|
||||
headers: { Authorization: "Bearer convex-session-token" },
|
||||
body: form,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(401);
|
||||
expect(await response.text()).toBe("Unauthorized");
|
||||
expect(runAction).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("multipart package publish ignores macOS junk files", async () => {
|
||||
@@ -9613,6 +9676,7 @@ describe("httpApiV1 handlers", () => {
|
||||
const runAction = vi
|
||||
.fn()
|
||||
.mockResolvedValue({ ok: true, packageId: "pkg:1", releaseId: "rel:1" });
|
||||
const storageStore = vi.fn(async () => "storage:plugin");
|
||||
const form = new FormData();
|
||||
form.set(
|
||||
"payload",
|
||||
@@ -9631,9 +9695,7 @@ describe("httpApiV1 handlers", () => {
|
||||
makeCtx({
|
||||
runAction,
|
||||
runMutation,
|
||||
storage: {
|
||||
store: vi.fn(async (entry: File) => `storage:${entry.name}`),
|
||||
},
|
||||
storage: { store: storageStore },
|
||||
}),
|
||||
new Request("https://example.com/api/v1/packages", {
|
||||
method: "POST",
|
||||
@@ -9643,18 +9705,30 @@ describe("httpApiV1 handlers", () => {
|
||||
);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(storageStore).toHaveBeenCalledTimes(1);
|
||||
expect(runAction).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
payload: expect.objectContaining({
|
||||
files: [
|
||||
expect.objectContaining({
|
||||
{
|
||||
path: "openclaw.plugin.json",
|
||||
}),
|
||||
size: 2,
|
||||
storageId: "storage:plugin",
|
||||
sha256: "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a",
|
||||
contentType: "application/json",
|
||||
},
|
||||
],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
const actionCall = runAction.mock.calls[0];
|
||||
expect(actionCall).toBeTruthy();
|
||||
expect(actionCall[1]).toEqual(
|
||||
expect.objectContaining({
|
||||
payload: expect.not.objectContaining({ artifact: expect.anything() }),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("multipart ClawPack publish stores the tarball and extracted file metadata", async () => {
|
||||
@@ -9736,6 +9810,234 @@ describe("httpApiV1 handlers", () => {
|
||||
expect(payload?.files?.map((file) => file.path)).toContain("dist/index.js");
|
||||
});
|
||||
|
||||
it("staged ClawPack publish derives artifact metadata from stored bytes", async () => {
|
||||
vi.mocked(getOptionalApiTokenUserId).mockResolvedValue("users:1" as never);
|
||||
vi.mocked(requirePackagePublishAuth).mockResolvedValue({
|
||||
kind: "user",
|
||||
userId: "users:1",
|
||||
user: { _id: "users:1", handle: "p" },
|
||||
} as never);
|
||||
const runMutation = vi.fn().mockResolvedValue(okRate());
|
||||
const runAction = vi
|
||||
.fn()
|
||||
.mockResolvedValue({ ok: true, packageId: "pkg:1", releaseId: "rel:1" });
|
||||
const pack = npmPackFixture({
|
||||
"package/package.json": JSON.stringify({ name: "demo-plugin", version: "1.0.0" }),
|
||||
"package/openclaw.plugin.json": JSON.stringify({ id: "demo.plugin" }),
|
||||
"package/dist/index.js": "export const demo = true;\n",
|
||||
});
|
||||
const storageGet = vi.fn(async (storageId: string) =>
|
||||
storageId === "storage:clawpack"
|
||||
? new Blob([bytesToArrayBuffer(pack)], { type: "application/octet-stream" })
|
||||
: null,
|
||||
);
|
||||
const storageStore = vi.fn(async (_entry: Blob) => `storage:${storageStore.mock.calls.length}`);
|
||||
const form = packagePublishForm(
|
||||
packagePublishMetadata({
|
||||
family: "code-plugin",
|
||||
}),
|
||||
);
|
||||
form.set("clawpack", "storage:clawpack");
|
||||
form.set("clawpackUploadTicket", "packagePublishUploadTickets:1");
|
||||
|
||||
const response = await __handlers.publishPackageV1Handler(
|
||||
makeCtx({
|
||||
runAction,
|
||||
runMutation,
|
||||
storage: { get: storageGet, store: storageStore },
|
||||
}),
|
||||
new Request("https://example.com/api/v1/packages", {
|
||||
method: "POST",
|
||||
headers: { Authorization: "Bearer clh_test" },
|
||||
body: form,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(runMutation).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
uploadTicket: "packagePublishUploadTickets:1",
|
||||
storageId: "storage:clawpack",
|
||||
auth: { kind: "user", userId: "users:1" },
|
||||
}),
|
||||
);
|
||||
expect(storageGet).toHaveBeenCalledWith("storage:clawpack");
|
||||
expect(storageStore).toHaveBeenCalledTimes(3);
|
||||
expect(runAction).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
payload: expect.objectContaining({
|
||||
artifact: expect.objectContaining({
|
||||
kind: "npm-pack",
|
||||
storageId: "storage:clawpack",
|
||||
size: pack.byteLength,
|
||||
npmFileCount: 3,
|
||||
}),
|
||||
files: [
|
||||
expect.objectContaining({ path: "package.json", storageId: "storage:1" }),
|
||||
expect.objectContaining({ path: "openclaw.plugin.json", storageId: "storage:2" }),
|
||||
expect.objectContaining({ path: "dist/index.js", storageId: "storage:3" }),
|
||||
],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("staged ClawPack publish rejects storage ids without upload tickets", async () => {
|
||||
vi.mocked(getOptionalApiTokenUserId).mockResolvedValue("users:1" as never);
|
||||
vi.mocked(requirePackagePublishAuth).mockResolvedValue({
|
||||
kind: "user",
|
||||
userId: "users:1",
|
||||
user: { _id: "users:1", handle: "p" },
|
||||
} as never);
|
||||
const runMutation = vi.fn().mockResolvedValue(okRate());
|
||||
const runAction = vi.fn();
|
||||
const storageGet = vi.fn();
|
||||
const form = packagePublishForm(packagePublishMetadata({ family: "code-plugin" }));
|
||||
form.set("clawpack", "storage:clawpack");
|
||||
|
||||
const response = await __handlers.publishPackageV1Handler(
|
||||
makeCtx({
|
||||
runAction,
|
||||
runMutation,
|
||||
storage: { get: storageGet, store: vi.fn() },
|
||||
}),
|
||||
new Request("https://example.com/api/v1/packages", {
|
||||
method: "POST",
|
||||
headers: { Authorization: "Bearer clh_test" },
|
||||
body: form,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
expect(await response.text()).toBe("Package tarball upload ticket required");
|
||||
expect(storageGet).not.toHaveBeenCalled();
|
||||
expect(runAction).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("multipart package publish rejects files and tarball together", async () => {
|
||||
vi.mocked(getOptionalApiTokenUserId).mockResolvedValue("users:1" as never);
|
||||
vi.mocked(requirePackagePublishAuth).mockResolvedValue({
|
||||
kind: "user",
|
||||
userId: "users:1",
|
||||
user: { _id: "users:1", handle: "p" },
|
||||
} as never);
|
||||
const runMutation = vi.fn().mockResolvedValue(okRate());
|
||||
const runAction = vi.fn();
|
||||
const pack = npmPackFixture({
|
||||
"package/package.json": JSON.stringify({ name: "demo-plugin", version: "1.0.0" }),
|
||||
"package/openclaw.plugin.json": JSON.stringify({ id: "demo.plugin" }),
|
||||
});
|
||||
const form = packagePublishForm(packagePublishMetadata({ family: "code-plugin" }));
|
||||
form.append("files", new File(["{}"], "openclaw.plugin.json", { type: "application/json" }));
|
||||
form.append(
|
||||
"clawpack",
|
||||
new File([bytesToArrayBuffer(pack)], "demo-plugin-1.0.0.tgz", {
|
||||
type: "application/octet-stream",
|
||||
}),
|
||||
);
|
||||
|
||||
const response = await __handlers.publishPackageV1Handler(
|
||||
makeCtx({ runAction, runMutation, storage: { store: vi.fn() } }),
|
||||
new Request("https://example.com/api/v1/packages", {
|
||||
method: "POST",
|
||||
headers: { Authorization: "Bearer clh_test" },
|
||||
body: form,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
expect(await response.text()).toBe(
|
||||
"Upload either a package tarball or individual files, not both",
|
||||
);
|
||||
expect(runAction).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(["files[]", "tarball", "artifact", "extraMetadata"])(
|
||||
"multipart package publish rejects unsupported field %s",
|
||||
async (field) => {
|
||||
vi.mocked(getOptionalApiTokenUserId).mockResolvedValue("users:1" as never);
|
||||
vi.mocked(requirePackagePublishAuth).mockResolvedValue({
|
||||
kind: "user",
|
||||
userId: "users:1",
|
||||
user: { _id: "users:1", handle: "p" },
|
||||
} as never);
|
||||
const runMutation = vi.fn().mockResolvedValue(okRate());
|
||||
const runAction = vi.fn();
|
||||
const form = packagePublishForm(packagePublishMetadata());
|
||||
form.append("files", new File(["{}"], "openclaw.plugin.json", { type: "application/json" }));
|
||||
form.append(field, new File(["{}"], "ignored.json", { type: "application/json" }));
|
||||
|
||||
const response = await __handlers.publishPackageV1Handler(
|
||||
makeCtx({ runAction, runMutation, storage: { store: vi.fn() } }),
|
||||
new Request("https://example.com/api/v1/packages", {
|
||||
method: "POST",
|
||||
headers: { Authorization: "Bearer clh_test" },
|
||||
body: form,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
expect(await response.text()).toBe(`Unsupported package publish form field: ${field}`);
|
||||
expect(runAction).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["files", "artifact"])(
|
||||
"multipart package publish rejects caller-supplied %s metadata",
|
||||
async (field) => {
|
||||
vi.mocked(getOptionalApiTokenUserId).mockResolvedValue("users:1" as never);
|
||||
vi.mocked(requirePackagePublishAuth).mockResolvedValue({
|
||||
kind: "user",
|
||||
userId: "users:1",
|
||||
user: { _id: "users:1", handle: "p" },
|
||||
} as never);
|
||||
const runMutation = vi.fn().mockResolvedValue(okRate());
|
||||
const runAction = vi.fn();
|
||||
const form = packagePublishForm(
|
||||
packagePublishMetadata({
|
||||
[field]:
|
||||
field === "files"
|
||||
? [
|
||||
{
|
||||
path: "openclaw.plugin.json",
|
||||
size: 2,
|
||||
storageId: "storage:attacker",
|
||||
sha256: "a".repeat(64),
|
||||
},
|
||||
]
|
||||
: {
|
||||
kind: "npm-pack",
|
||||
storageId: "storage:attacker",
|
||||
sha256: "a".repeat(64),
|
||||
size: 2,
|
||||
format: "tgz",
|
||||
npmIntegrity: "sha512-attacker",
|
||||
npmShasum: "a".repeat(40),
|
||||
npmTarballName: "demo-plugin-1.0.0.tgz",
|
||||
npmUnpackedSize: 2,
|
||||
npmFileCount: 1,
|
||||
},
|
||||
}),
|
||||
);
|
||||
form.append("files", new File(["{}"], "openclaw.plugin.json", { type: "application/json" }));
|
||||
|
||||
const response = await __handlers.publishPackageV1Handler(
|
||||
makeCtx({ runAction, runMutation, storage: { store: vi.fn() } }),
|
||||
new Request("https://example.com/api/v1/packages", {
|
||||
method: "POST",
|
||||
headers: { Authorization: "Bearer clh_test" },
|
||||
body: form,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
expect(await response.text()).toContain(`Package publish payload: ${field}`);
|
||||
expect(runAction).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it("package publish routes GitHub Actions auth through the trusted publisher action", async () => {
|
||||
vi.mocked(getOptionalApiTokenUserId).mockResolvedValue("users:1" as never);
|
||||
vi.mocked(requirePackagePublishAuth).mockResolvedValue({
|
||||
@@ -9746,36 +10048,35 @@ describe("httpApiV1 handlers", () => {
|
||||
const runAction = vi
|
||||
.fn()
|
||||
.mockResolvedValue({ ok: true, packageId: "pkg:1", releaseId: "rel:1" });
|
||||
const form = packagePublishForm(
|
||||
packagePublishMetadata({
|
||||
bundle: { hostTargets: ["desktop"] },
|
||||
}),
|
||||
);
|
||||
const pack = npmPackFixture({
|
||||
"package/package.json": JSON.stringify({ name: "demo-plugin", version: "1.0.0" }),
|
||||
"package/openclaw.plugin.json": JSON.stringify({ id: "demo.plugin" }),
|
||||
"package/dist/index.js": "export const demo = true;\n",
|
||||
});
|
||||
form.append(
|
||||
"clawpack",
|
||||
new File([bytesToArrayBuffer(pack)], "demo-plugin-1.0.0.tgz", {
|
||||
type: "application/octet-stream",
|
||||
}),
|
||||
);
|
||||
|
||||
const response = await __handlers.publishPackageV1Handler(
|
||||
makeCtx({ runAction, runMutation }),
|
||||
makeCtx({
|
||||
runAction,
|
||||
runMutation,
|
||||
storage: { store: vi.fn(async (_entry: Blob) => "storage:1") },
|
||||
}),
|
||||
new Request("https://example.com/api/v1/packages", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: "Bearer clh_publish",
|
||||
"content-type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
name: "demo-plugin",
|
||||
family: "bundle-plugin",
|
||||
version: "1.0.0",
|
||||
changelog: "init",
|
||||
bundle: { hostTargets: ["desktop"] },
|
||||
files: [
|
||||
{
|
||||
path: "openclaw.plugin.json",
|
||||
size: 2,
|
||||
storageId: "storage:1",
|
||||
sha256: "a".repeat(64),
|
||||
},
|
||||
{
|
||||
path: ".codex-plugin/plugin.json",
|
||||
size: 2,
|
||||
storageId: "storage:1",
|
||||
sha256: "a".repeat(64),
|
||||
},
|
||||
],
|
||||
}),
|
||||
body: form,
|
||||
}),
|
||||
);
|
||||
|
||||
|
||||
+245
-139
@@ -11,16 +11,18 @@ import {
|
||||
PackageReportRequestSchema,
|
||||
PackageReportTriageRequestSchema,
|
||||
PackageReleaseModerationRequestSchema,
|
||||
PackagePublishRequestSchema,
|
||||
PackagePublishMetadataSchema,
|
||||
PackageTransferRequestSchema,
|
||||
PackageTrustedPublisherUpsertRequestSchema,
|
||||
PublishTokenMintRequestSchema,
|
||||
isPluginCategorySlug,
|
||||
parseArk,
|
||||
type PackagePublishMetadata,
|
||||
type PackageAppealListStatus,
|
||||
type PackageModerationQueueStatus,
|
||||
type PackageOfficialMigrationListPhase,
|
||||
type PackageReportListStatus,
|
||||
type ServerPackagePublishRequest,
|
||||
} from "clawhub-schema";
|
||||
import { api, internal } from "../_generated/api";
|
||||
import type { Doc, Id } from "../_generated/dataModel";
|
||||
@@ -43,9 +45,13 @@ import {
|
||||
} from "../lib/packageSecurity";
|
||||
import {
|
||||
getClawPackSizeError,
|
||||
getPackageMultipartSizeError,
|
||||
getPublishFileSizeError,
|
||||
getPublishTotalSizeError,
|
||||
isPackageMultipartUploadTooLarge,
|
||||
MAX_CLAWPACK_BYTES,
|
||||
MAX_PUBLISH_FILE_BYTES,
|
||||
MAX_PUBLISH_TOTAL_BYTES,
|
||||
} from "../lib/publishLimits";
|
||||
import { getPublicSkillFileAccessBlock, isSkillVersionForSkill } from "../lib/skillFileAccess";
|
||||
import { isMacJunkPath, isTextFile } from "../lib/skills";
|
||||
@@ -120,6 +126,9 @@ const internalRefs = internal as unknown as {
|
||||
packagePublishTokens: {
|
||||
createInternal: unknown;
|
||||
};
|
||||
uploads: {
|
||||
consumePackagePublishUploadTicketInternal: unknown;
|
||||
};
|
||||
skills: {
|
||||
getSkillBySlugInternal: unknown;
|
||||
searchPackageCatalogForHttpInternal: unknown;
|
||||
@@ -1018,75 +1027,27 @@ function skillVersionTags(tags: Record<string, string>, version: string) {
|
||||
.map(([tag]) => tag);
|
||||
}
|
||||
|
||||
function parsePackagePublishBody(body: unknown) {
|
||||
const parsed = parseArk(PackagePublishRequestSchema, body, "Package publish payload") as {
|
||||
name: string;
|
||||
displayName?: string;
|
||||
ownerHandle?: string;
|
||||
family: "skill" | "code-plugin" | "bundle-plugin";
|
||||
version: string;
|
||||
changelog: string;
|
||||
manualOverrideReason?: string;
|
||||
channel?: "official" | "community" | "private";
|
||||
tags?: string[];
|
||||
source?: Record<string, unknown>;
|
||||
bundle?: Record<string, unknown>;
|
||||
files: Array<{
|
||||
path: string;
|
||||
size: number;
|
||||
storageId: string;
|
||||
sha256: string;
|
||||
contentType?: string;
|
||||
}>;
|
||||
artifact?: {
|
||||
kind: "npm-pack";
|
||||
storageId: string;
|
||||
sha256: string;
|
||||
size: number;
|
||||
format: "tgz";
|
||||
npmIntegrity: string;
|
||||
npmShasum: string;
|
||||
npmTarballName: string;
|
||||
npmUnpackedSize: number;
|
||||
npmFileCount: number;
|
||||
type StoredPackagePublishFile = ServerPackagePublishRequest["files"][number];
|
||||
type PackagePublishTarballArtifact = NonNullable<ServerPackagePublishRequest["artifact"]>;
|
||||
type ParsedPackageClawPack = Awaited<ReturnType<typeof parseClawPack>>;
|
||||
type PackagePublishAuth =
|
||||
| { kind: "user"; userId: Id<"users"> }
|
||||
| { kind: "github-actions"; publishToken: Doc<"packagePublishTokens"> };
|
||||
type PackagePublishTarballPart =
|
||||
| { kind: "file"; file: File }
|
||||
| {
|
||||
kind: "storage";
|
||||
storageId: Id<"_storage">;
|
||||
uploadTicket: Id<"packagePublishUploadTickets">;
|
||||
};
|
||||
};
|
||||
if (parsed.files.length === 0) throw new Error("files required");
|
||||
return {
|
||||
name: parsed.name,
|
||||
displayName: parsed.displayName ?? undefined,
|
||||
ownerHandle: parsed.ownerHandle?.trim().replace(/^@+/, "") || undefined,
|
||||
family: parsed.family,
|
||||
version: parsed.version,
|
||||
changelog: parsed.changelog,
|
||||
manualOverrideReason: parsed.manualOverrideReason?.trim() || undefined,
|
||||
channel: parsed.channel ?? undefined,
|
||||
tags: parsed.tags?.filter(Boolean) ?? undefined,
|
||||
source: parsed.source ?? undefined,
|
||||
bundle: parsed.bundle ?? undefined,
|
||||
files: parsed.files.map((file) => ({
|
||||
...file,
|
||||
storageId: file.storageId as Id<"_storage">,
|
||||
})),
|
||||
artifact: parsed.artifact
|
||||
? {
|
||||
...parsed.artifact,
|
||||
storageId: parsed.artifact.storageId as Id<"_storage">,
|
||||
}
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function inferStoredPackageContentType(path: string) {
|
||||
const lower = path.toLowerCase();
|
||||
if (lower.endsWith(".json")) return "application/json";
|
||||
if (lower.endsWith(".md") || lower.endsWith(".mdx") || lower.endsWith(".txt")) {
|
||||
return "text/plain; charset=utf-8";
|
||||
}
|
||||
if (lower.endsWith(".js") || lower.endsWith(".mjs") || lower.endsWith(".cjs")) {
|
||||
return "text/javascript; charset=utf-8";
|
||||
}
|
||||
if (lower.endsWith(".ts") || lower.endsWith(".tsx")) return "text/plain; charset=utf-8";
|
||||
if (isTextFile(path)) return "text/plain; charset=utf-8";
|
||||
return "application/octet-stream";
|
||||
}
|
||||
|
||||
@@ -1096,9 +1057,10 @@ function bytesToArrayBuffer(bytes: Uint8Array) {
|
||||
return copy.buffer;
|
||||
}
|
||||
|
||||
async function storeClawPackFile(ctx: ActionCtx, entry: { path: string; bytes: Uint8Array }) {
|
||||
// npm-pack artifacts are bounded by the tarball and total package limits; the
|
||||
// legacy per-file cap only applies to raw file uploads.
|
||||
async function storeClawPackFile(
|
||||
ctx: ActionCtx,
|
||||
entry: { path: string; bytes: Uint8Array },
|
||||
): Promise<StoredPackagePublishFile> {
|
||||
const contentType = inferStoredPackageContentType(entry.path);
|
||||
const storageId = await ctx.storage.store(
|
||||
new Blob([bytesToArrayBuffer(entry.bytes)], { type: contentType }),
|
||||
@@ -1116,91 +1078,234 @@ async function storeClawPackFiles(
|
||||
ctx: ActionCtx,
|
||||
entries: Array<{ path: string; bytes: Uint8Array }>,
|
||||
) {
|
||||
const files: Awaited<ReturnType<typeof storeClawPackFile>>[] = [];
|
||||
// Convex HTTP actions have a tight memory ceiling; concurrent Blob/storage
|
||||
// work can duplicate large npm-pack entries enough to OOM the action.
|
||||
const files: StoredPackagePublishFile[] = [];
|
||||
// Convex HTTP actions have a tight memory ceiling; avoid concurrent Blob work.
|
||||
for (const entry of entries) {
|
||||
files.push(await storeClawPackFile(ctx, entry));
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
async function parseMultipartPackagePublish(ctx: ActionCtx, request: Request) {
|
||||
async function storeUploadedPackageFile(
|
||||
ctx: ActionCtx,
|
||||
entry: File,
|
||||
): Promise<StoredPackagePublishFile> {
|
||||
if (entry.size > MAX_PUBLISH_FILE_BYTES) {
|
||||
throw new Error(getPublishFileSizeError(entry.name));
|
||||
}
|
||||
const buffer = new Uint8Array(await entry.arrayBuffer());
|
||||
const contentType = inferStoredPackageContentType(entry.name);
|
||||
const storageId = await ctx.storage.store(
|
||||
new Blob([bytesToArrayBuffer(buffer)], { type: contentType }),
|
||||
);
|
||||
return {
|
||||
path: entry.name,
|
||||
size: entry.size,
|
||||
storageId,
|
||||
sha256: await sha256Hex(buffer),
|
||||
contentType,
|
||||
};
|
||||
}
|
||||
|
||||
function getFileParts(form: FormData, fields: readonly string[], stringPartError: string) {
|
||||
const parts = fields.flatMap((field) => form.getAll(field));
|
||||
if (parts.some((entry) => typeof entry === "string")) {
|
||||
throw new Error(stringPartError);
|
||||
}
|
||||
return parts.filter((entry): entry is File => typeof entry !== "string");
|
||||
}
|
||||
|
||||
function getTarballPart(form: FormData): PackagePublishTarballPart | null {
|
||||
const parts = form.getAll("clawpack");
|
||||
if (parts.length > 1) throw new Error("Upload one package tarball");
|
||||
const ticketParts = form.getAll("clawpackUploadTicket");
|
||||
if (ticketParts.length > 1) throw new Error("Upload one package tarball ticket");
|
||||
const ticketPart = ticketParts[0];
|
||||
if (ticketPart && typeof ticketPart !== "string") {
|
||||
throw new Error("Package tarball upload ticket must be a string");
|
||||
}
|
||||
const part = parts[0];
|
||||
if (!part) {
|
||||
if (ticketPart) throw new Error("Package tarball upload ticket requires a staged ClawPack");
|
||||
return null;
|
||||
}
|
||||
if (typeof part !== "string") {
|
||||
if (ticketPart) throw new Error("Package tarball upload ticket requires a staged ClawPack");
|
||||
return { kind: "file", file: part };
|
||||
}
|
||||
|
||||
const storageId = part.trim();
|
||||
if (!storageId) throw new Error("Package tarball storage id required");
|
||||
const uploadTicket = ticketPart?.trim();
|
||||
if (!uploadTicket) throw new Error("Package tarball upload ticket required");
|
||||
return {
|
||||
kind: "storage",
|
||||
storageId: storageId as Id<"_storage">,
|
||||
uploadTicket: uploadTicket as Id<"packagePublishUploadTickets">,
|
||||
};
|
||||
}
|
||||
|
||||
async function consumePackageTarballUploadTicket(
|
||||
ctx: ActionCtx,
|
||||
auth: PackagePublishAuth,
|
||||
part: Extract<PackagePublishTarballPart, { kind: "storage" }>,
|
||||
) {
|
||||
await ctx.runMutation(
|
||||
internalRefs.uploads.consumePackagePublishUploadTicketInternal as never,
|
||||
{
|
||||
uploadTicket: part.uploadTicket,
|
||||
storageId: part.storageId,
|
||||
auth:
|
||||
auth.kind === "user"
|
||||
? { kind: "user", userId: auth.userId }
|
||||
: { kind: "github-actions", publishTokenId: auth.publishToken._id },
|
||||
} as never,
|
||||
);
|
||||
}
|
||||
|
||||
async function readStoredPackageTarball(ctx: ActionCtx, storageId: Id<"_storage">) {
|
||||
const blob = await ctx.storage.get(storageId);
|
||||
if (!blob) throw new Error("Package tarball upload no longer exists");
|
||||
if (blob.size > MAX_CLAWPACK_BYTES) {
|
||||
throw new Error(getClawPackSizeError("uploaded ClawPack"));
|
||||
}
|
||||
return new Uint8Array(await blob.arrayBuffer());
|
||||
}
|
||||
|
||||
async function buildPackagePublishRequestFromClawPack(
|
||||
ctx: ActionCtx,
|
||||
metadata: PackagePublishMetadata,
|
||||
parsed: ParsedPackageClawPack,
|
||||
artifactBytes: Uint8Array,
|
||||
artifactStorageId: Id<"_storage">,
|
||||
): Promise<ServerPackagePublishRequest> {
|
||||
if (parsed.unpackedSize > MAX_PUBLISH_TOTAL_BYTES) {
|
||||
throw new Error(getPublishTotalSizeError("package"));
|
||||
}
|
||||
const artifact: PackagePublishTarballArtifact = {
|
||||
kind: "npm-pack",
|
||||
storageId: artifactStorageId,
|
||||
sha256: parsed.artifactSha256,
|
||||
size: artifactBytes.byteLength,
|
||||
format: "tgz",
|
||||
npmIntegrity: parsed.npmIntegrity,
|
||||
npmShasum: parsed.npmShasum,
|
||||
npmTarballName: parsed.npmTarballName,
|
||||
npmUnpackedSize: parsed.unpackedSize,
|
||||
npmFileCount: parsed.fileCount,
|
||||
};
|
||||
const files = await storeClawPackFiles(ctx, parsed.entries);
|
||||
return { ...metadata, files, artifact };
|
||||
}
|
||||
|
||||
const PACKAGE_PUBLISH_FILE_FIELDS = ["files"] as const;
|
||||
const PACKAGE_PUBLISH_TARBALL_FIELDS = ["clawpack"] as const;
|
||||
const PACKAGE_PUBLISH_FORM_FIELDS = new Set([
|
||||
"payload",
|
||||
...PACKAGE_PUBLISH_FILE_FIELDS,
|
||||
...PACKAGE_PUBLISH_TARBALL_FIELDS,
|
||||
"clawpackUploadTicket",
|
||||
]);
|
||||
|
||||
function multipartUploadPart(file: File) {
|
||||
return {
|
||||
name: file.name,
|
||||
size: file.size,
|
||||
type: file.type || undefined,
|
||||
};
|
||||
}
|
||||
|
||||
async function parseMultipartPackagePublish(
|
||||
ctx: ActionCtx,
|
||||
auth: PackagePublishAuth,
|
||||
request: Request,
|
||||
): Promise<ServerPackagePublishRequest> {
|
||||
const form = await request.formData();
|
||||
const payloadRaw = form.get("payload");
|
||||
if (!payloadRaw || typeof payloadRaw !== "string") throw new Error("Missing payload");
|
||||
const payload = JSON.parse(payloadRaw) as Record<string, unknown>;
|
||||
const files: Array<{
|
||||
path: string;
|
||||
size: number;
|
||||
storageId: Id<"_storage">;
|
||||
sha256: string;
|
||||
contentType?: string;
|
||||
}> = [];
|
||||
let artifact:
|
||||
| {
|
||||
kind: "npm-pack";
|
||||
storageId: Id<"_storage">;
|
||||
sha256: string;
|
||||
size: number;
|
||||
format: "tgz";
|
||||
npmIntegrity: string;
|
||||
npmShasum: string;
|
||||
npmTarballName: string;
|
||||
npmUnpackedSize: number;
|
||||
npmFileCount: number;
|
||||
}
|
||||
| undefined;
|
||||
for (const field of form.keys()) {
|
||||
if (!PACKAGE_PUBLISH_FORM_FIELDS.has(field)) {
|
||||
throw new Error(`Unsupported package publish form field: ${field}`);
|
||||
}
|
||||
}
|
||||
|
||||
const clawpackEntry = form.get("clawpack") ?? form.get("artifact");
|
||||
if (clawpackEntry && typeof clawpackEntry !== "string") {
|
||||
if (form.getAll("files").some((entry) => typeof entry !== "string")) {
|
||||
throw new Error("Upload either a ClawPack tarball or individual files, not both");
|
||||
const payloadParts = form.getAll("payload");
|
||||
const payloadRaw = payloadParts[0];
|
||||
if (payloadParts.length !== 1 || typeof payloadRaw !== "string") {
|
||||
throw new Error("Package publish payload must be one JSON string");
|
||||
}
|
||||
const parsedPayload: unknown = JSON.parse(payloadRaw);
|
||||
const metadata: PackagePublishMetadata = parseArk(
|
||||
PackagePublishMetadataSchema,
|
||||
parsedPayload,
|
||||
"Package publish payload",
|
||||
);
|
||||
|
||||
const tarballPart = getTarballPart(form);
|
||||
const fileParts = getFileParts(
|
||||
form,
|
||||
PACKAGE_PUBLISH_FILE_FIELDS,
|
||||
"Package publish file uploads must be files",
|
||||
);
|
||||
|
||||
if (tarballPart) {
|
||||
if (fileParts.length > 0) {
|
||||
throw new Error("Upload either a package tarball or individual files, not both");
|
||||
}
|
||||
if (clawpackEntry.size > MAX_CLAWPACK_BYTES) {
|
||||
throw new Error(getClawPackSizeError(clawpackEntry.name));
|
||||
if (tarballPart.kind === "storage") {
|
||||
await consumePackageTarballUploadTicket(ctx, auth, tarballPart);
|
||||
const artifactBytes = await readStoredPackageTarball(ctx, tarballPart.storageId);
|
||||
const parsed = await parseClawPack(artifactBytes);
|
||||
return await buildPackagePublishRequestFromClawPack(
|
||||
ctx,
|
||||
metadata,
|
||||
parsed,
|
||||
artifactBytes,
|
||||
tarballPart.storageId,
|
||||
);
|
||||
}
|
||||
const artifactBytes = new Uint8Array(await clawpackEntry.arrayBuffer());
|
||||
|
||||
const tarballEntry = tarballPart.file;
|
||||
if (tarballEntry.size > MAX_CLAWPACK_BYTES) {
|
||||
throw new Error(getClawPackSizeError(tarballEntry.name));
|
||||
}
|
||||
if (
|
||||
isPackageMultipartUploadTooLarge({
|
||||
payloadJson: payloadRaw,
|
||||
fileFieldName: "clawpack",
|
||||
files: [multipartUploadPart(tarballEntry)],
|
||||
})
|
||||
) {
|
||||
throw new Error(getPackageMultipartSizeError());
|
||||
}
|
||||
const artifactBytes = new Uint8Array(await tarballEntry.arrayBuffer());
|
||||
const parsed = await parseClawPack(artifactBytes);
|
||||
const artifactBlob = new Blob([artifactBytes], { type: "application/octet-stream" });
|
||||
const artifactStorageId = await ctx.storage.store(artifactBlob);
|
||||
artifact = {
|
||||
kind: "npm-pack",
|
||||
storageId: artifactStorageId,
|
||||
sha256: parsed.artifactSha256,
|
||||
size: artifactBytes.byteLength,
|
||||
format: "tgz",
|
||||
npmIntegrity: parsed.npmIntegrity,
|
||||
npmShasum: parsed.npmShasum,
|
||||
npmTarballName: parsed.npmTarballName,
|
||||
npmUnpackedSize: parsed.unpackedSize,
|
||||
npmFileCount: parsed.fileCount,
|
||||
};
|
||||
files.push(...(await storeClawPackFiles(ctx, parsed.entries)));
|
||||
return parsePackagePublishBody({ ...payload, files, artifact });
|
||||
const artifactStorageId = await ctx.storage.store(
|
||||
new Blob([bytesToArrayBuffer(artifactBytes)], { type: "application/octet-stream" }),
|
||||
);
|
||||
return await buildPackagePublishRequestFromClawPack(
|
||||
ctx,
|
||||
metadata,
|
||||
parsed,
|
||||
artifactBytes,
|
||||
artifactStorageId,
|
||||
);
|
||||
}
|
||||
|
||||
for (const entry of form.getAll("files")) {
|
||||
if (typeof entry === "string") continue;
|
||||
if (isMacJunkPath(entry.name)) continue;
|
||||
if (entry.size > MAX_PUBLISH_FILE_BYTES) {
|
||||
throw new Error(getPublishFileSizeError(entry.name));
|
||||
}
|
||||
const buffer = new Uint8Array(await entry.arrayBuffer());
|
||||
const digest = await crypto.subtle.digest("SHA-256", buffer);
|
||||
const sha256 = Array.from(new Uint8Array(digest), (byte) =>
|
||||
byte.toString(16).padStart(2, "0"),
|
||||
).join("");
|
||||
const storageId = await ctx.storage.store(entry);
|
||||
files.push({
|
||||
path: entry.name,
|
||||
size: entry.size,
|
||||
storageId,
|
||||
sha256,
|
||||
contentType: entry.type || undefined,
|
||||
});
|
||||
if (
|
||||
isPackageMultipartUploadTooLarge({
|
||||
payloadJson: payloadRaw,
|
||||
fileFieldName: "files",
|
||||
files: fileParts.map(multipartUploadPart),
|
||||
})
|
||||
) {
|
||||
throw new Error(getPackageMultipartSizeError());
|
||||
}
|
||||
return parsePackagePublishBody({ ...payload, files });
|
||||
|
||||
const packageFileParts = fileParts.filter((entry) => !isMacJunkPath(entry.name));
|
||||
const files = await Promise.all(
|
||||
packageFileParts.map((entry) => storeUploadedPackageFile(ctx, entry)),
|
||||
);
|
||||
if (files.length === 0) throw new Error("files required");
|
||||
return { ...metadata, files };
|
||||
}
|
||||
|
||||
async function listPackages(
|
||||
@@ -1481,9 +1586,10 @@ export async function publishPackageV1Handler(ctx: ActionCtx, request: Request)
|
||||
|
||||
try {
|
||||
const contentType = request.headers.get("content-type") ?? "";
|
||||
const payload = contentType.includes("multipart/form-data")
|
||||
? await parseMultipartPackagePublish(ctx, request)
|
||||
: parsePackagePublishBody(await request.json());
|
||||
if (!contentType.includes("multipart/form-data")) {
|
||||
return text("Package publish requires multipart/form-data", 415, rate.headers);
|
||||
}
|
||||
const payload = await parseMultipartPackagePublish(ctx, auth.auth, request);
|
||||
const result =
|
||||
auth.auth.kind === "user"
|
||||
? await runActionRef(ctx, internalRefs.packages.publishPackageForUserInternal, {
|
||||
|
||||
@@ -37,9 +37,9 @@ function tarFile(path: string, content: string) {
|
||||
return [header, body];
|
||||
}
|
||||
|
||||
function npmPackFixture(files: Record<string, string>) {
|
||||
function npmPackFixtureEntries(files: Array<[string, string]>) {
|
||||
const parts: Uint8Array[] = [];
|
||||
for (const [path, content] of Object.entries(files)) {
|
||||
for (const [path, content] of files) {
|
||||
parts.push(...tarFile(path, content));
|
||||
}
|
||||
parts.push(new Uint8Array(BLOCK_SIZE), new Uint8Array(BLOCK_SIZE));
|
||||
@@ -53,6 +53,10 @@ function npmPackFixture(files: Record<string, string>) {
|
||||
return gzipSync(tar);
|
||||
}
|
||||
|
||||
function npmPackFixture(files: Record<string, string>) {
|
||||
return npmPackFixtureEntries(Object.entries(files));
|
||||
}
|
||||
|
||||
describe("clawpack", () => {
|
||||
it("parses npm pack tarballs and computes npm integrity fields", async () => {
|
||||
const pack = npmPackFixture({
|
||||
@@ -95,6 +99,18 @@ describe("clawpack", () => {
|
||||
await expect(parseClawPack(pack)).rejects.toThrow("rooted under package");
|
||||
});
|
||||
|
||||
it("rejects duplicate normalized archive paths", async () => {
|
||||
const pack = npmPackFixtureEntries([
|
||||
["package/package.json", JSON.stringify({ name: "demo", version: "1.0.0" })],
|
||||
["package/openclaw.plugin.json", JSON.stringify({ id: "demo" })],
|
||||
["package/package.json", JSON.stringify({ name: "other", version: "9.9.9" })],
|
||||
]);
|
||||
|
||||
await expect(parseClawPack(pack)).rejects.toThrow(
|
||||
"ClawPack contains duplicate path: package.json",
|
||||
);
|
||||
});
|
||||
|
||||
it("uses npm-style tarball names", () => {
|
||||
expect(npmTarballName("demo", "1.0.0")).toBe("demo-1.0.0.tgz");
|
||||
expect(npmTarballName("@scope/demo", "1.0.0")).toBe("scope-demo-1.0.0.tgz");
|
||||
|
||||
@@ -67,6 +67,7 @@ function isZeroBlock(block: Uint8Array) {
|
||||
|
||||
function parseTarEntries(bytes: Uint8Array): ClawPackEntry[] {
|
||||
const entries: ClawPackEntry[] = [];
|
||||
const paths = new Set<string>();
|
||||
let offset = 0;
|
||||
|
||||
while (offset + TAR_BLOCK_SIZE <= bytes.byteLength) {
|
||||
@@ -93,6 +94,10 @@ function parseTarEntries(bytes: Uint8Array): ClawPackEntry[] {
|
||||
offset = nextTarOffset(payloadOffset, size);
|
||||
continue;
|
||||
}
|
||||
if (paths.has(relPath)) {
|
||||
throw new Error(`ClawPack contains duplicate path: ${relPath}`);
|
||||
}
|
||||
paths.add(relPath);
|
||||
entries.push({
|
||||
path: relPath,
|
||||
bytes: Uint8Array.from(tarEntryPayload(bytes, payloadOffset, size)),
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
getPublishFileSizeError,
|
||||
getPublishTotalSizeError,
|
||||
MAX_CLAWPACK_BYTES,
|
||||
MAX_PACKAGE_MULTIPART_BYTES,
|
||||
MAX_PUBLISH_FILE_BYTES,
|
||||
} from "./publishLimits";
|
||||
|
||||
@@ -31,8 +32,9 @@ describe("publishLimits", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps the ClawPack tarball limit separate from legacy file limits", () => {
|
||||
it("keeps ClawPack capacity above the multipart request budget", () => {
|
||||
expect(MAX_CLAWPACK_BYTES).toBe(120 * 1024 * 1024);
|
||||
expect(MAX_CLAWPACK_BYTES).toBeGreaterThan(MAX_PACKAGE_MULTIPART_BYTES);
|
||||
expect(MAX_CLAWPACK_BYTES).toBeGreaterThan(MAX_PUBLISH_FILE_BYTES);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,17 @@
|
||||
import { MAX_PACKAGE_CLAWPACK_BYTES } from "clawhub-schema";
|
||||
|
||||
export {
|
||||
estimatePackageMultipartUploadBytes,
|
||||
getPackageMultipartSizeError,
|
||||
isPackageMultipartUploadTooLarge,
|
||||
MAX_PACKAGE_MULTIPART_BYTES,
|
||||
type PackageMultipartUploadField,
|
||||
type PackageMultipartUploadPart,
|
||||
} from "clawhub-schema";
|
||||
|
||||
export const MAX_PUBLISH_TOTAL_BYTES = 50 * 1024 * 1024;
|
||||
export const MAX_PUBLISH_FILE_BYTES = 10 * 1024 * 1024;
|
||||
export const MAX_CLAWPACK_BYTES = 120 * 1024 * 1024;
|
||||
export const MAX_CLAWPACK_BYTES = MAX_PACKAGE_CLAWPACK_BYTES;
|
||||
|
||||
type SizedPathLike = {
|
||||
path: string;
|
||||
|
||||
@@ -10,7 +10,6 @@ import {
|
||||
getPackageReleaseScanBackfillBatchInternal,
|
||||
getByName,
|
||||
list,
|
||||
publishPackage,
|
||||
publishPackageForTrustedPublisherInternal,
|
||||
publishPackageForUserInternal,
|
||||
listPackageReportsInternal,
|
||||
@@ -237,14 +236,6 @@ const searchForViewerInternalHandler = (
|
||||
Array<{ package: { name: string } }>
|
||||
>
|
||||
)._handler;
|
||||
const publishPackageHandler = (
|
||||
publishPackage as unknown as WrappedHandler<
|
||||
{
|
||||
payload: unknown;
|
||||
},
|
||||
unknown
|
||||
>
|
||||
)._handler;
|
||||
const publishPackageForUserInternalHandler = (
|
||||
publishPackageForUserInternal as unknown as WrappedHandler<
|
||||
{
|
||||
@@ -6300,20 +6291,6 @@ describe("packages public queries", () => {
|
||||
expect(result).toEqual([expect.objectContaining({ name: "demo-plugin" })]);
|
||||
});
|
||||
|
||||
it("requires auth inside the public publish action", async () => {
|
||||
await expect(
|
||||
publishPackageHandler({ runQuery: vi.fn(), runMutation: vi.fn() } as never, {
|
||||
payload: {
|
||||
name: "demo-plugin",
|
||||
family: "bundle-plugin",
|
||||
version: "1.0.0",
|
||||
changelog: "init",
|
||||
files: [],
|
||||
},
|
||||
}),
|
||||
).rejects.toThrow("Unauthorized");
|
||||
});
|
||||
|
||||
it("records package reports for moderation", async () => {
|
||||
const insert = vi.fn(async (table: string) =>
|
||||
table === "packageReports" ? "packageReports:1" : "auditLogs:1",
|
||||
|
||||
+8
-25
@@ -1,5 +1,5 @@
|
||||
import {
|
||||
PackagePublishRequestSchema,
|
||||
ServerPackagePublishRequestSchema,
|
||||
getPackageScopeOwnerMismatch,
|
||||
isPluginCategorySlug,
|
||||
parseArk,
|
||||
@@ -11,7 +11,7 @@ import {
|
||||
type PackageModerationQueueStatus,
|
||||
type PackageOfficialMigrationListPhase,
|
||||
type PackageOfficialMigrationPhase,
|
||||
type PackagePublishRequest,
|
||||
type ServerPackagePublishRequest,
|
||||
type PackageVerificationTier,
|
||||
} from "clawhub-schema";
|
||||
import { paginationOptsValidator } from "convex/server";
|
||||
@@ -33,7 +33,6 @@ import {
|
||||
assertModerator,
|
||||
getOptionalActiveAuthUserId,
|
||||
requireUser,
|
||||
requireUserFromAction,
|
||||
} from "./lib/access";
|
||||
import {
|
||||
assertArtifactAppealFinalAction,
|
||||
@@ -5027,9 +5026,9 @@ function buildGitHubActionsPublishActor(
|
||||
}
|
||||
|
||||
function resolveTrustedPublishSource(
|
||||
payload: PackagePublishRequest,
|
||||
payload: ServerPackagePublishRequest,
|
||||
publishToken: Doc<"packagePublishTokens">,
|
||||
): PackagePublishRequest["source"] {
|
||||
): ServerPackagePublishRequest["source"] {
|
||||
const source = payload.source;
|
||||
if (source && source.kind !== "github") {
|
||||
throw new ConvexError("Trusted publishes only support GitHub source metadata");
|
||||
@@ -5081,11 +5080,11 @@ async function publishPackageImpl(
|
||||
auth: PackagePublishAuthContext,
|
||||
rawPayload: unknown,
|
||||
) {
|
||||
const payload = parseArk(
|
||||
PackagePublishRequestSchema,
|
||||
const payload = parseArk<ServerPackagePublishRequest>(
|
||||
ServerPackagePublishRequestSchema,
|
||||
rawPayload,
|
||||
"Package publish payload",
|
||||
) as PackagePublishRequest;
|
||||
);
|
||||
if (payload.family === "skill") {
|
||||
throw new ConvexError("Skill packages must use the skills publish flow");
|
||||
}
|
||||
@@ -5204,7 +5203,7 @@ async function publishPackageImpl(
|
||||
}
|
||||
|
||||
const displayName = payload.displayName?.trim() || name;
|
||||
const files = normalizePublishFiles(payload.files as never);
|
||||
const files = normalizePublishFiles(payload.files);
|
||||
if (payload.artifact?.kind !== "npm-pack") {
|
||||
const oversizedFile = findOversizedPublishFile(files);
|
||||
if (oversizedFile) {
|
||||
@@ -5459,14 +5458,6 @@ async function publishPackageImpl(
|
||||
return publishResult;
|
||||
}
|
||||
|
||||
export const publishPackage = action({
|
||||
args: { payload: v.any() },
|
||||
handler: async (ctx, args) => {
|
||||
const { userId } = await requireUserFromAction(ctx);
|
||||
return await publishPackageImpl(ctx, { kind: "user", actorUserId: userId }, args.payload);
|
||||
},
|
||||
});
|
||||
|
||||
export const publishPackageForUserInternal = internalAction({
|
||||
args: {
|
||||
actorUserId: v.id("users"),
|
||||
@@ -5509,14 +5500,6 @@ export const publishPackageForTrustedPublisherInternal = internalAction({
|
||||
},
|
||||
});
|
||||
|
||||
export const publishRelease = action({
|
||||
args: { payload: v.any() },
|
||||
handler: async (ctx, args) => {
|
||||
const { userId } = await requireUserFromAction(ctx);
|
||||
return await publishPackageImpl(ctx, { kind: "user", actorUserId: userId }, args.payload);
|
||||
},
|
||||
});
|
||||
|
||||
export const reservePackageNameInternal = internalMutation({
|
||||
args: {
|
||||
actorUserId: v.id("users"),
|
||||
|
||||
@@ -1224,6 +1224,16 @@ const packagePublishTokens = defineTable({
|
||||
.index("by_package", ["packageId", "version", "createdAt"])
|
||||
.index("by_package_revoked_created", ["packageId", "revokedAt", "createdAt"]);
|
||||
|
||||
const packagePublishUploadTickets = defineTable({
|
||||
kind: v.union(v.literal("user"), v.literal("github-actions")),
|
||||
userId: v.optional(v.id("users")),
|
||||
publishTokenId: v.optional(v.id("packagePublishTokens")),
|
||||
createdAt: v.number(),
|
||||
expiresAt: v.number(),
|
||||
usedAt: v.optional(v.number()),
|
||||
storageId: v.optional(v.id("_storage")),
|
||||
});
|
||||
|
||||
const packageSearchDigest = defineTable({
|
||||
packageId: v.id("packages"),
|
||||
name: v.string(),
|
||||
@@ -2144,6 +2154,7 @@ export default defineSchema({
|
||||
packageStatEvents,
|
||||
packageTrustedPublishers,
|
||||
packagePublishTokens,
|
||||
packagePublishUploadTickets,
|
||||
packageBadges,
|
||||
packageSearchDigest,
|
||||
packageCapabilitySearchDigest,
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
/* @vitest-environment node */
|
||||
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { consumePackagePublishUploadTicketInternal } from "./uploads";
|
||||
|
||||
type ConsumeArgs = {
|
||||
uploadTicket: string;
|
||||
storageId: string;
|
||||
auth: { kind: "user"; userId: string } | { kind: "github-actions"; publishTokenId: string };
|
||||
};
|
||||
|
||||
type WrappedHandler<TArgs> = {
|
||||
_handler: (ctx: unknown, args: TArgs) => Promise<void>;
|
||||
};
|
||||
|
||||
const consumeHandler = (
|
||||
consumePackagePublishUploadTicketInternal as unknown as WrappedHandler<ConsumeArgs>
|
||||
)._handler;
|
||||
|
||||
function makeCtx(ticket: Record<string, unknown> | null, storage: Record<string, unknown> | null) {
|
||||
return {
|
||||
db: {
|
||||
get: vi.fn(async () => ticket),
|
||||
insert: vi.fn(),
|
||||
normalizeId: vi.fn(),
|
||||
patch: vi.fn(),
|
||||
query: vi.fn(),
|
||||
replace: vi.fn(),
|
||||
delete: vi.fn(),
|
||||
system: {
|
||||
get: vi.fn(async () => storage),
|
||||
query: vi.fn(),
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("package publish upload tickets", () => {
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("consumes a fresh upload ticket for the same user", async () => {
|
||||
vi.spyOn(Date, "now").mockReturnValue(2_000);
|
||||
const ctx = makeCtx(
|
||||
{
|
||||
_id: "packagePublishUploadTickets:1",
|
||||
kind: "user",
|
||||
userId: "users:1",
|
||||
createdAt: 1_000,
|
||||
expiresAt: 10_000,
|
||||
},
|
||||
{ _id: "storage:1", _creationTime: 1_500 },
|
||||
);
|
||||
|
||||
await consumeHandler(ctx, {
|
||||
uploadTicket: "packagePublishUploadTickets:1",
|
||||
storageId: "storage:1",
|
||||
auth: { kind: "user", userId: "users:1" },
|
||||
});
|
||||
|
||||
expect(ctx.db.system.get).toHaveBeenCalledWith("_storage", "storage:1");
|
||||
expect(ctx.db.patch).toHaveBeenCalledWith("packagePublishUploadTickets:1", {
|
||||
usedAt: 2_000,
|
||||
storageId: "storage:1",
|
||||
});
|
||||
});
|
||||
|
||||
it("allows retrying a used upload ticket for the same user and storage id", async () => {
|
||||
vi.spyOn(Date, "now").mockReturnValue(3_000);
|
||||
const ctx = makeCtx(
|
||||
{
|
||||
_id: "packagePublishUploadTickets:1",
|
||||
kind: "user",
|
||||
userId: "users:1",
|
||||
createdAt: 1_000,
|
||||
expiresAt: 10_000,
|
||||
usedAt: 2_000,
|
||||
storageId: "storage:1",
|
||||
},
|
||||
{ _id: "storage:1", _creationTime: 1_500 },
|
||||
);
|
||||
|
||||
await consumeHandler(ctx, {
|
||||
uploadTicket: "packagePublishUploadTickets:1",
|
||||
storageId: "storage:1",
|
||||
auth: { kind: "user", userId: "users:1" },
|
||||
});
|
||||
|
||||
expect(ctx.db.patch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects upload tickets from another auth context", async () => {
|
||||
vi.spyOn(Date, "now").mockReturnValue(2_000);
|
||||
const ctx = makeCtx(
|
||||
{
|
||||
_id: "packagePublishUploadTickets:1",
|
||||
kind: "user",
|
||||
userId: "users:1",
|
||||
createdAt: 1_000,
|
||||
expiresAt: 10_000,
|
||||
},
|
||||
{ _id: "storage:1", _creationTime: 1_500 },
|
||||
);
|
||||
|
||||
await expect(
|
||||
consumeHandler(ctx, {
|
||||
uploadTicket: "packagePublishUploadTickets:1",
|
||||
storageId: "storage:1",
|
||||
auth: { kind: "user", userId: "users:2" },
|
||||
}),
|
||||
).rejects.toThrow("Package tarball upload ticket does not match this publish token");
|
||||
|
||||
expect(ctx.db.patch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects storage created before the upload ticket", async () => {
|
||||
vi.spyOn(Date, "now").mockReturnValue(2_000);
|
||||
const ctx = makeCtx(
|
||||
{
|
||||
_id: "packagePublishUploadTickets:1",
|
||||
kind: "github-actions",
|
||||
publishTokenId: "packagePublishTokens:1",
|
||||
createdAt: 1_000,
|
||||
expiresAt: 10_000,
|
||||
},
|
||||
{ _id: "storage:1", _creationTime: 999 },
|
||||
);
|
||||
|
||||
await expect(
|
||||
consumeHandler(ctx, {
|
||||
uploadTicket: "packagePublishUploadTickets:1",
|
||||
storageId: "storage:1",
|
||||
auth: { kind: "github-actions", publishTokenId: "packagePublishTokens:1" },
|
||||
}),
|
||||
).rejects.toThrow("Package tarball upload must be created after its upload ticket");
|
||||
|
||||
expect(ctx.db.patch).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
+74
-2
@@ -2,6 +2,8 @@ import { v } from "convex/values";
|
||||
import { internalMutation, mutation } from "./functions";
|
||||
import { requireUser } from "./lib/access";
|
||||
|
||||
const PACKAGE_PUBLISH_UPLOAD_TICKET_TTL_MS = 15 * 60_000;
|
||||
|
||||
export const generateUploadUrl = mutation({
|
||||
args: {},
|
||||
handler: async (ctx) => {
|
||||
@@ -10,11 +12,81 @@ export const generateUploadUrl = mutation({
|
||||
},
|
||||
});
|
||||
|
||||
export const generateUploadUrlForUserInternal = internalMutation({
|
||||
export const createPackagePublishUploadForUserInternal = internalMutation({
|
||||
args: { userId: v.id("users") },
|
||||
handler: async (ctx, args) => {
|
||||
const user = await ctx.db.get(args.userId);
|
||||
if (!user || user.deletedAt || user.deactivatedAt) throw new Error("User not found");
|
||||
return ctx.storage.generateUploadUrl();
|
||||
const now = Date.now();
|
||||
const uploadTicket = await ctx.db.insert("packagePublishUploadTickets", {
|
||||
kind: "user",
|
||||
userId: args.userId,
|
||||
createdAt: now,
|
||||
expiresAt: now + PACKAGE_PUBLISH_UPLOAD_TICKET_TTL_MS,
|
||||
});
|
||||
const uploadUrl = await ctx.storage.generateUploadUrl();
|
||||
return { uploadUrl, uploadTicket };
|
||||
},
|
||||
});
|
||||
|
||||
export const createPackagePublishUploadForTokenInternal = internalMutation({
|
||||
args: { publishTokenId: v.id("packagePublishTokens") },
|
||||
handler: async (ctx, args) => {
|
||||
const publishToken = await ctx.db.get(args.publishTokenId);
|
||||
const now = Date.now();
|
||||
if (!publishToken || publishToken.revokedAt || publishToken.expiresAt <= now) {
|
||||
throw new Error("Trusted publish token is missing or expired");
|
||||
}
|
||||
const uploadTicket = await ctx.db.insert("packagePublishUploadTickets", {
|
||||
kind: "github-actions",
|
||||
publishTokenId: args.publishTokenId,
|
||||
createdAt: now,
|
||||
expiresAt: now + PACKAGE_PUBLISH_UPLOAD_TICKET_TTL_MS,
|
||||
});
|
||||
const uploadUrl = await ctx.storage.generateUploadUrl();
|
||||
return { uploadUrl, uploadTicket };
|
||||
},
|
||||
});
|
||||
|
||||
export const consumePackagePublishUploadTicketInternal = internalMutation({
|
||||
args: {
|
||||
uploadTicket: v.id("packagePublishUploadTickets"),
|
||||
storageId: v.id("_storage"),
|
||||
auth: v.union(
|
||||
v.object({ kind: v.literal("user"), userId: v.id("users") }),
|
||||
v.object({
|
||||
kind: v.literal("github-actions"),
|
||||
publishTokenId: v.id("packagePublishTokens"),
|
||||
}),
|
||||
),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const ticket = await ctx.db.get(args.uploadTicket);
|
||||
const now = Date.now();
|
||||
if (!ticket || ticket.expiresAt <= now) {
|
||||
throw new Error("Package tarball upload ticket is missing or expired");
|
||||
}
|
||||
if (
|
||||
args.auth.kind === "user"
|
||||
? ticket.kind !== "user" || ticket.userId !== args.auth.userId
|
||||
: ticket.kind !== "github-actions" || ticket.publishTokenId !== args.auth.publishTokenId
|
||||
) {
|
||||
throw new Error("Package tarball upload ticket does not match this publish token");
|
||||
}
|
||||
if (ticket.usedAt) {
|
||||
if (ticket.storageId === args.storageId) return;
|
||||
throw new Error("Package tarball upload ticket was already used");
|
||||
}
|
||||
|
||||
const metadata = await ctx.db.system.get("_storage", args.storageId);
|
||||
if (!metadata) throw new Error("Package tarball upload no longer exists");
|
||||
if (metadata._creationTime < ticket.createdAt) {
|
||||
throw new Error("Package tarball upload must be created after its upload ticket");
|
||||
}
|
||||
|
||||
await ctx.db.patch(ticket._id, {
|
||||
usedAt: now,
|
||||
storageId: args.storageId,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
+14
-2
@@ -1225,8 +1225,16 @@ Publishes a new version.
|
||||
Publishes a code-plugin or bundle-plugin release.
|
||||
|
||||
- Requires Bearer token auth.
|
||||
- Preferred: `multipart/form-data` with `payload` JSON + `files[]` blobs.
|
||||
- JSON body with `files` (storageId-based) is also accepted.
|
||||
- Requires `multipart/form-data`.
|
||||
- Allowed form fields are `payload`, repeated `files` blobs, or one `clawpack`
|
||||
tarball reference. `clawpack` may be a `.tgz` blob or a storage id returned by
|
||||
the upload-url flow. Staged storage-id publishes must also include the
|
||||
`clawpackUploadTicket` returned with that upload URL.
|
||||
- Use either `files` or `clawpack`, never both in the same request.
|
||||
- JSON bodies and caller-supplied `payload.files` / `payload.artifact`
|
||||
metadata are rejected.
|
||||
- Direct multipart publish requests are capped at 18MB. ClawPack tarballs may
|
||||
use the upload-url flow up to the 120MB tarball cap.
|
||||
- Optional payload field: `ownerHandle`. When present, only admins may publish on behalf of that owner.
|
||||
|
||||
Validation highlights:
|
||||
@@ -1479,6 +1487,10 @@ Still supported for older CLI versions:
|
||||
|
||||
See `DEPRECATIONS.md` for removal plan.
|
||||
|
||||
`POST /api/cli/upload-url` returns `uploadUrl` and `uploadTicket`. Package
|
||||
publishes that stage a ClawPack tarball must send the resulting storage id as
|
||||
`clawpack` and the returned ticket as `clawpackUploadTicket`.
|
||||
|
||||
## Registry discovery (`/.well-known/clawhub.json`)
|
||||
|
||||
The CLI can discover registry/auth settings from the site:
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
/* @vitest-environment node */
|
||||
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { gzipSync, zipSync } from "fflate";
|
||||
@@ -67,6 +67,10 @@ async function makeTmpWorkdir() {
|
||||
return await mkdtemp(join(tmpdir(), "clawhub-package-"));
|
||||
}
|
||||
|
||||
async function listClawPackTempDirs() {
|
||||
return new Set((await readdir(tmpdir())).filter((name) => name.startsWith("clawhub-clawpack-")));
|
||||
}
|
||||
|
||||
function runGit(cwd: string, args: string[]) {
|
||||
const result = spawnSync("git", ["-C", cwd, ...args], {
|
||||
encoding: "utf8",
|
||||
@@ -143,8 +147,8 @@ function tarOctal(value: number, width: number) {
|
||||
return value.toString(8).padStart(width - 1, "0") + "\0";
|
||||
}
|
||||
|
||||
function tarFile(path: string, content: string) {
|
||||
const bytes = new TextEncoder().encode(content);
|
||||
function tarFile(path: string, content: string | Uint8Array) {
|
||||
const bytes = typeof content === "string" ? new TextEncoder().encode(content) : content;
|
||||
const header = new Uint8Array(TAR_BLOCK_SIZE);
|
||||
writeTarString(header, 0, 100, path);
|
||||
writeTarString(header, 100, 8, tarOctal(0o644, 8));
|
||||
@@ -167,7 +171,7 @@ function tarFile(path: string, content: string) {
|
||||
return [header, body];
|
||||
}
|
||||
|
||||
function npmPackFixture(files: Record<string, string>) {
|
||||
function npmPackFixture(files: Record<string, string | Uint8Array>) {
|
||||
const parts: Uint8Array[] = [];
|
||||
for (const [path, content] of Object.entries(files)) {
|
||||
parts.push(...tarFile(path, content));
|
||||
@@ -1274,6 +1278,65 @@ describe("package commands", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("stages ClawPack tarballs over the multipart publish budget", async () => {
|
||||
const workdir = await makeTmpWorkdir();
|
||||
try {
|
||||
const packName = "oversized-plugin-1.0.0.tgz";
|
||||
const packBytes = npmPackFixture({
|
||||
"package/package.json": makeCodePluginPackageJson({
|
||||
name: "@scope/oversized-plugin",
|
||||
displayName: "Oversized Plugin",
|
||||
version: "1.0.0",
|
||||
}),
|
||||
"package/openclaw.plugin.json": JSON.stringify({ id: "oversized.plugin" }),
|
||||
"package/dist/index.js": "export const demo = true;\n",
|
||||
"package/dist/model.bin": randomBytes(24 * 1024 * 1024),
|
||||
});
|
||||
expect(packBytes.byteLength).toBeGreaterThan(18 * 1024 * 1024);
|
||||
await writeFile(join(workdir, packName), packBytes);
|
||||
httpMocks.apiRequest.mockResolvedValueOnce({
|
||||
uploadUrl: "https://upload.local",
|
||||
uploadTicket: "uploadTickets:clawpack",
|
||||
});
|
||||
httpMocks.uploadBinary.mockResolvedValueOnce({ storageId: "storage:clawpack" });
|
||||
httpMocks.apiRequestForm.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
packageId: "pkg_1",
|
||||
releaseId: "rel_1",
|
||||
});
|
||||
|
||||
await cmdPublishPackage(makeOpts(workdir), packName, {
|
||||
sourceRepo: "openclaw/oversized-plugin",
|
||||
sourceCommit: "abc123",
|
||||
});
|
||||
|
||||
expect(httpMocks.apiRequest).toHaveBeenCalledWith(
|
||||
"https://clawhub.ai",
|
||||
{
|
||||
method: "POST",
|
||||
path: "/api/cli/upload-url",
|
||||
token: "tkn",
|
||||
},
|
||||
expect.anything(),
|
||||
);
|
||||
expect(httpMocks.uploadBinary).toHaveBeenCalledWith(
|
||||
{
|
||||
url: "https://upload.local",
|
||||
bytes: expect.any(Uint8Array),
|
||||
contentType: "application/octet-stream",
|
||||
retryCount: 5,
|
||||
},
|
||||
expect.anything(),
|
||||
);
|
||||
expect(getPublishForm().get("clawpack")).toBe("storage:clawpack");
|
||||
expect(getPublishForm().get("clawpackUploadTicket")).toBe("uploadTickets:clawpack");
|
||||
expect(getPublishPayload()).not.toHaveProperty("artifact");
|
||||
expect(getPublishPayload()).not.toHaveProperty("files");
|
||||
} finally {
|
||||
await rm(workdir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("packs a plugin folder through npm pack and validates the ClawPack", async () => {
|
||||
const workdir = await makeTmpWorkdir();
|
||||
try {
|
||||
@@ -1315,6 +1378,88 @@ describe("package commands", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("packs local ClawPacks over the multipart publish upload budget", async () => {
|
||||
const workdir = await makeTmpWorkdir();
|
||||
try {
|
||||
const folder = join(workdir, "demo-heavy-plugin");
|
||||
await mkdir(join(folder, "dist"), { recursive: true });
|
||||
await mkdir(join(workdir, "packs"), { recursive: true });
|
||||
await writeFile(
|
||||
join(folder, "package.json"),
|
||||
makeCodePluginPackageJson({
|
||||
name: "demo-heavy-plugin",
|
||||
displayName: "Demo Heavy Plugin",
|
||||
version: "1.0.0",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(
|
||||
join(folder, "openclaw.plugin.json"),
|
||||
JSON.stringify({ id: "demo.heavy.plugin" }),
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(join(folder, "dist", "index.js"), "export const demo = true;\n", "utf8");
|
||||
await writeFile(join(folder, "dist", "model.bin"), randomBytes(24 * 1024 * 1024));
|
||||
|
||||
await cmdPackPackage(makeOpts(workdir), "demo-heavy-plugin", {
|
||||
packDestination: "packs",
|
||||
});
|
||||
|
||||
const packPath = join(workdir, "packs", "demo-heavy-plugin-1.0.0.tgz");
|
||||
const packed = await readFile(packPath);
|
||||
expect(packed.byteLength).toBeGreaterThan(18 * 1024 * 1024);
|
||||
expect(parseClawPack(new Uint8Array(packed)).packageName).toBe("demo-heavy-plugin");
|
||||
} finally {
|
||||
await rm(workdir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("cleans generated ClawPack temp dirs after staged publish failure", async () => {
|
||||
const workdir = await makeTmpWorkdir();
|
||||
const beforeTempDirs = await listClawPackTempDirs();
|
||||
try {
|
||||
const folder = join(workdir, "demo-heavy-plugin");
|
||||
await mkdir(join(folder, "dist"), { recursive: true });
|
||||
await writeFile(
|
||||
join(folder, "package.json"),
|
||||
makeCodePluginPackageJson({
|
||||
name: "demo-heavy-plugin",
|
||||
displayName: "Demo Heavy Plugin",
|
||||
version: "1.0.0",
|
||||
repository: "https://github.com/openclaw/demo-heavy-plugin.git",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(
|
||||
join(folder, "openclaw.plugin.json"),
|
||||
JSON.stringify({ id: "demo.heavy.plugin" }),
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(join(folder, "dist", "index.js"), "export const demo = true;\n", "utf8");
|
||||
await writeFile(join(folder, "dist", "model.bin"), randomBytes(24 * 1024 * 1024));
|
||||
httpMocks.apiRequest.mockResolvedValueOnce({
|
||||
uploadUrl: "https://upload.local",
|
||||
uploadTicket: "uploadTickets:clawpack",
|
||||
});
|
||||
httpMocks.uploadBinary.mockResolvedValueOnce({ storageId: "storage:clawpack" });
|
||||
httpMocks.apiRequestForm.mockRejectedValueOnce(new Error("Registry rejected upload"));
|
||||
|
||||
await expect(
|
||||
cmdPublishPackage(makeOpts(workdir), "demo-heavy-plugin", {
|
||||
sourceRepo: "openclaw/demo-heavy-plugin",
|
||||
sourceCommit: "abc123",
|
||||
}),
|
||||
).rejects.toThrow("Registry rejected upload");
|
||||
expect(getPublishForm().get("clawpack")).toBe("storage:clawpack");
|
||||
expect(getPublishForm().get("clawpackUploadTicket")).toBe("uploadTickets:clawpack");
|
||||
|
||||
const afterTempDirs = await listClawPackTempDirs();
|
||||
expect([...afterTempDirs].filter((name) => !beforeTempDirs.has(name))).toEqual([]);
|
||||
} finally {
|
||||
await rm(workdir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects a code plugin ClawPack with TypeScript entries and no compiled runtime", async () => {
|
||||
const workdir = await makeTmpWorkdir();
|
||||
try {
|
||||
|
||||
@@ -7,10 +7,20 @@ import ignore from "ignore";
|
||||
import mime from "mime";
|
||||
import semver from "semver";
|
||||
import { parseClawPack } from "../../clawpack.js";
|
||||
import { apiRequest, apiRequestForm, fetchBinary, fetchText, registryUrl } from "../../http.js";
|
||||
import {
|
||||
apiRequest,
|
||||
apiRequestForm,
|
||||
fetchBinary,
|
||||
fetchText,
|
||||
registryUrl,
|
||||
uploadBinary,
|
||||
} from "../../http.js";
|
||||
import {
|
||||
ApiCliUploadUrlResponseSchema,
|
||||
ApiRoutes,
|
||||
LegacyApiRoutes,
|
||||
ApiV1DeleteResponseSchema,
|
||||
ApiUploadFileResponseSchema,
|
||||
ApiV1PackageArtifactResponseSchema,
|
||||
ApiV1PackageListResponseSchema,
|
||||
ApiV1PackageModerationStatusResponseSchema,
|
||||
@@ -24,6 +34,10 @@ import {
|
||||
ApiV1PackageVersionListResponseSchema,
|
||||
ApiV1PackageVersionResponseSchema,
|
||||
ApiV1PublishTokenMintResponseSchema,
|
||||
estimatePackageMultipartUploadBytes,
|
||||
getPackageMultipartSizeError,
|
||||
MAX_PACKAGE_CLAWPACK_BYTES,
|
||||
MAX_PACKAGE_MULTIPART_BYTES,
|
||||
normalizeOpenClawExternalPluginCompatibility,
|
||||
type PackageArtifactSummary,
|
||||
type PackageCapabilitySummary,
|
||||
@@ -50,7 +64,6 @@ const DOT_DIR = ".clawhub";
|
||||
const LEGACY_DOT_DIR = ".clawdhub";
|
||||
const DOT_IGNORE = ".clawhubignore";
|
||||
const LEGACY_DOT_IGNORE = ".clawdhubignore";
|
||||
const MAX_CLAWPACK_BYTES = 120 * 1024 * 1024;
|
||||
const PACKAGE_PUBLISH_RETRY_COUNT = 5;
|
||||
|
||||
type PackageInspectOptions = {
|
||||
@@ -598,7 +611,6 @@ async function createClawPackFromFolder(options: {
|
||||
|
||||
const packPath = resolve(options.packDestination, filename);
|
||||
const bytes = new Uint8Array(await readFile(packPath));
|
||||
assertClawPackSize(bytes.byteLength, basename(packPath));
|
||||
const parsed = parseClawPack(bytes);
|
||||
return {
|
||||
path: packPath,
|
||||
@@ -665,14 +677,26 @@ export async function cmdPublishPackage(
|
||||
spinner,
|
||||
});
|
||||
const form = new FormData();
|
||||
form.set("payload", JSON.stringify(plan.payload));
|
||||
const payloadJson = JSON.stringify(plan.payload);
|
||||
form.set("payload", payloadJson);
|
||||
|
||||
if (plan.clawpackOnDisk) {
|
||||
if (spinner) spinner.text = `Uploading ${plan.clawpackOnDisk.relPath}`;
|
||||
const blob = new Blob([Buffer.from(plan.clawpackOnDisk.bytes)], {
|
||||
type: "application/octet-stream",
|
||||
});
|
||||
form.append("clawpack", blob, plan.clawpackOnDisk.relPath);
|
||||
if (isPackageMultipartTooLarge(payloadJson, "clawpack", [plan.clawpackOnDisk])) {
|
||||
const staged = await uploadClawPackToStorage(
|
||||
registry,
|
||||
publishToken,
|
||||
plan.clawpackOnDisk,
|
||||
spinner,
|
||||
);
|
||||
form.set("clawpack", staged.storageId);
|
||||
form.set("clawpackUploadTicket", staged.uploadTicket);
|
||||
} else {
|
||||
if (spinner) spinner.text = `Uploading ${plan.clawpackOnDisk.relPath}`;
|
||||
const blob = new Blob([Buffer.from(plan.clawpackOnDisk.bytes)], {
|
||||
type: "application/octet-stream",
|
||||
});
|
||||
form.append("clawpack", blob, plan.clawpackOnDisk.relPath);
|
||||
}
|
||||
} else {
|
||||
let index = 0;
|
||||
for (const file of plan.filesOnDisk) {
|
||||
@@ -1551,12 +1575,66 @@ function packageJsonString(value: Record<string, unknown> | null, key: string):
|
||||
return typeof candidate === "string" && candidate.trim() ? candidate.trim() : undefined;
|
||||
}
|
||||
|
||||
function assertClawPackSize(size: number, label: string) {
|
||||
if (size > MAX_CLAWPACK_BYTES) {
|
||||
fail(`ClawPack "${label}" exceeds 120MB limit`);
|
||||
function assertPackageMultipartSize(
|
||||
payloadJson: string,
|
||||
fileFieldName: "files" | "clawpack",
|
||||
files: PackageFile[],
|
||||
) {
|
||||
if (isPackageMultipartTooLarge(payloadJson, fileFieldName, files)) {
|
||||
fail(getPackageMultipartSizeError());
|
||||
}
|
||||
}
|
||||
|
||||
function getClawPackSizeError(path: string) {
|
||||
return `ClawPack "${path}" exceeds 120MB limit`;
|
||||
}
|
||||
|
||||
function isPackageMultipartTooLarge(
|
||||
payloadJson: string,
|
||||
fileFieldName: "files" | "clawpack",
|
||||
files: PackageFile[],
|
||||
) {
|
||||
return (
|
||||
estimatePackageMultipartUploadBytes({
|
||||
payloadJson,
|
||||
fileFieldName,
|
||||
files: files.map((file) => ({
|
||||
name: file.relPath,
|
||||
size: file.bytes.byteLength,
|
||||
type: file.contentType,
|
||||
})),
|
||||
}) > MAX_PACKAGE_MULTIPART_BYTES
|
||||
);
|
||||
}
|
||||
|
||||
async function uploadClawPackToStorage(
|
||||
registry: string,
|
||||
publishToken: string,
|
||||
file: PackageFile,
|
||||
spinner: ReturnType<typeof createSpinner> | null,
|
||||
) {
|
||||
if (spinner) spinner.text = `Uploading ${file.relPath}`;
|
||||
const { uploadUrl, uploadTicket } = await apiRequest(
|
||||
registry,
|
||||
{
|
||||
method: "POST",
|
||||
path: LegacyApiRoutes.cliUploadUrl,
|
||||
token: publishToken,
|
||||
},
|
||||
ApiCliUploadUrlResponseSchema,
|
||||
);
|
||||
const result = await uploadBinary(
|
||||
{
|
||||
url: uploadUrl,
|
||||
bytes: file.bytes,
|
||||
contentType: file.contentType ?? "application/octet-stream",
|
||||
retryCount: PACKAGE_PUBLISH_RETRY_COUNT,
|
||||
},
|
||||
ApiUploadFileResponseSchema,
|
||||
);
|
||||
return { storageId: result.storageId, uploadTicket };
|
||||
}
|
||||
|
||||
const REAL_BUNDLE_MANIFESTS = [
|
||||
{ path: ".codex-plugin/plugin.json", format: "codex" },
|
||||
{ path: ".claude-plugin/plugin.json", format: "claude" },
|
||||
@@ -1666,11 +1744,13 @@ async function preparePackagePublishPlan(
|
||||
}
|
||||
} else {
|
||||
const folderStat = await stat(folder).catch(() => null);
|
||||
if (!folderStat) fail("Path must be a folder or ClawPack .tgz");
|
||||
if (!folderStat) fail("Path must be a folder or package tarball .tgz");
|
||||
if (folderStat.isFile()) {
|
||||
if (!folder.endsWith(".tgz")) fail("ClawPack publish files must end in .tgz");
|
||||
if (!folder.endsWith(".tgz")) fail("Package publish files must end in .tgz");
|
||||
const bytes = new Uint8Array(await readFile(folder));
|
||||
assertClawPackSize(bytes.byteLength, basename(folder));
|
||||
if (bytes.byteLength > MAX_PACKAGE_CLAWPACK_BYTES) {
|
||||
fail(getClawPackSizeError(basename(folder)));
|
||||
}
|
||||
parsedClawpack = parseClawPack(bytes);
|
||||
clawpackOnDisk = {
|
||||
relPath: basename(folder),
|
||||
@@ -1678,7 +1758,7 @@ async function preparePackagePublishPlan(
|
||||
contentType: "application/octet-stream",
|
||||
};
|
||||
} else if (!folderStat.isDirectory()) {
|
||||
fail("Path must be a folder or ClawPack .tgz");
|
||||
fail("Path must be a folder or package tarball .tgz");
|
||||
}
|
||||
|
||||
const localGitInfo = folderStat.isDirectory() ? resolveLocalGitInfo(folder) : null;
|
||||
@@ -1782,7 +1862,9 @@ async function preparePackagePublishPlan(
|
||||
contentType: mime.getType(entry.path) ?? "application/octet-stream",
|
||||
}));
|
||||
}
|
||||
|
||||
const totalBytes = clawpackOnDisk
|
||||
? clawpackOnDisk.bytes.byteLength
|
||||
: filesOnDisk.reduce((sum, file) => sum + file.bytes.byteLength, 0);
|
||||
const payload: PackagePublishPayload = {
|
||||
name,
|
||||
displayName,
|
||||
@@ -1804,6 +1886,18 @@ async function preparePackagePublishPlan(
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
try {
|
||||
if (clawpackOnDisk) {
|
||||
if (clawpackOnDisk.bytes.byteLength > MAX_PACKAGE_CLAWPACK_BYTES) {
|
||||
fail(getClawPackSizeError(clawpackOnDisk.relPath));
|
||||
}
|
||||
} else {
|
||||
assertPackageMultipartSize(JSON.stringify(payload), "files", filesOnDisk);
|
||||
}
|
||||
} catch (error) {
|
||||
await cleanup?.();
|
||||
throw error;
|
||||
}
|
||||
const sourceLabel = describePublishSource(sourceForFetch, source, folder);
|
||||
|
||||
return {
|
||||
@@ -1826,9 +1920,7 @@ async function preparePackagePublishPlan(
|
||||
version,
|
||||
...(source?.commit ? { commit: source.commit } : {}),
|
||||
files: filesOnDisk.length,
|
||||
totalBytes: clawpackOnDisk
|
||||
? clawpackOnDisk.bytes.byteLength
|
||||
: filesOnDisk.reduce((sum, file) => sum + file.bytes.byteLength, 0),
|
||||
totalBytes,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -52,6 +52,12 @@ type FormRequestArgs =
|
||||
| { method: "POST"; url: string; token?: string; form: FormData; retryCount?: number };
|
||||
|
||||
type TextRequestArgs = { path: string; token?: string } | { url: string; token?: string };
|
||||
type BinaryUploadArgs = {
|
||||
url: string;
|
||||
bytes: Uint8Array;
|
||||
contentType?: string;
|
||||
retryCount?: number;
|
||||
};
|
||||
|
||||
type HeaderSource = Headers | Record<string, string> | null | undefined;
|
||||
|
||||
@@ -91,6 +97,7 @@ type HttpClient = {
|
||||
apiRequestForm<T>(registry: string, args: FormRequestArgs, schema: ArkValidator<T>): Promise<T>;
|
||||
fetchText(registry: string, args: TextRequestArgs): Promise<string>;
|
||||
fetchBinary(registry: string, args: TextRequestArgs): Promise<Uint8Array>;
|
||||
uploadBinary<T>(args: BinaryUploadArgs, schema?: ArkValidator<T>): Promise<T>;
|
||||
downloadZip(
|
||||
registry: string,
|
||||
args: { slug: string; version?: string; token?: string },
|
||||
@@ -264,6 +271,41 @@ export function createHttpClient(options: HttpClientOptions = {}): HttpClient {
|
||||
});
|
||||
}
|
||||
|
||||
async function uploadBinaryRequest<T>(
|
||||
args: BinaryUploadArgs,
|
||||
schema?: ArkValidator<T>,
|
||||
): Promise<T> {
|
||||
const json = await runWithRetries(async () => {
|
||||
if (deps.runtime === "bun") {
|
||||
return await uploadBinaryViaCurl(deps, args);
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {};
|
||||
if (args.contentType) headers["Content-Type"] = args.contentType;
|
||||
const response = await fetchWithTimeout(
|
||||
deps,
|
||||
args.url,
|
||||
{
|
||||
method: "POST",
|
||||
headers,
|
||||
body: bytesToArrayBuffer(args.bytes),
|
||||
},
|
||||
UPLOAD_TIMEOUT_MS,
|
||||
);
|
||||
if (!response.ok) {
|
||||
throwHttpStatusError(
|
||||
response.status,
|
||||
await readResponseTextSafe(response),
|
||||
response.headers,
|
||||
deps.now,
|
||||
);
|
||||
}
|
||||
return (await response.json()) as unknown;
|
||||
}, args.retryCount);
|
||||
if (schema) return parseArk(schema, json, "API response");
|
||||
return json as T;
|
||||
}
|
||||
|
||||
async function downloadZipRequest(
|
||||
registry: string,
|
||||
args: { slug: string; version?: string; token?: string },
|
||||
@@ -296,6 +338,7 @@ export function createHttpClient(options: HttpClientOptions = {}): HttpClient {
|
||||
apiRequestForm,
|
||||
fetchText: fetchTextRequest,
|
||||
fetchBinary: fetchBinaryRequest,
|
||||
uploadBinary: uploadBinaryRequest,
|
||||
downloadZip: downloadZipRequest,
|
||||
};
|
||||
}
|
||||
@@ -361,6 +404,15 @@ export async function fetchBinary(registry: string, args: TextRequestArgs): Prom
|
||||
return await defaultHttpClient.fetchBinary(registry, args);
|
||||
}
|
||||
|
||||
export async function uploadBinary<T>(args: BinaryUploadArgs): Promise<T>;
|
||||
export async function uploadBinary<T>(args: BinaryUploadArgs, schema: ArkValidator<T>): Promise<T>;
|
||||
export async function uploadBinary<T>(
|
||||
args: BinaryUploadArgs,
|
||||
schema?: ArkValidator<T>,
|
||||
): Promise<T> {
|
||||
return await defaultHttpClient.uploadBinary<T>(args, schema);
|
||||
}
|
||||
|
||||
export async function downloadZip(
|
||||
registry: string,
|
||||
args: { slug: string; version?: string; token?: string },
|
||||
@@ -388,6 +440,12 @@ function createRetryRunner(deps: Pick<HttpClientDeps, "setTimeoutImpl" | "random
|
||||
};
|
||||
}
|
||||
|
||||
function bytesToArrayBuffer(bytes: Uint8Array) {
|
||||
const copy = new Uint8Array(bytes.byteLength);
|
||||
copy.set(bytes);
|
||||
return copy.buffer;
|
||||
}
|
||||
|
||||
async function fetchWithTimeout(
|
||||
deps: Pick<HttpClientDeps, "fetchImpl" | "setTimeoutImpl" | "clearTimeoutImpl">,
|
||||
url: string,
|
||||
@@ -699,6 +757,45 @@ async function fetchJsonFormViaCurl(
|
||||
}
|
||||
}
|
||||
|
||||
async function uploadBinaryViaCurl(
|
||||
deps: Pick<
|
||||
HttpClientDeps,
|
||||
"spawnSyncImpl" | "mkdtempImpl" | "writeFileImpl" | "rmImpl" | "tmpdirPath" | "now"
|
||||
>,
|
||||
args: BinaryUploadArgs,
|
||||
) {
|
||||
const tempDir = await deps.mkdtempImpl(join(deps.tmpdirPath, "clawhub-upload-"));
|
||||
try {
|
||||
const filePath = join(tempDir, "upload.bin");
|
||||
await deps.writeFileImpl(filePath, args.bytes);
|
||||
const curlArgs = [
|
||||
"--silent",
|
||||
"--show-error",
|
||||
"--location",
|
||||
"--max-time",
|
||||
String(UPLOAD_TIMEOUT_SECONDS),
|
||||
"--write-out",
|
||||
CURL_WRITE_OUT_FORMAT,
|
||||
"-X",
|
||||
"POST",
|
||||
];
|
||||
if (args.contentType) curlArgs.push("-H", `Content-Type: ${args.contentType}`);
|
||||
curlArgs.push("--data-binary", `@${filePath}`, args.url);
|
||||
|
||||
const result = deps.spawnSyncImpl("curl", curlArgs, { encoding: "utf8" });
|
||||
if (result.status !== 0) {
|
||||
throw new Error(result.stderr || "curl failed");
|
||||
}
|
||||
const { body, status, headers: responseHeaders } = parseCurlBodyAndMeta(result.stdout ?? "");
|
||||
if (status < 200 || status >= 300) {
|
||||
throwHttpStatusError(status, body, responseHeaders, deps.now);
|
||||
}
|
||||
return JSON.parse(body || "null") as unknown;
|
||||
} finally {
|
||||
await deps.rmImpl(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchTextViaCurl(
|
||||
deps: Pick<HttpClientDeps, "spawnSyncImpl" | "now">,
|
||||
url: string,
|
||||
|
||||
@@ -229,7 +229,81 @@ export const PackageTrustedPublisherSchema = type({
|
||||
});
|
||||
export type PackageTrustedPublisher = (typeof PackageTrustedPublisherSchema)[inferred];
|
||||
|
||||
export const PackagePublishRequestSchema = type({
|
||||
export const MAX_PACKAGE_MULTIPART_BYTES = 18 * 1024 * 1024;
|
||||
export const MAX_PACKAGE_CLAWPACK_BYTES = 120 * 1024 * 1024;
|
||||
const PACKAGE_MULTIPART_FIXED_OVERHEAD_BYTES = 4096;
|
||||
const PACKAGE_MULTIPART_PART_OVERHEAD_BYTES = 1024;
|
||||
|
||||
export type PackageMultipartUploadField = "files" | "clawpack";
|
||||
export type PackageMultipartUploadPart = {
|
||||
name: string;
|
||||
size: number;
|
||||
type?: string;
|
||||
};
|
||||
export type PackageMultipartUploadSizeInput = {
|
||||
payloadJson: string;
|
||||
fileFieldName: PackageMultipartUploadField;
|
||||
files: readonly PackageMultipartUploadPart[];
|
||||
};
|
||||
|
||||
export function estimatePackageMultipartUploadBytes(
|
||||
input: PackageMultipartUploadSizeInput,
|
||||
): number {
|
||||
return (
|
||||
PACKAGE_MULTIPART_FIXED_OVERHEAD_BYTES +
|
||||
estimateMultipartStringPartBytes("payload", input.payloadJson) +
|
||||
input.files.reduce(
|
||||
(sum, file) => sum + estimateMultipartFilePartBytes(input.fileFieldName, file),
|
||||
0,
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
export function isPackageMultipartUploadTooLarge(input: PackageMultipartUploadSizeInput): boolean {
|
||||
return estimatePackageMultipartUploadBytes(input) > MAX_PACKAGE_MULTIPART_BYTES;
|
||||
}
|
||||
|
||||
export function getPackageMultipartSizeError(): string {
|
||||
return "Package upload exceeds 18MB multipart upload limit";
|
||||
}
|
||||
|
||||
function estimateMultipartStringPartBytes(fieldName: string, value: string): number {
|
||||
return PACKAGE_MULTIPART_PART_OVERHEAD_BYTES + utf8ByteLength(fieldName) + utf8ByteLength(value);
|
||||
}
|
||||
|
||||
function estimateMultipartFilePartBytes(
|
||||
fieldName: PackageMultipartUploadField,
|
||||
file: PackageMultipartUploadPart,
|
||||
): number {
|
||||
return (
|
||||
file.size +
|
||||
PACKAGE_MULTIPART_PART_OVERHEAD_BYTES +
|
||||
utf8ByteLength(fieldName) +
|
||||
utf8ByteLength(file.name) +
|
||||
utf8ByteLength(file.type ?? "")
|
||||
);
|
||||
}
|
||||
|
||||
function utf8ByteLength(value: string): number {
|
||||
let bytes = 0;
|
||||
for (let index = 0; index < value.length; index += 1) {
|
||||
const codePoint = value.codePointAt(index);
|
||||
if (codePoint === undefined) continue;
|
||||
if (codePoint > 0xffff) index += 1;
|
||||
if (codePoint <= 0x7f) {
|
||||
bytes += 1;
|
||||
} else if (codePoint <= 0x7ff) {
|
||||
bytes += 2;
|
||||
} else if (codePoint <= 0xffff) {
|
||||
bytes += 3;
|
||||
} else {
|
||||
bytes += 4;
|
||||
}
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
const PackagePublishMetadataFields = {
|
||||
name: "string",
|
||||
displayName: "string?",
|
||||
ownerHandle: "string?",
|
||||
@@ -241,10 +315,21 @@ export const PackagePublishRequestSchema = type({
|
||||
tags: "string[]?",
|
||||
source: PublishSourceSchema.optional(),
|
||||
bundle: BundlePublishMetadataSchema.optional(),
|
||||
} as const;
|
||||
|
||||
export const PackagePublishMetadataSchema = type({
|
||||
"+": "reject",
|
||||
...PackagePublishMetadataFields,
|
||||
});
|
||||
export type PackagePublishMetadata = (typeof PackagePublishMetadataSchema)[inferred];
|
||||
|
||||
export const ServerPackagePublishRequestSchema = type({
|
||||
"+": "reject",
|
||||
...PackagePublishMetadataFields,
|
||||
artifact: PackagePublishArtifactSchema.optional(),
|
||||
files: CliPublishFileSchema.array(),
|
||||
});
|
||||
export type PackagePublishRequest = (typeof PackagePublishRequestSchema)[inferred];
|
||||
export type ServerPackagePublishRequest = (typeof ServerPackagePublishRequestSchema)[inferred];
|
||||
|
||||
export const PackageListItemSchema = type({
|
||||
name: "string",
|
||||
|
||||
@@ -54,6 +54,7 @@ export const ApiSkillMetaResponseSchema = type({
|
||||
|
||||
export const ApiCliUploadUrlResponseSchema = type({
|
||||
uploadUrl: "string",
|
||||
uploadTicket: "string",
|
||||
});
|
||||
|
||||
export const ApiUploadFileResponseSchema = type({
|
||||
|
||||
@@ -24,6 +24,7 @@ export function createHttpModuleMocks() {
|
||||
const downloadZip = vi.fn();
|
||||
const fetchBinary = vi.fn();
|
||||
const fetchText = vi.fn();
|
||||
const uploadBinary = vi.fn();
|
||||
const registryUrl = vi.fn(buildRegistryUrl);
|
||||
|
||||
return {
|
||||
@@ -32,6 +33,7 @@ export function createHttpModuleMocks() {
|
||||
downloadZip,
|
||||
fetchBinary,
|
||||
fetchText,
|
||||
uploadBinary,
|
||||
registryUrl,
|
||||
moduleFactory: () => ({
|
||||
apiRequest: (registry: unknown, args: unknown, schema?: unknown) =>
|
||||
@@ -41,6 +43,7 @@ export function createHttpModuleMocks() {
|
||||
downloadZip: (registry: unknown, args: unknown) => downloadZip(registry, args),
|
||||
fetchBinary: (registry: unknown, args: unknown) => fetchBinary(registry, args),
|
||||
fetchText: (registry: unknown, args: unknown) => fetchText(registry, args),
|
||||
uploadBinary: (args: unknown, schema?: unknown) => uploadBinary(args, schema),
|
||||
registryUrl: (...args: [string, string]) => registryUrl(...args),
|
||||
}),
|
||||
};
|
||||
|
||||
Vendored
+51
-9
@@ -228,18 +228,27 @@ export declare const PackageTrustedPublisherSchema: import("arktype/internal/var
|
||||
environment?: string | undefined;
|
||||
}, {}>;
|
||||
export type PackageTrustedPublisher = (typeof PackageTrustedPublisherSchema)[inferred];
|
||||
export declare const PackagePublishRequestSchema: import("arktype/internal/variants/object.ts").ObjectType<{
|
||||
export declare const MAX_PACKAGE_MULTIPART_BYTES: number;
|
||||
export declare const MAX_PACKAGE_CLAWPACK_BYTES: number;
|
||||
export type PackageMultipartUploadField = "files" | "clawpack";
|
||||
export type PackageMultipartUploadPart = {
|
||||
name: string;
|
||||
size: number;
|
||||
type?: string;
|
||||
};
|
||||
export type PackageMultipartUploadSizeInput = {
|
||||
payloadJson: string;
|
||||
fileFieldName: PackageMultipartUploadField;
|
||||
files: readonly PackageMultipartUploadPart[];
|
||||
};
|
||||
export declare function estimatePackageMultipartUploadBytes(input: PackageMultipartUploadSizeInput): number;
|
||||
export declare function isPackageMultipartUploadTooLarge(input: PackageMultipartUploadSizeInput): boolean;
|
||||
export declare function getPackageMultipartSizeError(): string;
|
||||
export declare const PackagePublishMetadataSchema: import("arktype/internal/variants/object.ts").ObjectType<{
|
||||
name: string;
|
||||
family: "skill" | "code-plugin" | "bundle-plugin";
|
||||
version: string;
|
||||
changelog: string;
|
||||
files: {
|
||||
path: string;
|
||||
size: number;
|
||||
storageId: string;
|
||||
sha256: string;
|
||||
contentType?: string | undefined;
|
||||
}[];
|
||||
displayName?: string | undefined;
|
||||
ownerHandle?: string | undefined;
|
||||
manualOverrideReason?: string | undefined;
|
||||
@@ -259,6 +268,20 @@ export declare const PackagePublishRequestSchema: import("arktype/internal/varia
|
||||
format?: string | undefined;
|
||||
hostTargets?: string[] | undefined;
|
||||
} | undefined;
|
||||
}, {}>;
|
||||
export type PackagePublishMetadata = (typeof PackagePublishMetadataSchema)[inferred];
|
||||
export declare const ServerPackagePublishRequestSchema: import("arktype/internal/variants/object.ts").ObjectType<{
|
||||
files: {
|
||||
path: string;
|
||||
size: number;
|
||||
storageId: string;
|
||||
sha256: string;
|
||||
contentType?: string | undefined;
|
||||
}[];
|
||||
name: string;
|
||||
family: "skill" | "code-plugin" | "bundle-plugin";
|
||||
version: string;
|
||||
changelog: string;
|
||||
artifact?: {
|
||||
kind: "npm-pack";
|
||||
storageId: string;
|
||||
@@ -271,8 +294,27 @@ export declare const PackagePublishRequestSchema: import("arktype/internal/varia
|
||||
npmUnpackedSize: number;
|
||||
npmFileCount: number;
|
||||
} | undefined;
|
||||
displayName?: string | undefined;
|
||||
ownerHandle?: string | undefined;
|
||||
manualOverrideReason?: string | undefined;
|
||||
channel?: "official" | "community" | "private" | undefined;
|
||||
tags?: string[] | undefined;
|
||||
source?: {
|
||||
kind: "github";
|
||||
url: string;
|
||||
repo: string;
|
||||
ref: string;
|
||||
commit: string;
|
||||
path: string;
|
||||
importedAt: number;
|
||||
} | undefined;
|
||||
bundle?: {
|
||||
id?: string | undefined;
|
||||
format?: string | undefined;
|
||||
hostTargets?: string[] | undefined;
|
||||
} | undefined;
|
||||
}, {}>;
|
||||
export type PackagePublishRequest = (typeof PackagePublishRequestSchema)[inferred];
|
||||
export type ServerPackagePublishRequest = (typeof ServerPackagePublishRequestSchema)[inferred];
|
||||
export declare const PackageListItemSchema: import("arktype/internal/variants/object.ts").ObjectType<{
|
||||
name: string;
|
||||
displayName: string;
|
||||
|
||||
Vendored
+57
-1
@@ -188,7 +188,55 @@ export const PackageTrustedPublisherSchema = type({
|
||||
workflowFilename: "string",
|
||||
environment: "string?",
|
||||
});
|
||||
export const PackagePublishRequestSchema = type({
|
||||
export const MAX_PACKAGE_MULTIPART_BYTES = 18 * 1024 * 1024;
|
||||
export const MAX_PACKAGE_CLAWPACK_BYTES = 120 * 1024 * 1024;
|
||||
const PACKAGE_MULTIPART_FIXED_OVERHEAD_BYTES = 4096;
|
||||
const PACKAGE_MULTIPART_PART_OVERHEAD_BYTES = 1024;
|
||||
export function estimatePackageMultipartUploadBytes(input) {
|
||||
return (PACKAGE_MULTIPART_FIXED_OVERHEAD_BYTES +
|
||||
estimateMultipartStringPartBytes("payload", input.payloadJson) +
|
||||
input.files.reduce((sum, file) => sum + estimateMultipartFilePartBytes(input.fileFieldName, file), 0));
|
||||
}
|
||||
export function isPackageMultipartUploadTooLarge(input) {
|
||||
return estimatePackageMultipartUploadBytes(input) > MAX_PACKAGE_MULTIPART_BYTES;
|
||||
}
|
||||
export function getPackageMultipartSizeError() {
|
||||
return "Package upload exceeds 18MB multipart upload limit";
|
||||
}
|
||||
function estimateMultipartStringPartBytes(fieldName, value) {
|
||||
return PACKAGE_MULTIPART_PART_OVERHEAD_BYTES + utf8ByteLength(fieldName) + utf8ByteLength(value);
|
||||
}
|
||||
function estimateMultipartFilePartBytes(fieldName, file) {
|
||||
return (file.size +
|
||||
PACKAGE_MULTIPART_PART_OVERHEAD_BYTES +
|
||||
utf8ByteLength(fieldName) +
|
||||
utf8ByteLength(file.name) +
|
||||
utf8ByteLength(file.type ?? ""));
|
||||
}
|
||||
function utf8ByteLength(value) {
|
||||
let bytes = 0;
|
||||
for (let index = 0; index < value.length; index += 1) {
|
||||
const codePoint = value.codePointAt(index);
|
||||
if (codePoint === undefined)
|
||||
continue;
|
||||
if (codePoint > 0xffff)
|
||||
index += 1;
|
||||
if (codePoint <= 0x7f) {
|
||||
bytes += 1;
|
||||
}
|
||||
else if (codePoint <= 0x7ff) {
|
||||
bytes += 2;
|
||||
}
|
||||
else if (codePoint <= 0xffff) {
|
||||
bytes += 3;
|
||||
}
|
||||
else {
|
||||
bytes += 4;
|
||||
}
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
const PackagePublishMetadataFields = {
|
||||
name: "string",
|
||||
displayName: "string?",
|
||||
ownerHandle: "string?",
|
||||
@@ -200,6 +248,14 @@ export const PackagePublishRequestSchema = type({
|
||||
tags: "string[]?",
|
||||
source: PublishSourceSchema.optional(),
|
||||
bundle: BundlePublishMetadataSchema.optional(),
|
||||
};
|
||||
export const PackagePublishMetadataSchema = type({
|
||||
"+": "reject",
|
||||
...PackagePublishMetadataFields,
|
||||
});
|
||||
export const ServerPackagePublishRequestSchema = type({
|
||||
"+": "reject",
|
||||
...PackagePublishMetadataFields,
|
||||
artifact: PackagePublishArtifactSchema.optional(),
|
||||
files: CliPublishFileSchema.array(),
|
||||
});
|
||||
|
||||
Vendored
+1
-1
File diff suppressed because one or more lines are too long
Vendored
+1
@@ -47,6 +47,7 @@ export declare const ApiSkillMetaResponseSchema: import("arktype/internal/varian
|
||||
}, {}>;
|
||||
export declare const ApiCliUploadUrlResponseSchema: import("arktype/internal/variants/object.ts").ObjectType<{
|
||||
uploadUrl: string;
|
||||
uploadTicket: string;
|
||||
}, {}>;
|
||||
export declare const ApiUploadFileResponseSchema: import("arktype/internal/variants/object.ts").ObjectType<{
|
||||
storageId: string;
|
||||
|
||||
Vendored
+1
@@ -45,6 +45,7 @@ export const ApiSkillMetaResponseSchema = type({
|
||||
});
|
||||
export const ApiCliUploadUrlResponseSchema = type({
|
||||
uploadUrl: "string",
|
||||
uploadTicket: "string",
|
||||
});
|
||||
export const ApiUploadFileResponseSchema = type({
|
||||
storageId: "string",
|
||||
|
||||
Vendored
+1
-1
File diff suppressed because one or more lines are too long
@@ -253,7 +253,81 @@ export const PackageTrustedPublisherSchema = type({
|
||||
});
|
||||
export type PackageTrustedPublisher = (typeof PackageTrustedPublisherSchema)[inferred];
|
||||
|
||||
export const PackagePublishRequestSchema = type({
|
||||
export const MAX_PACKAGE_MULTIPART_BYTES = 18 * 1024 * 1024;
|
||||
export const MAX_PACKAGE_CLAWPACK_BYTES = 120 * 1024 * 1024;
|
||||
const PACKAGE_MULTIPART_FIXED_OVERHEAD_BYTES = 4096;
|
||||
const PACKAGE_MULTIPART_PART_OVERHEAD_BYTES = 1024;
|
||||
|
||||
export type PackageMultipartUploadField = "files" | "clawpack";
|
||||
export type PackageMultipartUploadPart = {
|
||||
name: string;
|
||||
size: number;
|
||||
type?: string;
|
||||
};
|
||||
export type PackageMultipartUploadSizeInput = {
|
||||
payloadJson: string;
|
||||
fileFieldName: PackageMultipartUploadField;
|
||||
files: readonly PackageMultipartUploadPart[];
|
||||
};
|
||||
|
||||
export function estimatePackageMultipartUploadBytes(
|
||||
input: PackageMultipartUploadSizeInput,
|
||||
): number {
|
||||
return (
|
||||
PACKAGE_MULTIPART_FIXED_OVERHEAD_BYTES +
|
||||
estimateMultipartStringPartBytes("payload", input.payloadJson) +
|
||||
input.files.reduce(
|
||||
(sum, file) => sum + estimateMultipartFilePartBytes(input.fileFieldName, file),
|
||||
0,
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
export function isPackageMultipartUploadTooLarge(input: PackageMultipartUploadSizeInput): boolean {
|
||||
return estimatePackageMultipartUploadBytes(input) > MAX_PACKAGE_MULTIPART_BYTES;
|
||||
}
|
||||
|
||||
export function getPackageMultipartSizeError(): string {
|
||||
return "Package upload exceeds 18MB multipart upload limit";
|
||||
}
|
||||
|
||||
function estimateMultipartStringPartBytes(fieldName: string, value: string): number {
|
||||
return PACKAGE_MULTIPART_PART_OVERHEAD_BYTES + utf8ByteLength(fieldName) + utf8ByteLength(value);
|
||||
}
|
||||
|
||||
function estimateMultipartFilePartBytes(
|
||||
fieldName: PackageMultipartUploadField,
|
||||
file: PackageMultipartUploadPart,
|
||||
): number {
|
||||
return (
|
||||
file.size +
|
||||
PACKAGE_MULTIPART_PART_OVERHEAD_BYTES +
|
||||
utf8ByteLength(fieldName) +
|
||||
utf8ByteLength(file.name) +
|
||||
utf8ByteLength(file.type ?? "")
|
||||
);
|
||||
}
|
||||
|
||||
function utf8ByteLength(value: string): number {
|
||||
let bytes = 0;
|
||||
for (let index = 0; index < value.length; index += 1) {
|
||||
const codePoint = value.codePointAt(index);
|
||||
if (codePoint === undefined) continue;
|
||||
if (codePoint > 0xffff) index += 1;
|
||||
if (codePoint <= 0x7f) {
|
||||
bytes += 1;
|
||||
} else if (codePoint <= 0x7ff) {
|
||||
bytes += 2;
|
||||
} else if (codePoint <= 0xffff) {
|
||||
bytes += 3;
|
||||
} else {
|
||||
bytes += 4;
|
||||
}
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
const PackagePublishMetadataFields = {
|
||||
name: "string",
|
||||
displayName: "string?",
|
||||
ownerHandle: "string?",
|
||||
@@ -265,10 +339,21 @@ export const PackagePublishRequestSchema = type({
|
||||
tags: "string[]?",
|
||||
source: PublishSourceSchema.optional(),
|
||||
bundle: BundlePublishMetadataSchema.optional(),
|
||||
} as const;
|
||||
|
||||
export const PackagePublishMetadataSchema = type({
|
||||
"+": "reject",
|
||||
...PackagePublishMetadataFields,
|
||||
});
|
||||
export type PackagePublishMetadata = (typeof PackagePublishMetadataSchema)[inferred];
|
||||
|
||||
export const ServerPackagePublishRequestSchema = type({
|
||||
"+": "reject",
|
||||
...PackagePublishMetadataFields,
|
||||
artifact: PackagePublishArtifactSchema.optional(),
|
||||
files: CliPublishFileSchema.array(),
|
||||
});
|
||||
export type PackagePublishRequest = (typeof PackagePublishRequestSchema)[inferred];
|
||||
export type ServerPackagePublishRequest = (typeof ServerPackagePublishRequestSchema)[inferred];
|
||||
|
||||
export const PackageListItemSchema = type({
|
||||
name: "string",
|
||||
|
||||
@@ -55,6 +55,7 @@ export const ApiSkillMetaResponseSchema = type({
|
||||
|
||||
export const ApiCliUploadUrlResponseSchema = type({
|
||||
uploadUrl: "string",
|
||||
uploadTicket: "string",
|
||||
});
|
||||
|
||||
export const ApiUploadFileResponseSchema = type({
|
||||
|
||||
@@ -116,6 +116,28 @@ See also: [acceptable-usage.md](./acceptable-usage.md) for the marketplace polic
|
||||
- Skills directory supports an optional "Hide suspicious" filter to exclude
|
||||
active-but-flagged (`flagged.suspicious`) entries from browse/search results.
|
||||
|
||||
## Package publish upload boundary
|
||||
|
||||
- Package publish is multipart-only. `POST /api/v1/packages` must reject JSON
|
||||
request bodies, including bodies that reference pre-existing storage IDs.
|
||||
- Public HTTP package publish payloads must not accept caller-supplied `files`
|
||||
or `artifact` metadata. Internal publish actions may receive that metadata
|
||||
only after the HTTP boundary derives it from uploaded multipart bytes or a
|
||||
staged ClawPack blob.
|
||||
- Package publish accepts either multipart `files` uploads or one `clawpack`
|
||||
tarball reference, never both in the same request. `clawpack` may be a direct
|
||||
`.tgz` file part or a Convex storage id created by the upload-url flow. The
|
||||
storage-id path must include the matching `clawpackUploadTicket`, and the
|
||||
server must reject tickets from a different auth context, expired or used
|
||||
tickets, and storage blobs created before the ticket.
|
||||
- Direct package publish multipart bytes are capped at 18MB so callers get a
|
||||
clear ClawHub validation error before hitting Convex's 20MB HTTP action body
|
||||
cap. ClawPack tarballs keep the 120MB package tarball cap through staged
|
||||
storage uploads.
|
||||
- For tarball uploads, ClawHub stores the uploaded tarball, derives its
|
||||
artifact hashes and npm metadata, and derives package file metadata from the
|
||||
tarball contents.
|
||||
|
||||
## Skill moderation pipeline
|
||||
|
||||
- New skill publishes now persist a deterministic static scan result on the version.
|
||||
|
||||
Reference in New Issue
Block a user