fix: require multipart package publishes

Require multipart package publishes so public clients cannot provide trusted file or artifact metadata.

Preserve released CLI multipart field names (`files` and `clawpack`), derive stored file and npm-pack artifact metadata from uploaded bytes, and keep large ClawPack tarballs working through ticketed staged uploads.

Tests:
- bunx vitest run convex/uploads.test.ts --reporter=dot
- bunx vitest run convex/httpApi.handlers.test.ts convex/httpApiV1.handlers.test.ts --testNamePattern "package publish|multipart package publish|multipart ClawPack|staged ClawPack|browser session auth|cliUploadUrl|upload tickets" --reporter=dot
- bun run --cwd packages/clawhub test:src -- src/cli/commands/packages.test.ts --testNamePattern "ClawPack tarballs over|stages ClawPack|publishes a ClawPack|cleans generated ClawPack"
- bun run format:check
- bun run lint
- bunx tsc -p packages/schema/tsconfig.json --noEmit --pretty false
- bunx tsc -p packages/clawhub/tsconfig.json --noEmit --pretty false
- bunx tsc --noEmit --pretty false
- bun run ci:unit
- git diff --check origin/main
This commit is contained in:
Jesse Merhi
2026-06-02 09:59:53 +10:00
committed by GitHub
parent 01aa28ccda
commit dcbc38999f
29 changed files with 1639 additions and 331 deletions
+41 -5
View File
@@ -4,13 +4,15 @@ import { afterEach, describe, expect, it, vi } from "vitest";
vi.mock("./lib/apiTokenAuth", () => ({
getOptionalApiTokenUser: vi.fn(),
requireApiTokenUser: vi.fn(),
requirePackagePublishAuth: vi.fn(),
}));
vi.mock("./skills", () => ({
publishVersionForUser: vi.fn(),
}));
const { getOptionalApiTokenUser, requireApiTokenUser } = await import("./lib/apiTokenAuth");
const { getOptionalApiTokenUser, requireApiTokenUser, requirePackagePublishAuth } =
await import("./lib/apiTokenAuth");
const { publishVersionForUser } = await import("./skills");
const { __handlers } = await import("./httpApi");
const { hashSkillFiles } = await import("./lib/skills");
@@ -23,6 +25,7 @@ describe("httpApi handlers", () => {
afterEach(() => {
vi.mocked(getOptionalApiTokenUser).mockReset();
vi.mocked(requireApiTokenUser).mockReset();
vi.mocked(requirePackagePublishAuth).mockReset();
vi.mocked(publishVersionForUser).mockReset();
});
@@ -444,18 +447,51 @@ describe("httpApi handlers", () => {
});
it("cliUploadUrlHttp returns uploadUrl", async () => {
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: "user1" } as never);
const runMutation = vi.fn().mockResolvedValue("https://upload.local");
vi.mocked(requirePackagePublishAuth).mockResolvedValueOnce({
kind: "user",
userId: "user1",
} as never);
const runMutation = vi.fn().mockResolvedValue({
uploadUrl: "https://upload.local",
uploadTicket: "packagePublishUploadTickets:1",
});
const response = await __handlers.cliUploadUrlHandler(
makeCtx({ runMutation }),
new Request("https://x/api/cli/upload-url", { method: "POST" }),
);
expect(response.status).toBe(200);
expect(await response.json()).toEqual({ uploadUrl: "https://upload.local" });
expect(await response.json()).toEqual({
uploadUrl: "https://upload.local",
uploadTicket: "packagePublishUploadTickets:1",
});
});
it("cliUploadUrlHttp accepts package publish tokens", async () => {
vi.mocked(requirePackagePublishAuth).mockResolvedValueOnce({
kind: "github-actions",
publishToken: { _id: "packagePublishTokens:1" },
} as never);
const runMutation = vi.fn().mockResolvedValue({
uploadUrl: "https://upload.local/package",
uploadTicket: "packagePublishUploadTickets:2",
});
const response = await __handlers.cliUploadUrlHandler(
makeCtx({ runMutation }),
new Request("https://x/api/cli/upload-url", { method: "POST" }),
);
expect(response.status).toBe(200);
expect(await response.json()).toEqual({
uploadUrl: "https://upload.local/package",
uploadTicket: "packagePublishUploadTickets:2",
});
expect(runMutation).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({ publishTokenId: "packagePublishTokens:1" }),
);
});
it("cliUploadUrlHttp returns 401 when unauthorized", async () => {
vi.mocked(requireApiTokenUser).mockRejectedValueOnce(new Error("Unauthorized"));
vi.mocked(requirePackagePublishAuth).mockRejectedValueOnce(new Error("Unauthorized"));
const response = await __handlers.cliUploadUrlHandler(
makeCtx({}),
new Request("https://x/api/cli/upload-url", { method: "POST" }),
+11 -6
View File
@@ -10,7 +10,7 @@ import { api, internal } from "./_generated/api";
import type { Id } from "./_generated/dataModel";
import type { ActionCtx } from "./_generated/server";
import { httpAction } from "./functions";
import { requireApiTokenUser } from "./lib/apiTokenAuth";
import { requireApiTokenUser, requirePackagePublishAuth } from "./lib/apiTokenAuth";
import { corsHeaders, mergeHeaders } from "./lib/httpHeaders";
import { applyRateLimit } from "./lib/httpRateLimit";
import { parseBooleanQueryParam, resolveBooleanQueryParam } from "./lib/httpUtils";
@@ -148,11 +148,16 @@ export const cliWhoamiHttp = httpAction(cliWhoamiHandler);
async function cliUploadUrlHandler(ctx: ActionCtx, request: Request) {
try {
const { userId } = await requireApiTokenUser(ctx, request);
const uploadUrl = await ctx.runMutation(internal.uploads.generateUploadUrlForUserInternal, {
userId,
});
return json({ uploadUrl });
const auth = await requirePackagePublishAuth(ctx, request);
const upload =
auth.kind === "user"
? await ctx.runMutation(internal.uploads.createPackagePublishUploadForUserInternal, {
userId: auth.userId,
})
: await ctx.runMutation(internal.uploads.createPackagePublishUploadForTokenInternal, {
publishTokenId: auth.publishToken._id,
});
return json(upload);
} catch (error) {
return text(formatAuthFailure(error), 401);
}
+385 -84
View File
@@ -99,8 +99,8 @@ function writeTarString(target: Uint8Array, offset: number, width: number, value
target.set(encoded.subarray(0, width), offset);
}
function tarFile(path: string, content: string) {
const bytes = new TextEncoder().encode(content);
function tarFile(path: string, content: string | Uint8Array) {
const bytes = typeof content === "string" ? new TextEncoder().encode(content) : content;
const header = new Uint8Array(TAR_BLOCK_SIZE);
writeTarString(header, 0, 100, path);
writeTarString(header, 100, 8, tarOctal(0o644, 8));
@@ -123,7 +123,7 @@ function tarFile(path: string, content: string) {
return [header, body];
}
function npmPackFixture(files: Record<string, string>) {
function npmPackFixture(files: Record<string, string | Uint8Array>) {
const parts: Uint8Array[] = [];
for (const [path, content] of Object.entries(files)) {
parts.push(...tarFile(path, content));
@@ -145,13 +145,35 @@ function bytesToArrayBuffer(bytes: Uint8Array): ArrayBuffer {
return copy.buffer;
}
function packagePublishMetadata(overrides: Record<string, unknown> = {}) {
return {
name: "demo-plugin",
family: "bundle-plugin",
version: "1.0.0",
changelog: "init",
...overrides,
};
}
function packagePublishForm(payload: Record<string, unknown>) {
const form = new FormData();
form.set("payload", JSON.stringify(payload));
return form;
}
function makeCtx(partial: Record<string, unknown>) {
const rateLimitStatus =
typeof partial.rateLimitStatus === "function"
? (partial.rateLimitStatus as (args: RateLimitArgs) => unknown)
: null;
const partialRunQuery =
typeof partial.runQuery === "function"
? (partial.runQuery as (query: unknown, args: Record<string, unknown>) => unknown)
: null;
const runQuery = vi.fn(async (query: unknown, args: Record<string, unknown>) => {
if (isRateLimitArgs(args)) return { ...okRate(), limit: args.limit };
if (isRateLimitArgs(args)) {
return rateLimitStatus?.(args) ?? { ...okRate(), limit: args.limit };
}
return partialRunQuery ? await partialRunQuery(query, args) : null;
});
const runMutation =
@@ -9495,37 +9517,26 @@ describe("httpApiV1 handlers", () => {
const runAction = vi
.fn()
.mockResolvedValue({ ok: true, packageId: "pkg:1", releaseId: "rel:1" });
const form = packagePublishForm(
packagePublishMetadata({
ownerHandle: "openclaw",
bundle: { hostTargets: ["desktop"] },
}),
);
form.append("files", new File(["{}"], "openclaw.plugin.json", { type: "application/json" }));
const response = await __handlers.publishPackageV1Handler(
makeCtx({ runAction, runMutation }),
makeCtx({
runAction,
runMutation,
storage: {
store: vi.fn(async (entry: File) => `storage:${entry.name}`),
},
}),
new Request("https://example.com/api/v1/packages", {
method: "POST",
headers: {
Authorization: "Bearer clh_test",
"content-type": "application/json",
},
body: JSON.stringify({
name: "demo-plugin",
ownerHandle: "openclaw",
family: "bundle-plugin",
version: "1.0.0",
changelog: "init",
bundle: { hostTargets: ["desktop"] },
files: [
{
path: "openclaw.plugin.json",
size: 2,
storageId: "storage:1",
sha256: "a".repeat(64),
},
{
path: ".codex-plugin/plugin.json",
size: 2,
storageId: "storage:1",
sha256: "a".repeat(64),
},
],
}),
headers: { Authorization: "Bearer clh_test" },
body: form,
}),
);
@@ -9563,6 +9574,53 @@ describe("httpApiV1 handlers", () => {
'Documents read from or written to the "publishers" table changed while this mutation was being run and on every subsequent retry.',
),
);
const form = packagePublishForm(
packagePublishMetadata({
ownerHandle: "openclaw",
bundle: { hostTargets: ["desktop"] },
}),
);
const pack = npmPackFixture({
"package/package.json": JSON.stringify({ name: "demo-plugin", version: "1.0.0" }),
"package/openclaw.plugin.json": JSON.stringify({ id: "demo.plugin" }),
"package/dist/index.js": "export const demo = true;\n",
});
form.append(
"clawpack",
new File([bytesToArrayBuffer(pack)], "demo-plugin-1.0.0.tgz", {
type: "application/octet-stream",
}),
);
const response = await __handlers.publishPackageV1Handler(
makeCtx({
runAction,
runMutation,
storage: { store: vi.fn(async (_entry: Blob) => "storage:1") },
}),
new Request("https://example.com/api/v1/packages", {
method: "POST",
headers: {
Authorization: "Bearer clh_test",
},
body: form,
}),
);
expect(response.status).toBe(503);
expect(response.headers.get("Retry-After")).toBe("1");
await expect(response.text()).resolves.toContain("Transient ClawHub write contention");
});
it("package publish rejects JSON request bodies before publish actions run", async () => {
vi.mocked(getOptionalApiTokenUserId).mockResolvedValue("users:1" as never);
vi.mocked(requirePackagePublishAuth).mockResolvedValue({
kind: "user",
userId: "users:1",
user: { _id: "users:1", handle: "p" },
} as never);
const runMutation = vi.fn().mockResolvedValue(okRate());
const runAction = vi.fn();
const response = await __handlers.publishPackageV1Handler(
makeCtx({ runAction, runMutation }),
@@ -9572,34 +9630,39 @@ describe("httpApiV1 handlers", () => {
Authorization: "Bearer clh_test",
"content-type": "application/json",
},
body: JSON.stringify({
name: "demo-plugin",
ownerHandle: "openclaw",
family: "bundle-plugin",
version: "1.0.0",
changelog: "init",
bundle: { hostTargets: ["desktop"] },
files: [
{
path: "openclaw.plugin.json",
size: 2,
storageId: "storage:1",
sha256: "a".repeat(64),
},
{
path: ".codex-plugin/plugin.json",
size: 2,
storageId: "storage:1",
sha256: "a".repeat(64),
},
],
}),
body: JSON.stringify(packagePublishMetadata()),
}),
);
expect(response.status).toBe(503);
expect(response.headers.get("Retry-After")).toBe("1");
await expect(response.text()).resolves.toContain("Transient ClawHub write contention");
expect(response.status).toBe(415);
expect(await response.text()).toBe("Package publish requires multipart/form-data");
expect(runAction).not.toHaveBeenCalled();
});
it("package publish rejects browser session auth when token auth is not an API token", async () => {
vi.mocked(getAuthUserId).mockResolvedValue("users:session" as never);
vi.mocked(requirePackagePublishAuth).mockRejectedValue(new Error("Unauthorized"));
const runMutation = vi.fn().mockResolvedValue(okRate());
const runAction = vi.fn();
const form = packagePublishForm(packagePublishMetadata());
form.append("files", new File(["{}"], "openclaw.plugin.json", { type: "application/json" }));
const response = await __handlers.publishPackageV1Handler(
makeCtx({
runAction,
runMutation,
storage: { store: vi.fn(async () => "storage:plugin") },
}),
new Request("https://example.com/api/v1/packages", {
method: "POST",
headers: { Authorization: "Bearer convex-session-token" },
body: form,
}),
);
expect(response.status).toBe(401);
expect(await response.text()).toBe("Unauthorized");
expect(runAction).not.toHaveBeenCalled();
});
it("multipart package publish ignores macOS junk files", async () => {
@@ -9613,6 +9676,7 @@ describe("httpApiV1 handlers", () => {
const runAction = vi
.fn()
.mockResolvedValue({ ok: true, packageId: "pkg:1", releaseId: "rel:1" });
const storageStore = vi.fn(async () => "storage:plugin");
const form = new FormData();
form.set(
"payload",
@@ -9631,9 +9695,7 @@ describe("httpApiV1 handlers", () => {
makeCtx({
runAction,
runMutation,
storage: {
store: vi.fn(async (entry: File) => `storage:${entry.name}`),
},
storage: { store: storageStore },
}),
new Request("https://example.com/api/v1/packages", {
method: "POST",
@@ -9643,18 +9705,30 @@ describe("httpApiV1 handlers", () => {
);
expect(response.status).toBe(200);
expect(storageStore).toHaveBeenCalledTimes(1);
expect(runAction).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
payload: expect.objectContaining({
files: [
expect.objectContaining({
{
path: "openclaw.plugin.json",
}),
size: 2,
storageId: "storage:plugin",
sha256: "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a",
contentType: "application/json",
},
],
}),
}),
);
const actionCall = runAction.mock.calls[0];
expect(actionCall).toBeTruthy();
expect(actionCall[1]).toEqual(
expect.objectContaining({
payload: expect.not.objectContaining({ artifact: expect.anything() }),
}),
);
});
it("multipart ClawPack publish stores the tarball and extracted file metadata", async () => {
@@ -9736,6 +9810,234 @@ describe("httpApiV1 handlers", () => {
expect(payload?.files?.map((file) => file.path)).toContain("dist/index.js");
});
it("staged ClawPack publish derives artifact metadata from stored bytes", async () => {
vi.mocked(getOptionalApiTokenUserId).mockResolvedValue("users:1" as never);
vi.mocked(requirePackagePublishAuth).mockResolvedValue({
kind: "user",
userId: "users:1",
user: { _id: "users:1", handle: "p" },
} as never);
const runMutation = vi.fn().mockResolvedValue(okRate());
const runAction = vi
.fn()
.mockResolvedValue({ ok: true, packageId: "pkg:1", releaseId: "rel:1" });
const pack = npmPackFixture({
"package/package.json": JSON.stringify({ name: "demo-plugin", version: "1.0.0" }),
"package/openclaw.plugin.json": JSON.stringify({ id: "demo.plugin" }),
"package/dist/index.js": "export const demo = true;\n",
});
const storageGet = vi.fn(async (storageId: string) =>
storageId === "storage:clawpack"
? new Blob([bytesToArrayBuffer(pack)], { type: "application/octet-stream" })
: null,
);
const storageStore = vi.fn(async (_entry: Blob) => `storage:${storageStore.mock.calls.length}`);
const form = packagePublishForm(
packagePublishMetadata({
family: "code-plugin",
}),
);
form.set("clawpack", "storage:clawpack");
form.set("clawpackUploadTicket", "packagePublishUploadTickets:1");
const response = await __handlers.publishPackageV1Handler(
makeCtx({
runAction,
runMutation,
storage: { get: storageGet, store: storageStore },
}),
new Request("https://example.com/api/v1/packages", {
method: "POST",
headers: { Authorization: "Bearer clh_test" },
body: form,
}),
);
expect(response.status).toBe(200);
expect(runMutation).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
uploadTicket: "packagePublishUploadTickets:1",
storageId: "storage:clawpack",
auth: { kind: "user", userId: "users:1" },
}),
);
expect(storageGet).toHaveBeenCalledWith("storage:clawpack");
expect(storageStore).toHaveBeenCalledTimes(3);
expect(runAction).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
payload: expect.objectContaining({
artifact: expect.objectContaining({
kind: "npm-pack",
storageId: "storage:clawpack",
size: pack.byteLength,
npmFileCount: 3,
}),
files: [
expect.objectContaining({ path: "package.json", storageId: "storage:1" }),
expect.objectContaining({ path: "openclaw.plugin.json", storageId: "storage:2" }),
expect.objectContaining({ path: "dist/index.js", storageId: "storage:3" }),
],
}),
}),
);
});
it("staged ClawPack publish rejects storage ids without upload tickets", async () => {
vi.mocked(getOptionalApiTokenUserId).mockResolvedValue("users:1" as never);
vi.mocked(requirePackagePublishAuth).mockResolvedValue({
kind: "user",
userId: "users:1",
user: { _id: "users:1", handle: "p" },
} as never);
const runMutation = vi.fn().mockResolvedValue(okRate());
const runAction = vi.fn();
const storageGet = vi.fn();
const form = packagePublishForm(packagePublishMetadata({ family: "code-plugin" }));
form.set("clawpack", "storage:clawpack");
const response = await __handlers.publishPackageV1Handler(
makeCtx({
runAction,
runMutation,
storage: { get: storageGet, store: vi.fn() },
}),
new Request("https://example.com/api/v1/packages", {
method: "POST",
headers: { Authorization: "Bearer clh_test" },
body: form,
}),
);
expect(response.status).toBe(400);
expect(await response.text()).toBe("Package tarball upload ticket required");
expect(storageGet).not.toHaveBeenCalled();
expect(runAction).not.toHaveBeenCalled();
});
it("multipart package publish rejects files and tarball together", async () => {
vi.mocked(getOptionalApiTokenUserId).mockResolvedValue("users:1" as never);
vi.mocked(requirePackagePublishAuth).mockResolvedValue({
kind: "user",
userId: "users:1",
user: { _id: "users:1", handle: "p" },
} as never);
const runMutation = vi.fn().mockResolvedValue(okRate());
const runAction = vi.fn();
const pack = npmPackFixture({
"package/package.json": JSON.stringify({ name: "demo-plugin", version: "1.0.0" }),
"package/openclaw.plugin.json": JSON.stringify({ id: "demo.plugin" }),
});
const form = packagePublishForm(packagePublishMetadata({ family: "code-plugin" }));
form.append("files", new File(["{}"], "openclaw.plugin.json", { type: "application/json" }));
form.append(
"clawpack",
new File([bytesToArrayBuffer(pack)], "demo-plugin-1.0.0.tgz", {
type: "application/octet-stream",
}),
);
const response = await __handlers.publishPackageV1Handler(
makeCtx({ runAction, runMutation, storage: { store: vi.fn() } }),
new Request("https://example.com/api/v1/packages", {
method: "POST",
headers: { Authorization: "Bearer clh_test" },
body: form,
}),
);
expect(response.status).toBe(400);
expect(await response.text()).toBe(
"Upload either a package tarball or individual files, not both",
);
expect(runAction).not.toHaveBeenCalled();
});
it.each(["files[]", "tarball", "artifact", "extraMetadata"])(
"multipart package publish rejects unsupported field %s",
async (field) => {
vi.mocked(getOptionalApiTokenUserId).mockResolvedValue("users:1" as never);
vi.mocked(requirePackagePublishAuth).mockResolvedValue({
kind: "user",
userId: "users:1",
user: { _id: "users:1", handle: "p" },
} as never);
const runMutation = vi.fn().mockResolvedValue(okRate());
const runAction = vi.fn();
const form = packagePublishForm(packagePublishMetadata());
form.append("files", new File(["{}"], "openclaw.plugin.json", { type: "application/json" }));
form.append(field, new File(["{}"], "ignored.json", { type: "application/json" }));
const response = await __handlers.publishPackageV1Handler(
makeCtx({ runAction, runMutation, storage: { store: vi.fn() } }),
new Request("https://example.com/api/v1/packages", {
method: "POST",
headers: { Authorization: "Bearer clh_test" },
body: form,
}),
);
expect(response.status).toBe(400);
expect(await response.text()).toBe(`Unsupported package publish form field: ${field}`);
expect(runAction).not.toHaveBeenCalled();
},
);
it.each(["files", "artifact"])(
"multipart package publish rejects caller-supplied %s metadata",
async (field) => {
vi.mocked(getOptionalApiTokenUserId).mockResolvedValue("users:1" as never);
vi.mocked(requirePackagePublishAuth).mockResolvedValue({
kind: "user",
userId: "users:1",
user: { _id: "users:1", handle: "p" },
} as never);
const runMutation = vi.fn().mockResolvedValue(okRate());
const runAction = vi.fn();
const form = packagePublishForm(
packagePublishMetadata({
[field]:
field === "files"
? [
{
path: "openclaw.plugin.json",
size: 2,
storageId: "storage:attacker",
sha256: "a".repeat(64),
},
]
: {
kind: "npm-pack",
storageId: "storage:attacker",
sha256: "a".repeat(64),
size: 2,
format: "tgz",
npmIntegrity: "sha512-attacker",
npmShasum: "a".repeat(40),
npmTarballName: "demo-plugin-1.0.0.tgz",
npmUnpackedSize: 2,
npmFileCount: 1,
},
}),
);
form.append("files", new File(["{}"], "openclaw.plugin.json", { type: "application/json" }));
const response = await __handlers.publishPackageV1Handler(
makeCtx({ runAction, runMutation, storage: { store: vi.fn() } }),
new Request("https://example.com/api/v1/packages", {
method: "POST",
headers: { Authorization: "Bearer clh_test" },
body: form,
}),
);
expect(response.status).toBe(400);
expect(await response.text()).toContain(`Package publish payload: ${field}`);
expect(runAction).not.toHaveBeenCalled();
},
);
it("package publish routes GitHub Actions auth through the trusted publisher action", async () => {
vi.mocked(getOptionalApiTokenUserId).mockResolvedValue("users:1" as never);
vi.mocked(requirePackagePublishAuth).mockResolvedValue({
@@ -9746,36 +10048,35 @@ describe("httpApiV1 handlers", () => {
const runAction = vi
.fn()
.mockResolvedValue({ ok: true, packageId: "pkg:1", releaseId: "rel:1" });
const form = packagePublishForm(
packagePublishMetadata({
bundle: { hostTargets: ["desktop"] },
}),
);
const pack = npmPackFixture({
"package/package.json": JSON.stringify({ name: "demo-plugin", version: "1.0.0" }),
"package/openclaw.plugin.json": JSON.stringify({ id: "demo.plugin" }),
"package/dist/index.js": "export const demo = true;\n",
});
form.append(
"clawpack",
new File([bytesToArrayBuffer(pack)], "demo-plugin-1.0.0.tgz", {
type: "application/octet-stream",
}),
);
const response = await __handlers.publishPackageV1Handler(
makeCtx({ runAction, runMutation }),
makeCtx({
runAction,
runMutation,
storage: { store: vi.fn(async (_entry: Blob) => "storage:1") },
}),
new Request("https://example.com/api/v1/packages", {
method: "POST",
headers: {
Authorization: "Bearer clh_publish",
"content-type": "application/json",
},
body: JSON.stringify({
name: "demo-plugin",
family: "bundle-plugin",
version: "1.0.0",
changelog: "init",
bundle: { hostTargets: ["desktop"] },
files: [
{
path: "openclaw.plugin.json",
size: 2,
storageId: "storage:1",
sha256: "a".repeat(64),
},
{
path: ".codex-plugin/plugin.json",
size: 2,
storageId: "storage:1",
sha256: "a".repeat(64),
},
],
}),
body: form,
}),
);
+245 -139
View File
@@ -11,16 +11,18 @@ import {
PackageReportRequestSchema,
PackageReportTriageRequestSchema,
PackageReleaseModerationRequestSchema,
PackagePublishRequestSchema,
PackagePublishMetadataSchema,
PackageTransferRequestSchema,
PackageTrustedPublisherUpsertRequestSchema,
PublishTokenMintRequestSchema,
isPluginCategorySlug,
parseArk,
type PackagePublishMetadata,
type PackageAppealListStatus,
type PackageModerationQueueStatus,
type PackageOfficialMigrationListPhase,
type PackageReportListStatus,
type ServerPackagePublishRequest,
} from "clawhub-schema";
import { api, internal } from "../_generated/api";
import type { Doc, Id } from "../_generated/dataModel";
@@ -43,9 +45,13 @@ import {
} from "../lib/packageSecurity";
import {
getClawPackSizeError,
getPackageMultipartSizeError,
getPublishFileSizeError,
getPublishTotalSizeError,
isPackageMultipartUploadTooLarge,
MAX_CLAWPACK_BYTES,
MAX_PUBLISH_FILE_BYTES,
MAX_PUBLISH_TOTAL_BYTES,
} from "../lib/publishLimits";
import { getPublicSkillFileAccessBlock, isSkillVersionForSkill } from "../lib/skillFileAccess";
import { isMacJunkPath, isTextFile } from "../lib/skills";
@@ -120,6 +126,9 @@ const internalRefs = internal as unknown as {
packagePublishTokens: {
createInternal: unknown;
};
uploads: {
consumePackagePublishUploadTicketInternal: unknown;
};
skills: {
getSkillBySlugInternal: unknown;
searchPackageCatalogForHttpInternal: unknown;
@@ -1018,75 +1027,27 @@ function skillVersionTags(tags: Record<string, string>, version: string) {
.map(([tag]) => tag);
}
function parsePackagePublishBody(body: unknown) {
const parsed = parseArk(PackagePublishRequestSchema, body, "Package publish payload") as {
name: string;
displayName?: string;
ownerHandle?: string;
family: "skill" | "code-plugin" | "bundle-plugin";
version: string;
changelog: string;
manualOverrideReason?: string;
channel?: "official" | "community" | "private";
tags?: string[];
source?: Record<string, unknown>;
bundle?: Record<string, unknown>;
files: Array<{
path: string;
size: number;
storageId: string;
sha256: string;
contentType?: string;
}>;
artifact?: {
kind: "npm-pack";
storageId: string;
sha256: string;
size: number;
format: "tgz";
npmIntegrity: string;
npmShasum: string;
npmTarballName: string;
npmUnpackedSize: number;
npmFileCount: number;
type StoredPackagePublishFile = ServerPackagePublishRequest["files"][number];
type PackagePublishTarballArtifact = NonNullable<ServerPackagePublishRequest["artifact"]>;
type ParsedPackageClawPack = Awaited<ReturnType<typeof parseClawPack>>;
type PackagePublishAuth =
| { kind: "user"; userId: Id<"users"> }
| { kind: "github-actions"; publishToken: Doc<"packagePublishTokens"> };
type PackagePublishTarballPart =
| { kind: "file"; file: File }
| {
kind: "storage";
storageId: Id<"_storage">;
uploadTicket: Id<"packagePublishUploadTickets">;
};
};
if (parsed.files.length === 0) throw new Error("files required");
return {
name: parsed.name,
displayName: parsed.displayName ?? undefined,
ownerHandle: parsed.ownerHandle?.trim().replace(/^@+/, "") || undefined,
family: parsed.family,
version: parsed.version,
changelog: parsed.changelog,
manualOverrideReason: parsed.manualOverrideReason?.trim() || undefined,
channel: parsed.channel ?? undefined,
tags: parsed.tags?.filter(Boolean) ?? undefined,
source: parsed.source ?? undefined,
bundle: parsed.bundle ?? undefined,
files: parsed.files.map((file) => ({
...file,
storageId: file.storageId as Id<"_storage">,
})),
artifact: parsed.artifact
? {
...parsed.artifact,
storageId: parsed.artifact.storageId as Id<"_storage">,
}
: undefined,
};
}
function inferStoredPackageContentType(path: string) {
const lower = path.toLowerCase();
if (lower.endsWith(".json")) return "application/json";
if (lower.endsWith(".md") || lower.endsWith(".mdx") || lower.endsWith(".txt")) {
return "text/plain; charset=utf-8";
}
if (lower.endsWith(".js") || lower.endsWith(".mjs") || lower.endsWith(".cjs")) {
return "text/javascript; charset=utf-8";
}
if (lower.endsWith(".ts") || lower.endsWith(".tsx")) return "text/plain; charset=utf-8";
if (isTextFile(path)) return "text/plain; charset=utf-8";
return "application/octet-stream";
}
@@ -1096,9 +1057,10 @@ function bytesToArrayBuffer(bytes: Uint8Array) {
return copy.buffer;
}
async function storeClawPackFile(ctx: ActionCtx, entry: { path: string; bytes: Uint8Array }) {
// npm-pack artifacts are bounded by the tarball and total package limits; the
// legacy per-file cap only applies to raw file uploads.
async function storeClawPackFile(
ctx: ActionCtx,
entry: { path: string; bytes: Uint8Array },
): Promise<StoredPackagePublishFile> {
const contentType = inferStoredPackageContentType(entry.path);
const storageId = await ctx.storage.store(
new Blob([bytesToArrayBuffer(entry.bytes)], { type: contentType }),
@@ -1116,91 +1078,234 @@ async function storeClawPackFiles(
ctx: ActionCtx,
entries: Array<{ path: string; bytes: Uint8Array }>,
) {
const files: Awaited<ReturnType<typeof storeClawPackFile>>[] = [];
// Convex HTTP actions have a tight memory ceiling; concurrent Blob/storage
// work can duplicate large npm-pack entries enough to OOM the action.
const files: StoredPackagePublishFile[] = [];
// Convex HTTP actions have a tight memory ceiling; avoid concurrent Blob work.
for (const entry of entries) {
files.push(await storeClawPackFile(ctx, entry));
}
return files;
}
async function parseMultipartPackagePublish(ctx: ActionCtx, request: Request) {
async function storeUploadedPackageFile(
ctx: ActionCtx,
entry: File,
): Promise<StoredPackagePublishFile> {
if (entry.size > MAX_PUBLISH_FILE_BYTES) {
throw new Error(getPublishFileSizeError(entry.name));
}
const buffer = new Uint8Array(await entry.arrayBuffer());
const contentType = inferStoredPackageContentType(entry.name);
const storageId = await ctx.storage.store(
new Blob([bytesToArrayBuffer(buffer)], { type: contentType }),
);
return {
path: entry.name,
size: entry.size,
storageId,
sha256: await sha256Hex(buffer),
contentType,
};
}
function getFileParts(form: FormData, fields: readonly string[], stringPartError: string) {
const parts = fields.flatMap((field) => form.getAll(field));
if (parts.some((entry) => typeof entry === "string")) {
throw new Error(stringPartError);
}
return parts.filter((entry): entry is File => typeof entry !== "string");
}
function getTarballPart(form: FormData): PackagePublishTarballPart | null {
const parts = form.getAll("clawpack");
if (parts.length > 1) throw new Error("Upload one package tarball");
const ticketParts = form.getAll("clawpackUploadTicket");
if (ticketParts.length > 1) throw new Error("Upload one package tarball ticket");
const ticketPart = ticketParts[0];
if (ticketPart && typeof ticketPart !== "string") {
throw new Error("Package tarball upload ticket must be a string");
}
const part = parts[0];
if (!part) {
if (ticketPart) throw new Error("Package tarball upload ticket requires a staged ClawPack");
return null;
}
if (typeof part !== "string") {
if (ticketPart) throw new Error("Package tarball upload ticket requires a staged ClawPack");
return { kind: "file", file: part };
}
const storageId = part.trim();
if (!storageId) throw new Error("Package tarball storage id required");
const uploadTicket = ticketPart?.trim();
if (!uploadTicket) throw new Error("Package tarball upload ticket required");
return {
kind: "storage",
storageId: storageId as Id<"_storage">,
uploadTicket: uploadTicket as Id<"packagePublishUploadTickets">,
};
}
async function consumePackageTarballUploadTicket(
ctx: ActionCtx,
auth: PackagePublishAuth,
part: Extract<PackagePublishTarballPart, { kind: "storage" }>,
) {
await ctx.runMutation(
internalRefs.uploads.consumePackagePublishUploadTicketInternal as never,
{
uploadTicket: part.uploadTicket,
storageId: part.storageId,
auth:
auth.kind === "user"
? { kind: "user", userId: auth.userId }
: { kind: "github-actions", publishTokenId: auth.publishToken._id },
} as never,
);
}
async function readStoredPackageTarball(ctx: ActionCtx, storageId: Id<"_storage">) {
const blob = await ctx.storage.get(storageId);
if (!blob) throw new Error("Package tarball upload no longer exists");
if (blob.size > MAX_CLAWPACK_BYTES) {
throw new Error(getClawPackSizeError("uploaded ClawPack"));
}
return new Uint8Array(await blob.arrayBuffer());
}
async function buildPackagePublishRequestFromClawPack(
ctx: ActionCtx,
metadata: PackagePublishMetadata,
parsed: ParsedPackageClawPack,
artifactBytes: Uint8Array,
artifactStorageId: Id<"_storage">,
): Promise<ServerPackagePublishRequest> {
if (parsed.unpackedSize > MAX_PUBLISH_TOTAL_BYTES) {
throw new Error(getPublishTotalSizeError("package"));
}
const artifact: PackagePublishTarballArtifact = {
kind: "npm-pack",
storageId: artifactStorageId,
sha256: parsed.artifactSha256,
size: artifactBytes.byteLength,
format: "tgz",
npmIntegrity: parsed.npmIntegrity,
npmShasum: parsed.npmShasum,
npmTarballName: parsed.npmTarballName,
npmUnpackedSize: parsed.unpackedSize,
npmFileCount: parsed.fileCount,
};
const files = await storeClawPackFiles(ctx, parsed.entries);
return { ...metadata, files, artifact };
}
const PACKAGE_PUBLISH_FILE_FIELDS = ["files"] as const;
const PACKAGE_PUBLISH_TARBALL_FIELDS = ["clawpack"] as const;
const PACKAGE_PUBLISH_FORM_FIELDS = new Set([
"payload",
...PACKAGE_PUBLISH_FILE_FIELDS,
...PACKAGE_PUBLISH_TARBALL_FIELDS,
"clawpackUploadTicket",
]);
function multipartUploadPart(file: File) {
return {
name: file.name,
size: file.size,
type: file.type || undefined,
};
}
async function parseMultipartPackagePublish(
ctx: ActionCtx,
auth: PackagePublishAuth,
request: Request,
): Promise<ServerPackagePublishRequest> {
const form = await request.formData();
const payloadRaw = form.get("payload");
if (!payloadRaw || typeof payloadRaw !== "string") throw new Error("Missing payload");
const payload = JSON.parse(payloadRaw) as Record<string, unknown>;
const files: Array<{
path: string;
size: number;
storageId: Id<"_storage">;
sha256: string;
contentType?: string;
}> = [];
let artifact:
| {
kind: "npm-pack";
storageId: Id<"_storage">;
sha256: string;
size: number;
format: "tgz";
npmIntegrity: string;
npmShasum: string;
npmTarballName: string;
npmUnpackedSize: number;
npmFileCount: number;
}
| undefined;
for (const field of form.keys()) {
if (!PACKAGE_PUBLISH_FORM_FIELDS.has(field)) {
throw new Error(`Unsupported package publish form field: ${field}`);
}
}
const clawpackEntry = form.get("clawpack") ?? form.get("artifact");
if (clawpackEntry && typeof clawpackEntry !== "string") {
if (form.getAll("files").some((entry) => typeof entry !== "string")) {
throw new Error("Upload either a ClawPack tarball or individual files, not both");
const payloadParts = form.getAll("payload");
const payloadRaw = payloadParts[0];
if (payloadParts.length !== 1 || typeof payloadRaw !== "string") {
throw new Error("Package publish payload must be one JSON string");
}
const parsedPayload: unknown = JSON.parse(payloadRaw);
const metadata: PackagePublishMetadata = parseArk(
PackagePublishMetadataSchema,
parsedPayload,
"Package publish payload",
);
const tarballPart = getTarballPart(form);
const fileParts = getFileParts(
form,
PACKAGE_PUBLISH_FILE_FIELDS,
"Package publish file uploads must be files",
);
if (tarballPart) {
if (fileParts.length > 0) {
throw new Error("Upload either a package tarball or individual files, not both");
}
if (clawpackEntry.size > MAX_CLAWPACK_BYTES) {
throw new Error(getClawPackSizeError(clawpackEntry.name));
if (tarballPart.kind === "storage") {
await consumePackageTarballUploadTicket(ctx, auth, tarballPart);
const artifactBytes = await readStoredPackageTarball(ctx, tarballPart.storageId);
const parsed = await parseClawPack(artifactBytes);
return await buildPackagePublishRequestFromClawPack(
ctx,
metadata,
parsed,
artifactBytes,
tarballPart.storageId,
);
}
const artifactBytes = new Uint8Array(await clawpackEntry.arrayBuffer());
const tarballEntry = tarballPart.file;
if (tarballEntry.size > MAX_CLAWPACK_BYTES) {
throw new Error(getClawPackSizeError(tarballEntry.name));
}
if (
isPackageMultipartUploadTooLarge({
payloadJson: payloadRaw,
fileFieldName: "clawpack",
files: [multipartUploadPart(tarballEntry)],
})
) {
throw new Error(getPackageMultipartSizeError());
}
const artifactBytes = new Uint8Array(await tarballEntry.arrayBuffer());
const parsed = await parseClawPack(artifactBytes);
const artifactBlob = new Blob([artifactBytes], { type: "application/octet-stream" });
const artifactStorageId = await ctx.storage.store(artifactBlob);
artifact = {
kind: "npm-pack",
storageId: artifactStorageId,
sha256: parsed.artifactSha256,
size: artifactBytes.byteLength,
format: "tgz",
npmIntegrity: parsed.npmIntegrity,
npmShasum: parsed.npmShasum,
npmTarballName: parsed.npmTarballName,
npmUnpackedSize: parsed.unpackedSize,
npmFileCount: parsed.fileCount,
};
files.push(...(await storeClawPackFiles(ctx, parsed.entries)));
return parsePackagePublishBody({ ...payload, files, artifact });
const artifactStorageId = await ctx.storage.store(
new Blob([bytesToArrayBuffer(artifactBytes)], { type: "application/octet-stream" }),
);
return await buildPackagePublishRequestFromClawPack(
ctx,
metadata,
parsed,
artifactBytes,
artifactStorageId,
);
}
for (const entry of form.getAll("files")) {
if (typeof entry === "string") continue;
if (isMacJunkPath(entry.name)) continue;
if (entry.size > MAX_PUBLISH_FILE_BYTES) {
throw new Error(getPublishFileSizeError(entry.name));
}
const buffer = new Uint8Array(await entry.arrayBuffer());
const digest = await crypto.subtle.digest("SHA-256", buffer);
const sha256 = Array.from(new Uint8Array(digest), (byte) =>
byte.toString(16).padStart(2, "0"),
).join("");
const storageId = await ctx.storage.store(entry);
files.push({
path: entry.name,
size: entry.size,
storageId,
sha256,
contentType: entry.type || undefined,
});
if (
isPackageMultipartUploadTooLarge({
payloadJson: payloadRaw,
fileFieldName: "files",
files: fileParts.map(multipartUploadPart),
})
) {
throw new Error(getPackageMultipartSizeError());
}
return parsePackagePublishBody({ ...payload, files });
const packageFileParts = fileParts.filter((entry) => !isMacJunkPath(entry.name));
const files = await Promise.all(
packageFileParts.map((entry) => storeUploadedPackageFile(ctx, entry)),
);
if (files.length === 0) throw new Error("files required");
return { ...metadata, files };
}
async function listPackages(
@@ -1481,9 +1586,10 @@ export async function publishPackageV1Handler(ctx: ActionCtx, request: Request)
try {
const contentType = request.headers.get("content-type") ?? "";
const payload = contentType.includes("multipart/form-data")
? await parseMultipartPackagePublish(ctx, request)
: parsePackagePublishBody(await request.json());
if (!contentType.includes("multipart/form-data")) {
return text("Package publish requires multipart/form-data", 415, rate.headers);
}
const payload = await parseMultipartPackagePublish(ctx, auth.auth, request);
const result =
auth.auth.kind === "user"
? await runActionRef(ctx, internalRefs.packages.publishPackageForUserInternal, {
+18 -2
View File
@@ -37,9 +37,9 @@ function tarFile(path: string, content: string) {
return [header, body];
}
function npmPackFixture(files: Record<string, string>) {
function npmPackFixtureEntries(files: Array<[string, string]>) {
const parts: Uint8Array[] = [];
for (const [path, content] of Object.entries(files)) {
for (const [path, content] of files) {
parts.push(...tarFile(path, content));
}
parts.push(new Uint8Array(BLOCK_SIZE), new Uint8Array(BLOCK_SIZE));
@@ -53,6 +53,10 @@ function npmPackFixture(files: Record<string, string>) {
return gzipSync(tar);
}
function npmPackFixture(files: Record<string, string>) {
return npmPackFixtureEntries(Object.entries(files));
}
describe("clawpack", () => {
it("parses npm pack tarballs and computes npm integrity fields", async () => {
const pack = npmPackFixture({
@@ -95,6 +99,18 @@ describe("clawpack", () => {
await expect(parseClawPack(pack)).rejects.toThrow("rooted under package");
});
it("rejects duplicate normalized archive paths", async () => {
const pack = npmPackFixtureEntries([
["package/package.json", JSON.stringify({ name: "demo", version: "1.0.0" })],
["package/openclaw.plugin.json", JSON.stringify({ id: "demo" })],
["package/package.json", JSON.stringify({ name: "other", version: "9.9.9" })],
]);
await expect(parseClawPack(pack)).rejects.toThrow(
"ClawPack contains duplicate path: package.json",
);
});
it("uses npm-style tarball names", () => {
expect(npmTarballName("demo", "1.0.0")).toBe("demo-1.0.0.tgz");
expect(npmTarballName("@scope/demo", "1.0.0")).toBe("scope-demo-1.0.0.tgz");
+5
View File
@@ -67,6 +67,7 @@ function isZeroBlock(block: Uint8Array) {
function parseTarEntries(bytes: Uint8Array): ClawPackEntry[] {
const entries: ClawPackEntry[] = [];
const paths = new Set<string>();
let offset = 0;
while (offset + TAR_BLOCK_SIZE <= bytes.byteLength) {
@@ -93,6 +94,10 @@ function parseTarEntries(bytes: Uint8Array): ClawPackEntry[] {
offset = nextTarOffset(payloadOffset, size);
continue;
}
if (paths.has(relPath)) {
throw new Error(`ClawPack contains duplicate path: ${relPath}`);
}
paths.add(relPath);
entries.push({
path: relPath,
bytes: Uint8Array.from(tarEntryPayload(bytes, payloadOffset, size)),
+3 -1
View File
@@ -5,6 +5,7 @@ import {
getPublishFileSizeError,
getPublishTotalSizeError,
MAX_CLAWPACK_BYTES,
MAX_PACKAGE_MULTIPART_BYTES,
MAX_PUBLISH_FILE_BYTES,
} from "./publishLimits";
@@ -31,8 +32,9 @@ describe("publishLimits", () => {
);
});
it("keeps the ClawPack tarball limit separate from legacy file limits", () => {
it("keeps ClawPack capacity above the multipart request budget", () => {
expect(MAX_CLAWPACK_BYTES).toBe(120 * 1024 * 1024);
expect(MAX_CLAWPACK_BYTES).toBeGreaterThan(MAX_PACKAGE_MULTIPART_BYTES);
expect(MAX_CLAWPACK_BYTES).toBeGreaterThan(MAX_PUBLISH_FILE_BYTES);
});
});
+12 -1
View File
@@ -1,6 +1,17 @@
import { MAX_PACKAGE_CLAWPACK_BYTES } from "clawhub-schema";
export {
estimatePackageMultipartUploadBytes,
getPackageMultipartSizeError,
isPackageMultipartUploadTooLarge,
MAX_PACKAGE_MULTIPART_BYTES,
type PackageMultipartUploadField,
type PackageMultipartUploadPart,
} from "clawhub-schema";
export const MAX_PUBLISH_TOTAL_BYTES = 50 * 1024 * 1024;
export const MAX_PUBLISH_FILE_BYTES = 10 * 1024 * 1024;
export const MAX_CLAWPACK_BYTES = 120 * 1024 * 1024;
export const MAX_CLAWPACK_BYTES = MAX_PACKAGE_CLAWPACK_BYTES;
type SizedPathLike = {
path: string;
-23
View File
@@ -10,7 +10,6 @@ import {
getPackageReleaseScanBackfillBatchInternal,
getByName,
list,
publishPackage,
publishPackageForTrustedPublisherInternal,
publishPackageForUserInternal,
listPackageReportsInternal,
@@ -237,14 +236,6 @@ const searchForViewerInternalHandler = (
Array<{ package: { name: string } }>
>
)._handler;
const publishPackageHandler = (
publishPackage as unknown as WrappedHandler<
{
payload: unknown;
},
unknown
>
)._handler;
const publishPackageForUserInternalHandler = (
publishPackageForUserInternal as unknown as WrappedHandler<
{
@@ -6300,20 +6291,6 @@ describe("packages public queries", () => {
expect(result).toEqual([expect.objectContaining({ name: "demo-plugin" })]);
});
it("requires auth inside the public publish action", async () => {
await expect(
publishPackageHandler({ runQuery: vi.fn(), runMutation: vi.fn() } as never, {
payload: {
name: "demo-plugin",
family: "bundle-plugin",
version: "1.0.0",
changelog: "init",
files: [],
},
}),
).rejects.toThrow("Unauthorized");
});
it("records package reports for moderation", async () => {
const insert = vi.fn(async (table: string) =>
table === "packageReports" ? "packageReports:1" : "auditLogs:1",
+8 -25
View File
@@ -1,5 +1,5 @@
import {
PackagePublishRequestSchema,
ServerPackagePublishRequestSchema,
getPackageScopeOwnerMismatch,
isPluginCategorySlug,
parseArk,
@@ -11,7 +11,7 @@ import {
type PackageModerationQueueStatus,
type PackageOfficialMigrationListPhase,
type PackageOfficialMigrationPhase,
type PackagePublishRequest,
type ServerPackagePublishRequest,
type PackageVerificationTier,
} from "clawhub-schema";
import { paginationOptsValidator } from "convex/server";
@@ -33,7 +33,6 @@ import {
assertModerator,
getOptionalActiveAuthUserId,
requireUser,
requireUserFromAction,
} from "./lib/access";
import {
assertArtifactAppealFinalAction,
@@ -5027,9 +5026,9 @@ function buildGitHubActionsPublishActor(
}
function resolveTrustedPublishSource(
payload: PackagePublishRequest,
payload: ServerPackagePublishRequest,
publishToken: Doc<"packagePublishTokens">,
): PackagePublishRequest["source"] {
): ServerPackagePublishRequest["source"] {
const source = payload.source;
if (source && source.kind !== "github") {
throw new ConvexError("Trusted publishes only support GitHub source metadata");
@@ -5081,11 +5080,11 @@ async function publishPackageImpl(
auth: PackagePublishAuthContext,
rawPayload: unknown,
) {
const payload = parseArk(
PackagePublishRequestSchema,
const payload = parseArk<ServerPackagePublishRequest>(
ServerPackagePublishRequestSchema,
rawPayload,
"Package publish payload",
) as PackagePublishRequest;
);
if (payload.family === "skill") {
throw new ConvexError("Skill packages must use the skills publish flow");
}
@@ -5204,7 +5203,7 @@ async function publishPackageImpl(
}
const displayName = payload.displayName?.trim() || name;
const files = normalizePublishFiles(payload.files as never);
const files = normalizePublishFiles(payload.files);
if (payload.artifact?.kind !== "npm-pack") {
const oversizedFile = findOversizedPublishFile(files);
if (oversizedFile) {
@@ -5459,14 +5458,6 @@ async function publishPackageImpl(
return publishResult;
}
export const publishPackage = action({
args: { payload: v.any() },
handler: async (ctx, args) => {
const { userId } = await requireUserFromAction(ctx);
return await publishPackageImpl(ctx, { kind: "user", actorUserId: userId }, args.payload);
},
});
export const publishPackageForUserInternal = internalAction({
args: {
actorUserId: v.id("users"),
@@ -5509,14 +5500,6 @@ export const publishPackageForTrustedPublisherInternal = internalAction({
},
});
export const publishRelease = action({
args: { payload: v.any() },
handler: async (ctx, args) => {
const { userId } = await requireUserFromAction(ctx);
return await publishPackageImpl(ctx, { kind: "user", actorUserId: userId }, args.payload);
},
});
export const reservePackageNameInternal = internalMutation({
args: {
actorUserId: v.id("users"),
+11
View File
@@ -1224,6 +1224,16 @@ const packagePublishTokens = defineTable({
.index("by_package", ["packageId", "version", "createdAt"])
.index("by_package_revoked_created", ["packageId", "revokedAt", "createdAt"]);
const packagePublishUploadTickets = defineTable({
kind: v.union(v.literal("user"), v.literal("github-actions")),
userId: v.optional(v.id("users")),
publishTokenId: v.optional(v.id("packagePublishTokens")),
createdAt: v.number(),
expiresAt: v.number(),
usedAt: v.optional(v.number()),
storageId: v.optional(v.id("_storage")),
});
const packageSearchDigest = defineTable({
packageId: v.id("packages"),
name: v.string(),
@@ -2144,6 +2154,7 @@ export default defineSchema({
packageStatEvents,
packageTrustedPublishers,
packagePublishTokens,
packagePublishUploadTickets,
packageBadges,
packageSearchDigest,
packageCapabilitySearchDigest,
+140
View File
@@ -0,0 +1,140 @@
/* @vitest-environment node */
import { afterEach, describe, expect, it, vi } from "vitest";
import { consumePackagePublishUploadTicketInternal } from "./uploads";
type ConsumeArgs = {
uploadTicket: string;
storageId: string;
auth: { kind: "user"; userId: string } | { kind: "github-actions"; publishTokenId: string };
};
type WrappedHandler<TArgs> = {
_handler: (ctx: unknown, args: TArgs) => Promise<void>;
};
const consumeHandler = (
consumePackagePublishUploadTicketInternal as unknown as WrappedHandler<ConsumeArgs>
)._handler;
function makeCtx(ticket: Record<string, unknown> | null, storage: Record<string, unknown> | null) {
return {
db: {
get: vi.fn(async () => ticket),
insert: vi.fn(),
normalizeId: vi.fn(),
patch: vi.fn(),
query: vi.fn(),
replace: vi.fn(),
delete: vi.fn(),
system: {
get: vi.fn(async () => storage),
query: vi.fn(),
},
},
};
}
describe("package publish upload tickets", () => {
afterEach(() => {
vi.restoreAllMocks();
});
it("consumes a fresh upload ticket for the same user", async () => {
vi.spyOn(Date, "now").mockReturnValue(2_000);
const ctx = makeCtx(
{
_id: "packagePublishUploadTickets:1",
kind: "user",
userId: "users:1",
createdAt: 1_000,
expiresAt: 10_000,
},
{ _id: "storage:1", _creationTime: 1_500 },
);
await consumeHandler(ctx, {
uploadTicket: "packagePublishUploadTickets:1",
storageId: "storage:1",
auth: { kind: "user", userId: "users:1" },
});
expect(ctx.db.system.get).toHaveBeenCalledWith("_storage", "storage:1");
expect(ctx.db.patch).toHaveBeenCalledWith("packagePublishUploadTickets:1", {
usedAt: 2_000,
storageId: "storage:1",
});
});
it("allows retrying a used upload ticket for the same user and storage id", async () => {
vi.spyOn(Date, "now").mockReturnValue(3_000);
const ctx = makeCtx(
{
_id: "packagePublishUploadTickets:1",
kind: "user",
userId: "users:1",
createdAt: 1_000,
expiresAt: 10_000,
usedAt: 2_000,
storageId: "storage:1",
},
{ _id: "storage:1", _creationTime: 1_500 },
);
await consumeHandler(ctx, {
uploadTicket: "packagePublishUploadTickets:1",
storageId: "storage:1",
auth: { kind: "user", userId: "users:1" },
});
expect(ctx.db.patch).not.toHaveBeenCalled();
});
it("rejects upload tickets from another auth context", async () => {
vi.spyOn(Date, "now").mockReturnValue(2_000);
const ctx = makeCtx(
{
_id: "packagePublishUploadTickets:1",
kind: "user",
userId: "users:1",
createdAt: 1_000,
expiresAt: 10_000,
},
{ _id: "storage:1", _creationTime: 1_500 },
);
await expect(
consumeHandler(ctx, {
uploadTicket: "packagePublishUploadTickets:1",
storageId: "storage:1",
auth: { kind: "user", userId: "users:2" },
}),
).rejects.toThrow("Package tarball upload ticket does not match this publish token");
expect(ctx.db.patch).not.toHaveBeenCalled();
});
it("rejects storage created before the upload ticket", async () => {
vi.spyOn(Date, "now").mockReturnValue(2_000);
const ctx = makeCtx(
{
_id: "packagePublishUploadTickets:1",
kind: "github-actions",
publishTokenId: "packagePublishTokens:1",
createdAt: 1_000,
expiresAt: 10_000,
},
{ _id: "storage:1", _creationTime: 999 },
);
await expect(
consumeHandler(ctx, {
uploadTicket: "packagePublishUploadTickets:1",
storageId: "storage:1",
auth: { kind: "github-actions", publishTokenId: "packagePublishTokens:1" },
}),
).rejects.toThrow("Package tarball upload must be created after its upload ticket");
expect(ctx.db.patch).not.toHaveBeenCalled();
});
});
+74 -2
View File
@@ -2,6 +2,8 @@ import { v } from "convex/values";
import { internalMutation, mutation } from "./functions";
import { requireUser } from "./lib/access";
const PACKAGE_PUBLISH_UPLOAD_TICKET_TTL_MS = 15 * 60_000;
export const generateUploadUrl = mutation({
args: {},
handler: async (ctx) => {
@@ -10,11 +12,81 @@ export const generateUploadUrl = mutation({
},
});
export const generateUploadUrlForUserInternal = internalMutation({
export const createPackagePublishUploadForUserInternal = internalMutation({
args: { userId: v.id("users") },
handler: async (ctx, args) => {
const user = await ctx.db.get(args.userId);
if (!user || user.deletedAt || user.deactivatedAt) throw new Error("User not found");
return ctx.storage.generateUploadUrl();
const now = Date.now();
const uploadTicket = await ctx.db.insert("packagePublishUploadTickets", {
kind: "user",
userId: args.userId,
createdAt: now,
expiresAt: now + PACKAGE_PUBLISH_UPLOAD_TICKET_TTL_MS,
});
const uploadUrl = await ctx.storage.generateUploadUrl();
return { uploadUrl, uploadTicket };
},
});
export const createPackagePublishUploadForTokenInternal = internalMutation({
args: { publishTokenId: v.id("packagePublishTokens") },
handler: async (ctx, args) => {
const publishToken = await ctx.db.get(args.publishTokenId);
const now = Date.now();
if (!publishToken || publishToken.revokedAt || publishToken.expiresAt <= now) {
throw new Error("Trusted publish token is missing or expired");
}
const uploadTicket = await ctx.db.insert("packagePublishUploadTickets", {
kind: "github-actions",
publishTokenId: args.publishTokenId,
createdAt: now,
expiresAt: now + PACKAGE_PUBLISH_UPLOAD_TICKET_TTL_MS,
});
const uploadUrl = await ctx.storage.generateUploadUrl();
return { uploadUrl, uploadTicket };
},
});
export const consumePackagePublishUploadTicketInternal = internalMutation({
args: {
uploadTicket: v.id("packagePublishUploadTickets"),
storageId: v.id("_storage"),
auth: v.union(
v.object({ kind: v.literal("user"), userId: v.id("users") }),
v.object({
kind: v.literal("github-actions"),
publishTokenId: v.id("packagePublishTokens"),
}),
),
},
handler: async (ctx, args) => {
const ticket = await ctx.db.get(args.uploadTicket);
const now = Date.now();
if (!ticket || ticket.expiresAt <= now) {
throw new Error("Package tarball upload ticket is missing or expired");
}
if (
args.auth.kind === "user"
? ticket.kind !== "user" || ticket.userId !== args.auth.userId
: ticket.kind !== "github-actions" || ticket.publishTokenId !== args.auth.publishTokenId
) {
throw new Error("Package tarball upload ticket does not match this publish token");
}
if (ticket.usedAt) {
if (ticket.storageId === args.storageId) return;
throw new Error("Package tarball upload ticket was already used");
}
const metadata = await ctx.db.system.get("_storage", args.storageId);
if (!metadata) throw new Error("Package tarball upload no longer exists");
if (metadata._creationTime < ticket.createdAt) {
throw new Error("Package tarball upload must be created after its upload ticket");
}
await ctx.db.patch(ticket._id, {
usedAt: now,
storageId: args.storageId,
});
},
});
+14 -2
View File
@@ -1225,8 +1225,16 @@ Publishes a new version.
Publishes a code-plugin or bundle-plugin release.
- Requires Bearer token auth.
- Preferred: `multipart/form-data` with `payload` JSON + `files[]` blobs.
- JSON body with `files` (storageId-based) is also accepted.
- Requires `multipart/form-data`.
- Allowed form fields are `payload`, repeated `files` blobs, or one `clawpack`
tarball reference. `clawpack` may be a `.tgz` blob or a storage id returned by
the upload-url flow. Staged storage-id publishes must also include the
`clawpackUploadTicket` returned with that upload URL.
- Use either `files` or `clawpack`, never both in the same request.
- JSON bodies and caller-supplied `payload.files` / `payload.artifact`
metadata are rejected.
- Direct multipart publish requests are capped at 18MB. ClawPack tarballs may
use the upload-url flow up to the 120MB tarball cap.
- Optional payload field: `ownerHandle`. When present, only admins may publish on behalf of that owner.
Validation highlights:
@@ -1479,6 +1487,10 @@ Still supported for older CLI versions:
See `DEPRECATIONS.md` for removal plan.
`POST /api/cli/upload-url` returns `uploadUrl` and `uploadTicket`. Package
publishes that stage a ClawPack tarball must send the resulting storage id as
`clawpack` and the returned ticket as `clawpackUploadTicket`.
## Registry discovery (`/.well-known/clawhub.json`)
The CLI can discover registry/auth settings from the site:
@@ -1,8 +1,8 @@
/* @vitest-environment node */
import { spawnSync } from "node:child_process";
import { createHash } from "node:crypto";
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { createHash, randomBytes } from "node:crypto";
import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { gzipSync, zipSync } from "fflate";
@@ -67,6 +67,10 @@ async function makeTmpWorkdir() {
return await mkdtemp(join(tmpdir(), "clawhub-package-"));
}
async function listClawPackTempDirs() {
return new Set((await readdir(tmpdir())).filter((name) => name.startsWith("clawhub-clawpack-")));
}
function runGit(cwd: string, args: string[]) {
const result = spawnSync("git", ["-C", cwd, ...args], {
encoding: "utf8",
@@ -143,8 +147,8 @@ function tarOctal(value: number, width: number) {
return value.toString(8).padStart(width - 1, "0") + "\0";
}
function tarFile(path: string, content: string) {
const bytes = new TextEncoder().encode(content);
function tarFile(path: string, content: string | Uint8Array) {
const bytes = typeof content === "string" ? new TextEncoder().encode(content) : content;
const header = new Uint8Array(TAR_BLOCK_SIZE);
writeTarString(header, 0, 100, path);
writeTarString(header, 100, 8, tarOctal(0o644, 8));
@@ -167,7 +171,7 @@ function tarFile(path: string, content: string) {
return [header, body];
}
function npmPackFixture(files: Record<string, string>) {
function npmPackFixture(files: Record<string, string | Uint8Array>) {
const parts: Uint8Array[] = [];
for (const [path, content] of Object.entries(files)) {
parts.push(...tarFile(path, content));
@@ -1274,6 +1278,65 @@ describe("package commands", () => {
}
});
it("stages ClawPack tarballs over the multipart publish budget", async () => {
const workdir = await makeTmpWorkdir();
try {
const packName = "oversized-plugin-1.0.0.tgz";
const packBytes = npmPackFixture({
"package/package.json": makeCodePluginPackageJson({
name: "@scope/oversized-plugin",
displayName: "Oversized Plugin",
version: "1.0.0",
}),
"package/openclaw.plugin.json": JSON.stringify({ id: "oversized.plugin" }),
"package/dist/index.js": "export const demo = true;\n",
"package/dist/model.bin": randomBytes(24 * 1024 * 1024),
});
expect(packBytes.byteLength).toBeGreaterThan(18 * 1024 * 1024);
await writeFile(join(workdir, packName), packBytes);
httpMocks.apiRequest.mockResolvedValueOnce({
uploadUrl: "https://upload.local",
uploadTicket: "uploadTickets:clawpack",
});
httpMocks.uploadBinary.mockResolvedValueOnce({ storageId: "storage:clawpack" });
httpMocks.apiRequestForm.mockResolvedValueOnce({
ok: true,
packageId: "pkg_1",
releaseId: "rel_1",
});
await cmdPublishPackage(makeOpts(workdir), packName, {
sourceRepo: "openclaw/oversized-plugin",
sourceCommit: "abc123",
});
expect(httpMocks.apiRequest).toHaveBeenCalledWith(
"https://clawhub.ai",
{
method: "POST",
path: "/api/cli/upload-url",
token: "tkn",
},
expect.anything(),
);
expect(httpMocks.uploadBinary).toHaveBeenCalledWith(
{
url: "https://upload.local",
bytes: expect.any(Uint8Array),
contentType: "application/octet-stream",
retryCount: 5,
},
expect.anything(),
);
expect(getPublishForm().get("clawpack")).toBe("storage:clawpack");
expect(getPublishForm().get("clawpackUploadTicket")).toBe("uploadTickets:clawpack");
expect(getPublishPayload()).not.toHaveProperty("artifact");
expect(getPublishPayload()).not.toHaveProperty("files");
} finally {
await rm(workdir, { recursive: true, force: true });
}
});
it("packs a plugin folder through npm pack and validates the ClawPack", async () => {
const workdir = await makeTmpWorkdir();
try {
@@ -1315,6 +1378,88 @@ describe("package commands", () => {
}
});
it("packs local ClawPacks over the multipart publish upload budget", async () => {
const workdir = await makeTmpWorkdir();
try {
const folder = join(workdir, "demo-heavy-plugin");
await mkdir(join(folder, "dist"), { recursive: true });
await mkdir(join(workdir, "packs"), { recursive: true });
await writeFile(
join(folder, "package.json"),
makeCodePluginPackageJson({
name: "demo-heavy-plugin",
displayName: "Demo Heavy Plugin",
version: "1.0.0",
}),
"utf8",
);
await writeFile(
join(folder, "openclaw.plugin.json"),
JSON.stringify({ id: "demo.heavy.plugin" }),
"utf8",
);
await writeFile(join(folder, "dist", "index.js"), "export const demo = true;\n", "utf8");
await writeFile(join(folder, "dist", "model.bin"), randomBytes(24 * 1024 * 1024));
await cmdPackPackage(makeOpts(workdir), "demo-heavy-plugin", {
packDestination: "packs",
});
const packPath = join(workdir, "packs", "demo-heavy-plugin-1.0.0.tgz");
const packed = await readFile(packPath);
expect(packed.byteLength).toBeGreaterThan(18 * 1024 * 1024);
expect(parseClawPack(new Uint8Array(packed)).packageName).toBe("demo-heavy-plugin");
} finally {
await rm(workdir, { recursive: true, force: true });
}
});
it("cleans generated ClawPack temp dirs after staged publish failure", async () => {
const workdir = await makeTmpWorkdir();
const beforeTempDirs = await listClawPackTempDirs();
try {
const folder = join(workdir, "demo-heavy-plugin");
await mkdir(join(folder, "dist"), { recursive: true });
await writeFile(
join(folder, "package.json"),
makeCodePluginPackageJson({
name: "demo-heavy-plugin",
displayName: "Demo Heavy Plugin",
version: "1.0.0",
repository: "https://github.com/openclaw/demo-heavy-plugin.git",
}),
"utf8",
);
await writeFile(
join(folder, "openclaw.plugin.json"),
JSON.stringify({ id: "demo.heavy.plugin" }),
"utf8",
);
await writeFile(join(folder, "dist", "index.js"), "export const demo = true;\n", "utf8");
await writeFile(join(folder, "dist", "model.bin"), randomBytes(24 * 1024 * 1024));
httpMocks.apiRequest.mockResolvedValueOnce({
uploadUrl: "https://upload.local",
uploadTicket: "uploadTickets:clawpack",
});
httpMocks.uploadBinary.mockResolvedValueOnce({ storageId: "storage:clawpack" });
httpMocks.apiRequestForm.mockRejectedValueOnce(new Error("Registry rejected upload"));
await expect(
cmdPublishPackage(makeOpts(workdir), "demo-heavy-plugin", {
sourceRepo: "openclaw/demo-heavy-plugin",
sourceCommit: "abc123",
}),
).rejects.toThrow("Registry rejected upload");
expect(getPublishForm().get("clawpack")).toBe("storage:clawpack");
expect(getPublishForm().get("clawpackUploadTicket")).toBe("uploadTickets:clawpack");
const afterTempDirs = await listClawPackTempDirs();
expect([...afterTempDirs].filter((name) => !beforeTempDirs.has(name))).toEqual([]);
} finally {
await rm(workdir, { recursive: true, force: true });
}
});
it("rejects a code plugin ClawPack with TypeScript entries and no compiled runtime", async () => {
const workdir = await makeTmpWorkdir();
try {
+112 -20
View File
@@ -7,10 +7,20 @@ import ignore from "ignore";
import mime from "mime";
import semver from "semver";
import { parseClawPack } from "../../clawpack.js";
import { apiRequest, apiRequestForm, fetchBinary, fetchText, registryUrl } from "../../http.js";
import {
apiRequest,
apiRequestForm,
fetchBinary,
fetchText,
registryUrl,
uploadBinary,
} from "../../http.js";
import {
ApiCliUploadUrlResponseSchema,
ApiRoutes,
LegacyApiRoutes,
ApiV1DeleteResponseSchema,
ApiUploadFileResponseSchema,
ApiV1PackageArtifactResponseSchema,
ApiV1PackageListResponseSchema,
ApiV1PackageModerationStatusResponseSchema,
@@ -24,6 +34,10 @@ import {
ApiV1PackageVersionListResponseSchema,
ApiV1PackageVersionResponseSchema,
ApiV1PublishTokenMintResponseSchema,
estimatePackageMultipartUploadBytes,
getPackageMultipartSizeError,
MAX_PACKAGE_CLAWPACK_BYTES,
MAX_PACKAGE_MULTIPART_BYTES,
normalizeOpenClawExternalPluginCompatibility,
type PackageArtifactSummary,
type PackageCapabilitySummary,
@@ -50,7 +64,6 @@ const DOT_DIR = ".clawhub";
const LEGACY_DOT_DIR = ".clawdhub";
const DOT_IGNORE = ".clawhubignore";
const LEGACY_DOT_IGNORE = ".clawdhubignore";
const MAX_CLAWPACK_BYTES = 120 * 1024 * 1024;
const PACKAGE_PUBLISH_RETRY_COUNT = 5;
type PackageInspectOptions = {
@@ -598,7 +611,6 @@ async function createClawPackFromFolder(options: {
const packPath = resolve(options.packDestination, filename);
const bytes = new Uint8Array(await readFile(packPath));
assertClawPackSize(bytes.byteLength, basename(packPath));
const parsed = parseClawPack(bytes);
return {
path: packPath,
@@ -665,14 +677,26 @@ export async function cmdPublishPackage(
spinner,
});
const form = new FormData();
form.set("payload", JSON.stringify(plan.payload));
const payloadJson = JSON.stringify(plan.payload);
form.set("payload", payloadJson);
if (plan.clawpackOnDisk) {
if (spinner) spinner.text = `Uploading ${plan.clawpackOnDisk.relPath}`;
const blob = new Blob([Buffer.from(plan.clawpackOnDisk.bytes)], {
type: "application/octet-stream",
});
form.append("clawpack", blob, plan.clawpackOnDisk.relPath);
if (isPackageMultipartTooLarge(payloadJson, "clawpack", [plan.clawpackOnDisk])) {
const staged = await uploadClawPackToStorage(
registry,
publishToken,
plan.clawpackOnDisk,
spinner,
);
form.set("clawpack", staged.storageId);
form.set("clawpackUploadTicket", staged.uploadTicket);
} else {
if (spinner) spinner.text = `Uploading ${plan.clawpackOnDisk.relPath}`;
const blob = new Blob([Buffer.from(plan.clawpackOnDisk.bytes)], {
type: "application/octet-stream",
});
form.append("clawpack", blob, plan.clawpackOnDisk.relPath);
}
} else {
let index = 0;
for (const file of plan.filesOnDisk) {
@@ -1551,12 +1575,66 @@ function packageJsonString(value: Record<string, unknown> | null, key: string):
return typeof candidate === "string" && candidate.trim() ? candidate.trim() : undefined;
}
function assertClawPackSize(size: number, label: string) {
if (size > MAX_CLAWPACK_BYTES) {
fail(`ClawPack "${label}" exceeds 120MB limit`);
function assertPackageMultipartSize(
payloadJson: string,
fileFieldName: "files" | "clawpack",
files: PackageFile[],
) {
if (isPackageMultipartTooLarge(payloadJson, fileFieldName, files)) {
fail(getPackageMultipartSizeError());
}
}
function getClawPackSizeError(path: string) {
return `ClawPack "${path}" exceeds 120MB limit`;
}
function isPackageMultipartTooLarge(
payloadJson: string,
fileFieldName: "files" | "clawpack",
files: PackageFile[],
) {
return (
estimatePackageMultipartUploadBytes({
payloadJson,
fileFieldName,
files: files.map((file) => ({
name: file.relPath,
size: file.bytes.byteLength,
type: file.contentType,
})),
}) > MAX_PACKAGE_MULTIPART_BYTES
);
}
async function uploadClawPackToStorage(
registry: string,
publishToken: string,
file: PackageFile,
spinner: ReturnType<typeof createSpinner> | null,
) {
if (spinner) spinner.text = `Uploading ${file.relPath}`;
const { uploadUrl, uploadTicket } = await apiRequest(
registry,
{
method: "POST",
path: LegacyApiRoutes.cliUploadUrl,
token: publishToken,
},
ApiCliUploadUrlResponseSchema,
);
const result = await uploadBinary(
{
url: uploadUrl,
bytes: file.bytes,
contentType: file.contentType ?? "application/octet-stream",
retryCount: PACKAGE_PUBLISH_RETRY_COUNT,
},
ApiUploadFileResponseSchema,
);
return { storageId: result.storageId, uploadTicket };
}
const REAL_BUNDLE_MANIFESTS = [
{ path: ".codex-plugin/plugin.json", format: "codex" },
{ path: ".claude-plugin/plugin.json", format: "claude" },
@@ -1666,11 +1744,13 @@ async function preparePackagePublishPlan(
}
} else {
const folderStat = await stat(folder).catch(() => null);
if (!folderStat) fail("Path must be a folder or ClawPack .tgz");
if (!folderStat) fail("Path must be a folder or package tarball .tgz");
if (folderStat.isFile()) {
if (!folder.endsWith(".tgz")) fail("ClawPack publish files must end in .tgz");
if (!folder.endsWith(".tgz")) fail("Package publish files must end in .tgz");
const bytes = new Uint8Array(await readFile(folder));
assertClawPackSize(bytes.byteLength, basename(folder));
if (bytes.byteLength > MAX_PACKAGE_CLAWPACK_BYTES) {
fail(getClawPackSizeError(basename(folder)));
}
parsedClawpack = parseClawPack(bytes);
clawpackOnDisk = {
relPath: basename(folder),
@@ -1678,7 +1758,7 @@ async function preparePackagePublishPlan(
contentType: "application/octet-stream",
};
} else if (!folderStat.isDirectory()) {
fail("Path must be a folder or ClawPack .tgz");
fail("Path must be a folder or package tarball .tgz");
}
const localGitInfo = folderStat.isDirectory() ? resolveLocalGitInfo(folder) : null;
@@ -1782,7 +1862,9 @@ async function preparePackagePublishPlan(
contentType: mime.getType(entry.path) ?? "application/octet-stream",
}));
}
const totalBytes = clawpackOnDisk
? clawpackOnDisk.bytes.byteLength
: filesOnDisk.reduce((sum, file) => sum + file.bytes.byteLength, 0);
const payload: PackagePublishPayload = {
name,
displayName,
@@ -1804,6 +1886,18 @@ async function preparePackagePublishPlan(
}
: {}),
};
try {
if (clawpackOnDisk) {
if (clawpackOnDisk.bytes.byteLength > MAX_PACKAGE_CLAWPACK_BYTES) {
fail(getClawPackSizeError(clawpackOnDisk.relPath));
}
} else {
assertPackageMultipartSize(JSON.stringify(payload), "files", filesOnDisk);
}
} catch (error) {
await cleanup?.();
throw error;
}
const sourceLabel = describePublishSource(sourceForFetch, source, folder);
return {
@@ -1826,9 +1920,7 @@ async function preparePackagePublishPlan(
version,
...(source?.commit ? { commit: source.commit } : {}),
files: filesOnDisk.length,
totalBytes: clawpackOnDisk
? clawpackOnDisk.bytes.byteLength
: filesOnDisk.reduce((sum, file) => sum + file.bytes.byteLength, 0),
totalBytes,
},
};
}
+97
View File
@@ -52,6 +52,12 @@ type FormRequestArgs =
| { method: "POST"; url: string; token?: string; form: FormData; retryCount?: number };
type TextRequestArgs = { path: string; token?: string } | { url: string; token?: string };
type BinaryUploadArgs = {
url: string;
bytes: Uint8Array;
contentType?: string;
retryCount?: number;
};
type HeaderSource = Headers | Record<string, string> | null | undefined;
@@ -91,6 +97,7 @@ type HttpClient = {
apiRequestForm<T>(registry: string, args: FormRequestArgs, schema: ArkValidator<T>): Promise<T>;
fetchText(registry: string, args: TextRequestArgs): Promise<string>;
fetchBinary(registry: string, args: TextRequestArgs): Promise<Uint8Array>;
uploadBinary<T>(args: BinaryUploadArgs, schema?: ArkValidator<T>): Promise<T>;
downloadZip(
registry: string,
args: { slug: string; version?: string; token?: string },
@@ -264,6 +271,41 @@ export function createHttpClient(options: HttpClientOptions = {}): HttpClient {
});
}
async function uploadBinaryRequest<T>(
args: BinaryUploadArgs,
schema?: ArkValidator<T>,
): Promise<T> {
const json = await runWithRetries(async () => {
if (deps.runtime === "bun") {
return await uploadBinaryViaCurl(deps, args);
}
const headers: Record<string, string> = {};
if (args.contentType) headers["Content-Type"] = args.contentType;
const response = await fetchWithTimeout(
deps,
args.url,
{
method: "POST",
headers,
body: bytesToArrayBuffer(args.bytes),
},
UPLOAD_TIMEOUT_MS,
);
if (!response.ok) {
throwHttpStatusError(
response.status,
await readResponseTextSafe(response),
response.headers,
deps.now,
);
}
return (await response.json()) as unknown;
}, args.retryCount);
if (schema) return parseArk(schema, json, "API response");
return json as T;
}
async function downloadZipRequest(
registry: string,
args: { slug: string; version?: string; token?: string },
@@ -296,6 +338,7 @@ export function createHttpClient(options: HttpClientOptions = {}): HttpClient {
apiRequestForm,
fetchText: fetchTextRequest,
fetchBinary: fetchBinaryRequest,
uploadBinary: uploadBinaryRequest,
downloadZip: downloadZipRequest,
};
}
@@ -361,6 +404,15 @@ export async function fetchBinary(registry: string, args: TextRequestArgs): Prom
return await defaultHttpClient.fetchBinary(registry, args);
}
export async function uploadBinary<T>(args: BinaryUploadArgs): Promise<T>;
export async function uploadBinary<T>(args: BinaryUploadArgs, schema: ArkValidator<T>): Promise<T>;
export async function uploadBinary<T>(
args: BinaryUploadArgs,
schema?: ArkValidator<T>,
): Promise<T> {
return await defaultHttpClient.uploadBinary<T>(args, schema);
}
export async function downloadZip(
registry: string,
args: { slug: string; version?: string; token?: string },
@@ -388,6 +440,12 @@ function createRetryRunner(deps: Pick<HttpClientDeps, "setTimeoutImpl" | "random
};
}
function bytesToArrayBuffer(bytes: Uint8Array) {
const copy = new Uint8Array(bytes.byteLength);
copy.set(bytes);
return copy.buffer;
}
async function fetchWithTimeout(
deps: Pick<HttpClientDeps, "fetchImpl" | "setTimeoutImpl" | "clearTimeoutImpl">,
url: string,
@@ -699,6 +757,45 @@ async function fetchJsonFormViaCurl(
}
}
async function uploadBinaryViaCurl(
deps: Pick<
HttpClientDeps,
"spawnSyncImpl" | "mkdtempImpl" | "writeFileImpl" | "rmImpl" | "tmpdirPath" | "now"
>,
args: BinaryUploadArgs,
) {
const tempDir = await deps.mkdtempImpl(join(deps.tmpdirPath, "clawhub-upload-"));
try {
const filePath = join(tempDir, "upload.bin");
await deps.writeFileImpl(filePath, args.bytes);
const curlArgs = [
"--silent",
"--show-error",
"--location",
"--max-time",
String(UPLOAD_TIMEOUT_SECONDS),
"--write-out",
CURL_WRITE_OUT_FORMAT,
"-X",
"POST",
];
if (args.contentType) curlArgs.push("-H", `Content-Type: ${args.contentType}`);
curlArgs.push("--data-binary", `@${filePath}`, args.url);
const result = deps.spawnSyncImpl("curl", curlArgs, { encoding: "utf8" });
if (result.status !== 0) {
throw new Error(result.stderr || "curl failed");
}
const { body, status, headers: responseHeaders } = parseCurlBodyAndMeta(result.stdout ?? "");
if (status < 200 || status >= 300) {
throwHttpStatusError(status, body, responseHeaders, deps.now);
}
return JSON.parse(body || "null") as unknown;
} finally {
await deps.rmImpl(tempDir, { recursive: true, force: true });
}
}
async function fetchTextViaCurl(
deps: Pick<HttpClientDeps, "spawnSyncImpl" | "now">,
url: string,
+87 -2
View File
@@ -229,7 +229,81 @@ export const PackageTrustedPublisherSchema = type({
});
export type PackageTrustedPublisher = (typeof PackageTrustedPublisherSchema)[inferred];
export const PackagePublishRequestSchema = type({
export const MAX_PACKAGE_MULTIPART_BYTES = 18 * 1024 * 1024;
export const MAX_PACKAGE_CLAWPACK_BYTES = 120 * 1024 * 1024;
const PACKAGE_MULTIPART_FIXED_OVERHEAD_BYTES = 4096;
const PACKAGE_MULTIPART_PART_OVERHEAD_BYTES = 1024;
export type PackageMultipartUploadField = "files" | "clawpack";
export type PackageMultipartUploadPart = {
name: string;
size: number;
type?: string;
};
export type PackageMultipartUploadSizeInput = {
payloadJson: string;
fileFieldName: PackageMultipartUploadField;
files: readonly PackageMultipartUploadPart[];
};
export function estimatePackageMultipartUploadBytes(
input: PackageMultipartUploadSizeInput,
): number {
return (
PACKAGE_MULTIPART_FIXED_OVERHEAD_BYTES +
estimateMultipartStringPartBytes("payload", input.payloadJson) +
input.files.reduce(
(sum, file) => sum + estimateMultipartFilePartBytes(input.fileFieldName, file),
0,
)
);
}
export function isPackageMultipartUploadTooLarge(input: PackageMultipartUploadSizeInput): boolean {
return estimatePackageMultipartUploadBytes(input) > MAX_PACKAGE_MULTIPART_BYTES;
}
export function getPackageMultipartSizeError(): string {
return "Package upload exceeds 18MB multipart upload limit";
}
function estimateMultipartStringPartBytes(fieldName: string, value: string): number {
return PACKAGE_MULTIPART_PART_OVERHEAD_BYTES + utf8ByteLength(fieldName) + utf8ByteLength(value);
}
function estimateMultipartFilePartBytes(
fieldName: PackageMultipartUploadField,
file: PackageMultipartUploadPart,
): number {
return (
file.size +
PACKAGE_MULTIPART_PART_OVERHEAD_BYTES +
utf8ByteLength(fieldName) +
utf8ByteLength(file.name) +
utf8ByteLength(file.type ?? "")
);
}
function utf8ByteLength(value: string): number {
let bytes = 0;
for (let index = 0; index < value.length; index += 1) {
const codePoint = value.codePointAt(index);
if (codePoint === undefined) continue;
if (codePoint > 0xffff) index += 1;
if (codePoint <= 0x7f) {
bytes += 1;
} else if (codePoint <= 0x7ff) {
bytes += 2;
} else if (codePoint <= 0xffff) {
bytes += 3;
} else {
bytes += 4;
}
}
return bytes;
}
const PackagePublishMetadataFields = {
name: "string",
displayName: "string?",
ownerHandle: "string?",
@@ -241,10 +315,21 @@ export const PackagePublishRequestSchema = type({
tags: "string[]?",
source: PublishSourceSchema.optional(),
bundle: BundlePublishMetadataSchema.optional(),
} as const;
export const PackagePublishMetadataSchema = type({
"+": "reject",
...PackagePublishMetadataFields,
});
export type PackagePublishMetadata = (typeof PackagePublishMetadataSchema)[inferred];
export const ServerPackagePublishRequestSchema = type({
"+": "reject",
...PackagePublishMetadataFields,
artifact: PackagePublishArtifactSchema.optional(),
files: CliPublishFileSchema.array(),
});
export type PackagePublishRequest = (typeof PackagePublishRequestSchema)[inferred];
export type ServerPackagePublishRequest = (typeof ServerPackagePublishRequestSchema)[inferred];
export const PackageListItemSchema = type({
name: "string",
+1
View File
@@ -54,6 +54,7 @@ export const ApiSkillMetaResponseSchema = type({
export const ApiCliUploadUrlResponseSchema = type({
uploadUrl: "string",
uploadTicket: "string",
});
export const ApiUploadFileResponseSchema = type({
@@ -24,6 +24,7 @@ export function createHttpModuleMocks() {
const downloadZip = vi.fn();
const fetchBinary = vi.fn();
const fetchText = vi.fn();
const uploadBinary = vi.fn();
const registryUrl = vi.fn(buildRegistryUrl);
return {
@@ -32,6 +33,7 @@ export function createHttpModuleMocks() {
downloadZip,
fetchBinary,
fetchText,
uploadBinary,
registryUrl,
moduleFactory: () => ({
apiRequest: (registry: unknown, args: unknown, schema?: unknown) =>
@@ -41,6 +43,7 @@ export function createHttpModuleMocks() {
downloadZip: (registry: unknown, args: unknown) => downloadZip(registry, args),
fetchBinary: (registry: unknown, args: unknown) => fetchBinary(registry, args),
fetchText: (registry: unknown, args: unknown) => fetchText(registry, args),
uploadBinary: (args: unknown, schema?: unknown) => uploadBinary(args, schema),
registryUrl: (...args: [string, string]) => registryUrl(...args),
}),
};
+51 -9
View File
@@ -228,18 +228,27 @@ export declare const PackageTrustedPublisherSchema: import("arktype/internal/var
environment?: string | undefined;
}, {}>;
export type PackageTrustedPublisher = (typeof PackageTrustedPublisherSchema)[inferred];
export declare const PackagePublishRequestSchema: import("arktype/internal/variants/object.ts").ObjectType<{
export declare const MAX_PACKAGE_MULTIPART_BYTES: number;
export declare const MAX_PACKAGE_CLAWPACK_BYTES: number;
export type PackageMultipartUploadField = "files" | "clawpack";
export type PackageMultipartUploadPart = {
name: string;
size: number;
type?: string;
};
export type PackageMultipartUploadSizeInput = {
payloadJson: string;
fileFieldName: PackageMultipartUploadField;
files: readonly PackageMultipartUploadPart[];
};
export declare function estimatePackageMultipartUploadBytes(input: PackageMultipartUploadSizeInput): number;
export declare function isPackageMultipartUploadTooLarge(input: PackageMultipartUploadSizeInput): boolean;
export declare function getPackageMultipartSizeError(): string;
export declare const PackagePublishMetadataSchema: import("arktype/internal/variants/object.ts").ObjectType<{
name: string;
family: "skill" | "code-plugin" | "bundle-plugin";
version: string;
changelog: string;
files: {
path: string;
size: number;
storageId: string;
sha256: string;
contentType?: string | undefined;
}[];
displayName?: string | undefined;
ownerHandle?: string | undefined;
manualOverrideReason?: string | undefined;
@@ -259,6 +268,20 @@ export declare const PackagePublishRequestSchema: import("arktype/internal/varia
format?: string | undefined;
hostTargets?: string[] | undefined;
} | undefined;
}, {}>;
export type PackagePublishMetadata = (typeof PackagePublishMetadataSchema)[inferred];
export declare const ServerPackagePublishRequestSchema: import("arktype/internal/variants/object.ts").ObjectType<{
files: {
path: string;
size: number;
storageId: string;
sha256: string;
contentType?: string | undefined;
}[];
name: string;
family: "skill" | "code-plugin" | "bundle-plugin";
version: string;
changelog: string;
artifact?: {
kind: "npm-pack";
storageId: string;
@@ -271,8 +294,27 @@ export declare const PackagePublishRequestSchema: import("arktype/internal/varia
npmUnpackedSize: number;
npmFileCount: number;
} | undefined;
displayName?: string | undefined;
ownerHandle?: string | undefined;
manualOverrideReason?: string | undefined;
channel?: "official" | "community" | "private" | undefined;
tags?: string[] | undefined;
source?: {
kind: "github";
url: string;
repo: string;
ref: string;
commit: string;
path: string;
importedAt: number;
} | undefined;
bundle?: {
id?: string | undefined;
format?: string | undefined;
hostTargets?: string[] | undefined;
} | undefined;
}, {}>;
export type PackagePublishRequest = (typeof PackagePublishRequestSchema)[inferred];
export type ServerPackagePublishRequest = (typeof ServerPackagePublishRequestSchema)[inferred];
export declare const PackageListItemSchema: import("arktype/internal/variants/object.ts").ObjectType<{
name: string;
displayName: string;
+57 -1
View File
@@ -188,7 +188,55 @@ export const PackageTrustedPublisherSchema = type({
workflowFilename: "string",
environment: "string?",
});
export const PackagePublishRequestSchema = type({
export const MAX_PACKAGE_MULTIPART_BYTES = 18 * 1024 * 1024;
export const MAX_PACKAGE_CLAWPACK_BYTES = 120 * 1024 * 1024;
const PACKAGE_MULTIPART_FIXED_OVERHEAD_BYTES = 4096;
const PACKAGE_MULTIPART_PART_OVERHEAD_BYTES = 1024;
export function estimatePackageMultipartUploadBytes(input) {
return (PACKAGE_MULTIPART_FIXED_OVERHEAD_BYTES +
estimateMultipartStringPartBytes("payload", input.payloadJson) +
input.files.reduce((sum, file) => sum + estimateMultipartFilePartBytes(input.fileFieldName, file), 0));
}
export function isPackageMultipartUploadTooLarge(input) {
return estimatePackageMultipartUploadBytes(input) > MAX_PACKAGE_MULTIPART_BYTES;
}
export function getPackageMultipartSizeError() {
return "Package upload exceeds 18MB multipart upload limit";
}
function estimateMultipartStringPartBytes(fieldName, value) {
return PACKAGE_MULTIPART_PART_OVERHEAD_BYTES + utf8ByteLength(fieldName) + utf8ByteLength(value);
}
function estimateMultipartFilePartBytes(fieldName, file) {
return (file.size +
PACKAGE_MULTIPART_PART_OVERHEAD_BYTES +
utf8ByteLength(fieldName) +
utf8ByteLength(file.name) +
utf8ByteLength(file.type ?? ""));
}
function utf8ByteLength(value) {
let bytes = 0;
for (let index = 0; index < value.length; index += 1) {
const codePoint = value.codePointAt(index);
if (codePoint === undefined)
continue;
if (codePoint > 0xffff)
index += 1;
if (codePoint <= 0x7f) {
bytes += 1;
}
else if (codePoint <= 0x7ff) {
bytes += 2;
}
else if (codePoint <= 0xffff) {
bytes += 3;
}
else {
bytes += 4;
}
}
return bytes;
}
const PackagePublishMetadataFields = {
name: "string",
displayName: "string?",
ownerHandle: "string?",
@@ -200,6 +248,14 @@ export const PackagePublishRequestSchema = type({
tags: "string[]?",
source: PublishSourceSchema.optional(),
bundle: BundlePublishMetadataSchema.optional(),
};
export const PackagePublishMetadataSchema = type({
"+": "reject",
...PackagePublishMetadataFields,
});
export const ServerPackagePublishRequestSchema = type({
"+": "reject",
...PackagePublishMetadataFields,
artifact: PackagePublishArtifactSchema.optional(),
files: CliPublishFileSchema.array(),
});
File diff suppressed because one or more lines are too long
+1
View File
@@ -47,6 +47,7 @@ export declare const ApiSkillMetaResponseSchema: import("arktype/internal/varian
}, {}>;
export declare const ApiCliUploadUrlResponseSchema: import("arktype/internal/variants/object.ts").ObjectType<{
uploadUrl: string;
uploadTicket: string;
}, {}>;
export declare const ApiUploadFileResponseSchema: import("arktype/internal/variants/object.ts").ObjectType<{
storageId: string;
+1
View File
@@ -45,6 +45,7 @@ export const ApiSkillMetaResponseSchema = type({
});
export const ApiCliUploadUrlResponseSchema = type({
uploadUrl: "string",
uploadTicket: "string",
});
export const ApiUploadFileResponseSchema = type({
storageId: "string",
File diff suppressed because one or more lines are too long
+87 -2
View File
@@ -253,7 +253,81 @@ export const PackageTrustedPublisherSchema = type({
});
export type PackageTrustedPublisher = (typeof PackageTrustedPublisherSchema)[inferred];
export const PackagePublishRequestSchema = type({
export const MAX_PACKAGE_MULTIPART_BYTES = 18 * 1024 * 1024;
export const MAX_PACKAGE_CLAWPACK_BYTES = 120 * 1024 * 1024;
const PACKAGE_MULTIPART_FIXED_OVERHEAD_BYTES = 4096;
const PACKAGE_MULTIPART_PART_OVERHEAD_BYTES = 1024;
export type PackageMultipartUploadField = "files" | "clawpack";
export type PackageMultipartUploadPart = {
name: string;
size: number;
type?: string;
};
export type PackageMultipartUploadSizeInput = {
payloadJson: string;
fileFieldName: PackageMultipartUploadField;
files: readonly PackageMultipartUploadPart[];
};
export function estimatePackageMultipartUploadBytes(
input: PackageMultipartUploadSizeInput,
): number {
return (
PACKAGE_MULTIPART_FIXED_OVERHEAD_BYTES +
estimateMultipartStringPartBytes("payload", input.payloadJson) +
input.files.reduce(
(sum, file) => sum + estimateMultipartFilePartBytes(input.fileFieldName, file),
0,
)
);
}
export function isPackageMultipartUploadTooLarge(input: PackageMultipartUploadSizeInput): boolean {
return estimatePackageMultipartUploadBytes(input) > MAX_PACKAGE_MULTIPART_BYTES;
}
export function getPackageMultipartSizeError(): string {
return "Package upload exceeds 18MB multipart upload limit";
}
function estimateMultipartStringPartBytes(fieldName: string, value: string): number {
return PACKAGE_MULTIPART_PART_OVERHEAD_BYTES + utf8ByteLength(fieldName) + utf8ByteLength(value);
}
function estimateMultipartFilePartBytes(
fieldName: PackageMultipartUploadField,
file: PackageMultipartUploadPart,
): number {
return (
file.size +
PACKAGE_MULTIPART_PART_OVERHEAD_BYTES +
utf8ByteLength(fieldName) +
utf8ByteLength(file.name) +
utf8ByteLength(file.type ?? "")
);
}
function utf8ByteLength(value: string): number {
let bytes = 0;
for (let index = 0; index < value.length; index += 1) {
const codePoint = value.codePointAt(index);
if (codePoint === undefined) continue;
if (codePoint > 0xffff) index += 1;
if (codePoint <= 0x7f) {
bytes += 1;
} else if (codePoint <= 0x7ff) {
bytes += 2;
} else if (codePoint <= 0xffff) {
bytes += 3;
} else {
bytes += 4;
}
}
return bytes;
}
const PackagePublishMetadataFields = {
name: "string",
displayName: "string?",
ownerHandle: "string?",
@@ -265,10 +339,21 @@ export const PackagePublishRequestSchema = type({
tags: "string[]?",
source: PublishSourceSchema.optional(),
bundle: BundlePublishMetadataSchema.optional(),
} as const;
export const PackagePublishMetadataSchema = type({
"+": "reject",
...PackagePublishMetadataFields,
});
export type PackagePublishMetadata = (typeof PackagePublishMetadataSchema)[inferred];
export const ServerPackagePublishRequestSchema = type({
"+": "reject",
...PackagePublishMetadataFields,
artifact: PackagePublishArtifactSchema.optional(),
files: CliPublishFileSchema.array(),
});
export type PackagePublishRequest = (typeof PackagePublishRequestSchema)[inferred];
export type ServerPackagePublishRequest = (typeof ServerPackagePublishRequestSchema)[inferred];
export const PackageListItemSchema = type({
name: "string",
+1
View File
@@ -55,6 +55,7 @@ export const ApiSkillMetaResponseSchema = type({
export const ApiCliUploadUrlResponseSchema = type({
uploadUrl: "string",
uploadTicket: "string",
});
export const ApiUploadFileResponseSchema = type({
+22
View File
@@ -116,6 +116,28 @@ See also: [acceptable-usage.md](./acceptable-usage.md) for the marketplace polic
- Skills directory supports an optional "Hide suspicious" filter to exclude
active-but-flagged (`flagged.suspicious`) entries from browse/search results.
## Package publish upload boundary
- Package publish is multipart-only. `POST /api/v1/packages` must reject JSON
request bodies, including bodies that reference pre-existing storage IDs.
- Public HTTP package publish payloads must not accept caller-supplied `files`
or `artifact` metadata. Internal publish actions may receive that metadata
only after the HTTP boundary derives it from uploaded multipart bytes or a
staged ClawPack blob.
- Package publish accepts either multipart `files` uploads or one `clawpack`
tarball reference, never both in the same request. `clawpack` may be a direct
`.tgz` file part or a Convex storage id created by the upload-url flow. The
storage-id path must include the matching `clawpackUploadTicket`, and the
server must reject tickets from a different auth context, expired or used
tickets, and storage blobs created before the ticket.
- Direct package publish multipart bytes are capped at 18MB so callers get a
clear ClawHub validation error before hitting Convex's 20MB HTTP action body
cap. ClawPack tarballs keep the 120MB package tarball cap through staged
storage uploads.
- For tarball uploads, ClawHub stores the uploaded tarball, derives its
artifact hashes and npm metadata, and derives package file metadata from the
tarball contents.
## Skill moderation pipeline
- New skill publishes now persist a deterministic static scan result on the version.