fix: upload skill files directly to Convex (#3391)

* fix: upload skill files directly to Convex

* chore: prepare clawhub CLI 0.23.3
This commit is contained in:
Patrick Erichsen
2026-08-03 20:37:01 -07:00
committed by GitHub
parent 7571488ab3
commit 87ca030c30
34 changed files with 1014 additions and 128 deletions
+6
View File
@@ -21,6 +21,12 @@
- CLI: accept npm 12's package-keyed `npm pack --json` output when building ClawPacks while retaining compatibility with earlier npm array output.
- Web/API: preserve JSON, SSR, and OG responses through the Convex proxy after the H3 response-wrapper update.
## 0.23.3 - 2026-08-03
### Fixes
- CLI/API: stage skill files directly in Convex storage before publishing metadata, so bundles within the documented 50MB total limit no longer hit Vercel's smaller request-body limit.
## 0.23.2 - 2026-08-03
### Changes
+1 -1
View File
@@ -101,7 +101,7 @@
},
"packages/clawhub": {
"name": "clawhub",
"version": "0.23.2",
"version": "0.23.3",
"bin": {
"clawdhub": "bin/clawdhub.js",
"clawhub": "bin/clawdhub.js",
+2
View File
@@ -192,6 +192,7 @@ import type * as skillPresentationAssets from "../skillPresentationAssets.js";
import type * as skillPresentationAssetsHttp from "../skillPresentationAssetsHttp.js";
import type * as skillPresentationBackfill from "../skillPresentationBackfill.js";
import type * as skillPresentationImageNode from "../skillPresentationImageNode.js";
import type * as skillPublishUploads from "../skillPublishUploads.js";
import type * as skillStatEvents from "../skillStatEvents.js";
import type * as skillTransfers from "../skillTransfers.js";
import type * as skills from "../skills.js";
@@ -401,6 +402,7 @@ declare const fullApi: ApiFromModules<{
skillPresentationAssetsHttp: typeof skillPresentationAssetsHttp;
skillPresentationBackfill: typeof skillPresentationBackfill;
skillPresentationImageNode: typeof skillPresentationImageNode;
skillPublishUploads: typeof skillPublishUploads;
skillStatEvents: typeof skillStatEvents;
skillTransfers: typeof skillTransfers;
skills: typeof skills;
+7
View File
@@ -245,6 +245,13 @@ if (process.env.CLAWHUB_DISABLE_CRONS !== "1" && process.env.CLAWHUB_PREVIEW !==
{ batchSize: RETENTION_STANDARD_BATCH_SIZE },
);
crons.interval(
"skill-publish-upload-retention-prune",
{ hours: 1 },
internal.retention.pruneExpiredSkillPublishUploadsInternal,
{ batchSize: RETENTION_STANDARD_BATCH_SIZE },
);
crons.interval(
"http-rate-limit-keys-prune",
{ hours: 1 },
+136 -3
View File
@@ -7228,6 +7228,124 @@ describe("httpApiV1 handlers", () => {
expect(new Uint8Array(await response.arrayBuffer())).toEqual(fileBytes);
});
it("creates a direct skill upload URL for an authenticated API user", async () => {
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({
userId: "users:1",
user: { handle: "p" },
} as never);
const runMutation = vi
.fn()
.mockResolvedValueOnce(okRate())
.mockResolvedValueOnce({ uploadTicket: "skillPublishUploadTickets:1" });
const response = await __handlers.skillsPostRouterV1Handler(
makeCtx({ runMutation }),
new Request("https://example.com/api/v1/skills/-/upload-url", {
method: "POST",
headers: {
Authorization: "Bearer clh_test",
"Content-Type": "application/json",
},
body: JSON.stringify({
path: "SKILL.md",
size: 5,
sha256: "a".repeat(64),
contentType: "text/markdown",
}),
}),
);
expect(response.status).toBe(200);
expect(await response.json()).toEqual({
uploadUrl: "https://example.com/api/v1/skills/-/upload/skillPublishUploadTickets%3A1",
uploadTicket: "skillPublishUploadTickets:1",
});
});
it("stores a bounded direct skill upload and attaches it to its ticket", async () => {
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({
userId: "users:1",
user: { handle: "p" },
} as never);
const bytes = new TextEncoder().encode("hello");
const runQuery = vi.fn().mockResolvedValue({
path: "SKILL.md",
size: bytes.byteLength,
sha256: "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824",
contentType: "text/markdown",
});
const runMutation = vi.fn().mockResolvedValue(okRate());
const store = vi.fn().mockResolvedValue("storage:1");
const response = await __handlers.skillsPostRouterV1Handler(
makeCtx({ runQuery, runMutation, storage: { store, delete: vi.fn() } }),
new Request(
"https://example.convex.site/api/v1/skills/-/upload/skillPublishUploadTickets%3A1",
{
method: "POST",
headers: {
Authorization: "Bearer clh_test",
"Content-Type": "text/markdown",
},
body: bytes,
},
),
);
expect(response.status).toBe(200);
expect(await response.json()).toEqual({ storageId: "storage:1" });
expect(store).toHaveBeenCalledTimes(1);
expect(runMutation).toHaveBeenCalledWith(
internal.skillPublishUploads.attachSkillPublishUploadInternal,
{
userId: "users:1",
uploadTicket: "skillPublishUploadTickets:1",
storageId: "storage:1",
},
);
});
it("stops reading a direct skill upload once it exceeds the ticket size", async () => {
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({
userId: "users:1",
user: { handle: "p" },
} as never);
const runQuery = vi.fn().mockResolvedValue({
path: "SKILL.md",
size: 5,
sha256: "a".repeat(64),
contentType: "text/markdown",
});
const store = vi.fn();
const body = new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(new Uint8Array([1, 2, 3]));
controller.enqueue(new Uint8Array([4, 5, 6]));
controller.close();
},
});
const response = await __handlers.skillsPostRouterV1Handler(
makeCtx({
runQuery,
runMutation: vi.fn().mockResolvedValue(okRate()),
storage: { store, delete: vi.fn() },
}),
new Request(
"https://example.convex.site/api/v1/skills/-/upload/skillPublishUploadTickets%3A1",
{
method: "POST",
headers: { Authorization: "Bearer clh_test" },
body,
duplex: "half",
} as RequestInit & { duplex: "half" },
),
);
expect(response.status).toBe(413);
expect(store).not.toHaveBeenCalled();
});
it("publish json succeeds", async () => {
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({
userId: "users:1",
@@ -7254,6 +7372,7 @@ describe("httpApiV1 handlers", () => {
storageId: "storage:1",
sha256: "abc",
contentType: "text/plain",
uploadTicket: "skillPublishUploadTickets:1",
},
],
});
@@ -7312,6 +7431,7 @@ describe("httpApiV1 handlers", () => {
storageId: "storage:1",
sha256: "abc",
contentType: "text/plain",
uploadTicket: "skillPublishUploadTickets:1",
},
],
});
@@ -7376,6 +7496,7 @@ describe("httpApiV1 handlers", () => {
storageId: "storage:1",
sha256: "abc",
contentType: "text/plain",
uploadTicket: "skillPublishUploadTickets:1",
},
],
});
@@ -7394,7 +7515,9 @@ describe("httpApiV1 handlers", () => {
expect.anything(),
"users:1",
expect.objectContaining({ source }),
{},
expect.objectContaining({
skillPublishUploadTickets: ["skillPublishUploadTickets:1"],
}),
);
expect(vi.mocked(publishVersionForUser).mock.calls[0]?.[3]).not.toHaveProperty(
"sourceProvenance",
@@ -7431,6 +7554,7 @@ describe("httpApiV1 handlers", () => {
storageId: "storage:1",
sha256: "abc",
contentType: "text/plain",
uploadTicket: "skillPublishUploadTickets:1",
},
],
});
@@ -7455,7 +7579,10 @@ describe("httpApiV1 handlers", () => {
expect.anything(),
"users:1",
expect.not.objectContaining({ ownerHandle: expect.anything() }),
{ ownerPublisherId: "publishers:openclaw" },
expect.objectContaining({
ownerPublisherId: "publishers:openclaw",
skillPublishUploadTickets: ["skillPublishUploadTickets:1"],
}),
);
});
@@ -7490,6 +7617,7 @@ describe("httpApiV1 handlers", () => {
storageId: "storage:1",
sha256: "abc",
contentType: "text/plain",
uploadTicket: "skillPublishUploadTickets:1",
},
],
});
@@ -7506,7 +7634,10 @@ describe("httpApiV1 handlers", () => {
expect.anything(),
"users:1",
expect.not.objectContaining({ ownerHandle: expect.anything() }),
{ ownerPublisherId: "publishers:openclaw" },
expect.objectContaining({
ownerPublisherId: "publishers:openclaw",
skillPublishUploadTickets: ["skillPublishUploadTickets:1"],
}),
);
});
@@ -7533,6 +7664,7 @@ describe("httpApiV1 handlers", () => {
storageId: "storage:1",
sha256: "abc",
contentType: "text/plain",
uploadTicket: "skillPublishUploadTickets:1",
},
],
});
@@ -7568,6 +7700,7 @@ describe("httpApiV1 handlers", () => {
storageId: "storage:1",
sha256: "abc",
contentType: "text/plain",
uploadTicket: "skillPublishUploadTickets:1",
},
],
});
+1
View File
@@ -537,6 +537,7 @@ export function parsePublishBody(body: unknown) {
files: parsed.files.map((file) => ({
...file,
storageId: file.storageId as Id<"_storage">,
uploadTicket: file.uploadTicket as Id<"skillPublishUploadTickets"> | undefined,
})),
};
}
+126 -5
View File
@@ -1,5 +1,6 @@
import {
ApiRoutes,
ApiV1SkillUploadUrlRequestSchema,
ApiV1SkillBulkRescanBatchRequestSchema,
ApiV1SkillBulkRescanStatusRequestSchema,
ApiV1SkillHardDeleteRequestSchema,
@@ -95,6 +96,30 @@ const DEFAULT_EXPORT_PAGE_LIMIT = 250;
const MAX_EXPORT_TOTAL_BYTES = 256 * 1024 * 1024;
const MAX_SECURITY_VERDICT_ITEMS = 100;
async function readRequestBodyWithinLimit(request: Request, maxBytes: number) {
if (!request.body) return new Uint8Array();
const reader = request.body.getReader();
const chunks: Uint8Array[] = [];
let totalBytes = 0;
while (true) {
const { done, value } = await reader.read();
if (done) break;
totalBytes += value.byteLength;
if (totalBytes > maxBytes) {
await reader.cancel("Upload exceeds its declared size").catch(() => undefined);
return null;
}
chunks.push(value);
}
const bytes = new Uint8Array(totalBytes);
let offset = 0;
for (const chunk of chunks) {
bytes.set(chunk, offset);
offset += chunk.byteLength;
}
return bytes;
}
type ListSkillsResult = {
items: Array<{
skill: {
@@ -2625,6 +2650,13 @@ export async function publishSkillV1Handler(ctx: ActionCtx, request: Request) {
if (!hasAcceptedLegacyLicenseTerms(payload.acceptLicenseTerms)) {
return text("MIT-0 license terms must be accepted to publish skills", 400, rate.headers);
}
if (payload.files.some((file) => !file.uploadTicket)) {
return text(
"Every directly uploaded skill file requires an upload ticket",
400,
rate.headers,
);
}
const result = await publishSkillPayloadForApiUser(ctx, auth.userId, payload);
return json({ ok: true, ...result }, 200, rate.headers);
}
@@ -2651,6 +2683,13 @@ async function publishSkillPayloadForApiUser(
payload: ReturnType<typeof parsePublishBody>,
) {
const { ownerHandle, sourceOwnerHandle, migrateOwner, ...publishPayload } = payload;
const uploadTickets = publishPayload.files.flatMap((file) =>
file.uploadTicket ? [file.uploadTicket] : [],
);
if (uploadTickets.length > 0 && uploadTickets.length !== publishPayload.files.length) {
throw new Error("Every directly uploaded skill file requires an upload ticket");
}
const files = publishPayload.files.map(({ uploadTicket: _uploadTicket, ...file }) => file);
const target = ownerHandle
? ((await ctx.runMutation(internal.publishers.resolvePublishTargetForUserInternal, {
actorUserId: userId,
@@ -2667,11 +2706,17 @@ async function publishSkillPayloadForApiUser(
})) as { publisherId: Id<"publishers"> })
: null;
const shouldMigrateOwner = Boolean(target && source);
return await publishVersionForUser(ctx, userId, publishPayload, {
...(target ? { ownerPublisherId: target.publisherId } : {}),
...(source ? { sourceOwnerPublisherId: source.publisherId } : {}),
...(shouldMigrateOwner ? { migrateOwner: true } : {}),
});
return await publishVersionForUser(
ctx,
userId,
{ ...publishPayload, files },
{
...(target ? { ownerPublisherId: target.publisherId } : {}),
...(source ? { sourceOwnerPublisherId: source.publisherId } : {}),
...(shouldMigrateOwner ? { migrateOwner: true } : {}),
...(uploadTickets.length > 0 ? { skillPublishUploadTickets: uploadTickets } : {}),
},
);
}
function hasAcceptedLegacyLicenseTerms(acceptLicenseTerms: boolean | undefined) {
@@ -2942,6 +2987,82 @@ export async function skillsPostRouterV1Handler(ctx: ActionCtx, request: Request
const action = segments[1] ?? "";
const slug = segments[0]?.trim().toLowerCase() ?? "";
if (segments.length === 2 && slug === "-" && action === "upload-url") {
const auth = await requireApiTokenUserOrResponse(ctx, request, rate.headers);
if (!auth.ok) return auth.response;
try {
const expected = parseArk(
ApiV1SkillUploadUrlRequestSchema,
await request.json(),
"Skill upload request",
);
const { uploadTicket } = await ctx.runMutation(
internal.skillPublishUploads.createSkillPublishUploadInternal,
{
userId: auth.userId,
path: expected.path,
size: expected.size,
sha256: expected.sha256,
...(expected.contentType ? { contentType: expected.contentType } : {}),
},
);
// The request reaching this Convex action has the direct Convex origin even
// when clawhub.ai forwarded it. Keep the file body off the Vercel request path.
const uploadUrl = new URL(
`/api/v1/skills/-/upload/${encodeURIComponent(uploadTicket)}`,
request.url,
).toString();
return json({ uploadUrl, uploadTicket }, 200, rate.headers);
} catch (error) {
const message = error instanceof Error ? error.message : "Invalid skill upload request";
return text(message, 400, rate.headers);
}
}
if (segments.length === 3 && slug === "-" && action === "upload" && segments[2]) {
const auth = await requireApiTokenUserOrResponse(ctx, request, rate.headers);
if (!auth.ok) return auth.response;
const uploadTicket = segments[2] as Id<"skillPublishUploadTickets">;
let storageId: Id<"_storage"> | undefined;
try {
const expected = await ctx.runQuery(
internal.skillPublishUploads.getSkillPublishUploadForUserInternal,
{ userId: auth.userId, uploadTicket },
);
const declaredLength = Number(request.headers.get("content-length"));
if (Number.isFinite(declaredLength) && declaredLength > expected.size) {
return text("Uploaded file exceeds its declared size", 413, rate.headers);
}
const bytes = await readRequestBodyWithinLimit(request, expected.size);
if (!bytes) {
return text("Uploaded file exceeds its declared size", 413, rate.headers);
}
if (bytes.byteLength !== expected.size) {
return text("Uploaded file size does not match its upload ticket", 400, rate.headers);
}
const digest = await crypto.subtle.digest("SHA-256", bytes);
const sha256 = Array.from(new Uint8Array(digest), (byte) =>
byte.toString(16).padStart(2, "0"),
).join("");
if (sha256 !== expected.sha256) {
return text("Uploaded file SHA-256 does not match its upload ticket", 400, rate.headers);
}
storageId = await ctx.storage.store(
new Blob([bytes], expected.contentType ? { type: expected.contentType } : undefined),
);
await ctx.runMutation(internal.skillPublishUploads.attachSkillPublishUploadInternal, {
userId: auth.userId,
uploadTicket,
storageId,
});
return json({ storageId }, 200, rate.headers);
} catch (error) {
if (storageId) await ctx.storage.delete(storageId).catch(() => undefined);
const message = error instanceof Error ? error.message : "Skill upload failed";
return text(message, 400, rate.headers);
}
}
if (segments.length === 3 && segments[1] === "tags" && segments[2]) {
if (!slug) return text("Slug required", 400, rate.headers);
const auth = await requireApiTokenUserOrResponse(ctx, request, rate.headers);
+6
View File
@@ -45,6 +45,12 @@ describe("retention policies", () => {
expirationIndex: "by_expires_at",
prune: "retention.pruneExpiredPublisherInvitesInternal",
});
expect(getRetentionPolicy("skillPublishUploadTickets")).toMatchObject({
classification: "ephemeral",
expirationField: "expiresAt",
expirationIndex: "by_expires_at",
prune: "retention.pruneExpiredSkillPublishUploadsInternal",
});
});
it("documents package daily stats as durable analytics", () => {
+9
View File
@@ -105,6 +105,15 @@ export const RETENTION_POLICIES = {
retention: "Organization logo upload ticket TTL.",
},
),
skillPublishUploadTickets: ephemeral(
"Direct skill file upload tickets are deleted after their short staging window.",
{
expirationField: "expiresAt",
expirationIndex: "by_expires_at",
prune: "retention.pruneExpiredSkillPublishUploadsInternal",
retention: "1 hour.",
},
),
officialPublishers: permanent("Manual official publisher assignments."),
githubSkillSources: permanent("Tracked GitHub source configuration."),
githubSkillContents: derived("Cached GitHub source content snapshots.", "githubSkillSources"),
+2
View File
@@ -154,6 +154,7 @@ export type PublishOptions = {
// accidentally transfer ownership.
migrateOwner?: boolean;
stagePrePublicationChecks?: boolean;
skillPublishUploadTickets?: Id<"skillPublishUploadTickets">[];
};
type InternalPublishOptions = PublishOptions;
@@ -489,6 +490,7 @@ async function publishVersionForUserInternal(
const skillInsertArgs = {
userId,
skillPublishUploadTickets: options.skillPublishUploadTickets,
ownerPublisherId: options.ownerPublisherId,
sourceOwnerPublisherId: options.sourceOwnerPublisherId,
migrateOwner: options.migrateOwner,
+50
View File
@@ -5,6 +5,7 @@ const retentionRefs = vi.hoisted(() => ({
pruneExpiredAuthSessionsInternal: Symbol("pruneExpiredAuthSessionsInternal"),
pruneExpiredAuthRefreshTokensInternal: Symbol("pruneExpiredAuthRefreshTokensInternal"),
pruneExpiredPublisherInvitesInternal: Symbol("pruneExpiredPublisherInvitesInternal"),
pruneExpiredSkillPublishUploadsInternal: Symbol("pruneExpiredSkillPublishUploadsInternal"),
}));
vi.mock("./_generated/api", () => ({
@@ -17,6 +18,7 @@ const {
pruneExpiredAuthRefreshTokensInternal,
pruneExpiredAuthSessionsInternal,
pruneExpiredPublisherInvitesInternal,
pruneExpiredSkillPublishUploadsInternal,
} = await import("./retention");
type WrappedHandler<TArgs, TResult> = {
@@ -41,6 +43,12 @@ const prunePublisherInvitesHandler = (
{ deleted: number; hasMore: boolean }
>
)._handler;
const pruneSkillUploadsHandler = (
pruneExpiredSkillPublishUploadsInternal as unknown as WrappedHandler<
{ batchSize?: number },
{ deletedTickets: number; deletedStorage: number; hasMore: boolean }
>
)._handler;
function makeDb(base: { query: ReturnType<typeof vi.fn>; delete: ReturnType<typeof vi.fn> }) {
return {
@@ -233,4 +241,46 @@ describe("auth retention", () => {
batchSize: 1,
});
});
it("deletes expired unconsumed skill upload storage in bounded batches", async () => {
const now = 4_000_000;
vi.spyOn(Date, "now").mockReturnValue(now);
const rows = [
{
_id: "skillPublishUploadTickets:one",
expiresAt: now - 1,
storageId: "storage:one",
},
];
const deleteDoc = vi.fn();
const deleteStorage = vi.fn();
const runAfter = vi.fn();
const lt = vi.fn(() => ({}));
const ctx = {
db: makeDb({
query: vi.fn(() => ({
withIndex: vi.fn((indexName: string, build: (q: unknown) => unknown) => {
expect(indexName).toBe("by_expires_at");
build({ lt });
return { take: vi.fn(async () => rows) };
}),
})),
delete: deleteDoc,
}),
storage: { delete: deleteStorage },
scheduler: { runAfter },
};
const result = await pruneSkillUploadsHandler(ctx as never, { batchSize: 1 });
expect(result).toEqual({ deletedTickets: 1, deletedStorage: 1, hasMore: true });
expect(lt).toHaveBeenCalledWith("expiresAt", now);
expect(deleteStorage).toHaveBeenCalledWith("storage:one");
expect(deleteDoc).toHaveBeenCalledWith("skillPublishUploadTickets:one");
expect(runAfter).toHaveBeenCalledWith(
0,
retentionRefs.pruneExpiredSkillPublishUploadsInternal,
{ batchSize: 1 },
);
});
});
+31
View File
@@ -109,3 +109,34 @@ export const pruneExpiredPublisherInvitesInternal = internalMutation({
return { deleted: stale.length, hasMore };
},
});
export const pruneExpiredSkillPublishUploadsInternal = internalMutation({
args: {
batchSize: v.optional(v.number()),
},
handler: async (ctx, args) => {
const batchSize = normalizeRetentionBatchSize(args.batchSize);
const stale = await ctx.db
.query("skillPublishUploadTickets")
.withIndex("by_expires_at", (q) => q.lt("expiresAt", Date.now()))
.take(batchSize);
let deletedStorage = 0;
for (const ticket of stale) {
if (!ticket.usedAt && ticket.storageId) {
await ctx.storage.delete(ticket.storageId);
deletedStorage += 1;
}
await ctx.db.delete(ticket._id);
}
const hasMore = stale.length === batchSize;
if (hasMore) {
await ctx.scheduler.runAfter(0, internal.retention.pruneExpiredSkillPublishUploadsInternal, {
batchSize,
});
}
return { deletedTickets: stale.length, deletedStorage, hasMore };
},
});
+15
View File
@@ -336,6 +336,20 @@ const publisherImageUploadTickets = defineTable({
storageId: v.optional(v.id("_storage")),
}).index("by_publisher_user", ["publisherId", "userId"]);
const skillPublishUploadTickets = defineTable({
userId: v.id("users"),
path: v.string(),
size: v.number(),
sha256: v.string(),
contentType: v.optional(v.string()),
storageId: v.optional(v.id("_storage")),
createdAt: v.number(),
expiresAt: v.number(),
usedAt: v.optional(v.number()),
})
.index("by_user", ["userId"])
.index("by_expires_at", ["expiresAt"]);
const officialPublishers = defineTable({
publisherId: v.id("publishers"),
reason: v.optional(v.string()),
@@ -4266,6 +4280,7 @@ export default defineSchema({
publisherMembers,
publisherInvites,
publisherImageUploadTickets,
skillPublishUploadTickets,
officialPublishers,
githubSkillSources,
githubSkillContents,
+122
View File
@@ -0,0 +1,122 @@
/* @vitest-environment node */
import { afterEach, describe, expect, it, vi } from "vitest";
import {
cleanupSkillPublishUploadInternal,
consumeSkillPublishUploads,
} from "./skillPublishUploads";
type CleanupHandler = {
_handler: (
ctx: unknown,
args: { uploadTicket: "skillPublishUploadTickets:1" },
) => Promise<unknown>;
};
const cleanupHandler = (cleanupSkillPublishUploadInternal as unknown as CleanupHandler)._handler;
function makeCtx(ticket: Record<string, unknown> | null) {
return {
db: {
get: vi.fn(async () => ticket),
patch: vi.fn(),
delete: vi.fn(),
normalizeId: vi.fn(),
query: vi.fn(),
replace: vi.fn(),
insert: vi.fn(),
system: { get: vi.fn(), query: vi.fn() },
},
storage: { delete: vi.fn() },
};
}
describe("skill publish upload tickets", () => {
afterEach(() => {
vi.restoreAllMocks();
});
it("atomically consumes a matching staged upload", async () => {
vi.spyOn(Date, "now").mockReturnValue(2_000);
const ctx = makeCtx({
_id: "skillPublishUploadTickets:1",
userId: "users:1",
path: "SKILL.md",
size: 5,
sha256: "a".repeat(64),
contentType: "text/markdown",
storageId: "storage:1",
expiresAt: 10_000,
});
await consumeSkillPublishUploads(ctx as never, {
userId: "users:1" as never,
uploadTickets: ["skillPublishUploadTickets:1" as never],
files: [
{
path: "SKILL.md",
size: 5,
sha256: "a".repeat(64),
contentType: "text/markdown",
storageId: "storage:1" as never,
},
],
});
expect(ctx.db.patch).toHaveBeenCalledWith("skillPublishUploadTickets:1", { usedAt: 2_000 });
});
it("rejects a ticket whose staged file does not match the publish", async () => {
vi.spyOn(Date, "now").mockReturnValue(2_000);
const ctx = makeCtx({
_id: "skillPublishUploadTickets:1",
userId: "users:1",
path: "SKILL.md",
size: 5,
sha256: "a".repeat(64),
storageId: "storage:1",
expiresAt: 10_000,
});
await expect(
consumeSkillPublishUploads(ctx as never, {
userId: "users:1" as never,
uploadTickets: ["skillPublishUploadTickets:1" as never],
files: [
{
path: "SKILL.md",
size: 6,
sha256: "a".repeat(64),
storageId: "storage:1" as never,
},
],
}),
).rejects.toThrow("does not match this publish");
expect(ctx.db.patch).not.toHaveBeenCalled();
});
it("deletes an abandoned staged file when its ticket expires", async () => {
const ctx = makeCtx({
_id: "skillPublishUploadTickets:1",
storageId: "storage:1",
});
await cleanupHandler(ctx, { uploadTicket: "skillPublishUploadTickets:1" });
expect(ctx.storage.delete).toHaveBeenCalledWith("storage:1");
expect(ctx.db.delete).toHaveBeenCalledWith("skillPublishUploadTickets:1");
});
it("keeps published storage when deleting a consumed ticket", async () => {
const ctx = makeCtx({
_id: "skillPublishUploadTickets:1",
storageId: "storage:1",
usedAt: 2_000,
});
await cleanupHandler(ctx, { uploadTicket: "skillPublishUploadTickets:1" });
expect(ctx.storage.delete).not.toHaveBeenCalled();
expect(ctx.db.delete).toHaveBeenCalledWith("skillPublishUploadTickets:1");
});
});
+170
View File
@@ -0,0 +1,170 @@
import { normalizeContentType } from "clawhub-schema";
import { v } from "convex/values";
import { internal } from "./_generated/api";
import type { Id } from "./_generated/dataModel";
import type { MutationCtx } from "./_generated/server";
import { internalMutation, internalQuery } from "./functions";
import { MAX_PUBLISH_FILE_BYTES } from "./lib/publishLimits";
import { validateFilePath } from "./lib/skillZip";
const SKILL_PUBLISH_UPLOAD_TTL_MS = 60 * 60_000;
function assertExpectedUpload(args: {
path: string;
size: number;
sha256: string;
contentType?: string;
}) {
if (!validateFilePath(args.path)) throw new Error("Invalid upload path");
if (!Number.isSafeInteger(args.size) || args.size < 0 || args.size > MAX_PUBLISH_FILE_BYTES) {
throw new Error(`Upload must be at most ${MAX_PUBLISH_FILE_BYTES} bytes`);
}
if (!/^[a-f0-9]{64}$/i.test(args.sha256)) throw new Error("Invalid upload SHA-256");
}
export const createSkillPublishUploadInternal = internalMutation({
args: {
userId: v.id("users"),
path: v.string(),
size: v.number(),
sha256: v.string(),
contentType: v.optional(v.string()),
},
handler: async (ctx, args) => {
const user = await ctx.db.get(args.userId);
if (!user || user.deletedAt || user.deactivatedAt) throw new Error("User not found");
assertExpectedUpload(args);
const now = Date.now();
const expiresAt = now + SKILL_PUBLISH_UPLOAD_TTL_MS;
const uploadTicket = await ctx.db.insert("skillPublishUploadTickets", {
userId: args.userId,
path: args.path,
size: args.size,
sha256: args.sha256.toLowerCase(),
contentType: normalizeContentType(args.contentType),
createdAt: now,
expiresAt,
});
await ctx.scheduler.runAt(
expiresAt,
internal.skillPublishUploads.cleanupSkillPublishUploadInternal,
{ uploadTicket },
);
return { uploadTicket };
},
});
export const getSkillPublishUploadForUserInternal = internalQuery({
args: {
userId: v.id("users"),
uploadTicket: v.id("skillPublishUploadTickets"),
},
handler: async (ctx, args) => {
const ticket = await ctx.db.get(args.uploadTicket);
if (
!ticket ||
ticket.userId !== args.userId ||
ticket.usedAt ||
ticket.storageId ||
ticket.expiresAt <= Date.now()
) {
throw new Error("Skill upload ticket is missing, used, or expired");
}
return {
path: ticket.path,
size: ticket.size,
sha256: ticket.sha256,
contentType: ticket.contentType,
};
},
});
export const attachSkillPublishUploadInternal = internalMutation({
args: {
userId: v.id("users"),
uploadTicket: v.id("skillPublishUploadTickets"),
storageId: v.id("_storage"),
},
handler: async (ctx, args) => {
const ticket = await ctx.db.get(args.uploadTicket);
const now = Date.now();
if (
!ticket ||
ticket.userId !== args.userId ||
ticket.usedAt ||
ticket.storageId ||
ticket.expiresAt <= now
) {
throw new Error("Skill upload ticket is missing, used, or expired");
}
const metadata = await ctx.db.system.get("_storage", args.storageId);
if (
!metadata ||
metadata._creationTime < ticket.createdAt ||
metadata.size !== ticket.size ||
metadata.sha256.toLowerCase() !== ticket.sha256 ||
normalizeContentType(metadata.contentType) !== ticket.contentType
) {
throw new Error("Uploaded file does not match its skill upload ticket");
}
await ctx.db.patch(ticket._id, { storageId: args.storageId });
},
});
type SkillPublishFile = {
path: string;
size: number;
storageId: Id<"_storage">;
sha256: string;
contentType?: string;
};
export async function consumeSkillPublishUploads(
ctx: MutationCtx,
args: {
userId: Id<"users">;
uploadTickets: Id<"skillPublishUploadTickets">[];
files: SkillPublishFile[];
},
) {
if (args.uploadTickets.length !== args.files.length) {
throw new Error("Every directly uploaded skill file requires an upload ticket");
}
if (new Set(args.uploadTickets).size !== args.uploadTickets.length) {
throw new Error("Skill upload tickets cannot be reused");
}
const now = Date.now();
for (let index = 0; index < args.files.length; index += 1) {
const uploadTicket = args.uploadTickets[index];
const file = args.files[index];
if (!uploadTicket || !file) throw new Error("Skill upload ticket mismatch");
const ticket = await ctx.db.get(uploadTicket);
if (
!ticket ||
ticket.userId !== args.userId ||
ticket.usedAt ||
ticket.expiresAt <= now ||
ticket.storageId !== file.storageId ||
ticket.path !== file.path ||
ticket.size !== file.size ||
ticket.sha256 !== file.sha256.toLowerCase() ||
ticket.contentType !== normalizeContentType(file.contentType)
) {
throw new Error("Skill upload ticket does not match this publish");
}
await ctx.db.patch(ticket._id, { usedAt: now });
}
}
export const cleanupSkillPublishUploadInternal = internalMutation({
args: { uploadTicket: v.id("skillPublishUploadTickets") },
handler: async (ctx, args) => {
const ticket = await ctx.db.get(args.uploadTicket);
if (!ticket) return { deleted: false };
if (!ticket.usedAt && ticket.storageId) {
await ctx.storage.delete(ticket.storageId);
}
await ctx.db.delete(ticket._id);
return { deleted: true };
},
});
+9
View File
@@ -173,6 +173,7 @@ import { normalizeSkillTags } from "./lib/skillTags";
import { runStaticPublishScan } from "./lib/staticPublishScan";
import { adjustUserSkillStatsForSkillChange } from "./lib/userSkillStats";
import schema from "./schema";
import { consumeSkillPublishUploads } from "./skillPublishUploads";
const MAX_OWNER_SUMMARY_LENGTH = 500;
const MAX_POINTERLESS_VERSION_SURVIVOR_SCAN = 100;
@@ -12454,6 +12455,7 @@ function stripUndefinedForStoredPublication(value: unknown): unknown {
export const insertVersion = internalMutation({
args: {
userId: v.id("users"),
skillPublishUploadTickets: v.optional(v.array(v.id("skillPublishUploadTickets"))),
ownerPublisherId: v.optional(v.id("publishers")),
sourceOwnerPublisherId: v.optional(v.id("publishers")),
// Explicit opt-in to owner migration. When an existing skill row already has
@@ -12587,6 +12589,13 @@ export const insertVersion = internalMutation({
if (!normalizedSlug) throw new ConvexError("Slug is required.");
const user = await ctx.db.get(userId);
if (!user || user.deletedAt || user.deactivatedAt) throw new Error("User not found");
if (args.skillPublishUploadTickets) {
await consumeSkillPublishUploads(ctx, {
userId,
uploadTickets: args.skillPublishUploadTickets,
files: args.files,
});
}
const personalPublisher = await ensurePersonalPublisherForUser(ctx, user, {
actorUserId: userId,
source: "skill.publish",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "clawhub",
"version": "0.23.2",
"version": "0.23.3",
"description": "ClawHub CLI \\u2014 install, update, search, and publish skills plus OpenClaw packages.",
"homepage": "https://clawhub.ai",
"bugs": {
+154 -89
View File
@@ -16,6 +16,7 @@ const authTokenMocks = createAuthTokenModuleMocks();
const registryMocks = createRegistryModuleMocks();
const httpMocks = createHttpModuleMocks();
const uiMocks = createUiModuleMocks();
let publishResponse: Record<string, unknown>;
vi.mock("../authToken.js", () => authTokenMocks.moduleFactory());
vi.mock("../registry.js", () => registryMocks.moduleFactory());
@@ -78,7 +79,7 @@ describe("cmdPublish", () => {
match: { version: "1.2.3" },
latestVersion: { version: "1.2.3" },
});
httpMocks.apiRequestForm.mockResolvedValueOnce({
mockPublishResponse({
ok: true,
skillId: "skill_1",
versionId: "ver_2",
@@ -114,7 +115,7 @@ describe("cmdPublish", () => {
httpMocks.apiRequest.mockRejectedValueOnce(
new Error("Skill not found or unavailable to this account."),
);
httpMocks.apiRequestForm.mockResolvedValueOnce({
mockPublishResponse({
ok: true,
skillId: "skill_1",
versionId: "ver_1",
@@ -143,7 +144,7 @@ describe("cmdPublish", () => {
httpMocks.apiRequest.mockRejectedValueOnce(
new Error("Skill not found or unavailable to this account."),
);
httpMocks.apiRequestForm.mockResolvedValueOnce({
mockPublishResponse({
ok: true,
skillId: "skill_1",
versionId: "ver_pending",
@@ -179,7 +180,7 @@ describe("cmdPublish", () => {
httpMocks.apiRequest.mockRejectedValueOnce(
new Error("Skill not found or unavailable to this account."),
);
httpMocks.apiRequestForm.mockResolvedValueOnce({
mockPublishResponse({
ok: true,
skillId: "skill_1",
versionId: "ver_pending",
@@ -212,7 +213,7 @@ describe("cmdPublish", () => {
const folder = join(workdir, "unknown-status-skill");
await mkdir(folder, { recursive: true });
await writeFile(join(folder, "SKILL.md"), "# Skill\n", "utf8");
httpMocks.apiRequestForm.mockResolvedValueOnce({
mockPublishResponse({
ok: true,
skillId: "skill_1",
versionId: "ver_1",
@@ -239,7 +240,7 @@ describe("cmdPublish", () => {
match: null,
latestVersion: { version: "1.2.3" },
});
httpMocks.apiRequestForm.mockResolvedValueOnce({
mockPublishResponse({
ok: true,
skillId: "skill_1",
versionId: "ver_2",
@@ -269,7 +270,7 @@ describe("cmdPublish", () => {
match: { version: "1.2.3" },
latestVersion: { version: "1.2.3" },
});
httpMocks.apiRequestForm.mockResolvedValueOnce({
mockPublishResponse({
ok: true,
skillId: "skill_1",
versionId: "ver_2",
@@ -316,6 +317,85 @@ describe("cmdPublish", () => {
}
});
it("uploads each skill file separately before sending the publish metadata", async () => {
const workdir = await makeTmpWorkdir();
try {
const folder = join(workdir, "staged-skill");
await mkdir(join(folder, "assets"), { recursive: true });
await writeFile(join(folder, "SKILL.md"), "# Staged skill\n", "utf8");
await writeFile(join(folder, "assets", "payload.bin"), Uint8Array.from([0, 1, 2, 255]));
httpMocks.uploadBinary
.mockResolvedValueOnce({ storageId: "storage:skill" })
.mockResolvedValueOnce({ storageId: "storage:payload" });
mockPublishResponse({
ok: true,
skillId: "skill_1",
versionId: "ver_1",
publicationStatus: "published",
});
await cmdPublish(makeOpts(workdir), "staged-skill", {
version: "1.0.0",
});
expect(httpMocks.apiRequestForm).not.toHaveBeenCalled();
expect(httpMocks.uploadBinary).toHaveBeenCalledTimes(2);
const uploadTicketCalls = httpMocks.apiRequest.mock.calls.filter((call) => {
const args = call[1] as { method?: string; path?: string };
return args.method === "POST" && args.path === "/api/v1/skills/-/upload-url";
});
expect(uploadTicketCalls).toHaveLength(2);
expect(uploadTicketCalls[0]?.[1]).toMatchObject({
token: "tkn",
body: {
path: "SKILL.md",
size: 15,
sha256: "90b735dd867ee1111738fb1397982e1dbf7e6bda451d60d60fc10623926adcfc",
},
});
expect(httpMocks.uploadBinary).toHaveBeenNthCalledWith(
1,
expect.objectContaining({
url: "https://upload.local/1",
token: "tkn",
}),
expect.anything(),
);
const publishCall = httpMocks.apiRequest.mock.calls.find((call) => {
const args = call[1] as { method?: string; path?: string };
return args.method === "POST" && args.path === "/api/v1/skills";
});
expect(publishCall?.[1]).toMatchObject({
method: "POST",
path: "/api/v1/skills",
token: "tkn",
body: {
slug: "staged-skill",
files: [
expect.objectContaining({
path: "SKILL.md",
size: 15,
storageId: "storage:skill",
sha256: "90b735dd867ee1111738fb1397982e1dbf7e6bda451d60d60fc10623926adcfc",
uploadTicket: "skillPublishUploadTickets:1",
}),
expect.objectContaining({
path: "assets/payload.bin",
size: 4,
storageId: "storage:payload",
sha256: "3d1f57c984978ef98a18378c8166c1cb8ede02c03eeb6aee7e2f121dfeee3e56",
contentType: "application/octet-stream",
uploadTicket: "skillPublishUploadTickets:2",
}),
],
},
});
} finally {
await rm(workdir, { recursive: true, force: true });
}
});
it("publishes Terraform and opaque files with exact bytes (mocked HTTP)", async () => {
const workdir = await makeTmpWorkdir();
try {
@@ -333,7 +413,7 @@ describe("cmdPublish", () => {
await writeFile(join(folder, "terraform.tfvars"), variablesContent, "utf8");
await writeFile(join(folder, "assets", "payload.bin"), opaqueBytes);
httpMocks.apiRequestForm.mockResolvedValueOnce({
mockPublishResponse({
ok: true,
skillId: "skill_1",
versionId: "ver_1",
@@ -351,15 +431,7 @@ describe("cmdPublish", () => {
await cmdPublish(makeOpts(workdir), "my-skill", options);
const publishCall = httpMocks.apiRequestForm.mock.calls.find((call) => {
const req = call[1] as { path?: string } | undefined;
return req?.path === "/api/v1/skills";
});
if (!publishCall) throw new Error("Missing publish call");
const publishForm = (publishCall[1] as { form?: FormData }).form as FormData;
const payloadEntry = publishForm.get("payload");
if (typeof payloadEntry !== "string") throw new Error("Missing publish payload");
const payload = JSON.parse(payloadEntry);
const payload = publishPayload();
expect(payload.slug).toBe("my-skill");
expect(payload.displayName).toBe("My Skill");
expect(payload.ownerHandle).toBe("me");
@@ -369,22 +441,26 @@ describe("cmdPublish", () => {
expect(payload.tags).toEqual(["latest"]);
expect(payload.categories).toEqual(["automation", "development"]);
expect(payload.topics).toEqual(["React", "GPU development"]);
const files = publishForm.getAll("files") as Array<Blob & { name?: string }>;
expect(files.map((file) => file.name ?? "").sort()).toEqual([
const files = payload.files as Array<{ path: string }>;
expect(files.map((file) => file.path).sort()).toEqual([
"SKILL.md",
"assets/payload.bin",
"main.tf",
"notes.md",
"terraform.tfvars",
]);
const byName = new Map(files.map((file) => [file.name ?? "", file]));
expect(await byName.get("main.tf")?.text()).toBe(terraformContent);
expect(await byName.get("terraform.tfvars")?.text()).toBe(variablesContent);
expect(
new Uint8Array(
(await byName.get("assets/payload.bin")?.arrayBuffer()) ?? new ArrayBuffer(0),
),
).toEqual(opaqueBytes);
const uploadedBytes = new Map(
files.map((file, index) => {
const uploadCall = httpMocks.uploadBinary.mock.calls[index];
if (!uploadCall) throw new Error(`Missing upload call for ${file.path}`);
return [file.path, (uploadCall[0] as { bytes: Uint8Array }).bytes] as const;
}),
);
expect(new TextDecoder().decode(uploadedBytes.get("main.tf"))).toBe(terraformContent);
expect(new TextDecoder().decode(uploadedBytes.get("terraform.tfvars"))).toBe(
variablesContent,
);
expect(uploadedBytes.get("assets/payload.bin")).toEqual(opaqueBytes);
} finally {
await rm(workdir, { recursive: true, force: true });
}
@@ -397,7 +473,7 @@ describe("cmdPublish", () => {
await mkdir(folder, { recursive: true });
await writeFile(join(folder, "SKILL.md"), "# Clear topics\n", "utf8");
httpMocks.apiRequestForm.mockResolvedValueOnce({
mockPublishResponse({
ok: true,
skillId: "skill_1",
versionId: "ver_1",
@@ -418,7 +494,7 @@ describe("cmdPublish", () => {
await mkdir(folder, { recursive: true });
await writeFile(join(folder, "SKILL.md"), "# Skill\n", "utf8");
httpMocks.apiRequestForm.mockResolvedValueOnce({
mockPublishResponse({
ok: true,
skillId: "skill_1",
versionId: "ver_1",
@@ -432,15 +508,7 @@ describe("cmdPublish", () => {
forkOf: "@openclaw/demo@1.2.3",
});
const publishCall = httpMocks.apiRequestForm.mock.calls.find((call) => {
const req = call[1] as { path?: string } | undefined;
return req?.path === "/api/v1/skills";
});
if (!publishCall) throw new Error("Missing publish call");
const publishForm = (publishCall[1] as { form?: FormData }).form as FormData;
const payloadEntry = publishForm.get("payload");
if (typeof payloadEntry !== "string") throw new Error("Missing publish payload");
expect(JSON.parse(payloadEntry).forkOf).toEqual({
expect(publishPayload().forkOf).toEqual({
slug: "demo",
ownerHandle: "openclaw",
version: "1.2.3",
@@ -459,7 +527,7 @@ describe("cmdPublish", () => {
await writeFile(join(folder, "notes.md"), "notes\n", "utf8");
await writeFile(join(folder, "skill-card.md"), "# Generated card\n", "utf8");
httpMocks.apiRequestForm.mockResolvedValueOnce({
mockPublishResponse({
ok: true,
skillId: "skill_1",
versionId: "ver_1",
@@ -473,14 +541,8 @@ describe("cmdPublish", () => {
tags: "latest",
});
const publishCall = httpMocks.apiRequestForm.mock.calls.find((call) => {
const req = call[1] as { path?: string } | undefined;
return req?.path === "/api/v1/skills";
});
if (!publishCall) throw new Error("Missing publish call");
const publishForm = (publishCall[1] as { form?: FormData }).form as FormData;
const files = publishForm.getAll("files") as Array<Blob & { name?: string }>;
expect(files.map((file) => file.name ?? "").sort()).toEqual(["SKILL.md", "notes.md"]);
const files = publishPayload().files as Array<{ path: string }>;
expect(files.map((file) => file.path).sort()).toEqual(["SKILL.md", "notes.md"]);
} finally {
await rm(workdir, { recursive: true, force: true });
}
@@ -493,7 +555,7 @@ describe("cmdPublish", () => {
await mkdir(folder, { recursive: true });
await writeFile(join(folder, "SKILL.md"), "# Skill\n", "utf8");
httpMocks.apiRequestForm.mockResolvedValueOnce({
mockPublishResponse({
ok: true,
skillId: "skill_1",
versionId: "ver_2",
@@ -505,11 +567,7 @@ describe("cmdPublish", () => {
tags: "latest",
});
expect(httpMocks.apiRequestForm).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({ path: "/api/v1/skills", method: "POST" }),
expect.anything(),
);
expect(publishPayload()).toMatchObject({ changelog: "" });
} finally {
await rm(workdir, { recursive: true, force: true });
}
@@ -524,7 +582,7 @@ describe("cmdPublish", () => {
await writeFile(join(folder, "SKILL.md"), "# Skill\n", "utf8");
await writeFile(join(folder, "notes.md"), "ignored notes\n", "utf8");
httpMocks.apiRequestForm.mockResolvedValueOnce({
mockPublishResponse({
ok: true,
skillId: "skill_1",
versionId: "ver_1",
@@ -538,14 +596,8 @@ describe("cmdPublish", () => {
tags: "latest",
});
const publishCall = httpMocks.apiRequestForm.mock.calls.find((call) => {
const req = call[1] as { path?: string } | undefined;
return req?.path === "/api/v1/skills";
});
if (!publishCall) throw new Error("Missing publish call");
const publishForm = (publishCall[1] as { form?: FormData }).form as FormData;
const files = publishForm.getAll("files") as Array<Blob & { name?: string }>;
expect(files.map((file) => file.name ?? "")).toEqual(["SKILL.md"]);
const files = publishPayload().files as Array<{ path: string }>;
expect(files.map((file) => file.path)).toEqual(["SKILL.md"]);
} finally {
await rm(workdir, { recursive: true, force: true });
}
@@ -558,7 +610,7 @@ describe("cmdPublish", () => {
await mkdir(folder, { recursive: true });
await writeFile(join(folder, "SKILL.md"), "# Skill\n", "utf8");
httpMocks.apiRequestForm.mockResolvedValueOnce({
mockPublishResponse({
ok: true,
skillId: "skill_1",
versionId: "ver_2",
@@ -572,15 +624,7 @@ describe("cmdPublish", () => {
tags: "latest",
});
const publishCall = httpMocks.apiRequestForm.mock.calls.find((call) => {
const req = call[1] as { path?: string } | undefined;
return req?.path === "/api/v1/skills";
});
if (!publishCall) throw new Error("Missing publish call");
const publishForm = (publishCall[1] as { form?: FormData }).form as FormData;
const payloadEntry = publishForm.get("payload");
if (typeof payloadEntry !== "string") throw new Error("Missing publish payload");
const payload = JSON.parse(payloadEntry);
const payload = publishPayload();
expect(payload.ownerHandle).toBe("openclaw");
expect(payload.sourceOwnerHandle).toBe("me");
expect(payload.migrateOwner).toBe(true);
@@ -620,7 +664,7 @@ describe("cmdPublish", () => {
await writeFile(join(folder, "SKILL.md"), "# Skill\n", "utf8");
mockDefaultApiRequest("steipete");
httpMocks.apiRequestForm.mockResolvedValueOnce({
mockPublishResponse({
ok: true,
skillId: "skill_1",
versionId: "ver_1",
@@ -636,15 +680,7 @@ describe("cmdPublish", () => {
sourcePath: "skills/source-skill",
});
const publishCall = httpMocks.apiRequestForm.mock.calls.find((call) => {
const req = call[1] as { path?: string } | undefined;
return req?.path === "/api/v1/skills";
});
if (!publishCall) throw new Error("Missing publish call");
const publishForm = (publishCall[1] as { form?: FormData }).form as FormData;
const payloadEntry = publishForm.get("payload");
if (typeof payloadEntry !== "string") throw new Error("Missing publish payload");
const payload = JSON.parse(payloadEntry);
const payload = publishPayload();
expect(payload.source).toEqual({
kind: "github",
url: "https://github.com/NVIDIA/skills",
@@ -691,28 +727,57 @@ describe("cmdPublish", () => {
});
function publishPayload() {
const publishCall = httpMocks.apiRequestForm.mock.calls.find((call) => {
const request = call[1] as { path?: string } | undefined;
return request?.path === "/api/v1/skills";
const publishCall = httpMocks.apiRequest.mock.calls.find((call) => {
const request = call[1] as { method?: string; path?: string } | undefined;
return request?.method === "POST" && request.path === "/api/v1/skills";
});
if (!publishCall) throw new Error("Missing publish call");
const form = (publishCall[1] as { form?: FormData }).form;
const payload = form?.get("payload");
if (typeof payload !== "string") throw new Error("Missing publish payload");
return JSON.parse(payload) as Record<string, unknown>;
const body = (publishCall[1] as { body?: unknown }).body;
if (!body || typeof body !== "object" || Array.isArray(body)) {
throw new Error("Missing publish payload");
}
return body as Record<string, unknown>;
}
function mockPublishResponse(response: Record<string, unknown>) {
publishResponse = response;
}
function mockDefaultApiRequest(whoamiHandle: string | null = "me") {
publishResponse = {
ok: true,
skillId: "skill_1",
versionId: "ver_1",
publicationStatus: "published",
};
let uploadIndex = 0;
httpMocks.apiRequest.mockReset();
httpMocks.apiRequest.mockImplementation(async (_registry: unknown, request: unknown) => {
if (isWhoamiRequest(request)) {
return { user: { handle: whoamiHandle } };
}
const args = request as { method?: string; path?: string };
if (args.method === "POST" && args.path === "/api/v1/skills/-/upload-url") {
uploadIndex += 1;
return {
uploadUrl: `https://upload.local/${uploadIndex}`,
uploadTicket: `skillPublishUploadTickets:${uploadIndex}`,
};
}
if (args.method === "POST" && args.path === "/api/v1/skills") {
return publishResponse;
}
return {
match: null,
latestVersion: null,
};
});
let storageIndex = 0;
httpMocks.uploadBinary.mockReset();
httpMocks.uploadBinary.mockImplementation(async () => {
storageIndex += 1;
return { storageId: `storage:${storageIndex}` };
});
}
function isWhoamiRequest(request: unknown) {
+68 -25
View File
@@ -1,11 +1,13 @@
import { readFile, readdir, stat } from "node:fs/promises";
import { basename, join, resolve } from "node:path";
import semver from "semver";
import { apiRequest, apiRequestForm, registryUrl } from "../../http.js";
import { apiRequest, registryUrl, uploadBinary } from "../../http.js";
import {
ApiRoutes,
ApiUploadFileResponseSchema,
ApiV1PublishResponseSchema,
ApiV1SkillResolveResponseSchema,
ApiV1SkillUploadUrlResponseSchema,
ApiV1WhoamiResponseSchema,
} from "../../schema/index.js";
import { hashSkillFiles, listSkillFiles } from "../../skills.js";
@@ -168,40 +170,81 @@ export async function cmdPublish(
options.migrateOwner && publishOwnerHandle
? sourceOwnerHandle || explicitSourceOwnerHandle || (await getDefaultOwnerHandle(token))
: undefined;
const form = new FormData();
form.set(
"payload",
JSON.stringify({
slug,
displayName,
ownerHandle: publishOwnerHandle,
...(publishSourceOwnerHandle ? { sourceOwnerHandle: publishSourceOwnerHandle } : {}),
...(options.migrateOwner ? { migrateOwner: true } : {}),
version,
changelog,
acceptLicenseTerms: true,
tags,
...(options.categories !== undefined ? { categories } : {}),
...(options.topics !== undefined ? { topics } : {}),
...(source ? { source } : {}),
...(forkOf ? { forkOf } : {}),
}),
);
const fileHashes = new Map(hashed.files.map((file) => [file.path, file]));
const uploadedFiles = [] as Array<{
path: string;
size: number;
storageId: string;
sha256: string;
contentType?: string;
uploadTicket: string;
}>;
let index = 0;
for (const file of filesOnDisk) {
index += 1;
if (spinner) spinner.text = `Uploading ${file.relPath} (${index}/${filesOnDisk.length})`;
const blob = new Blob([Buffer.from(file.bytes)], {
type: file.contentType ?? "application/octet-stream",
const hash = fileHashes.get(file.relPath);
if (!hash) fail(`Unable to hash ${file.relPath}`);
const contentType = file.contentType ?? "application/octet-stream";
const { uploadUrl, uploadTicket } = await apiRequest(
registry,
{
method: "POST",
path: ApiRoutes.skillUploadUrl,
token,
body: {
path: file.relPath,
size: hash.size,
sha256: hash.sha256,
contentType,
},
},
ApiV1SkillUploadUrlResponseSchema,
);
const { storageId } = await uploadBinary(
{
url: uploadUrl,
bytes: file.bytes,
contentType,
token,
},
ApiUploadFileResponseSchema,
);
uploadedFiles.push({
path: file.relPath,
size: hash.size,
storageId,
sha256: hash.sha256,
contentType,
uploadTicket,
});
form.append("files", blob, file.relPath);
}
if (spinner) spinner.text = `Publishing ${slug}@${version}`;
const result = await apiRequestForm(
const result = await apiRequest(
registry,
{ method: "POST", path: ApiRoutes.skills, token, form },
{
method: "POST",
path: ApiRoutes.skills,
token,
body: {
slug,
displayName,
ownerHandle: publishOwnerHandle,
...(publishSourceOwnerHandle ? { sourceOwnerHandle: publishSourceOwnerHandle } : {}),
...(options.migrateOwner ? { migrateOwner: true } : {}),
version,
changelog,
acceptLicenseTerms: true,
tags,
...(options.categories !== undefined ? { categories } : {}),
...(options.topics !== undefined ? { topics } : {}),
...(source ? { source } : {}),
...(forkOf ? { forkOf } : {}),
files: uploadedFiles,
},
},
ApiV1PublishResponseSchema,
);
+19
View File
@@ -210,4 +210,23 @@ describe("bun http client", () => {
true,
);
});
it("keeps binary upload bearer tokens out of curl arguments", async () => {
const { client, spawnImpl } = createBunClient({
spawnImpl: () => ({ status: 0, stdout: '{"storageId":"storage:1"}\n200', stderr: "" }),
mkdtempValue: "/tmp/clawhub-binary-upload",
});
await client.uploadBinary({
url: "https://upload.example/file",
bytes: new Uint8Array([1, 2, 3]),
contentType: "application/octet-stream",
token: "clh_secret",
});
const [, args, options] = spawnImpl.mock.calls[0] as [string, string[], { input?: string }];
expect(args).toContain("@-");
expect(args.join(" ")).not.toContain("clh_secret");
expect(options.input).toBe("Authorization: Bearer clh_secret\n");
});
});
+8 -1
View File
@@ -58,6 +58,7 @@ type BinaryUploadArgs = {
url: string;
bytes: Uint8Array;
contentType?: string;
token?: string;
retryCount?: number;
};
@@ -310,6 +311,7 @@ export function createHttpClient(options: HttpClientOptions = {}): HttpClient {
const headers: Record<string, string> = {};
if (args.contentType) headers["Content-Type"] = args.contentType;
if (args.token) headers.Authorization = `Bearer ${args.token}`;
const response = await fetchWithTimeout(
deps,
args.url,
@@ -842,6 +844,7 @@ async function uploadBinaryViaCurl(
>,
args: BinaryUploadArgs,
) {
if (args.token && /[\r\n]/.test(args.token)) throw new Error("Invalid API token");
const tempDir = await deps.mkdtempImpl(join(deps.tmpdirPath, "clawhub-upload-"));
try {
const filePath = join(tempDir, "upload.bin");
@@ -858,9 +861,13 @@ async function uploadBinaryViaCurl(
"POST",
];
if (args.contentType) curlArgs.push("-H", `Content-Type: ${args.contentType}`);
if (args.token) curlArgs.push("-H", "@-");
curlArgs.push("--data-binary", `@${filePath}`, args.url);
const result = deps.spawnSyncImpl("curl", curlArgs, { encoding: "utf8" });
const result = deps.spawnSyncImpl("curl", curlArgs, {
encoding: "utf8",
input: args.token ? `Authorization: Bearer ${args.token}\n` : undefined,
});
if (result.status !== 0) {
throw new Error(result.stderr || "curl failed");
}
+1
View File
@@ -18,6 +18,7 @@ export const ApiRoutes = {
publishTokenMint: "/api/v1/publish/token/mint",
publishAttempts: "/api/v1/publish/attempts",
skills: "/api/v1/skills",
skillUploadUrl: "/api/v1/skills/-/upload-url",
trending: "/api/v1/trending",
skillsSh: "/api/v1/skills-sh",
skillScans: "/api/v1/skills/-/scan",
+13
View File
@@ -71,12 +71,25 @@ export const ApiUploadFileResponseSchema = type({
storageId: "string",
});
export const ApiV1SkillUploadUrlRequestSchema = type({
path: "string",
size: "number",
sha256: "string",
contentType: "string?",
});
export const ApiV1SkillUploadUrlResponseSchema = type({
uploadUrl: "string",
uploadTicket: "string",
});
export const CliPublishFileSchema = type({
path: "string",
size: "number",
storageId: "string",
sha256: "string",
contentType: "string?",
uploadTicket: "string?",
});
export type CliPublishFile = (typeof CliPublishFileSchema)[inferred];
+1
View File
@@ -333,6 +333,7 @@ export declare const ServerPackagePublishRequestSchema: import("arktype/internal
storageId: string;
sha256: string;
contentType?: string | undefined;
uploadTicket?: string | undefined;
}[];
}, {}>;
export type ServerPackagePublishRequest = (typeof ServerPackagePublishRequestSchema)[inferred];
+1
View File
@@ -17,6 +17,7 @@ export declare const ApiRoutes: {
readonly publishTokenMint: "/api/v1/publish/token/mint";
readonly publishAttempts: "/api/v1/publish/attempts";
readonly skills: "/api/v1/skills";
readonly skillUploadUrl: "/api/v1/skills/-/upload-url";
readonly trending: "/api/v1/trending";
readonly skillsSh: "/api/v1/skills-sh";
readonly skillScans: "/api/v1/skills/-/scan";
+1
View File
@@ -17,6 +17,7 @@ export const ApiRoutes = {
publishTokenMint: "/api/v1/publish/token/mint",
publishAttempts: "/api/v1/publish/attempts",
skills: "/api/v1/skills",
skillUploadUrl: "/api/v1/skills/-/upload-url",
trending: "/api/v1/trending",
skillsSh: "/api/v1/skills-sh",
skillScans: "/api/v1/skills/-/scan",
+1 -1
View File
@@ -1 +1 @@
{"version":3,"file":"routes.js","sourceRoot":"","sources":["../src/routes.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,QAAQ,EAAE,eAAe;IACzB,MAAM,EAAE,aAAa;IACrB,KAAK,EAAE,YAAY;IACnB,YAAY,EAAE,oBAAoB;IAClC,SAAS,EAAE,iBAAiB;IAC5B,YAAY,EAAE,qBAAqB;IACnC,UAAU,EAAE,kBAAkB;IAC9B,mBAAmB,EAAE,4BAA4B;IACjD,cAAc,EAAE,uBAAuB;IACvC,gBAAgB,EAAE,yBAAyB;CACnC,CAAC;AAEX,MAAM,CAAC,MAAM,SAAS,GAAG;IACvB,MAAM,EAAE,gBAAgB;IACxB,OAAO,EAAE,iBAAiB;IAC1B,QAAQ,EAAE,kBAAkB;IAC5B,gBAAgB,EAAE,4BAA4B;IAC9C,eAAe,EAAE,0BAA0B;IAC3C,MAAM,EAAE,gBAAgB;IACxB,QAAQ,EAAE,kBAAkB;IAC5B,QAAQ,EAAE,mBAAmB;IAC7B,UAAU,EAAE,uBAAuB;IACnC,OAAO,EAAE,iBAAiB;IAC1B,aAAa,EAAE,wBAAwB;IACvC,QAAQ,EAAE,kBAAkB;IAC5B,WAAW,EAAE,sBAAsB;IACnC,aAAa,EAAE,wBAAwB;IACvC,UAAU,EAAE,oBAAoB;IAChC,WAAW,EAAE,uBAAuB;IACpC,iBAAiB,EAAE,sBAAsB;IACzC,gBAAgB,EAAE,qBAAqB;IACvC,cAAc,EAAE,0BAA0B;IAC1C,KAAK,EAAE,eAAe;IACtB,SAAS,EAAE,mBAAmB;IAC9B,UAAU,EAAE,oBAAoB;IAChC,KAAK,EAAE,eAAe;IACtB,aAAa,EAAE,wBAAwB;IACvC,MAAM,EAAE,gBAAgB;IACxB,YAAY,EAAE,uBAAuB;CAC7B,CAAC"}
{"version":3,"file":"routes.js","sourceRoot":"","sources":["../src/routes.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,QAAQ,EAAE,eAAe;IACzB,MAAM,EAAE,aAAa;IACrB,KAAK,EAAE,YAAY;IACnB,YAAY,EAAE,oBAAoB;IAClC,SAAS,EAAE,iBAAiB;IAC5B,YAAY,EAAE,qBAAqB;IACnC,UAAU,EAAE,kBAAkB;IAC9B,mBAAmB,EAAE,4BAA4B;IACjD,cAAc,EAAE,uBAAuB;IACvC,gBAAgB,EAAE,yBAAyB;CACnC,CAAC;AAEX,MAAM,CAAC,MAAM,SAAS,GAAG;IACvB,MAAM,EAAE,gBAAgB;IACxB,OAAO,EAAE,iBAAiB;IAC1B,QAAQ,EAAE,kBAAkB;IAC5B,gBAAgB,EAAE,4BAA4B;IAC9C,eAAe,EAAE,0BAA0B;IAC3C,MAAM,EAAE,gBAAgB;IACxB,cAAc,EAAE,6BAA6B;IAC7C,QAAQ,EAAE,kBAAkB;IAC5B,QAAQ,EAAE,mBAAmB;IAC7B,UAAU,EAAE,uBAAuB;IACnC,OAAO,EAAE,iBAAiB;IAC1B,aAAa,EAAE,wBAAwB;IACvC,QAAQ,EAAE,kBAAkB;IAC5B,WAAW,EAAE,sBAAsB;IACnC,aAAa,EAAE,wBAAwB;IACvC,UAAU,EAAE,oBAAoB;IAChC,WAAW,EAAE,uBAAuB;IACpC,iBAAiB,EAAE,sBAAsB;IACzC,gBAAgB,EAAE,qBAAqB;IACvC,cAAc,EAAE,0BAA0B;IAC1C,KAAK,EAAE,eAAe;IACtB,SAAS,EAAE,mBAAmB;IAC9B,UAAU,EAAE,oBAAoB;IAChC,KAAK,EAAE,eAAe;IACtB,aAAa,EAAE,wBAAwB;IACvC,MAAM,EAAE,gBAAgB;IACxB,YAAY,EAAE,uBAAuB;CAC7B,CAAC"}
+12
View File
@@ -55,12 +55,23 @@ export declare const ApiCliUploadUrlResponseSchema: import("arktype/internal/var
export declare const ApiUploadFileResponseSchema: import("arktype/internal/variants/object.ts").ObjectType<{
storageId: string;
}, {}>;
export declare const ApiV1SkillUploadUrlRequestSchema: import("arktype/internal/variants/object.ts").ObjectType<{
path: string;
size: number;
sha256: string;
contentType?: string | undefined;
}, {}>;
export declare const ApiV1SkillUploadUrlResponseSchema: import("arktype/internal/variants/object.ts").ObjectType<{
uploadUrl: string;
uploadTicket: string;
}, {}>;
export declare const CliPublishFileSchema: import("arktype/internal/variants/object.ts").ObjectType<{
path: string;
size: number;
storageId: string;
sha256: string;
contentType?: string | undefined;
uploadTicket?: string | undefined;
}, {}>;
export type CliPublishFile = (typeof CliPublishFileSchema)[inferred];
export declare const PublishSourceSchema: import("arktype/internal/variants/object.ts").ObjectType<{
@@ -104,6 +115,7 @@ export declare const CliPublishRequestSchema: import("arktype/internal/variants/
storageId: string;
sha256: string;
contentType?: string | undefined;
uploadTicket?: string | undefined;
}[];
}, {}>;
export type CliPublishRequest = (typeof CliPublishRequestSchema)[inferred];
+11
View File
@@ -53,12 +53,23 @@ export const ApiCliUploadUrlResponseSchema = type({
export const ApiUploadFileResponseSchema = type({
storageId: "string",
});
export const ApiV1SkillUploadUrlRequestSchema = type({
path: "string",
size: "number",
sha256: "string",
contentType: "string?",
});
export const ApiV1SkillUploadUrlResponseSchema = type({
uploadUrl: "string",
uploadTicket: "string",
});
export const CliPublishFileSchema = type({
path: "string",
size: "number",
storageId: "string",
sha256: "string",
contentType: "string?",
uploadTicket: "string?",
});
export const PublishSourceSchema = type({
kind: '"github"',
File diff suppressed because one or more lines are too long
+1
View File
@@ -18,6 +18,7 @@ export const ApiRoutes = {
publishTokenMint: "/api/v1/publish/token/mint",
publishAttempts: "/api/v1/publish/attempts",
skills: "/api/v1/skills",
skillUploadUrl: "/api/v1/skills/-/upload-url",
trending: "/api/v1/trending",
skillsSh: "/api/v1/skills-sh",
skillScans: "/api/v1/skills/-/scan",
+13
View File
@@ -65,12 +65,25 @@ export const ApiUploadFileResponseSchema = type({
storageId: "string",
});
export const ApiV1SkillUploadUrlRequestSchema = type({
path: "string",
size: "number",
sha256: "string",
contentType: "string?",
});
export const ApiV1SkillUploadUrlResponseSchema = type({
uploadUrl: "string",
uploadTicket: "string",
});
export const CliPublishFileSchema = type({
path: "string",
size: "number",
storageId: "string",
sha256: "string",
contentType: "string?",
uploadTicket: "string?",
});
export type CliPublishFile = (typeof CliPublishFileSchema)[inferred];
@@ -4,7 +4,7 @@ import { spawnSync } from "node:child_process";
import { describe, expect, it } from "vitest";
describe("clawhub CLI npm release metadata check", () => {
const releaseTag = "v0.23.2";
const releaseTag = "v0.23.3";
function runCheck(args) {
const env = { ...process.env };
+14
View File
@@ -265,6 +265,20 @@ See also: [acceptable-usage.md](./acceptable-usage.md) for the marketplace polic
- Skills directory supports an optional "Hide suspicious" filter to exclude
active-but-flagged (`flagged.suspicious`) entries from browse/search results.
## Skill publish upload boundary
- CLI skill publishing stages each file through the direct Convex HTTP surface,
keeping file bodies off the Vercel request path. Each file remains capped at
10MB and the complete publish remains capped at 50MB.
- The server creates a short-lived, user-bound ticket before accepting a file.
The upload action enforces the declared path, byte size, content type, and
SHA-256 before attaching the resulting storage id to that ticket.
- JSON skill publishes must present the matching ticket for every staged file.
Ticket ownership and file metadata are revalidated, and ticket consumption is
committed atomically with the new skill version.
- Unconsumed staged files are deleted when their tickets expire. Failed
attachment attempts delete the just-created storage blob immediately.
## Package publish upload boundary
- Package publish is multipart-only. `POST /api/v1/packages` must reject JSON