mirror of
https://github.com/openclaw/clawhub.git
synced 2026-08-14 00:47:57 +00:00
fix: upload skill files directly to Convex (#3391)
* fix: upload skill files directly to Convex * chore: prepare clawhub CLI 0.23.3
This commit is contained in:
@@ -21,6 +21,12 @@
|
||||
- CLI: accept npm 12's package-keyed `npm pack --json` output when building ClawPacks while retaining compatibility with earlier npm array output.
|
||||
- Web/API: preserve JSON, SSR, and OG responses through the Convex proxy after the H3 response-wrapper update.
|
||||
|
||||
## 0.23.3 - 2026-08-03
|
||||
|
||||
### Fixes
|
||||
|
||||
- CLI/API: stage skill files directly in Convex storage before publishing metadata, so bundles within the documented 50MB total limit no longer hit Vercel's smaller request-body limit.
|
||||
|
||||
## 0.23.2 - 2026-08-03
|
||||
|
||||
### Changes
|
||||
|
||||
@@ -101,7 +101,7 @@
|
||||
},
|
||||
"packages/clawhub": {
|
||||
"name": "clawhub",
|
||||
"version": "0.23.2",
|
||||
"version": "0.23.3",
|
||||
"bin": {
|
||||
"clawdhub": "bin/clawdhub.js",
|
||||
"clawhub": "bin/clawdhub.js",
|
||||
|
||||
Vendored
+2
@@ -192,6 +192,7 @@ import type * as skillPresentationAssets from "../skillPresentationAssets.js";
|
||||
import type * as skillPresentationAssetsHttp from "../skillPresentationAssetsHttp.js";
|
||||
import type * as skillPresentationBackfill from "../skillPresentationBackfill.js";
|
||||
import type * as skillPresentationImageNode from "../skillPresentationImageNode.js";
|
||||
import type * as skillPublishUploads from "../skillPublishUploads.js";
|
||||
import type * as skillStatEvents from "../skillStatEvents.js";
|
||||
import type * as skillTransfers from "../skillTransfers.js";
|
||||
import type * as skills from "../skills.js";
|
||||
@@ -401,6 +402,7 @@ declare const fullApi: ApiFromModules<{
|
||||
skillPresentationAssetsHttp: typeof skillPresentationAssetsHttp;
|
||||
skillPresentationBackfill: typeof skillPresentationBackfill;
|
||||
skillPresentationImageNode: typeof skillPresentationImageNode;
|
||||
skillPublishUploads: typeof skillPublishUploads;
|
||||
skillStatEvents: typeof skillStatEvents;
|
||||
skillTransfers: typeof skillTransfers;
|
||||
skills: typeof skills;
|
||||
|
||||
@@ -245,6 +245,13 @@ if (process.env.CLAWHUB_DISABLE_CRONS !== "1" && process.env.CLAWHUB_PREVIEW !==
|
||||
{ batchSize: RETENTION_STANDARD_BATCH_SIZE },
|
||||
);
|
||||
|
||||
crons.interval(
|
||||
"skill-publish-upload-retention-prune",
|
||||
{ hours: 1 },
|
||||
internal.retention.pruneExpiredSkillPublishUploadsInternal,
|
||||
{ batchSize: RETENTION_STANDARD_BATCH_SIZE },
|
||||
);
|
||||
|
||||
crons.interval(
|
||||
"http-rate-limit-keys-prune",
|
||||
{ hours: 1 },
|
||||
|
||||
@@ -7228,6 +7228,124 @@ describe("httpApiV1 handlers", () => {
|
||||
expect(new Uint8Array(await response.arrayBuffer())).toEqual(fileBytes);
|
||||
});
|
||||
|
||||
it("creates a direct skill upload URL for an authenticated API user", async () => {
|
||||
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({
|
||||
userId: "users:1",
|
||||
user: { handle: "p" },
|
||||
} as never);
|
||||
const runMutation = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(okRate())
|
||||
.mockResolvedValueOnce({ uploadTicket: "skillPublishUploadTickets:1" });
|
||||
|
||||
const response = await __handlers.skillsPostRouterV1Handler(
|
||||
makeCtx({ runMutation }),
|
||||
new Request("https://example.com/api/v1/skills/-/upload-url", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: "Bearer clh_test",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
path: "SKILL.md",
|
||||
size: 5,
|
||||
sha256: "a".repeat(64),
|
||||
contentType: "text/markdown",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(await response.json()).toEqual({
|
||||
uploadUrl: "https://example.com/api/v1/skills/-/upload/skillPublishUploadTickets%3A1",
|
||||
uploadTicket: "skillPublishUploadTickets:1",
|
||||
});
|
||||
});
|
||||
|
||||
it("stores a bounded direct skill upload and attaches it to its ticket", async () => {
|
||||
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({
|
||||
userId: "users:1",
|
||||
user: { handle: "p" },
|
||||
} as never);
|
||||
const bytes = new TextEncoder().encode("hello");
|
||||
const runQuery = vi.fn().mockResolvedValue({
|
||||
path: "SKILL.md",
|
||||
size: bytes.byteLength,
|
||||
sha256: "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824",
|
||||
contentType: "text/markdown",
|
||||
});
|
||||
const runMutation = vi.fn().mockResolvedValue(okRate());
|
||||
const store = vi.fn().mockResolvedValue("storage:1");
|
||||
|
||||
const response = await __handlers.skillsPostRouterV1Handler(
|
||||
makeCtx({ runQuery, runMutation, storage: { store, delete: vi.fn() } }),
|
||||
new Request(
|
||||
"https://example.convex.site/api/v1/skills/-/upload/skillPublishUploadTickets%3A1",
|
||||
{
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: "Bearer clh_test",
|
||||
"Content-Type": "text/markdown",
|
||||
},
|
||||
body: bytes,
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(await response.json()).toEqual({ storageId: "storage:1" });
|
||||
expect(store).toHaveBeenCalledTimes(1);
|
||||
expect(runMutation).toHaveBeenCalledWith(
|
||||
internal.skillPublishUploads.attachSkillPublishUploadInternal,
|
||||
{
|
||||
userId: "users:1",
|
||||
uploadTicket: "skillPublishUploadTickets:1",
|
||||
storageId: "storage:1",
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("stops reading a direct skill upload once it exceeds the ticket size", async () => {
|
||||
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({
|
||||
userId: "users:1",
|
||||
user: { handle: "p" },
|
||||
} as never);
|
||||
const runQuery = vi.fn().mockResolvedValue({
|
||||
path: "SKILL.md",
|
||||
size: 5,
|
||||
sha256: "a".repeat(64),
|
||||
contentType: "text/markdown",
|
||||
});
|
||||
const store = vi.fn();
|
||||
const body = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(new Uint8Array([1, 2, 3]));
|
||||
controller.enqueue(new Uint8Array([4, 5, 6]));
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
|
||||
const response = await __handlers.skillsPostRouterV1Handler(
|
||||
makeCtx({
|
||||
runQuery,
|
||||
runMutation: vi.fn().mockResolvedValue(okRate()),
|
||||
storage: { store, delete: vi.fn() },
|
||||
}),
|
||||
new Request(
|
||||
"https://example.convex.site/api/v1/skills/-/upload/skillPublishUploadTickets%3A1",
|
||||
{
|
||||
method: "POST",
|
||||
headers: { Authorization: "Bearer clh_test" },
|
||||
body,
|
||||
duplex: "half",
|
||||
} as RequestInit & { duplex: "half" },
|
||||
),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(413);
|
||||
expect(store).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("publish json succeeds", async () => {
|
||||
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({
|
||||
userId: "users:1",
|
||||
@@ -7254,6 +7372,7 @@ describe("httpApiV1 handlers", () => {
|
||||
storageId: "storage:1",
|
||||
sha256: "abc",
|
||||
contentType: "text/plain",
|
||||
uploadTicket: "skillPublishUploadTickets:1",
|
||||
},
|
||||
],
|
||||
});
|
||||
@@ -7312,6 +7431,7 @@ describe("httpApiV1 handlers", () => {
|
||||
storageId: "storage:1",
|
||||
sha256: "abc",
|
||||
contentType: "text/plain",
|
||||
uploadTicket: "skillPublishUploadTickets:1",
|
||||
},
|
||||
],
|
||||
});
|
||||
@@ -7376,6 +7496,7 @@ describe("httpApiV1 handlers", () => {
|
||||
storageId: "storage:1",
|
||||
sha256: "abc",
|
||||
contentType: "text/plain",
|
||||
uploadTicket: "skillPublishUploadTickets:1",
|
||||
},
|
||||
],
|
||||
});
|
||||
@@ -7394,7 +7515,9 @@ describe("httpApiV1 handlers", () => {
|
||||
expect.anything(),
|
||||
"users:1",
|
||||
expect.objectContaining({ source }),
|
||||
{},
|
||||
expect.objectContaining({
|
||||
skillPublishUploadTickets: ["skillPublishUploadTickets:1"],
|
||||
}),
|
||||
);
|
||||
expect(vi.mocked(publishVersionForUser).mock.calls[0]?.[3]).not.toHaveProperty(
|
||||
"sourceProvenance",
|
||||
@@ -7431,6 +7554,7 @@ describe("httpApiV1 handlers", () => {
|
||||
storageId: "storage:1",
|
||||
sha256: "abc",
|
||||
contentType: "text/plain",
|
||||
uploadTicket: "skillPublishUploadTickets:1",
|
||||
},
|
||||
],
|
||||
});
|
||||
@@ -7455,7 +7579,10 @@ describe("httpApiV1 handlers", () => {
|
||||
expect.anything(),
|
||||
"users:1",
|
||||
expect.not.objectContaining({ ownerHandle: expect.anything() }),
|
||||
{ ownerPublisherId: "publishers:openclaw" },
|
||||
expect.objectContaining({
|
||||
ownerPublisherId: "publishers:openclaw",
|
||||
skillPublishUploadTickets: ["skillPublishUploadTickets:1"],
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
@@ -7490,6 +7617,7 @@ describe("httpApiV1 handlers", () => {
|
||||
storageId: "storage:1",
|
||||
sha256: "abc",
|
||||
contentType: "text/plain",
|
||||
uploadTicket: "skillPublishUploadTickets:1",
|
||||
},
|
||||
],
|
||||
});
|
||||
@@ -7506,7 +7634,10 @@ describe("httpApiV1 handlers", () => {
|
||||
expect.anything(),
|
||||
"users:1",
|
||||
expect.not.objectContaining({ ownerHandle: expect.anything() }),
|
||||
{ ownerPublisherId: "publishers:openclaw" },
|
||||
expect.objectContaining({
|
||||
ownerPublisherId: "publishers:openclaw",
|
||||
skillPublishUploadTickets: ["skillPublishUploadTickets:1"],
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
@@ -7533,6 +7664,7 @@ describe("httpApiV1 handlers", () => {
|
||||
storageId: "storage:1",
|
||||
sha256: "abc",
|
||||
contentType: "text/plain",
|
||||
uploadTicket: "skillPublishUploadTickets:1",
|
||||
},
|
||||
],
|
||||
});
|
||||
@@ -7568,6 +7700,7 @@ describe("httpApiV1 handlers", () => {
|
||||
storageId: "storage:1",
|
||||
sha256: "abc",
|
||||
contentType: "text/plain",
|
||||
uploadTicket: "skillPublishUploadTickets:1",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
@@ -537,6 +537,7 @@ export function parsePublishBody(body: unknown) {
|
||||
files: parsed.files.map((file) => ({
|
||||
...file,
|
||||
storageId: file.storageId as Id<"_storage">,
|
||||
uploadTicket: file.uploadTicket as Id<"skillPublishUploadTickets"> | undefined,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import {
|
||||
ApiRoutes,
|
||||
ApiV1SkillUploadUrlRequestSchema,
|
||||
ApiV1SkillBulkRescanBatchRequestSchema,
|
||||
ApiV1SkillBulkRescanStatusRequestSchema,
|
||||
ApiV1SkillHardDeleteRequestSchema,
|
||||
@@ -95,6 +96,30 @@ const DEFAULT_EXPORT_PAGE_LIMIT = 250;
|
||||
const MAX_EXPORT_TOTAL_BYTES = 256 * 1024 * 1024;
|
||||
const MAX_SECURITY_VERDICT_ITEMS = 100;
|
||||
|
||||
async function readRequestBodyWithinLimit(request: Request, maxBytes: number) {
|
||||
if (!request.body) return new Uint8Array();
|
||||
const reader = request.body.getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
let totalBytes = 0;
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
totalBytes += value.byteLength;
|
||||
if (totalBytes > maxBytes) {
|
||||
await reader.cancel("Upload exceeds its declared size").catch(() => undefined);
|
||||
return null;
|
||||
}
|
||||
chunks.push(value);
|
||||
}
|
||||
const bytes = new Uint8Array(totalBytes);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
bytes.set(chunk, offset);
|
||||
offset += chunk.byteLength;
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
type ListSkillsResult = {
|
||||
items: Array<{
|
||||
skill: {
|
||||
@@ -2625,6 +2650,13 @@ export async function publishSkillV1Handler(ctx: ActionCtx, request: Request) {
|
||||
if (!hasAcceptedLegacyLicenseTerms(payload.acceptLicenseTerms)) {
|
||||
return text("MIT-0 license terms must be accepted to publish skills", 400, rate.headers);
|
||||
}
|
||||
if (payload.files.some((file) => !file.uploadTicket)) {
|
||||
return text(
|
||||
"Every directly uploaded skill file requires an upload ticket",
|
||||
400,
|
||||
rate.headers,
|
||||
);
|
||||
}
|
||||
const result = await publishSkillPayloadForApiUser(ctx, auth.userId, payload);
|
||||
return json({ ok: true, ...result }, 200, rate.headers);
|
||||
}
|
||||
@@ -2651,6 +2683,13 @@ async function publishSkillPayloadForApiUser(
|
||||
payload: ReturnType<typeof parsePublishBody>,
|
||||
) {
|
||||
const { ownerHandle, sourceOwnerHandle, migrateOwner, ...publishPayload } = payload;
|
||||
const uploadTickets = publishPayload.files.flatMap((file) =>
|
||||
file.uploadTicket ? [file.uploadTicket] : [],
|
||||
);
|
||||
if (uploadTickets.length > 0 && uploadTickets.length !== publishPayload.files.length) {
|
||||
throw new Error("Every directly uploaded skill file requires an upload ticket");
|
||||
}
|
||||
const files = publishPayload.files.map(({ uploadTicket: _uploadTicket, ...file }) => file);
|
||||
const target = ownerHandle
|
||||
? ((await ctx.runMutation(internal.publishers.resolvePublishTargetForUserInternal, {
|
||||
actorUserId: userId,
|
||||
@@ -2667,11 +2706,17 @@ async function publishSkillPayloadForApiUser(
|
||||
})) as { publisherId: Id<"publishers"> })
|
||||
: null;
|
||||
const shouldMigrateOwner = Boolean(target && source);
|
||||
return await publishVersionForUser(ctx, userId, publishPayload, {
|
||||
...(target ? { ownerPublisherId: target.publisherId } : {}),
|
||||
...(source ? { sourceOwnerPublisherId: source.publisherId } : {}),
|
||||
...(shouldMigrateOwner ? { migrateOwner: true } : {}),
|
||||
});
|
||||
return await publishVersionForUser(
|
||||
ctx,
|
||||
userId,
|
||||
{ ...publishPayload, files },
|
||||
{
|
||||
...(target ? { ownerPublisherId: target.publisherId } : {}),
|
||||
...(source ? { sourceOwnerPublisherId: source.publisherId } : {}),
|
||||
...(shouldMigrateOwner ? { migrateOwner: true } : {}),
|
||||
...(uploadTickets.length > 0 ? { skillPublishUploadTickets: uploadTickets } : {}),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function hasAcceptedLegacyLicenseTerms(acceptLicenseTerms: boolean | undefined) {
|
||||
@@ -2942,6 +2987,82 @@ export async function skillsPostRouterV1Handler(ctx: ActionCtx, request: Request
|
||||
const action = segments[1] ?? "";
|
||||
const slug = segments[0]?.trim().toLowerCase() ?? "";
|
||||
|
||||
if (segments.length === 2 && slug === "-" && action === "upload-url") {
|
||||
const auth = await requireApiTokenUserOrResponse(ctx, request, rate.headers);
|
||||
if (!auth.ok) return auth.response;
|
||||
try {
|
||||
const expected = parseArk(
|
||||
ApiV1SkillUploadUrlRequestSchema,
|
||||
await request.json(),
|
||||
"Skill upload request",
|
||||
);
|
||||
const { uploadTicket } = await ctx.runMutation(
|
||||
internal.skillPublishUploads.createSkillPublishUploadInternal,
|
||||
{
|
||||
userId: auth.userId,
|
||||
path: expected.path,
|
||||
size: expected.size,
|
||||
sha256: expected.sha256,
|
||||
...(expected.contentType ? { contentType: expected.contentType } : {}),
|
||||
},
|
||||
);
|
||||
// The request reaching this Convex action has the direct Convex origin even
|
||||
// when clawhub.ai forwarded it. Keep the file body off the Vercel request path.
|
||||
const uploadUrl = new URL(
|
||||
`/api/v1/skills/-/upload/${encodeURIComponent(uploadTicket)}`,
|
||||
request.url,
|
||||
).toString();
|
||||
return json({ uploadUrl, uploadTicket }, 200, rate.headers);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "Invalid skill upload request";
|
||||
return text(message, 400, rate.headers);
|
||||
}
|
||||
}
|
||||
|
||||
if (segments.length === 3 && slug === "-" && action === "upload" && segments[2]) {
|
||||
const auth = await requireApiTokenUserOrResponse(ctx, request, rate.headers);
|
||||
if (!auth.ok) return auth.response;
|
||||
const uploadTicket = segments[2] as Id<"skillPublishUploadTickets">;
|
||||
let storageId: Id<"_storage"> | undefined;
|
||||
try {
|
||||
const expected = await ctx.runQuery(
|
||||
internal.skillPublishUploads.getSkillPublishUploadForUserInternal,
|
||||
{ userId: auth.userId, uploadTicket },
|
||||
);
|
||||
const declaredLength = Number(request.headers.get("content-length"));
|
||||
if (Number.isFinite(declaredLength) && declaredLength > expected.size) {
|
||||
return text("Uploaded file exceeds its declared size", 413, rate.headers);
|
||||
}
|
||||
const bytes = await readRequestBodyWithinLimit(request, expected.size);
|
||||
if (!bytes) {
|
||||
return text("Uploaded file exceeds its declared size", 413, rate.headers);
|
||||
}
|
||||
if (bytes.byteLength !== expected.size) {
|
||||
return text("Uploaded file size does not match its upload ticket", 400, rate.headers);
|
||||
}
|
||||
const digest = await crypto.subtle.digest("SHA-256", bytes);
|
||||
const sha256 = Array.from(new Uint8Array(digest), (byte) =>
|
||||
byte.toString(16).padStart(2, "0"),
|
||||
).join("");
|
||||
if (sha256 !== expected.sha256) {
|
||||
return text("Uploaded file SHA-256 does not match its upload ticket", 400, rate.headers);
|
||||
}
|
||||
storageId = await ctx.storage.store(
|
||||
new Blob([bytes], expected.contentType ? { type: expected.contentType } : undefined),
|
||||
);
|
||||
await ctx.runMutation(internal.skillPublishUploads.attachSkillPublishUploadInternal, {
|
||||
userId: auth.userId,
|
||||
uploadTicket,
|
||||
storageId,
|
||||
});
|
||||
return json({ storageId }, 200, rate.headers);
|
||||
} catch (error) {
|
||||
if (storageId) await ctx.storage.delete(storageId).catch(() => undefined);
|
||||
const message = error instanceof Error ? error.message : "Skill upload failed";
|
||||
return text(message, 400, rate.headers);
|
||||
}
|
||||
}
|
||||
|
||||
if (segments.length === 3 && segments[1] === "tags" && segments[2]) {
|
||||
if (!slug) return text("Slug required", 400, rate.headers);
|
||||
const auth = await requireApiTokenUserOrResponse(ctx, request, rate.headers);
|
||||
|
||||
@@ -45,6 +45,12 @@ describe("retention policies", () => {
|
||||
expirationIndex: "by_expires_at",
|
||||
prune: "retention.pruneExpiredPublisherInvitesInternal",
|
||||
});
|
||||
expect(getRetentionPolicy("skillPublishUploadTickets")).toMatchObject({
|
||||
classification: "ephemeral",
|
||||
expirationField: "expiresAt",
|
||||
expirationIndex: "by_expires_at",
|
||||
prune: "retention.pruneExpiredSkillPublishUploadsInternal",
|
||||
});
|
||||
});
|
||||
|
||||
it("documents package daily stats as durable analytics", () => {
|
||||
|
||||
@@ -105,6 +105,15 @@ export const RETENTION_POLICIES = {
|
||||
retention: "Organization logo upload ticket TTL.",
|
||||
},
|
||||
),
|
||||
skillPublishUploadTickets: ephemeral(
|
||||
"Direct skill file upload tickets are deleted after their short staging window.",
|
||||
{
|
||||
expirationField: "expiresAt",
|
||||
expirationIndex: "by_expires_at",
|
||||
prune: "retention.pruneExpiredSkillPublishUploadsInternal",
|
||||
retention: "1 hour.",
|
||||
},
|
||||
),
|
||||
officialPublishers: permanent("Manual official publisher assignments."),
|
||||
githubSkillSources: permanent("Tracked GitHub source configuration."),
|
||||
githubSkillContents: derived("Cached GitHub source content snapshots.", "githubSkillSources"),
|
||||
|
||||
@@ -154,6 +154,7 @@ export type PublishOptions = {
|
||||
// accidentally transfer ownership.
|
||||
migrateOwner?: boolean;
|
||||
stagePrePublicationChecks?: boolean;
|
||||
skillPublishUploadTickets?: Id<"skillPublishUploadTickets">[];
|
||||
};
|
||||
|
||||
type InternalPublishOptions = PublishOptions;
|
||||
@@ -489,6 +490,7 @@ async function publishVersionForUserInternal(
|
||||
|
||||
const skillInsertArgs = {
|
||||
userId,
|
||||
skillPublishUploadTickets: options.skillPublishUploadTickets,
|
||||
ownerPublisherId: options.ownerPublisherId,
|
||||
sourceOwnerPublisherId: options.sourceOwnerPublisherId,
|
||||
migrateOwner: options.migrateOwner,
|
||||
|
||||
@@ -5,6 +5,7 @@ const retentionRefs = vi.hoisted(() => ({
|
||||
pruneExpiredAuthSessionsInternal: Symbol("pruneExpiredAuthSessionsInternal"),
|
||||
pruneExpiredAuthRefreshTokensInternal: Symbol("pruneExpiredAuthRefreshTokensInternal"),
|
||||
pruneExpiredPublisherInvitesInternal: Symbol("pruneExpiredPublisherInvitesInternal"),
|
||||
pruneExpiredSkillPublishUploadsInternal: Symbol("pruneExpiredSkillPublishUploadsInternal"),
|
||||
}));
|
||||
|
||||
vi.mock("./_generated/api", () => ({
|
||||
@@ -17,6 +18,7 @@ const {
|
||||
pruneExpiredAuthRefreshTokensInternal,
|
||||
pruneExpiredAuthSessionsInternal,
|
||||
pruneExpiredPublisherInvitesInternal,
|
||||
pruneExpiredSkillPublishUploadsInternal,
|
||||
} = await import("./retention");
|
||||
|
||||
type WrappedHandler<TArgs, TResult> = {
|
||||
@@ -41,6 +43,12 @@ const prunePublisherInvitesHandler = (
|
||||
{ deleted: number; hasMore: boolean }
|
||||
>
|
||||
)._handler;
|
||||
const pruneSkillUploadsHandler = (
|
||||
pruneExpiredSkillPublishUploadsInternal as unknown as WrappedHandler<
|
||||
{ batchSize?: number },
|
||||
{ deletedTickets: number; deletedStorage: number; hasMore: boolean }
|
||||
>
|
||||
)._handler;
|
||||
|
||||
function makeDb(base: { query: ReturnType<typeof vi.fn>; delete: ReturnType<typeof vi.fn> }) {
|
||||
return {
|
||||
@@ -233,4 +241,46 @@ describe("auth retention", () => {
|
||||
batchSize: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it("deletes expired unconsumed skill upload storage in bounded batches", async () => {
|
||||
const now = 4_000_000;
|
||||
vi.spyOn(Date, "now").mockReturnValue(now);
|
||||
const rows = [
|
||||
{
|
||||
_id: "skillPublishUploadTickets:one",
|
||||
expiresAt: now - 1,
|
||||
storageId: "storage:one",
|
||||
},
|
||||
];
|
||||
const deleteDoc = vi.fn();
|
||||
const deleteStorage = vi.fn();
|
||||
const runAfter = vi.fn();
|
||||
const lt = vi.fn(() => ({}));
|
||||
const ctx = {
|
||||
db: makeDb({
|
||||
query: vi.fn(() => ({
|
||||
withIndex: vi.fn((indexName: string, build: (q: unknown) => unknown) => {
|
||||
expect(indexName).toBe("by_expires_at");
|
||||
build({ lt });
|
||||
return { take: vi.fn(async () => rows) };
|
||||
}),
|
||||
})),
|
||||
delete: deleteDoc,
|
||||
}),
|
||||
storage: { delete: deleteStorage },
|
||||
scheduler: { runAfter },
|
||||
};
|
||||
|
||||
const result = await pruneSkillUploadsHandler(ctx as never, { batchSize: 1 });
|
||||
|
||||
expect(result).toEqual({ deletedTickets: 1, deletedStorage: 1, hasMore: true });
|
||||
expect(lt).toHaveBeenCalledWith("expiresAt", now);
|
||||
expect(deleteStorage).toHaveBeenCalledWith("storage:one");
|
||||
expect(deleteDoc).toHaveBeenCalledWith("skillPublishUploadTickets:one");
|
||||
expect(runAfter).toHaveBeenCalledWith(
|
||||
0,
|
||||
retentionRefs.pruneExpiredSkillPublishUploadsInternal,
|
||||
{ batchSize: 1 },
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -109,3 +109,34 @@ export const pruneExpiredPublisherInvitesInternal = internalMutation({
|
||||
return { deleted: stale.length, hasMore };
|
||||
},
|
||||
});
|
||||
|
||||
export const pruneExpiredSkillPublishUploadsInternal = internalMutation({
|
||||
args: {
|
||||
batchSize: v.optional(v.number()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const batchSize = normalizeRetentionBatchSize(args.batchSize);
|
||||
const stale = await ctx.db
|
||||
.query("skillPublishUploadTickets")
|
||||
.withIndex("by_expires_at", (q) => q.lt("expiresAt", Date.now()))
|
||||
.take(batchSize);
|
||||
|
||||
let deletedStorage = 0;
|
||||
for (const ticket of stale) {
|
||||
if (!ticket.usedAt && ticket.storageId) {
|
||||
await ctx.storage.delete(ticket.storageId);
|
||||
deletedStorage += 1;
|
||||
}
|
||||
await ctx.db.delete(ticket._id);
|
||||
}
|
||||
|
||||
const hasMore = stale.length === batchSize;
|
||||
if (hasMore) {
|
||||
await ctx.scheduler.runAfter(0, internal.retention.pruneExpiredSkillPublishUploadsInternal, {
|
||||
batchSize,
|
||||
});
|
||||
}
|
||||
|
||||
return { deletedTickets: stale.length, deletedStorage, hasMore };
|
||||
},
|
||||
});
|
||||
|
||||
@@ -336,6 +336,20 @@ const publisherImageUploadTickets = defineTable({
|
||||
storageId: v.optional(v.id("_storage")),
|
||||
}).index("by_publisher_user", ["publisherId", "userId"]);
|
||||
|
||||
const skillPublishUploadTickets = defineTable({
|
||||
userId: v.id("users"),
|
||||
path: v.string(),
|
||||
size: v.number(),
|
||||
sha256: v.string(),
|
||||
contentType: v.optional(v.string()),
|
||||
storageId: v.optional(v.id("_storage")),
|
||||
createdAt: v.number(),
|
||||
expiresAt: v.number(),
|
||||
usedAt: v.optional(v.number()),
|
||||
})
|
||||
.index("by_user", ["userId"])
|
||||
.index("by_expires_at", ["expiresAt"]);
|
||||
|
||||
const officialPublishers = defineTable({
|
||||
publisherId: v.id("publishers"),
|
||||
reason: v.optional(v.string()),
|
||||
@@ -4266,6 +4280,7 @@ export default defineSchema({
|
||||
publisherMembers,
|
||||
publisherInvites,
|
||||
publisherImageUploadTickets,
|
||||
skillPublishUploadTickets,
|
||||
officialPublishers,
|
||||
githubSkillSources,
|
||||
githubSkillContents,
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
/* @vitest-environment node */
|
||||
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
cleanupSkillPublishUploadInternal,
|
||||
consumeSkillPublishUploads,
|
||||
} from "./skillPublishUploads";
|
||||
|
||||
type CleanupHandler = {
|
||||
_handler: (
|
||||
ctx: unknown,
|
||||
args: { uploadTicket: "skillPublishUploadTickets:1" },
|
||||
) => Promise<unknown>;
|
||||
};
|
||||
|
||||
const cleanupHandler = (cleanupSkillPublishUploadInternal as unknown as CleanupHandler)._handler;
|
||||
|
||||
function makeCtx(ticket: Record<string, unknown> | null) {
|
||||
return {
|
||||
db: {
|
||||
get: vi.fn(async () => ticket),
|
||||
patch: vi.fn(),
|
||||
delete: vi.fn(),
|
||||
normalizeId: vi.fn(),
|
||||
query: vi.fn(),
|
||||
replace: vi.fn(),
|
||||
insert: vi.fn(),
|
||||
system: { get: vi.fn(), query: vi.fn() },
|
||||
},
|
||||
storage: { delete: vi.fn() },
|
||||
};
|
||||
}
|
||||
|
||||
describe("skill publish upload tickets", () => {
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("atomically consumes a matching staged upload", async () => {
|
||||
vi.spyOn(Date, "now").mockReturnValue(2_000);
|
||||
const ctx = makeCtx({
|
||||
_id: "skillPublishUploadTickets:1",
|
||||
userId: "users:1",
|
||||
path: "SKILL.md",
|
||||
size: 5,
|
||||
sha256: "a".repeat(64),
|
||||
contentType: "text/markdown",
|
||||
storageId: "storage:1",
|
||||
expiresAt: 10_000,
|
||||
});
|
||||
|
||||
await consumeSkillPublishUploads(ctx as never, {
|
||||
userId: "users:1" as never,
|
||||
uploadTickets: ["skillPublishUploadTickets:1" as never],
|
||||
files: [
|
||||
{
|
||||
path: "SKILL.md",
|
||||
size: 5,
|
||||
sha256: "a".repeat(64),
|
||||
contentType: "text/markdown",
|
||||
storageId: "storage:1" as never,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(ctx.db.patch).toHaveBeenCalledWith("skillPublishUploadTickets:1", { usedAt: 2_000 });
|
||||
});
|
||||
|
||||
it("rejects a ticket whose staged file does not match the publish", async () => {
|
||||
vi.spyOn(Date, "now").mockReturnValue(2_000);
|
||||
const ctx = makeCtx({
|
||||
_id: "skillPublishUploadTickets:1",
|
||||
userId: "users:1",
|
||||
path: "SKILL.md",
|
||||
size: 5,
|
||||
sha256: "a".repeat(64),
|
||||
storageId: "storage:1",
|
||||
expiresAt: 10_000,
|
||||
});
|
||||
|
||||
await expect(
|
||||
consumeSkillPublishUploads(ctx as never, {
|
||||
userId: "users:1" as never,
|
||||
uploadTickets: ["skillPublishUploadTickets:1" as never],
|
||||
files: [
|
||||
{
|
||||
path: "SKILL.md",
|
||||
size: 6,
|
||||
sha256: "a".repeat(64),
|
||||
storageId: "storage:1" as never,
|
||||
},
|
||||
],
|
||||
}),
|
||||
).rejects.toThrow("does not match this publish");
|
||||
expect(ctx.db.patch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("deletes an abandoned staged file when its ticket expires", async () => {
|
||||
const ctx = makeCtx({
|
||||
_id: "skillPublishUploadTickets:1",
|
||||
storageId: "storage:1",
|
||||
});
|
||||
|
||||
await cleanupHandler(ctx, { uploadTicket: "skillPublishUploadTickets:1" });
|
||||
|
||||
expect(ctx.storage.delete).toHaveBeenCalledWith("storage:1");
|
||||
expect(ctx.db.delete).toHaveBeenCalledWith("skillPublishUploadTickets:1");
|
||||
});
|
||||
|
||||
it("keeps published storage when deleting a consumed ticket", async () => {
|
||||
const ctx = makeCtx({
|
||||
_id: "skillPublishUploadTickets:1",
|
||||
storageId: "storage:1",
|
||||
usedAt: 2_000,
|
||||
});
|
||||
|
||||
await cleanupHandler(ctx, { uploadTicket: "skillPublishUploadTickets:1" });
|
||||
|
||||
expect(ctx.storage.delete).not.toHaveBeenCalled();
|
||||
expect(ctx.db.delete).toHaveBeenCalledWith("skillPublishUploadTickets:1");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,170 @@
|
||||
import { normalizeContentType } from "clawhub-schema";
|
||||
import { v } from "convex/values";
|
||||
import { internal } from "./_generated/api";
|
||||
import type { Id } from "./_generated/dataModel";
|
||||
import type { MutationCtx } from "./_generated/server";
|
||||
import { internalMutation, internalQuery } from "./functions";
|
||||
import { MAX_PUBLISH_FILE_BYTES } from "./lib/publishLimits";
|
||||
import { validateFilePath } from "./lib/skillZip";
|
||||
|
||||
const SKILL_PUBLISH_UPLOAD_TTL_MS = 60 * 60_000;
|
||||
|
||||
function assertExpectedUpload(args: {
|
||||
path: string;
|
||||
size: number;
|
||||
sha256: string;
|
||||
contentType?: string;
|
||||
}) {
|
||||
if (!validateFilePath(args.path)) throw new Error("Invalid upload path");
|
||||
if (!Number.isSafeInteger(args.size) || args.size < 0 || args.size > MAX_PUBLISH_FILE_BYTES) {
|
||||
throw new Error(`Upload must be at most ${MAX_PUBLISH_FILE_BYTES} bytes`);
|
||||
}
|
||||
if (!/^[a-f0-9]{64}$/i.test(args.sha256)) throw new Error("Invalid upload SHA-256");
|
||||
}
|
||||
|
||||
export const createSkillPublishUploadInternal = internalMutation({
|
||||
args: {
|
||||
userId: v.id("users"),
|
||||
path: v.string(),
|
||||
size: v.number(),
|
||||
sha256: v.string(),
|
||||
contentType: v.optional(v.string()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const user = await ctx.db.get(args.userId);
|
||||
if (!user || user.deletedAt || user.deactivatedAt) throw new Error("User not found");
|
||||
assertExpectedUpload(args);
|
||||
const now = Date.now();
|
||||
const expiresAt = now + SKILL_PUBLISH_UPLOAD_TTL_MS;
|
||||
const uploadTicket = await ctx.db.insert("skillPublishUploadTickets", {
|
||||
userId: args.userId,
|
||||
path: args.path,
|
||||
size: args.size,
|
||||
sha256: args.sha256.toLowerCase(),
|
||||
contentType: normalizeContentType(args.contentType),
|
||||
createdAt: now,
|
||||
expiresAt,
|
||||
});
|
||||
await ctx.scheduler.runAt(
|
||||
expiresAt,
|
||||
internal.skillPublishUploads.cleanupSkillPublishUploadInternal,
|
||||
{ uploadTicket },
|
||||
);
|
||||
return { uploadTicket };
|
||||
},
|
||||
});
|
||||
|
||||
export const getSkillPublishUploadForUserInternal = internalQuery({
|
||||
args: {
|
||||
userId: v.id("users"),
|
||||
uploadTicket: v.id("skillPublishUploadTickets"),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const ticket = await ctx.db.get(args.uploadTicket);
|
||||
if (
|
||||
!ticket ||
|
||||
ticket.userId !== args.userId ||
|
||||
ticket.usedAt ||
|
||||
ticket.storageId ||
|
||||
ticket.expiresAt <= Date.now()
|
||||
) {
|
||||
throw new Error("Skill upload ticket is missing, used, or expired");
|
||||
}
|
||||
return {
|
||||
path: ticket.path,
|
||||
size: ticket.size,
|
||||
sha256: ticket.sha256,
|
||||
contentType: ticket.contentType,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
export const attachSkillPublishUploadInternal = internalMutation({
|
||||
args: {
|
||||
userId: v.id("users"),
|
||||
uploadTicket: v.id("skillPublishUploadTickets"),
|
||||
storageId: v.id("_storage"),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const ticket = await ctx.db.get(args.uploadTicket);
|
||||
const now = Date.now();
|
||||
if (
|
||||
!ticket ||
|
||||
ticket.userId !== args.userId ||
|
||||
ticket.usedAt ||
|
||||
ticket.storageId ||
|
||||
ticket.expiresAt <= now
|
||||
) {
|
||||
throw new Error("Skill upload ticket is missing, used, or expired");
|
||||
}
|
||||
const metadata = await ctx.db.system.get("_storage", args.storageId);
|
||||
if (
|
||||
!metadata ||
|
||||
metadata._creationTime < ticket.createdAt ||
|
||||
metadata.size !== ticket.size ||
|
||||
metadata.sha256.toLowerCase() !== ticket.sha256 ||
|
||||
normalizeContentType(metadata.contentType) !== ticket.contentType
|
||||
) {
|
||||
throw new Error("Uploaded file does not match its skill upload ticket");
|
||||
}
|
||||
await ctx.db.patch(ticket._id, { storageId: args.storageId });
|
||||
},
|
||||
});
|
||||
|
||||
type SkillPublishFile = {
|
||||
path: string;
|
||||
size: number;
|
||||
storageId: Id<"_storage">;
|
||||
sha256: string;
|
||||
contentType?: string;
|
||||
};
|
||||
|
||||
export async function consumeSkillPublishUploads(
|
||||
ctx: MutationCtx,
|
||||
args: {
|
||||
userId: Id<"users">;
|
||||
uploadTickets: Id<"skillPublishUploadTickets">[];
|
||||
files: SkillPublishFile[];
|
||||
},
|
||||
) {
|
||||
if (args.uploadTickets.length !== args.files.length) {
|
||||
throw new Error("Every directly uploaded skill file requires an upload ticket");
|
||||
}
|
||||
if (new Set(args.uploadTickets).size !== args.uploadTickets.length) {
|
||||
throw new Error("Skill upload tickets cannot be reused");
|
||||
}
|
||||
const now = Date.now();
|
||||
for (let index = 0; index < args.files.length; index += 1) {
|
||||
const uploadTicket = args.uploadTickets[index];
|
||||
const file = args.files[index];
|
||||
if (!uploadTicket || !file) throw new Error("Skill upload ticket mismatch");
|
||||
const ticket = await ctx.db.get(uploadTicket);
|
||||
if (
|
||||
!ticket ||
|
||||
ticket.userId !== args.userId ||
|
||||
ticket.usedAt ||
|
||||
ticket.expiresAt <= now ||
|
||||
ticket.storageId !== file.storageId ||
|
||||
ticket.path !== file.path ||
|
||||
ticket.size !== file.size ||
|
||||
ticket.sha256 !== file.sha256.toLowerCase() ||
|
||||
ticket.contentType !== normalizeContentType(file.contentType)
|
||||
) {
|
||||
throw new Error("Skill upload ticket does not match this publish");
|
||||
}
|
||||
await ctx.db.patch(ticket._id, { usedAt: now });
|
||||
}
|
||||
}
|
||||
|
||||
export const cleanupSkillPublishUploadInternal = internalMutation({
|
||||
args: { uploadTicket: v.id("skillPublishUploadTickets") },
|
||||
handler: async (ctx, args) => {
|
||||
const ticket = await ctx.db.get(args.uploadTicket);
|
||||
if (!ticket) return { deleted: false };
|
||||
if (!ticket.usedAt && ticket.storageId) {
|
||||
await ctx.storage.delete(ticket.storageId);
|
||||
}
|
||||
await ctx.db.delete(ticket._id);
|
||||
return { deleted: true };
|
||||
},
|
||||
});
|
||||
@@ -173,6 +173,7 @@ import { normalizeSkillTags } from "./lib/skillTags";
|
||||
import { runStaticPublishScan } from "./lib/staticPublishScan";
|
||||
import { adjustUserSkillStatsForSkillChange } from "./lib/userSkillStats";
|
||||
import schema from "./schema";
|
||||
import { consumeSkillPublishUploads } from "./skillPublishUploads";
|
||||
|
||||
const MAX_OWNER_SUMMARY_LENGTH = 500;
|
||||
const MAX_POINTERLESS_VERSION_SURVIVOR_SCAN = 100;
|
||||
@@ -12454,6 +12455,7 @@ function stripUndefinedForStoredPublication(value: unknown): unknown {
|
||||
export const insertVersion = internalMutation({
|
||||
args: {
|
||||
userId: v.id("users"),
|
||||
skillPublishUploadTickets: v.optional(v.array(v.id("skillPublishUploadTickets"))),
|
||||
ownerPublisherId: v.optional(v.id("publishers")),
|
||||
sourceOwnerPublisherId: v.optional(v.id("publishers")),
|
||||
// Explicit opt-in to owner migration. When an existing skill row already has
|
||||
@@ -12587,6 +12589,13 @@ export const insertVersion = internalMutation({
|
||||
if (!normalizedSlug) throw new ConvexError("Slug is required.");
|
||||
const user = await ctx.db.get(userId);
|
||||
if (!user || user.deletedAt || user.deactivatedAt) throw new Error("User not found");
|
||||
if (args.skillPublishUploadTickets) {
|
||||
await consumeSkillPublishUploads(ctx, {
|
||||
userId,
|
||||
uploadTickets: args.skillPublishUploadTickets,
|
||||
files: args.files,
|
||||
});
|
||||
}
|
||||
const personalPublisher = await ensurePersonalPublisherForUser(ctx, user, {
|
||||
actorUserId: userId,
|
||||
source: "skill.publish",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "clawhub",
|
||||
"version": "0.23.2",
|
||||
"version": "0.23.3",
|
||||
"description": "ClawHub CLI \\u2014 install, update, search, and publish skills plus OpenClaw packages.",
|
||||
"homepage": "https://clawhub.ai",
|
||||
"bugs": {
|
||||
|
||||
@@ -16,6 +16,7 @@ const authTokenMocks = createAuthTokenModuleMocks();
|
||||
const registryMocks = createRegistryModuleMocks();
|
||||
const httpMocks = createHttpModuleMocks();
|
||||
const uiMocks = createUiModuleMocks();
|
||||
let publishResponse: Record<string, unknown>;
|
||||
|
||||
vi.mock("../authToken.js", () => authTokenMocks.moduleFactory());
|
||||
vi.mock("../registry.js", () => registryMocks.moduleFactory());
|
||||
@@ -78,7 +79,7 @@ describe("cmdPublish", () => {
|
||||
match: { version: "1.2.3" },
|
||||
latestVersion: { version: "1.2.3" },
|
||||
});
|
||||
httpMocks.apiRequestForm.mockResolvedValueOnce({
|
||||
mockPublishResponse({
|
||||
ok: true,
|
||||
skillId: "skill_1",
|
||||
versionId: "ver_2",
|
||||
@@ -114,7 +115,7 @@ describe("cmdPublish", () => {
|
||||
httpMocks.apiRequest.mockRejectedValueOnce(
|
||||
new Error("Skill not found or unavailable to this account."),
|
||||
);
|
||||
httpMocks.apiRequestForm.mockResolvedValueOnce({
|
||||
mockPublishResponse({
|
||||
ok: true,
|
||||
skillId: "skill_1",
|
||||
versionId: "ver_1",
|
||||
@@ -143,7 +144,7 @@ describe("cmdPublish", () => {
|
||||
httpMocks.apiRequest.mockRejectedValueOnce(
|
||||
new Error("Skill not found or unavailable to this account."),
|
||||
);
|
||||
httpMocks.apiRequestForm.mockResolvedValueOnce({
|
||||
mockPublishResponse({
|
||||
ok: true,
|
||||
skillId: "skill_1",
|
||||
versionId: "ver_pending",
|
||||
@@ -179,7 +180,7 @@ describe("cmdPublish", () => {
|
||||
httpMocks.apiRequest.mockRejectedValueOnce(
|
||||
new Error("Skill not found or unavailable to this account."),
|
||||
);
|
||||
httpMocks.apiRequestForm.mockResolvedValueOnce({
|
||||
mockPublishResponse({
|
||||
ok: true,
|
||||
skillId: "skill_1",
|
||||
versionId: "ver_pending",
|
||||
@@ -212,7 +213,7 @@ describe("cmdPublish", () => {
|
||||
const folder = join(workdir, "unknown-status-skill");
|
||||
await mkdir(folder, { recursive: true });
|
||||
await writeFile(join(folder, "SKILL.md"), "# Skill\n", "utf8");
|
||||
httpMocks.apiRequestForm.mockResolvedValueOnce({
|
||||
mockPublishResponse({
|
||||
ok: true,
|
||||
skillId: "skill_1",
|
||||
versionId: "ver_1",
|
||||
@@ -239,7 +240,7 @@ describe("cmdPublish", () => {
|
||||
match: null,
|
||||
latestVersion: { version: "1.2.3" },
|
||||
});
|
||||
httpMocks.apiRequestForm.mockResolvedValueOnce({
|
||||
mockPublishResponse({
|
||||
ok: true,
|
||||
skillId: "skill_1",
|
||||
versionId: "ver_2",
|
||||
@@ -269,7 +270,7 @@ describe("cmdPublish", () => {
|
||||
match: { version: "1.2.3" },
|
||||
latestVersion: { version: "1.2.3" },
|
||||
});
|
||||
httpMocks.apiRequestForm.mockResolvedValueOnce({
|
||||
mockPublishResponse({
|
||||
ok: true,
|
||||
skillId: "skill_1",
|
||||
versionId: "ver_2",
|
||||
@@ -316,6 +317,85 @@ describe("cmdPublish", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("uploads each skill file separately before sending the publish metadata", async () => {
|
||||
const workdir = await makeTmpWorkdir();
|
||||
try {
|
||||
const folder = join(workdir, "staged-skill");
|
||||
await mkdir(join(folder, "assets"), { recursive: true });
|
||||
await writeFile(join(folder, "SKILL.md"), "# Staged skill\n", "utf8");
|
||||
await writeFile(join(folder, "assets", "payload.bin"), Uint8Array.from([0, 1, 2, 255]));
|
||||
|
||||
httpMocks.uploadBinary
|
||||
.mockResolvedValueOnce({ storageId: "storage:skill" })
|
||||
.mockResolvedValueOnce({ storageId: "storage:payload" });
|
||||
mockPublishResponse({
|
||||
ok: true,
|
||||
skillId: "skill_1",
|
||||
versionId: "ver_1",
|
||||
publicationStatus: "published",
|
||||
});
|
||||
|
||||
await cmdPublish(makeOpts(workdir), "staged-skill", {
|
||||
version: "1.0.0",
|
||||
});
|
||||
|
||||
expect(httpMocks.apiRequestForm).not.toHaveBeenCalled();
|
||||
expect(httpMocks.uploadBinary).toHaveBeenCalledTimes(2);
|
||||
const uploadTicketCalls = httpMocks.apiRequest.mock.calls.filter((call) => {
|
||||
const args = call[1] as { method?: string; path?: string };
|
||||
return args.method === "POST" && args.path === "/api/v1/skills/-/upload-url";
|
||||
});
|
||||
expect(uploadTicketCalls).toHaveLength(2);
|
||||
expect(uploadTicketCalls[0]?.[1]).toMatchObject({
|
||||
token: "tkn",
|
||||
body: {
|
||||
path: "SKILL.md",
|
||||
size: 15,
|
||||
sha256: "90b735dd867ee1111738fb1397982e1dbf7e6bda451d60d60fc10623926adcfc",
|
||||
},
|
||||
});
|
||||
expect(httpMocks.uploadBinary).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({
|
||||
url: "https://upload.local/1",
|
||||
token: "tkn",
|
||||
}),
|
||||
expect.anything(),
|
||||
);
|
||||
const publishCall = httpMocks.apiRequest.mock.calls.find((call) => {
|
||||
const args = call[1] as { method?: string; path?: string };
|
||||
return args.method === "POST" && args.path === "/api/v1/skills";
|
||||
});
|
||||
expect(publishCall?.[1]).toMatchObject({
|
||||
method: "POST",
|
||||
path: "/api/v1/skills",
|
||||
token: "tkn",
|
||||
body: {
|
||||
slug: "staged-skill",
|
||||
files: [
|
||||
expect.objectContaining({
|
||||
path: "SKILL.md",
|
||||
size: 15,
|
||||
storageId: "storage:skill",
|
||||
sha256: "90b735dd867ee1111738fb1397982e1dbf7e6bda451d60d60fc10623926adcfc",
|
||||
uploadTicket: "skillPublishUploadTickets:1",
|
||||
}),
|
||||
expect.objectContaining({
|
||||
path: "assets/payload.bin",
|
||||
size: 4,
|
||||
storageId: "storage:payload",
|
||||
sha256: "3d1f57c984978ef98a18378c8166c1cb8ede02c03eeb6aee7e2f121dfeee3e56",
|
||||
contentType: "application/octet-stream",
|
||||
uploadTicket: "skillPublishUploadTickets:2",
|
||||
}),
|
||||
],
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
await rm(workdir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("publishes Terraform and opaque files with exact bytes (mocked HTTP)", async () => {
|
||||
const workdir = await makeTmpWorkdir();
|
||||
try {
|
||||
@@ -333,7 +413,7 @@ describe("cmdPublish", () => {
|
||||
await writeFile(join(folder, "terraform.tfvars"), variablesContent, "utf8");
|
||||
await writeFile(join(folder, "assets", "payload.bin"), opaqueBytes);
|
||||
|
||||
httpMocks.apiRequestForm.mockResolvedValueOnce({
|
||||
mockPublishResponse({
|
||||
ok: true,
|
||||
skillId: "skill_1",
|
||||
versionId: "ver_1",
|
||||
@@ -351,15 +431,7 @@ describe("cmdPublish", () => {
|
||||
|
||||
await cmdPublish(makeOpts(workdir), "my-skill", options);
|
||||
|
||||
const publishCall = httpMocks.apiRequestForm.mock.calls.find((call) => {
|
||||
const req = call[1] as { path?: string } | undefined;
|
||||
return req?.path === "/api/v1/skills";
|
||||
});
|
||||
if (!publishCall) throw new Error("Missing publish call");
|
||||
const publishForm = (publishCall[1] as { form?: FormData }).form as FormData;
|
||||
const payloadEntry = publishForm.get("payload");
|
||||
if (typeof payloadEntry !== "string") throw new Error("Missing publish payload");
|
||||
const payload = JSON.parse(payloadEntry);
|
||||
const payload = publishPayload();
|
||||
expect(payload.slug).toBe("my-skill");
|
||||
expect(payload.displayName).toBe("My Skill");
|
||||
expect(payload.ownerHandle).toBe("me");
|
||||
@@ -369,22 +441,26 @@ describe("cmdPublish", () => {
|
||||
expect(payload.tags).toEqual(["latest"]);
|
||||
expect(payload.categories).toEqual(["automation", "development"]);
|
||||
expect(payload.topics).toEqual(["React", "GPU development"]);
|
||||
const files = publishForm.getAll("files") as Array<Blob & { name?: string }>;
|
||||
expect(files.map((file) => file.name ?? "").sort()).toEqual([
|
||||
const files = payload.files as Array<{ path: string }>;
|
||||
expect(files.map((file) => file.path).sort()).toEqual([
|
||||
"SKILL.md",
|
||||
"assets/payload.bin",
|
||||
"main.tf",
|
||||
"notes.md",
|
||||
"terraform.tfvars",
|
||||
]);
|
||||
const byName = new Map(files.map((file) => [file.name ?? "", file]));
|
||||
expect(await byName.get("main.tf")?.text()).toBe(terraformContent);
|
||||
expect(await byName.get("terraform.tfvars")?.text()).toBe(variablesContent);
|
||||
expect(
|
||||
new Uint8Array(
|
||||
(await byName.get("assets/payload.bin")?.arrayBuffer()) ?? new ArrayBuffer(0),
|
||||
),
|
||||
).toEqual(opaqueBytes);
|
||||
const uploadedBytes = new Map(
|
||||
files.map((file, index) => {
|
||||
const uploadCall = httpMocks.uploadBinary.mock.calls[index];
|
||||
if (!uploadCall) throw new Error(`Missing upload call for ${file.path}`);
|
||||
return [file.path, (uploadCall[0] as { bytes: Uint8Array }).bytes] as const;
|
||||
}),
|
||||
);
|
||||
expect(new TextDecoder().decode(uploadedBytes.get("main.tf"))).toBe(terraformContent);
|
||||
expect(new TextDecoder().decode(uploadedBytes.get("terraform.tfvars"))).toBe(
|
||||
variablesContent,
|
||||
);
|
||||
expect(uploadedBytes.get("assets/payload.bin")).toEqual(opaqueBytes);
|
||||
} finally {
|
||||
await rm(workdir, { recursive: true, force: true });
|
||||
}
|
||||
@@ -397,7 +473,7 @@ describe("cmdPublish", () => {
|
||||
await mkdir(folder, { recursive: true });
|
||||
await writeFile(join(folder, "SKILL.md"), "# Clear topics\n", "utf8");
|
||||
|
||||
httpMocks.apiRequestForm.mockResolvedValueOnce({
|
||||
mockPublishResponse({
|
||||
ok: true,
|
||||
skillId: "skill_1",
|
||||
versionId: "ver_1",
|
||||
@@ -418,7 +494,7 @@ describe("cmdPublish", () => {
|
||||
await mkdir(folder, { recursive: true });
|
||||
await writeFile(join(folder, "SKILL.md"), "# Skill\n", "utf8");
|
||||
|
||||
httpMocks.apiRequestForm.mockResolvedValueOnce({
|
||||
mockPublishResponse({
|
||||
ok: true,
|
||||
skillId: "skill_1",
|
||||
versionId: "ver_1",
|
||||
@@ -432,15 +508,7 @@ describe("cmdPublish", () => {
|
||||
forkOf: "@openclaw/demo@1.2.3",
|
||||
});
|
||||
|
||||
const publishCall = httpMocks.apiRequestForm.mock.calls.find((call) => {
|
||||
const req = call[1] as { path?: string } | undefined;
|
||||
return req?.path === "/api/v1/skills";
|
||||
});
|
||||
if (!publishCall) throw new Error("Missing publish call");
|
||||
const publishForm = (publishCall[1] as { form?: FormData }).form as FormData;
|
||||
const payloadEntry = publishForm.get("payload");
|
||||
if (typeof payloadEntry !== "string") throw new Error("Missing publish payload");
|
||||
expect(JSON.parse(payloadEntry).forkOf).toEqual({
|
||||
expect(publishPayload().forkOf).toEqual({
|
||||
slug: "demo",
|
||||
ownerHandle: "openclaw",
|
||||
version: "1.2.3",
|
||||
@@ -459,7 +527,7 @@ describe("cmdPublish", () => {
|
||||
await writeFile(join(folder, "notes.md"), "notes\n", "utf8");
|
||||
await writeFile(join(folder, "skill-card.md"), "# Generated card\n", "utf8");
|
||||
|
||||
httpMocks.apiRequestForm.mockResolvedValueOnce({
|
||||
mockPublishResponse({
|
||||
ok: true,
|
||||
skillId: "skill_1",
|
||||
versionId: "ver_1",
|
||||
@@ -473,14 +541,8 @@ describe("cmdPublish", () => {
|
||||
tags: "latest",
|
||||
});
|
||||
|
||||
const publishCall = httpMocks.apiRequestForm.mock.calls.find((call) => {
|
||||
const req = call[1] as { path?: string } | undefined;
|
||||
return req?.path === "/api/v1/skills";
|
||||
});
|
||||
if (!publishCall) throw new Error("Missing publish call");
|
||||
const publishForm = (publishCall[1] as { form?: FormData }).form as FormData;
|
||||
const files = publishForm.getAll("files") as Array<Blob & { name?: string }>;
|
||||
expect(files.map((file) => file.name ?? "").sort()).toEqual(["SKILL.md", "notes.md"]);
|
||||
const files = publishPayload().files as Array<{ path: string }>;
|
||||
expect(files.map((file) => file.path).sort()).toEqual(["SKILL.md", "notes.md"]);
|
||||
} finally {
|
||||
await rm(workdir, { recursive: true, force: true });
|
||||
}
|
||||
@@ -493,7 +555,7 @@ describe("cmdPublish", () => {
|
||||
await mkdir(folder, { recursive: true });
|
||||
await writeFile(join(folder, "SKILL.md"), "# Skill\n", "utf8");
|
||||
|
||||
httpMocks.apiRequestForm.mockResolvedValueOnce({
|
||||
mockPublishResponse({
|
||||
ok: true,
|
||||
skillId: "skill_1",
|
||||
versionId: "ver_2",
|
||||
@@ -505,11 +567,7 @@ describe("cmdPublish", () => {
|
||||
tags: "latest",
|
||||
});
|
||||
|
||||
expect(httpMocks.apiRequestForm).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({ path: "/api/v1/skills", method: "POST" }),
|
||||
expect.anything(),
|
||||
);
|
||||
expect(publishPayload()).toMatchObject({ changelog: "" });
|
||||
} finally {
|
||||
await rm(workdir, { recursive: true, force: true });
|
||||
}
|
||||
@@ -524,7 +582,7 @@ describe("cmdPublish", () => {
|
||||
await writeFile(join(folder, "SKILL.md"), "# Skill\n", "utf8");
|
||||
await writeFile(join(folder, "notes.md"), "ignored notes\n", "utf8");
|
||||
|
||||
httpMocks.apiRequestForm.mockResolvedValueOnce({
|
||||
mockPublishResponse({
|
||||
ok: true,
|
||||
skillId: "skill_1",
|
||||
versionId: "ver_1",
|
||||
@@ -538,14 +596,8 @@ describe("cmdPublish", () => {
|
||||
tags: "latest",
|
||||
});
|
||||
|
||||
const publishCall = httpMocks.apiRequestForm.mock.calls.find((call) => {
|
||||
const req = call[1] as { path?: string } | undefined;
|
||||
return req?.path === "/api/v1/skills";
|
||||
});
|
||||
if (!publishCall) throw new Error("Missing publish call");
|
||||
const publishForm = (publishCall[1] as { form?: FormData }).form as FormData;
|
||||
const files = publishForm.getAll("files") as Array<Blob & { name?: string }>;
|
||||
expect(files.map((file) => file.name ?? "")).toEqual(["SKILL.md"]);
|
||||
const files = publishPayload().files as Array<{ path: string }>;
|
||||
expect(files.map((file) => file.path)).toEqual(["SKILL.md"]);
|
||||
} finally {
|
||||
await rm(workdir, { recursive: true, force: true });
|
||||
}
|
||||
@@ -558,7 +610,7 @@ describe("cmdPublish", () => {
|
||||
await mkdir(folder, { recursive: true });
|
||||
await writeFile(join(folder, "SKILL.md"), "# Skill\n", "utf8");
|
||||
|
||||
httpMocks.apiRequestForm.mockResolvedValueOnce({
|
||||
mockPublishResponse({
|
||||
ok: true,
|
||||
skillId: "skill_1",
|
||||
versionId: "ver_2",
|
||||
@@ -572,15 +624,7 @@ describe("cmdPublish", () => {
|
||||
tags: "latest",
|
||||
});
|
||||
|
||||
const publishCall = httpMocks.apiRequestForm.mock.calls.find((call) => {
|
||||
const req = call[1] as { path?: string } | undefined;
|
||||
return req?.path === "/api/v1/skills";
|
||||
});
|
||||
if (!publishCall) throw new Error("Missing publish call");
|
||||
const publishForm = (publishCall[1] as { form?: FormData }).form as FormData;
|
||||
const payloadEntry = publishForm.get("payload");
|
||||
if (typeof payloadEntry !== "string") throw new Error("Missing publish payload");
|
||||
const payload = JSON.parse(payloadEntry);
|
||||
const payload = publishPayload();
|
||||
expect(payload.ownerHandle).toBe("openclaw");
|
||||
expect(payload.sourceOwnerHandle).toBe("me");
|
||||
expect(payload.migrateOwner).toBe(true);
|
||||
@@ -620,7 +664,7 @@ describe("cmdPublish", () => {
|
||||
await writeFile(join(folder, "SKILL.md"), "# Skill\n", "utf8");
|
||||
|
||||
mockDefaultApiRequest("steipete");
|
||||
httpMocks.apiRequestForm.mockResolvedValueOnce({
|
||||
mockPublishResponse({
|
||||
ok: true,
|
||||
skillId: "skill_1",
|
||||
versionId: "ver_1",
|
||||
@@ -636,15 +680,7 @@ describe("cmdPublish", () => {
|
||||
sourcePath: "skills/source-skill",
|
||||
});
|
||||
|
||||
const publishCall = httpMocks.apiRequestForm.mock.calls.find((call) => {
|
||||
const req = call[1] as { path?: string } | undefined;
|
||||
return req?.path === "/api/v1/skills";
|
||||
});
|
||||
if (!publishCall) throw new Error("Missing publish call");
|
||||
const publishForm = (publishCall[1] as { form?: FormData }).form as FormData;
|
||||
const payloadEntry = publishForm.get("payload");
|
||||
if (typeof payloadEntry !== "string") throw new Error("Missing publish payload");
|
||||
const payload = JSON.parse(payloadEntry);
|
||||
const payload = publishPayload();
|
||||
expect(payload.source).toEqual({
|
||||
kind: "github",
|
||||
url: "https://github.com/NVIDIA/skills",
|
||||
@@ -691,28 +727,57 @@ describe("cmdPublish", () => {
|
||||
});
|
||||
|
||||
function publishPayload() {
|
||||
const publishCall = httpMocks.apiRequestForm.mock.calls.find((call) => {
|
||||
const request = call[1] as { path?: string } | undefined;
|
||||
return request?.path === "/api/v1/skills";
|
||||
const publishCall = httpMocks.apiRequest.mock.calls.find((call) => {
|
||||
const request = call[1] as { method?: string; path?: string } | undefined;
|
||||
return request?.method === "POST" && request.path === "/api/v1/skills";
|
||||
});
|
||||
if (!publishCall) throw new Error("Missing publish call");
|
||||
const form = (publishCall[1] as { form?: FormData }).form;
|
||||
const payload = form?.get("payload");
|
||||
if (typeof payload !== "string") throw new Error("Missing publish payload");
|
||||
return JSON.parse(payload) as Record<string, unknown>;
|
||||
const body = (publishCall[1] as { body?: unknown }).body;
|
||||
if (!body || typeof body !== "object" || Array.isArray(body)) {
|
||||
throw new Error("Missing publish payload");
|
||||
}
|
||||
return body as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function mockPublishResponse(response: Record<string, unknown>) {
|
||||
publishResponse = response;
|
||||
}
|
||||
|
||||
function mockDefaultApiRequest(whoamiHandle: string | null = "me") {
|
||||
publishResponse = {
|
||||
ok: true,
|
||||
skillId: "skill_1",
|
||||
versionId: "ver_1",
|
||||
publicationStatus: "published",
|
||||
};
|
||||
let uploadIndex = 0;
|
||||
httpMocks.apiRequest.mockReset();
|
||||
httpMocks.apiRequest.mockImplementation(async (_registry: unknown, request: unknown) => {
|
||||
if (isWhoamiRequest(request)) {
|
||||
return { user: { handle: whoamiHandle } };
|
||||
}
|
||||
const args = request as { method?: string; path?: string };
|
||||
if (args.method === "POST" && args.path === "/api/v1/skills/-/upload-url") {
|
||||
uploadIndex += 1;
|
||||
return {
|
||||
uploadUrl: `https://upload.local/${uploadIndex}`,
|
||||
uploadTicket: `skillPublishUploadTickets:${uploadIndex}`,
|
||||
};
|
||||
}
|
||||
if (args.method === "POST" && args.path === "/api/v1/skills") {
|
||||
return publishResponse;
|
||||
}
|
||||
return {
|
||||
match: null,
|
||||
latestVersion: null,
|
||||
};
|
||||
});
|
||||
let storageIndex = 0;
|
||||
httpMocks.uploadBinary.mockReset();
|
||||
httpMocks.uploadBinary.mockImplementation(async () => {
|
||||
storageIndex += 1;
|
||||
return { storageId: `storage:${storageIndex}` };
|
||||
});
|
||||
}
|
||||
|
||||
function isWhoamiRequest(request: unknown) {
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
import { readFile, readdir, stat } from "node:fs/promises";
|
||||
import { basename, join, resolve } from "node:path";
|
||||
import semver from "semver";
|
||||
import { apiRequest, apiRequestForm, registryUrl } from "../../http.js";
|
||||
import { apiRequest, registryUrl, uploadBinary } from "../../http.js";
|
||||
import {
|
||||
ApiRoutes,
|
||||
ApiUploadFileResponseSchema,
|
||||
ApiV1PublishResponseSchema,
|
||||
ApiV1SkillResolveResponseSchema,
|
||||
ApiV1SkillUploadUrlResponseSchema,
|
||||
ApiV1WhoamiResponseSchema,
|
||||
} from "../../schema/index.js";
|
||||
import { hashSkillFiles, listSkillFiles } from "../../skills.js";
|
||||
@@ -168,40 +170,81 @@ export async function cmdPublish(
|
||||
options.migrateOwner && publishOwnerHandle
|
||||
? sourceOwnerHandle || explicitSourceOwnerHandle || (await getDefaultOwnerHandle(token))
|
||||
: undefined;
|
||||
const form = new FormData();
|
||||
form.set(
|
||||
"payload",
|
||||
JSON.stringify({
|
||||
slug,
|
||||
displayName,
|
||||
ownerHandle: publishOwnerHandle,
|
||||
...(publishSourceOwnerHandle ? { sourceOwnerHandle: publishSourceOwnerHandle } : {}),
|
||||
...(options.migrateOwner ? { migrateOwner: true } : {}),
|
||||
version,
|
||||
changelog,
|
||||
acceptLicenseTerms: true,
|
||||
tags,
|
||||
...(options.categories !== undefined ? { categories } : {}),
|
||||
...(options.topics !== undefined ? { topics } : {}),
|
||||
...(source ? { source } : {}),
|
||||
...(forkOf ? { forkOf } : {}),
|
||||
}),
|
||||
);
|
||||
const fileHashes = new Map(hashed.files.map((file) => [file.path, file]));
|
||||
const uploadedFiles = [] as Array<{
|
||||
path: string;
|
||||
size: number;
|
||||
storageId: string;
|
||||
sha256: string;
|
||||
contentType?: string;
|
||||
uploadTicket: string;
|
||||
}>;
|
||||
|
||||
let index = 0;
|
||||
for (const file of filesOnDisk) {
|
||||
index += 1;
|
||||
if (spinner) spinner.text = `Uploading ${file.relPath} (${index}/${filesOnDisk.length})`;
|
||||
const blob = new Blob([Buffer.from(file.bytes)], {
|
||||
type: file.contentType ?? "application/octet-stream",
|
||||
const hash = fileHashes.get(file.relPath);
|
||||
if (!hash) fail(`Unable to hash ${file.relPath}`);
|
||||
const contentType = file.contentType ?? "application/octet-stream";
|
||||
const { uploadUrl, uploadTicket } = await apiRequest(
|
||||
registry,
|
||||
{
|
||||
method: "POST",
|
||||
path: ApiRoutes.skillUploadUrl,
|
||||
token,
|
||||
body: {
|
||||
path: file.relPath,
|
||||
size: hash.size,
|
||||
sha256: hash.sha256,
|
||||
contentType,
|
||||
},
|
||||
},
|
||||
ApiV1SkillUploadUrlResponseSchema,
|
||||
);
|
||||
const { storageId } = await uploadBinary(
|
||||
{
|
||||
url: uploadUrl,
|
||||
bytes: file.bytes,
|
||||
contentType,
|
||||
token,
|
||||
},
|
||||
ApiUploadFileResponseSchema,
|
||||
);
|
||||
uploadedFiles.push({
|
||||
path: file.relPath,
|
||||
size: hash.size,
|
||||
storageId,
|
||||
sha256: hash.sha256,
|
||||
contentType,
|
||||
uploadTicket,
|
||||
});
|
||||
form.append("files", blob, file.relPath);
|
||||
}
|
||||
|
||||
if (spinner) spinner.text = `Publishing ${slug}@${version}`;
|
||||
const result = await apiRequestForm(
|
||||
const result = await apiRequest(
|
||||
registry,
|
||||
{ method: "POST", path: ApiRoutes.skills, token, form },
|
||||
{
|
||||
method: "POST",
|
||||
path: ApiRoutes.skills,
|
||||
token,
|
||||
body: {
|
||||
slug,
|
||||
displayName,
|
||||
ownerHandle: publishOwnerHandle,
|
||||
...(publishSourceOwnerHandle ? { sourceOwnerHandle: publishSourceOwnerHandle } : {}),
|
||||
...(options.migrateOwner ? { migrateOwner: true } : {}),
|
||||
version,
|
||||
changelog,
|
||||
acceptLicenseTerms: true,
|
||||
tags,
|
||||
...(options.categories !== undefined ? { categories } : {}),
|
||||
...(options.topics !== undefined ? { topics } : {}),
|
||||
...(source ? { source } : {}),
|
||||
...(forkOf ? { forkOf } : {}),
|
||||
files: uploadedFiles,
|
||||
},
|
||||
},
|
||||
ApiV1PublishResponseSchema,
|
||||
);
|
||||
|
||||
|
||||
@@ -210,4 +210,23 @@ describe("bun http client", () => {
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps binary upload bearer tokens out of curl arguments", async () => {
|
||||
const { client, spawnImpl } = createBunClient({
|
||||
spawnImpl: () => ({ status: 0, stdout: '{"storageId":"storage:1"}\n200', stderr: "" }),
|
||||
mkdtempValue: "/tmp/clawhub-binary-upload",
|
||||
});
|
||||
|
||||
await client.uploadBinary({
|
||||
url: "https://upload.example/file",
|
||||
bytes: new Uint8Array([1, 2, 3]),
|
||||
contentType: "application/octet-stream",
|
||||
token: "clh_secret",
|
||||
});
|
||||
|
||||
const [, args, options] = spawnImpl.mock.calls[0] as [string, string[], { input?: string }];
|
||||
expect(args).toContain("@-");
|
||||
expect(args.join(" ")).not.toContain("clh_secret");
|
||||
expect(options.input).toBe("Authorization: Bearer clh_secret\n");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -58,6 +58,7 @@ type BinaryUploadArgs = {
|
||||
url: string;
|
||||
bytes: Uint8Array;
|
||||
contentType?: string;
|
||||
token?: string;
|
||||
retryCount?: number;
|
||||
};
|
||||
|
||||
@@ -310,6 +311,7 @@ export function createHttpClient(options: HttpClientOptions = {}): HttpClient {
|
||||
|
||||
const headers: Record<string, string> = {};
|
||||
if (args.contentType) headers["Content-Type"] = args.contentType;
|
||||
if (args.token) headers.Authorization = `Bearer ${args.token}`;
|
||||
const response = await fetchWithTimeout(
|
||||
deps,
|
||||
args.url,
|
||||
@@ -842,6 +844,7 @@ async function uploadBinaryViaCurl(
|
||||
>,
|
||||
args: BinaryUploadArgs,
|
||||
) {
|
||||
if (args.token && /[\r\n]/.test(args.token)) throw new Error("Invalid API token");
|
||||
const tempDir = await deps.mkdtempImpl(join(deps.tmpdirPath, "clawhub-upload-"));
|
||||
try {
|
||||
const filePath = join(tempDir, "upload.bin");
|
||||
@@ -858,9 +861,13 @@ async function uploadBinaryViaCurl(
|
||||
"POST",
|
||||
];
|
||||
if (args.contentType) curlArgs.push("-H", `Content-Type: ${args.contentType}`);
|
||||
if (args.token) curlArgs.push("-H", "@-");
|
||||
curlArgs.push("--data-binary", `@${filePath}`, args.url);
|
||||
|
||||
const result = deps.spawnSyncImpl("curl", curlArgs, { encoding: "utf8" });
|
||||
const result = deps.spawnSyncImpl("curl", curlArgs, {
|
||||
encoding: "utf8",
|
||||
input: args.token ? `Authorization: Bearer ${args.token}\n` : undefined,
|
||||
});
|
||||
if (result.status !== 0) {
|
||||
throw new Error(result.stderr || "curl failed");
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ export const ApiRoutes = {
|
||||
publishTokenMint: "/api/v1/publish/token/mint",
|
||||
publishAttempts: "/api/v1/publish/attempts",
|
||||
skills: "/api/v1/skills",
|
||||
skillUploadUrl: "/api/v1/skills/-/upload-url",
|
||||
trending: "/api/v1/trending",
|
||||
skillsSh: "/api/v1/skills-sh",
|
||||
skillScans: "/api/v1/skills/-/scan",
|
||||
|
||||
@@ -71,12 +71,25 @@ export const ApiUploadFileResponseSchema = type({
|
||||
storageId: "string",
|
||||
});
|
||||
|
||||
export const ApiV1SkillUploadUrlRequestSchema = type({
|
||||
path: "string",
|
||||
size: "number",
|
||||
sha256: "string",
|
||||
contentType: "string?",
|
||||
});
|
||||
|
||||
export const ApiV1SkillUploadUrlResponseSchema = type({
|
||||
uploadUrl: "string",
|
||||
uploadTicket: "string",
|
||||
});
|
||||
|
||||
export const CliPublishFileSchema = type({
|
||||
path: "string",
|
||||
size: "number",
|
||||
storageId: "string",
|
||||
sha256: "string",
|
||||
contentType: "string?",
|
||||
uploadTicket: "string?",
|
||||
});
|
||||
export type CliPublishFile = (typeof CliPublishFileSchema)[inferred];
|
||||
|
||||
|
||||
Vendored
+1
@@ -333,6 +333,7 @@ export declare const ServerPackagePublishRequestSchema: import("arktype/internal
|
||||
storageId: string;
|
||||
sha256: string;
|
||||
contentType?: string | undefined;
|
||||
uploadTicket?: string | undefined;
|
||||
}[];
|
||||
}, {}>;
|
||||
export type ServerPackagePublishRequest = (typeof ServerPackagePublishRequestSchema)[inferred];
|
||||
|
||||
Vendored
+1
@@ -17,6 +17,7 @@ export declare const ApiRoutes: {
|
||||
readonly publishTokenMint: "/api/v1/publish/token/mint";
|
||||
readonly publishAttempts: "/api/v1/publish/attempts";
|
||||
readonly skills: "/api/v1/skills";
|
||||
readonly skillUploadUrl: "/api/v1/skills/-/upload-url";
|
||||
readonly trending: "/api/v1/trending";
|
||||
readonly skillsSh: "/api/v1/skills-sh";
|
||||
readonly skillScans: "/api/v1/skills/-/scan";
|
||||
|
||||
Vendored
+1
@@ -17,6 +17,7 @@ export const ApiRoutes = {
|
||||
publishTokenMint: "/api/v1/publish/token/mint",
|
||||
publishAttempts: "/api/v1/publish/attempts",
|
||||
skills: "/api/v1/skills",
|
||||
skillUploadUrl: "/api/v1/skills/-/upload-url",
|
||||
trending: "/api/v1/trending",
|
||||
skillsSh: "/api/v1/skills-sh",
|
||||
skillScans: "/api/v1/skills/-/scan",
|
||||
|
||||
Vendored
+1
-1
@@ -1 +1 @@
|
||||
{"version":3,"file":"routes.js","sourceRoot":"","sources":["../src/routes.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,QAAQ,EAAE,eAAe;IACzB,MAAM,EAAE,aAAa;IACrB,KAAK,EAAE,YAAY;IACnB,YAAY,EAAE,oBAAoB;IAClC,SAAS,EAAE,iBAAiB;IAC5B,YAAY,EAAE,qBAAqB;IACnC,UAAU,EAAE,kBAAkB;IAC9B,mBAAmB,EAAE,4BAA4B;IACjD,cAAc,EAAE,uBAAuB;IACvC,gBAAgB,EAAE,yBAAyB;CACnC,CAAC;AAEX,MAAM,CAAC,MAAM,SAAS,GAAG;IACvB,MAAM,EAAE,gBAAgB;IACxB,OAAO,EAAE,iBAAiB;IAC1B,QAAQ,EAAE,kBAAkB;IAC5B,gBAAgB,EAAE,4BAA4B;IAC9C,eAAe,EAAE,0BAA0B;IAC3C,MAAM,EAAE,gBAAgB;IACxB,QAAQ,EAAE,kBAAkB;IAC5B,QAAQ,EAAE,mBAAmB;IAC7B,UAAU,EAAE,uBAAuB;IACnC,OAAO,EAAE,iBAAiB;IAC1B,aAAa,EAAE,wBAAwB;IACvC,QAAQ,EAAE,kBAAkB;IAC5B,WAAW,EAAE,sBAAsB;IACnC,aAAa,EAAE,wBAAwB;IACvC,UAAU,EAAE,oBAAoB;IAChC,WAAW,EAAE,uBAAuB;IACpC,iBAAiB,EAAE,sBAAsB;IACzC,gBAAgB,EAAE,qBAAqB;IACvC,cAAc,EAAE,0BAA0B;IAC1C,KAAK,EAAE,eAAe;IACtB,SAAS,EAAE,mBAAmB;IAC9B,UAAU,EAAE,oBAAoB;IAChC,KAAK,EAAE,eAAe;IACtB,aAAa,EAAE,wBAAwB;IACvC,MAAM,EAAE,gBAAgB;IACxB,YAAY,EAAE,uBAAuB;CAC7B,CAAC"}
|
||||
{"version":3,"file":"routes.js","sourceRoot":"","sources":["../src/routes.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,QAAQ,EAAE,eAAe;IACzB,MAAM,EAAE,aAAa;IACrB,KAAK,EAAE,YAAY;IACnB,YAAY,EAAE,oBAAoB;IAClC,SAAS,EAAE,iBAAiB;IAC5B,YAAY,EAAE,qBAAqB;IACnC,UAAU,EAAE,kBAAkB;IAC9B,mBAAmB,EAAE,4BAA4B;IACjD,cAAc,EAAE,uBAAuB;IACvC,gBAAgB,EAAE,yBAAyB;CACnC,CAAC;AAEX,MAAM,CAAC,MAAM,SAAS,GAAG;IACvB,MAAM,EAAE,gBAAgB;IACxB,OAAO,EAAE,iBAAiB;IAC1B,QAAQ,EAAE,kBAAkB;IAC5B,gBAAgB,EAAE,4BAA4B;IAC9C,eAAe,EAAE,0BAA0B;IAC3C,MAAM,EAAE,gBAAgB;IACxB,cAAc,EAAE,6BAA6B;IAC7C,QAAQ,EAAE,kBAAkB;IAC5B,QAAQ,EAAE,mBAAmB;IAC7B,UAAU,EAAE,uBAAuB;IACnC,OAAO,EAAE,iBAAiB;IAC1B,aAAa,EAAE,wBAAwB;IACvC,QAAQ,EAAE,kBAAkB;IAC5B,WAAW,EAAE,sBAAsB;IACnC,aAAa,EAAE,wBAAwB;IACvC,UAAU,EAAE,oBAAoB;IAChC,WAAW,EAAE,uBAAuB;IACpC,iBAAiB,EAAE,sBAAsB;IACzC,gBAAgB,EAAE,qBAAqB;IACvC,cAAc,EAAE,0BAA0B;IAC1C,KAAK,EAAE,eAAe;IACtB,SAAS,EAAE,mBAAmB;IAC9B,UAAU,EAAE,oBAAoB;IAChC,KAAK,EAAE,eAAe;IACtB,aAAa,EAAE,wBAAwB;IACvC,MAAM,EAAE,gBAAgB;IACxB,YAAY,EAAE,uBAAuB;CAC7B,CAAC"}
|
||||
Vendored
+12
@@ -55,12 +55,23 @@ export declare const ApiCliUploadUrlResponseSchema: import("arktype/internal/var
|
||||
export declare const ApiUploadFileResponseSchema: import("arktype/internal/variants/object.ts").ObjectType<{
|
||||
storageId: string;
|
||||
}, {}>;
|
||||
export declare const ApiV1SkillUploadUrlRequestSchema: import("arktype/internal/variants/object.ts").ObjectType<{
|
||||
path: string;
|
||||
size: number;
|
||||
sha256: string;
|
||||
contentType?: string | undefined;
|
||||
}, {}>;
|
||||
export declare const ApiV1SkillUploadUrlResponseSchema: import("arktype/internal/variants/object.ts").ObjectType<{
|
||||
uploadUrl: string;
|
||||
uploadTicket: string;
|
||||
}, {}>;
|
||||
export declare const CliPublishFileSchema: import("arktype/internal/variants/object.ts").ObjectType<{
|
||||
path: string;
|
||||
size: number;
|
||||
storageId: string;
|
||||
sha256: string;
|
||||
contentType?: string | undefined;
|
||||
uploadTicket?: string | undefined;
|
||||
}, {}>;
|
||||
export type CliPublishFile = (typeof CliPublishFileSchema)[inferred];
|
||||
export declare const PublishSourceSchema: import("arktype/internal/variants/object.ts").ObjectType<{
|
||||
@@ -104,6 +115,7 @@ export declare const CliPublishRequestSchema: import("arktype/internal/variants/
|
||||
storageId: string;
|
||||
sha256: string;
|
||||
contentType?: string | undefined;
|
||||
uploadTicket?: string | undefined;
|
||||
}[];
|
||||
}, {}>;
|
||||
export type CliPublishRequest = (typeof CliPublishRequestSchema)[inferred];
|
||||
|
||||
Vendored
+11
@@ -53,12 +53,23 @@ export const ApiCliUploadUrlResponseSchema = type({
|
||||
export const ApiUploadFileResponseSchema = type({
|
||||
storageId: "string",
|
||||
});
|
||||
export const ApiV1SkillUploadUrlRequestSchema = type({
|
||||
path: "string",
|
||||
size: "number",
|
||||
sha256: "string",
|
||||
contentType: "string?",
|
||||
});
|
||||
export const ApiV1SkillUploadUrlResponseSchema = type({
|
||||
uploadUrl: "string",
|
||||
uploadTicket: "string",
|
||||
});
|
||||
export const CliPublishFileSchema = type({
|
||||
path: "string",
|
||||
size: "number",
|
||||
storageId: "string",
|
||||
sha256: "string",
|
||||
contentType: "string?",
|
||||
uploadTicket: "string?",
|
||||
});
|
||||
export const PublishSourceSchema = type({
|
||||
kind: '"github"',
|
||||
|
||||
Vendored
+1
-1
File diff suppressed because one or more lines are too long
@@ -18,6 +18,7 @@ export const ApiRoutes = {
|
||||
publishTokenMint: "/api/v1/publish/token/mint",
|
||||
publishAttempts: "/api/v1/publish/attempts",
|
||||
skills: "/api/v1/skills",
|
||||
skillUploadUrl: "/api/v1/skills/-/upload-url",
|
||||
trending: "/api/v1/trending",
|
||||
skillsSh: "/api/v1/skills-sh",
|
||||
skillScans: "/api/v1/skills/-/scan",
|
||||
|
||||
@@ -65,12 +65,25 @@ export const ApiUploadFileResponseSchema = type({
|
||||
storageId: "string",
|
||||
});
|
||||
|
||||
export const ApiV1SkillUploadUrlRequestSchema = type({
|
||||
path: "string",
|
||||
size: "number",
|
||||
sha256: "string",
|
||||
contentType: "string?",
|
||||
});
|
||||
|
||||
export const ApiV1SkillUploadUrlResponseSchema = type({
|
||||
uploadUrl: "string",
|
||||
uploadTicket: "string",
|
||||
});
|
||||
|
||||
export const CliPublishFileSchema = type({
|
||||
path: "string",
|
||||
size: "number",
|
||||
storageId: "string",
|
||||
sha256: "string",
|
||||
contentType: "string?",
|
||||
uploadTicket: "string?",
|
||||
});
|
||||
export type CliPublishFile = (typeof CliPublishFileSchema)[inferred];
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ import { spawnSync } from "node:child_process";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
describe("clawhub CLI npm release metadata check", () => {
|
||||
const releaseTag = "v0.23.2";
|
||||
const releaseTag = "v0.23.3";
|
||||
|
||||
function runCheck(args) {
|
||||
const env = { ...process.env };
|
||||
|
||||
@@ -265,6 +265,20 @@ See also: [acceptable-usage.md](./acceptable-usage.md) for the marketplace polic
|
||||
- Skills directory supports an optional "Hide suspicious" filter to exclude
|
||||
active-but-flagged (`flagged.suspicious`) entries from browse/search results.
|
||||
|
||||
## Skill publish upload boundary
|
||||
|
||||
- CLI skill publishing stages each file through the direct Convex HTTP surface,
|
||||
keeping file bodies off the Vercel request path. Each file remains capped at
|
||||
10MB and the complete publish remains capped at 50MB.
|
||||
- The server creates a short-lived, user-bound ticket before accepting a file.
|
||||
The upload action enforces the declared path, byte size, content type, and
|
||||
SHA-256 before attaching the resulting storage id to that ticket.
|
||||
- JSON skill publishes must present the matching ticket for every staged file.
|
||||
Ticket ownership and file metadata are revalidated, and ticket consumption is
|
||||
committed atomically with the new skill version.
|
||||
- Unconsumed staged files are deleted when their tickets expire. Failed
|
||||
attachment attempts delete the just-created storage blob immediately.
|
||||
|
||||
## Package publish upload boundary
|
||||
|
||||
- Package publish is multipart-only. `POST /api/v1/packages` must reject JSON
|
||||
|
||||
Reference in New Issue
Block a user