Compare commits

...
75 Commits
Author SHA1 Message Date
Peter Steinberger 2b552c4803 feat: default workdir from clawdbot config 2026-01-13 06:04:38 +00:00
Peter Steinberger ece0b830fb test: raise branch coverage 2026-01-13 01:25:06 +00:00
Peter Steinberger 57afea9de5 chore: ignore test-results in biome 2026-01-13 01:24:56 +00:00
Peter Steinberger 1e788b2e0c test: add playwright smoke suite 2026-01-13 00:46:13 +00:00
Peter Steinberger f025721261 fix: prevent skills index crash 2026-01-13 00:40:04 +00:00
Shadow c2cc61df24 feat: add skills lazy loading 2026-01-12 15:52:52 -06:00
Shadow 7a8a3f75ce fix: paginate skills index 2026-01-12 13:36:28 -06:00
Peter Steinberger ac0bdf0375 fix: hide onlycrabs branding 2026-01-11 05:20:20 +01:00
Peter Steinberger 6c7dd2ec6d fix: hide onlycrabs link and code pill 2026-01-11 05:15:43 +01:00
Peter Steinberger c72e3007a1 test: guard skills list query limit 2026-01-10 22:04:42 +01:00
Peter Steinberger b66d28a184 fix: lower skills list query limit 2026-01-10 22:01:06 +01:00
Shadow dddee828af Change ClawdBot link to new URL 2026-01-10 14:01:27 -06:00
Peter Steinberger 106cb1896a Merge pull request #1 from clawdbot/nix-plugin-metadata
Add nix-clawdbot plugin pointers to skill metadata
2026-01-10 19:33:11 +00:00
Peter Steinberger e29ec88afd fix: restore backups + fork lineage (#1) (thanks @joshp123) 2026-01-10 20:32:47 +01:00
Josh Palmer 0eb3047ca6 feat: add nix plugin bundles
- include nix plugin metadata, config requirements, and CLI help
- add config examples and format bundle code blocks
- refresh bundle UI styling and layout
2026-01-10 20:27:22 +01:00
Peter Steinberger b34c7261bd feat: add v1 public api 2026-01-10 20:25:50 +01:00
DB Hurley e0d553602a feat: refresh skill detail layout and dashboard
- add dashboard with skill management and upload prefill
- redesign skill detail layout with full-width panels
- refactor modules and format dashboard/upload routes
2026-01-10 20:23:01 +01:00
Peter Steinberger f80fa90e01 fix(seed): harden SoulHub auto-seed 2026-01-10 20:18:19 +01:00
Josh Palmer 0cc0bdcd50 feat: SoulHub registry + auto-seed
SoulHub SOUL.md registry (souls table, versions, search, OG) + first-run auto-seed; fixes seed concurrency and GitHub backup owner handle.
2026-01-10 18:25:11 +00:00
Peter Steinberger cc0027a094 test(og): update OG layout version 2026-01-09 19:16:39 +01:00
Peter Steinberger dddbd3a78e fix(og): prevent OG title clipping 2026-01-09 19:13:19 +01:00
Peter Steinberger f350442002 feat: import skills from public GitHub 2026-01-09 09:10:49 +01:00
Peter Steinberger 826c60f4da test(cli): expand clawdbot sync coverage 2026-01-09 02:34:09 +01:00
Peter Steinberger a679c3a999 docs: note clawdbot sync roots 2026-01-09 02:05:34 +01:00
Peter Steinberger d5d8e6ae5b feat(cli): auto-scan clawdbot skill roots 2026-01-09 01:57:56 +01:00
Peter Steinberger f0772e7215 test: cover OG text clamping 2026-01-08 23:07:10 +01:00
Peter Steinberger 770bb3aeb8 fix: clamp OG description width 2026-01-08 23:02:33 +01:00
Peter Steinberger 6811691055 fix: prevent OG text bleed 2026-01-08 22:58:53 +01:00
Peter Steinberger 26b46d9f6e docs: note OG image runtime fix 2026-01-08 06:15:09 +01:00
Peter Steinberger d145c186a7 fix: resolve OG api base on all runtimes 2026-01-08 06:12:01 +01:00
Peter Steinberger 57af81d054 refactor: modularize skill OG images 2026-01-08 06:07:36 +01:00
Peter Steinberger 0131229843 fix: embed fonts in OG images 2026-01-08 05:54:49 +01:00
Peter Steinberger d7650583dc feat: dynamic skill OG images 2026-01-08 05:47:27 +01:00
Peter Steinberger cf2ad58e86 chore: remove docs page link 2026-01-08 04:20:25 +01:00
Peter Steinberger 153c3f5b9e style: soften markdown block styling 2026-01-07 22:41:57 +01:00
Peter Steinberger 243ca9ca2b feat: link docs and clarify cli usage 2026-01-07 21:11:04 +01:00
Peter Steinberger 8216c73c9b test: stabilize upload route mocks 2026-01-07 20:18:32 +01:00
Peter Steinberger 860902a574 fix: harden upload utils 2026-01-07 20:14:18 +01:00
Peter Steinberger 18af63b630 fix: silence lint/build warnings 2026-01-07 20:08:13 +01:00
Peter Steinberger 96ac7567ba chore: prepare 0.1.0 release notes 2026-01-07 20:07:06 +01:00
Peter Steinberger b55e266457 fix: harden GitHub backups 2026-01-07 18:48:09 +00:00
Shadowandvercel[bot] <35613825+vercel[bot]@users.noreply.github.com> 2492a52ca3 Update skillPublish.ts
Co-authored-by: vercel[bot] <35613825+vercel[bot]@users.noreply.github.com>
2026-01-07 18:48:09 +00:00
Shadow 9b6a4c7b6f chore: format lint fixes 2026-01-07 18:48:09 +00:00
Shadow 232d06debd fix: preserve previous commit in github history 2026-01-07 18:48:09 +00:00
Shadow e6ba699b18 feat: back up skills to github 2026-01-07 18:48:09 +00:00
Peter Steinberger 7444c22d57 chore: bump cli version to 0.1.0 2026-01-07 19:13:00 +01:00
Peter Steinberger cf26cd143a fix: address deploy typecheck 2026-01-07 18:49:49 +01:00
Peter Steinberger 82b426cc8e fix: harden multipart publish parsing 2026-01-07 18:42:44 +01:00
Peter Steinberger ba7e82ba02 feat: add v1 public api 2026-01-07 18:28:51 +01:00
Peter Steinberger 9f83114dee chore: add docs:list helper 2026-01-07 17:58:00 +01:00
Peter Steinberger a98b51a2a1 test: fix upload route mocks 2026-01-07 17:53:01 +01:00
Peter Steinberger 4d6987097d style: format dashboard and upload routes 2026-01-07 17:52:57 +01:00
Peter Steinberger 1bf3bcf152 docs: add Mintlify-ready docs set 2026-01-07 17:52:55 +01:00
Peter Steinberger 974338fb97 docs: thank dbhurley for dashboard 2026-01-07 10:08:20 +01:00
Peter Steinberger 927cd4d425 Merge pull request #2 from dbhurley/feat/user-dashboard
feat: Add user dashboard with skill management
2026-01-07 09:07:42 +00:00
Peter Steinberger bfa59cabf5 merge main into feat/user-dashboard 2026-01-07 10:06:14 +01:00
Peter Steinberger 2a28f75fc6 docs: thank dbhurley in changelog 2026-01-07 09:50:31 +01:00
Peter Steinberger fff263ee68 Merge pull request #4 from dbhurley/fix/token-color-error
fix: handle shorthand hex colors in SkillDiffCard
2026-01-07 08:50:02 +00:00
Peter Steinberger af40c53bc1 style: pad diff file list 2026-01-07 07:52:37 +01:00
Peter Steinberger 74a0ca9bf6 style: remove diff hover lift 2026-01-07 07:42:10 +01:00
Peter Steinberger 3f85eccd1a refactor: make skill detail full width 2026-01-07 07:17:14 +01:00
Peter Steinberger 9513bd9f44 style: remove divider before versions 2026-01-07 07:04:55 +01:00
Peter Steinberger 27958bbb69 fix: type skill origin in tests 2026-01-07 06:18:41 +01:00
Peter Steinberger 7a62bec162 style: format skill tests 2026-01-07 06:16:21 +01:00
Peter Steinberger e895840411 test: cover skill utilities 2026-01-07 06:13:42 +01:00
Peter Steinberger 9c574a7709 feat: redesign skill detail layout 2026-01-07 05:56:12 +01:00
Peter Steinberger 6af5cf11ec docs: note Convex run --env-file auth gotcha 2026-01-07 05:50:42 +01:00
Peter Steinberger 237a965673 feat: dedupe skills via canonical forks 2026-01-07 05:20:19 +01:00
DB Hurley 219e3257ec fix: normalize hex colors in SkillDiffCard to prevent Monaco crash 2026-01-06 21:44:36 -05:00
Peter Steinberger 5e1598954f feat: add Vercel Analytics 2026-01-07 03:23:12 +01:00
Peter Steinberger 0a059e337f style: align version rows 2026-01-07 02:21:13 +01:00
Peter Steinberger 3d24969f1e style: refine comment form sizing 2026-01-07 02:20:00 +01:00
Peter Steinberger cc530d8a69 test: raise coverage for diffing 2026-01-07 01:46:03 +01:00
Peter Steinberger 6049863b8f feat: add skill diff viewer 2026-01-06 23:29:03 +01:00
DB Hurley d4b947af47 feat: Add user dashboard with skill management
- Add /dashboard route showing user's published skills
- Add 'Dashboard' link to user dropdown menu in header
- Skills display name, slug, description, stats (downloads, stars, versions)
- 'New Version' button links to upload with pre-populated slug
- Upload route accepts ?updateSlug param to pre-fill form for updates
- Auto-bumps version number when updating existing skill
- Responsive design for mobile
- Empty state with call-to-action for new users
2026-01-05 17:34:54 -05:00
150 changed files with 16976 additions and 1023 deletions
+3
View File
@@ -1,6 +1,9 @@
# Frontend
VITE_CONVEX_URL=
VITE_CONVEX_SITE_URL=
VITE_SOULHUB_SITE_URL=
VITE_SOULHUB_HOST=
VITE_SITE_MODE=
SITE_URL=http://localhost:3000
CONVEX_SITE_URL=
+6
View File
@@ -1,5 +1,8 @@
node_modules
.DS_Store
.bun-build
*.bun-build
bin/docs-list
dist
dist-ssr
!packages/schema/dist
@@ -18,3 +21,6 @@ todos.json
.vscode
.env*.local
coverage
playwright-report
test-results
.playwright
+4
View File
@@ -38,3 +38,7 @@
- Local env: `.env.local` (never commit secrets).
- Convex env holds JWT keys; Vercel only needs `VITE_CONVEX_URL` + `VITE_CONVEX_SITE_URL`.
- OAuth: GitHub OAuth App credentials required for login.
## Convex Ops (Gotchas)
- New Convex functions must be pushed before `convex run`: use `bunx convex dev --once` (dev) or `bunx convex deploy --prod` (prod).
- If `bunx convex run --env-file .env.local ...` returns `401 MissingAccessToken` despite `bunx convex login`, workaround: omit `--env-file` and use `--deployment-name <name>` / `--prod`.
+31
View File
@@ -1,5 +1,32 @@
# Changelog
## 0.2.0 - 2026-01-13
### Added
- Web: dynamic OG image cards for skills (name, description, version).
- CLI: auto-scan Clawdbot skill roots (per-agent workspaces, shared skills, extraDirs).
- Web: import skills from public GitHub URLs (auto-detect `SKILL.md`, smart file selection, provenance).
- Web/API: SoulHub (SOUL.md registry) with v1 endpoints and first-run auto-seed.
### Fixed
- Web: stabilize skill OG image generation on server runtimes.
- Web: prevent skill OG text overflow outside the card.
- Registry: make SoulHub auto-seed idempotent and non-user-owned.
- Registry: keep GitHub backup state + publish backups intact (thanks @joshp123, #1).
- CLI/Registry: restore fork lineage on sync + clamp bulk list queries (thanks @joshp123, #1).
- CLI: default workdir falls back to Clawdbot workspace (override with `--workdir` / `CLAWDHUB_WORKDIR`).
## 0.0.6 - 2026-01-07
### Added
- API: v1 public REST endpoints with rate limits, raw file fetch, and OpenAPI spec.
- Docs: `docs/api.md` and `DEPRECATIONS.md` for the v1 cutover plan.
### Changed
- CLI: publish now uses single multipart `POST /api/v1/skills`.
- Registry: legacy `/api/*` + `/api/cli/*` marked for deprecation (kept for now).
## 0.0.5 - 2026-01-06
### Added
@@ -8,11 +35,15 @@
- Profile: private "Installed" tab with JSON export + delete telemetry controls.
- Docs: add `docs/telemetry.md` (what we track + how to opt out).
- Web: custom Open Graph image (`/og.png`) + richer OG/Twitter tags.
- Web: dashboard for managing your published skills (thanks @dbhurley!).
### Changed
- CLI: telemetry opt-out via `CLAWDHUB_DISABLE_TELEMETRY=1`.
- Web: move theme picker into mobile menu.
### Fixed
- Web: handle shorthand hex colors in diff theme (thanks @dbhurley!).
## 0.0.5 - 2026-01-06
### Added
+7
View File
@@ -0,0 +1,7 @@
# Deprecations
## Legacy /api routes (pre-v1)
- Deprecated: 2026-01-07
- TODO: remove legacy `/api/*` and `/api/cli/*` routes after clients migrate to `/api/v1`.
- Legacy handlers live in `convex/http.ts` and `convex/httpApi.ts`.
+63 -2
View File
@@ -9,14 +9,26 @@
ClawdHub is the **public skill registry for Clawdbot**: publish, version, and search text-based agent skills (a `SKILL.md` plus supporting files).
Its designed for fast browsing + a CLI-friendly API, with moderation hooks and vector search.
onlycrabs.ai is the **SOUL.md registry**: publish and share system lore the same way you publish skills.
Live: `https://clawdhub.com`
onlycrabs.ai: `https://onlycrabs.ai`
## What you can do
- Browse skills + render their `SKILL.md`.
- Publish new versions with changelogs + tags (including `latest`).
- Publish new skill versions with changelogs + tags (including `latest`).
- Browse souls + render their `SOUL.md`.
- Publish new soul versions with changelogs + tags.
- Search via embeddings (vector index) instead of brittle keywords.
- Star + comment; admins/mods can curate and approve.
- Star + comment; admins/mods can curate and approve skills.
## onlycrabs.ai (SOUL.md registry)
- Entry point is host-based: `onlycrabs.ai`.
- On the onlycrabs.ai host, the home page and nav default to souls.
- On ClawdHub, souls live under `/souls`.
- Soul bundles only accept `SOUL.md` for now (no extra files).
## How it works (high level)
@@ -72,12 +84,61 @@ This writes `JWT_PRIVATE_KEY` + `JWKS` to the deployment and prints values for y
- `VITE_CONVEX_URL`: Convex deployment URL (`https://<deployment>.convex.cloud`).
- `VITE_CONVEX_SITE_URL`: Convex site URL (`https://<deployment>.convex.site`).
- `VITE_SOULHUB_SITE_URL`: onlycrabs.ai site URL (`https://onlycrabs.ai`).
- `VITE_SOULHUB_HOST`: onlycrabs.ai host match (`onlycrabs.ai`).
- `VITE_SITE_MODE`: Optional override (`skills` or `souls`) for SSR builds.
- `CONVEX_SITE_URL`: same as `VITE_CONVEX_SITE_URL` (auth + cookies).
- `SITE_URL`: App URL (local: `http://localhost:3000`).
- `AUTH_GITHUB_ID` / `AUTH_GITHUB_SECRET`: GitHub OAuth App.
- `JWT_PRIVATE_KEY` / `JWKS`: Convex Auth keys.
- `OPENAI_API_KEY`: embeddings for search + indexing.
## Nix plugins (nixmode skills)
ClawdHub can store a nix-clawdbot plugin pointer in SKILL frontmatter so the registry knows which
Nix package bundle to install. A nix plugin is different from a regular skill pack: it bundles the
skill pack, the CLI binary, and its config flags/requirements together.
Add this to `SKILL.md`:
```yaml
---
name: peekaboo
description: Capture and automate macOS UI with the Peekaboo CLI.
metadata: {"clawdbot":{"nix":{"plugin":"github:clawdbot/nix-steipete-tools?dir=tools/peekaboo","systems":["aarch64-darwin"]}}}
---
```
Install via nix-clawdbot:
```nix
programs.clawdbot.plugins = [
{ source = "github:clawdbot/nix-steipete-tools?dir=tools/peekaboo"; }
];
```
You can also declare config requirements + an example snippet:
```yaml
---
name: padel
description: Check padel court availability and manage bookings via Playtomic.
metadata: {"clawdbot":{"config":{"requiredEnv":["PADEL_AUTH_FILE"],"stateDirs":[".config/padel"],"example":"config = { env = { PADEL_AUTH_FILE = \\\"/run/agenix/padel-auth\\\"; }; };"}}}
---
```
To show CLI help (recommended for nix plugins), include the `cli --help` output:
```yaml
---
name: padel
description: Check padel court availability and manage bookings via Playtomic.
metadata: {"clawdbot":{"cliHelp":"padel --help\\nUsage: padel [command]\\n"}}
---
```
`metadata.clawdbot` is preferred, but `metadata.clawdis` is accepted as an alias for compatibility.
## Scripts
```bash
+4 -1
View File
@@ -10,9 +10,12 @@
"!**/.output",
"!**/coverage",
"!**/convex/_generated",
"!**/test-results",
"!**/src/routeTree.gen.ts",
"!**/.tanstack",
"!**/public"
"!**/public",
"!**/.devenv",
"!**/.devenv"
]
},
"assist": { "actions": { "source": { "organizeImports": "on" } } },
+35 -2
View File
@@ -1,6 +1,6 @@
{
"lockfileVersion": 1,
"configVersion": 1,
"configVersion": 0,
"workspaces": {
"": {
"name": "clawdhub",
@@ -10,19 +10,24 @@
"@fontsource/bricolage-grotesque": "^5.2.10",
"@fontsource/ibm-plex-mono": "^5.2.7",
"@fontsource/manrope": "^5.2.8",
"@monaco-editor/react": "^4.7.0",
"@radix-ui/react-dropdown-menu": "^2.1.16",
"@radix-ui/react-toggle-group": "^1.1.11",
"@resvg/resvg-wasm": "^2.6.2",
"@tailwindcss/vite": "^4.1.18",
"@tanstack/react-devtools": "^0.9.0",
"@tanstack/react-router": "^1.144.0",
"@tanstack/react-router-devtools": "^1.144.0",
"@tanstack/react-start": "^1.145.3",
"@tanstack/router-plugin": "^1.145.2",
"@vercel/analytics": "^1.6.1",
"clawdhub-schema": "^0.0.2",
"clsx": "^2.1.1",
"convex": "^1.31.2",
"fflate": "^0.8.2",
"h3": "2.0.1-rc.5",
"lucide-react": "^0.562.0",
"monaco-editor": "^0.55.1",
"nitro": "^3.0.1-alpha.1",
"react": "^19.2.3",
"react-dom": "^19.2.3",
@@ -36,6 +41,7 @@
},
"devDependencies": {
"@biomejs/biome": "^2.3.11",
"@playwright/test": "^1.57.0",
"@tanstack/devtools-vite": "^0.4.0",
"@testing-library/dom": "^10.4.1",
"@testing-library/react": "^16.3.1",
@@ -55,7 +61,7 @@
},
"packages/clawdhub": {
"name": "clawdhub",
"version": "0.0.4",
"version": "0.1.0",
"bin": {
"clawdhub": "bin/clawdhub.js",
},
@@ -65,6 +71,7 @@
"commander": "^14.0.2",
"fflate": "^0.8.2",
"ignore": "^7.0.5",
"json5": "^2.2.3",
"mime": "^4.1.0",
"ora": "^9.0.0",
"p-retry": "^7.1.1",
@@ -267,6 +274,10 @@
"@jridgewell/trace-mapping": ["@jridgewell/trace-mapping@0.3.31", "", { "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", "@jridgewell/sourcemap-codec": "^1.4.14" } }, "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw=="],
"@monaco-editor/loader": ["@monaco-editor/loader@1.7.0", "", { "dependencies": { "state-local": "^1.0.6" } }, "sha512-gIwR1HrJrrx+vfyOhYmCZ0/JcWqG5kbfG7+d3f/C1LXk2EvzAbHSg3MQ5lO2sMlo9izoAZ04shohfKLVT6crVA=="],
"@monaco-editor/react": ["@monaco-editor/react@4.7.0", "", { "dependencies": { "@monaco-editor/loader": "^1.5.0" }, "peerDependencies": { "monaco-editor": ">= 0.25.0 < 1", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-cyzXQCtO47ydzxpQtCGSQGOC8Gk3ZUeBXFAxD+CWXYFo5OqZyZUonFl0DwUlTyAfRHntBfw2p3w4s9R6oe1eCA=="],
"@napi-rs/wasm-runtime": ["@napi-rs/wasm-runtime@1.1.1", "", { "dependencies": { "@emnapi/core": "^1.7.1", "@emnapi/runtime": "^1.7.1", "@tybys/wasm-util": "^0.10.1" } }, "sha512-p64ah1M1ld8xjWv3qbvFwHiFVWrq1yFvV4f7w+mzaqiR4IlSgkqhcRdHwsGgomwzBH51sRY4NEowLxnaBjcW/A=="],
"@oozcitak/dom": ["@oozcitak/dom@2.0.2", "", { "dependencies": { "@oozcitak/infra": "^2.0.2", "@oozcitak/url": "^3.0.0", "@oozcitak/util": "^10.0.0" } }, "sha512-GjpKhkSYC3Mj4+lfwEyI1dqnsKTgwGy48ytZEhm4A/xnH/8z9M3ZVXKr/YGQi3uCLs1AEBS+x5T2JPiueEDW8w=="],
@@ -375,6 +386,8 @@
"@panva/hkdf": ["@panva/hkdf@1.2.1", "", {}, "sha512-6oclG6Y3PiDFcoyk8srjLfVKyMfVCKJ27JwNPViuXziFpmdz+MZnZN/aKY0JGXgYuO/VghU0jcOAZgWXZ1Dmrw=="],
"@playwright/test": ["@playwright/test@1.57.0", "", { "dependencies": { "playwright": "1.57.0" }, "bin": { "playwright": "cli.js" } }, "sha512-6TyEnHgd6SArQO8UO2OMTxshln3QMWBtPGrOCgs3wVEmQmwyuNtB10IZMfmYDE0riwNR1cu4q+pPcxMVtaG3TA=="],
"@radix-ui/primitive": ["@radix-ui/primitive@1.1.3", "", {}, "sha512-JTF99U/6XIjCBo0wqkU5sK10glYe27MRRsfwoiq5zzOEZLHU3A3KCMa5X/azekYRCJ0HlwI0crAXS/5dEHTzDg=="],
"@radix-ui/react-arrow": ["@radix-ui/react-arrow@1.1.7", "", { "dependencies": { "@radix-ui/react-primitive": "2.1.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-F+M1tLhO+mlQaOWspE8Wstg+z6PwxwRd8oQ8IXceWz92kfAmalTRf0EjrouQeo7QssEPfCn05B4Ihs1K9WQ/7w=="],
@@ -431,6 +444,8 @@
"@radix-ui/rect": ["@radix-ui/rect@1.1.1", "", {}, "sha512-HPwpGIzkl28mWyZqG52jiqDJ12waP11Pa1lGoiyUkIEuMLBP0oeK/C89esbXrxsky5we7dfd8U58nm0SgAWpVw=="],
"@resvg/resvg-wasm": ["@resvg/resvg-wasm@2.6.2", "", {}, "sha512-FqALmHI8D4o6lk/LRWDnhw95z5eO+eAa6ORjVg09YRR7BkcM6oPHU9uyC0gtQG5vpFLvgpeU4+zEAz2H8APHNw=="],
"@rolldown/pluginutils": ["@rolldown/pluginutils@1.0.0-beta.53", "", {}, "sha512-vENRlFU4YbrwVqNDZ7fLvy+JR1CRkyr01jhSiDpE1u6py3OMzQfztQU2jxykW3ALNxO4kSlqIDeYyD0Y9RcQeQ=="],
"@rollup/rollup-android-arm-eabi": ["@rollup/rollup-android-arm-eabi@4.54.0", "", { "os": "android", "cpu": "arm" }, "sha512-OywsdRHrFvCdvsewAInDKCNyR3laPA2mc9bRYJ6LBp5IyvF3fvXbbNR0bSzHlZVFtn6E0xw2oZlyjg4rKCVcng=="],
@@ -613,10 +628,14 @@
"@types/semver": ["@types/semver@7.7.1", "", {}, "sha512-FmgJfu+MOcQ370SD0ev7EI8TlCAfKYU+B4m5T3yXc1CiRN94g/SZPtsCkk506aUDtlMnFZvasDwHHUcZUEaYuA=="],
"@types/trusted-types": ["@types/trusted-types@2.0.7", "", {}, "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw=="],
"@types/unist": ["@types/unist@3.0.3", "", {}, "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q=="],
"@ungap/structured-clone": ["@ungap/structured-clone@1.3.0", "", {}, "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g=="],
"@vercel/analytics": ["@vercel/analytics@1.6.1", "", { "peerDependencies": { "@remix-run/react": "^2", "@sveltejs/kit": "^1 || ^2", "next": ">= 13", "react": "^18 || ^19 || ^19.0.0-rc", "svelte": ">= 4", "vue": "^3", "vue-router": "^4" }, "optionalPeers": ["@remix-run/react", "@sveltejs/kit", "next", "react", "svelte", "vue", "vue-router"] }, "sha512-oH9He/bEM+6oKlv3chWuOOcp8Y6fo6/PSro8hEkgCW3pu9/OiCXiUpRUogDh3Fs3LH2sosDrx8CxeOLBEE+afg=="],
"@vitejs/plugin-react": ["@vitejs/plugin-react@5.1.2", "", { "dependencies": { "@babel/core": "^7.28.5", "@babel/plugin-transform-react-jsx-self": "^7.27.1", "@babel/plugin-transform-react-jsx-source": "^7.27.1", "@rolldown/pluginutils": "1.0.0-beta.53", "@types/babel__core": "^7.20.5", "react-refresh": "^0.18.0" }, "peerDependencies": { "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0" } }, "sha512-EcA07pHJouywpzsoTUqNh5NwGayl2PPVEJKUSinGGSxFGYn+shYbqMGBg6FXDqgXum9Ou/ecb+411ssw8HImJQ=="],
"@vitest/coverage-v8": ["@vitest/coverage-v8@4.0.16", "", { "dependencies": { "@bcoe/v8-coverage": "^1.0.2", "@vitest/utils": "4.0.16", "ast-v8-to-istanbul": "^0.3.8", "istanbul-lib-coverage": "^3.2.2", "istanbul-lib-report": "^3.0.1", "istanbul-lib-source-maps": "^5.0.6", "istanbul-reports": "^3.2.0", "magicast": "^0.5.1", "obug": "^2.1.1", "std-env": "^3.10.0", "tinyrainbow": "^3.0.3" }, "peerDependencies": { "@vitest/browser": "4.0.16", "vitest": "4.0.16" }, "optionalPeers": ["@vitest/browser"] }, "sha512-2rNdjEIsPRzsdu6/9Eq0AYAzYdpP6Bx9cje9tL3FE5XzXRQF1fNU9pe/1yE8fCrS0HD+fBtt6gLPh6LI57tX7A=="],
@@ -765,6 +784,8 @@
"domhandler": ["domhandler@5.0.3", "", { "dependencies": { "domelementtype": "^2.3.0" } }, "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w=="],
"dompurify": ["dompurify@3.2.7", "", { "optionalDependencies": { "@types/trusted-types": "^2.0.7" } }, "sha512-WhL/YuveyGXJaerVlMYGWhvQswa7myDG17P7Vu65EWC05o8vfeNbvNf4d/BOvH99+ZW+LlQsc1GDKMa1vNK6dw=="],
"domutils": ["domutils@3.2.2", "", { "dependencies": { "dom-serializer": "^2.0.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3" } }, "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw=="],
"electron-to-chromium": ["electron-to-chromium@1.5.267", "", {}, "sha512-0Drusm6MVRXSOJpGbaSVgcQsuB4hEkMpHXaVstcPmhu5LIedxs1xNK/nIxmQIU/RPC0+1/o0AVZfBTkTNJOdUw=="],
@@ -945,6 +966,8 @@
"markdown-table": ["markdown-table@3.0.4", "", {}, "sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw=="],
"marked": ["marked@14.0.0", "", { "bin": { "marked": "bin/marked.js" } }, "sha512-uIj4+faQ+MgHgwUW1l2PsPglZLOLOT1uErt06dAPtx2kjteLAkbsd/0FiYg/MGS+i7ZKLb7w2WClxHkzOOuryQ=="],
"mdast-util-find-and-replace": ["mdast-util-find-and-replace@3.0.2", "", { "dependencies": { "@types/mdast": "^4.0.0", "escape-string-regexp": "^5.0.0", "unist-util-is": "^6.0.0", "unist-util-visit-parents": "^6.0.0" } }, "sha512-Tmd1Vg/m3Xz43afeNxDIhWRtFZgM2VLyaf4vSTYwudTyeuTneoL3qtWMA5jeLyz/O1vDJmmV4QuScFCA2tBPwg=="],
"mdast-util-from-markdown": ["mdast-util-from-markdown@2.0.2", "", { "dependencies": { "@types/mdast": "^4.0.0", "@types/unist": "^3.0.0", "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "mdast-util-to-string": "^4.0.0", "micromark": "^4.0.0", "micromark-util-decode-numeric-character-reference": "^2.0.0", "micromark-util-decode-string": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0", "unist-util-stringify-position": "^4.0.0" } }, "sha512-uZhTV/8NBuw0WHkPTrCqDOl0zVe1BIng5ZtHoDk49ME1qqcjYmmLmOf0gELgcRMxN4w2iuIeVso5/6QymSrgmA=="],
@@ -1037,6 +1060,8 @@
"mimic-function": ["mimic-function@5.0.1", "", {}, "sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA=="],
"monaco-editor": ["monaco-editor@0.55.1", "", { "dependencies": { "dompurify": "3.2.7", "marked": "14.0.0" } }, "sha512-jz4x+TJNFHwHtwuV9vA9rMujcZRb0CEilTEwG2rRSpe/A7Jdkuj8xPKttCgOh+v/lkHy7HsZ64oj+q3xoAFl9A=="],
"ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="],
"nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="],
@@ -1089,6 +1114,10 @@
"picomatch": ["picomatch@4.0.3", "", {}, "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q=="],
"playwright": ["playwright@1.57.0", "", { "dependencies": { "playwright-core": "1.57.0" }, "optionalDependencies": { "fsevents": "2.3.2" }, "bin": { "playwright": "cli.js" } }, "sha512-ilYQj1s8sr2ppEJ2YVadYBN0Mb3mdo9J0wQ+UuDhzYqURwSoW4n1Xs5vs7ORwgDGmyEh33tRMeS8KhdkMoLXQw=="],
"playwright-core": ["playwright-core@1.57.0", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-agTcKlMw/mjBWOnD6kFZttAAGHgi/Nw0CZ2o6JqWSbMlI219lAFLZZCyqByTsvVAJq5XA5H8cA6PrvBRpBWEuQ=="],
"postcss": ["postcss@8.5.6", "", { "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg=="],
"preact": ["preact@10.24.3", "", {}, "sha512-Z2dPnBnMUfyQfSQ+GBdsGa16hz35YmLmtTLhM169uW944hYL6xzTYkJjC07j+Wosz733pMWx0fgON3JNw1jJQA=="],
@@ -1175,6 +1204,8 @@
"stackback": ["stackback@0.0.2", "", {}, "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw=="],
"state-local": ["state-local@1.0.7", "", {}, "sha512-HTEHMNieakEnoe33shBYcZ7NX83ACUjCu8c40iOGEZsngj9zRnkqS9j1pqQPXwobB0ZcVTk27REb7COQ0UR59w=="],
"std-env": ["std-env@3.10.0", "", {}, "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg=="],
"stdin-discarder": ["stdin-discarder@0.2.2", "", {}, "sha512-UhDfHmA92YAlNnCfhmq0VeNL5bDbiZGg7sZ2IvPsXubGkiNa9EC+tUTsjBRsYUAz87btI6/1wf4XoVvQ3uRnmQ=="],
@@ -1351,6 +1382,8 @@
"parse5-parser-stream/parse5": ["parse5@7.3.0", "", { "dependencies": { "entities": "^6.0.0" } }, "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw=="],
"playwright/fsevents": ["fsevents@2.3.2", "", { "os": "darwin" }, "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA=="],
"readdirp/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="],
"recast/source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="],
+42
View File
@@ -10,20 +10,41 @@
import type * as auth from "../auth.js";
import type * as comments from "../comments.js";
import type * as crons from "../crons.js";
import type * as devSeed from "../devSeed.js";
import type * as downloads from "../downloads.js";
import type * as githubBackups from "../githubBackups.js";
import type * as githubBackupsNode from "../githubBackupsNode.js";
import type * as githubImport from "../githubImport.js";
import type * as githubSoulBackups from "../githubSoulBackups.js";
import type * as githubSoulBackupsNode from "../githubSoulBackupsNode.js";
import type * as http from "../http.js";
import type * as httpApi from "../httpApi.js";
import type * as httpApiV1 from "../httpApiV1.js";
import type * as lib_access from "../lib/access.js";
import type * as lib_apiTokenAuth from "../lib/apiTokenAuth.js";
import type * as lib_changelog from "../lib/changelog.js";
import type * as lib_embeddings from "../lib/embeddings.js";
import type * as lib_githubBackup from "../lib/githubBackup.js";
import type * as lib_githubImport from "../lib/githubImport.js";
import type * as lib_githubSoulBackup from "../lib/githubSoulBackup.js";
import type * as lib_skillBackfill from "../lib/skillBackfill.js";
import type * as lib_skillPublish from "../lib/skillPublish.js";
import type * as lib_skills from "../lib/skills.js";
import type * as lib_soulChangelog from "../lib/soulChangelog.js";
import type * as lib_soulPublish from "../lib/soulPublish.js";
import type * as lib_tokens from "../lib/tokens.js";
import type * as lib_webhooks from "../lib/webhooks.js";
import type * as maintenance from "../maintenance.js";
import type * as rateLimits from "../rateLimits.js";
import type * as search from "../search.js";
import type * as seed from "../seed.js";
import type * as seedSouls from "../seedSouls.js";
import type * as skills from "../skills.js";
import type * as soulComments from "../soulComments.js";
import type * as soulDownloads from "../soulDownloads.js";
import type * as soulStars from "../soulStars.js";
import type * as souls from "../souls.js";
import type * as stars from "../stars.js";
import type * as telemetry from "../telemetry.js";
import type * as tokens from "../tokens.js";
@@ -40,20 +61,41 @@ import type {
declare const fullApi: ApiFromModules<{
auth: typeof auth;
comments: typeof comments;
crons: typeof crons;
devSeed: typeof devSeed;
downloads: typeof downloads;
githubBackups: typeof githubBackups;
githubBackupsNode: typeof githubBackupsNode;
githubImport: typeof githubImport;
githubSoulBackups: typeof githubSoulBackups;
githubSoulBackupsNode: typeof githubSoulBackupsNode;
http: typeof http;
httpApi: typeof httpApi;
httpApiV1: typeof httpApiV1;
"lib/access": typeof lib_access;
"lib/apiTokenAuth": typeof lib_apiTokenAuth;
"lib/changelog": typeof lib_changelog;
"lib/embeddings": typeof lib_embeddings;
"lib/githubBackup": typeof lib_githubBackup;
"lib/githubImport": typeof lib_githubImport;
"lib/githubSoulBackup": typeof lib_githubSoulBackup;
"lib/skillBackfill": typeof lib_skillBackfill;
"lib/skillPublish": typeof lib_skillPublish;
"lib/skills": typeof lib_skills;
"lib/soulChangelog": typeof lib_soulChangelog;
"lib/soulPublish": typeof lib_soulPublish;
"lib/tokens": typeof lib_tokens;
"lib/webhooks": typeof lib_webhooks;
maintenance: typeof maintenance;
rateLimits: typeof rateLimits;
search: typeof search;
seed: typeof seed;
seedSouls: typeof seedSouls;
skills: typeof skills;
soulComments: typeof soulComments;
soulDownloads: typeof soulDownloads;
soulStars: typeof soulStars;
souls: typeof souls;
stars: typeof stars;
telemetry: typeof telemetry;
tokens: typeof tokens;
+13
View File
@@ -0,0 +1,13 @@
import { cronJobs } from 'convex/server'
import { internal } from './_generated/api'
const crons = cronJobs()
crons.interval(
'github-backup-sync',
{ minutes: 30 },
internal.githubBackupsNode.syncGitHubBackupsInternal,
{ batchSize: 50, maxBatches: 5 },
)
export default crons
+429
View File
@@ -0,0 +1,429 @@
import { v } from 'convex/values'
import { internal } from './_generated/api'
import { internalAction, internalMutation } from './_generated/server'
import { EMBEDDING_DIMENSIONS } from './lib/embeddings'
import { parseClawdisMetadata, parseFrontmatter } from './lib/skills'
type SeedSkillSpec = {
slug: string
displayName: string
summary: string
version: string
metadata: Record<string, unknown>
rawSkillMd: string
}
const SEED_SKILLS: SeedSkillSpec[] = [
{
slug: 'padel',
displayName: 'Padel',
summary: 'Check padel court availability and manage bookings via Playtomic.',
version: '0.1.0',
metadata: {
clawdbot: {
nix: {
plugin: 'github:joshp123/padel-cli',
systems: ['aarch64-darwin', 'x86_64-linux'],
},
config: {
requiredEnv: ['PADEL_AUTH_FILE'],
stateDirs: ['.config/padel'],
example:
'config = { env = { PADEL_AUTH_FILE = "/run/agenix/padel-auth"; }; stateDirs = [ ".config/padel" ]; };',
},
cliHelp: `Padel CLI for availability
Usage:
padel [command]
Available Commands:
auth Manage authentication
availability Show availability for a club on a date
book Book a court
bookings Manage bookings history
search Search for available courts
venues Manage saved venues
Flags:
-h, --help help for padel
--json Output JSON
Use "padel [command] --help" for more information about a command.
`,
},
},
rawSkillMd: `---
name: padel
description: Check padel court availability and manage bookings via the padel CLI.
---
# Padel Booking Skill
## CLI
\`\`\`bash
padel # On PATH (clawdbot plugin bundle)
\`\`\`
## Venues
Use the configured venue list in order of preference. If no venues are configured, ask for a venue name or location.
## Commands
### Check next booking
\`\`\`bash
padel bookings list 2>&1 | head -3
\`\`\`
### Search availability
\`\`\`bash
padel search --venues VENUE1,VENUE2 --date YYYY-MM-DD --time 09:00-12:00
\`\`\`
## Response guidelines
- Keep responses concise.
- Use 🎾 emoji.
- End with a call to action.
## Authorization
Only the authorized booker can confirm bookings. If the requester is not authorized, ask the authorized user to confirm.
`,
},
{
slug: 'gohome',
displayName: 'GoHome',
summary: 'Operate GoHome via gRPC discovery, metrics, and Grafana dashboards.',
version: '0.1.0',
metadata: {
clawdbot: {
nix: {
plugin: 'github:joshp123/gohome',
systems: ['x86_64-linux', 'aarch64-linux'],
},
config: {
requiredEnv: ['GOHOME_GRPC_ADDR', 'GOHOME_HTTP_BASE'],
example:
'config = { env = { GOHOME_GRPC_ADDR = "gohome:9000"; GOHOME_HTTP_BASE = "http://gohome:8080"; }; };',
},
cliHelp: `GoHome CLI
Usage:
gohome-cli [command]
Available Commands:
services List registered services
plugins Inspect loaded plugins
methods List RPC methods
call Call an RPC method
roborock Manage roborock devices
tado Manage tado zones
Flags:
--grpc-addr string gRPC endpoint (host:port)
-h, --help help for gohome-cli
`,
},
},
rawSkillMd: `---
name: gohome
description: Use when Clawdbot needs to test or operate GoHome via gRPC discovery, metrics, and Grafana.
---
# GoHome Skill
## Quick start
\`\`\`bash
export GOHOME_HTTP_BASE="http://gohome:8080"
export GOHOME_GRPC_ADDR="gohome:9000"
\`\`\`
## CLI
\`\`\`bash
gohome-cli services
\`\`\`
## Discovery flow (read-only)
1) List plugins.
2) Describe a plugin.
3) List RPC methods.
4) Call a read-only RPC.
## Metrics validation
\`\`\`bash
curl -s "\${GOHOME_HTTP_BASE}/gohome/metrics" | rg -n "gohome_"
\`\`\`
## Stateful actions
Only call write RPCs after explicit user approval.
`,
},
{
slug: 'xuezh',
displayName: 'Xuezh',
summary: 'Teach Mandarin with the xuezh engine for review, speaking, and audits.',
version: '0.1.0',
metadata: {
clawdbot: {
nix: {
plugin: 'github:joshp123/xuezh',
systems: ['aarch64-darwin', 'x86_64-linux'],
},
config: {
requiredEnv: ['XUEZH_AZURE_SPEECH_KEY_FILE', 'XUEZH_AZURE_SPEECH_REGION'],
stateDirs: ['.config/xuezh'],
example:
'config = { env = { XUEZH_AZURE_SPEECH_KEY_FILE = "/run/agenix/xuezh-azure-speech-key"; XUEZH_AZURE_SPEECH_REGION = "westeurope"; }; stateDirs = [ ".config/xuezh" ]; };',
},
cliHelp: `xuezh - Chinese learning engine
Usage:
xuezh [command]
Available Commands:
snapshot Fetch learner state snapshot
review Review due items
audio Process speech audio
items Manage learning items
events Log learning events
Flags:
-h, --help help for xuezh
--json Output JSON
`,
},
},
rawSkillMd: `---
name: xuezh
description: Teach Mandarin using the xuezh engine for review, speaking, and audits.
---
# Xuezh Skill
## Contract
Use the xuezh CLI exactly as specified. If a command is missing, ask for implementation instead of guessing.
## Default loop
1) Call \`xuezh snapshot\`.
2) Pick a tiny plan (1-2 bullets).
3) Run a short activity.
4) Log outcomes.
## CLI examples
\`\`\`bash
xuezh snapshot --profile default
xuezh review next --limit 10
xuezh audio process-voice --file ./utterance.wav
\`\`\`
`,
},
]
function injectMetadata(rawSkillMd: string, metadata: Record<string, unknown>) {
const frontmatterEnd = rawSkillMd.indexOf('\n---', 3)
if (frontmatterEnd === -1) return rawSkillMd
return `${rawSkillMd.slice(0, frontmatterEnd)}\nmetadata: ${JSON.stringify(
metadata,
)}${rawSkillMd.slice(frontmatterEnd)}`
}
export const seedNixSkills = internalAction({
args: {
reset: v.optional(v.boolean()),
},
handler: async (ctx, args) => {
const results = []
for (const spec of SEED_SKILLS) {
const skillMd = injectMetadata(spec.rawSkillMd, spec.metadata)
const frontmatter = parseFrontmatter(skillMd)
const clawdis = parseClawdisMetadata(frontmatter)
const storageId = await ctx.storage.store(new Blob([skillMd], { type: 'text/markdown' }))
const result = await ctx.runMutation(internal.devSeed.seedSkillMutation, {
reset: args.reset,
storageId,
metadata: spec.metadata,
frontmatter,
clawdis,
skillMd,
slug: spec.slug,
displayName: spec.displayName,
summary: spec.summary,
version: spec.version,
})
results.push({ slug: spec.slug, ...result })
}
return { ok: true, results }
},
})
export const seedPadelSkill = internalAction({
args: {
reset: v.optional(v.boolean()),
},
handler: async (ctx, args) => {
const spec = SEED_SKILLS.find((entry) => entry.slug === 'padel')
if (!spec) throw new Error('padel seed spec missing')
const skillMd = injectMetadata(spec.rawSkillMd, spec.metadata)
const frontmatter = parseFrontmatter(skillMd)
const clawdis = parseClawdisMetadata(frontmatter)
const storageId = await ctx.storage.store(new Blob([skillMd], { type: 'text/markdown' }))
return ctx.runMutation(internal.devSeed.seedSkillMutation, {
reset: args.reset,
storageId,
metadata: spec.metadata,
frontmatter,
clawdis,
skillMd,
slug: spec.slug,
displayName: spec.displayName,
summary: spec.summary,
version: spec.version,
})
},
})
export const seedSkillMutation = internalMutation({
args: {
reset: v.optional(v.boolean()),
storageId: v.id('_storage'),
metadata: v.any(),
frontmatter: v.any(),
clawdis: v.any(),
skillMd: v.string(),
slug: v.string(),
displayName: v.string(),
summary: v.optional(v.string()),
version: v.string(),
},
handler: async (ctx, args) => {
const existing = await ctx.db
.query('skills')
.withIndex('by_slug', (q) => q.eq('slug', args.slug))
.unique()
if (existing && !args.reset) {
return { ok: true, skipped: true, skillId: existing._id }
}
if (existing && args.reset) {
const versions = await ctx.db
.query('skillVersions')
.withIndex('by_skill', (q) => q.eq('skillId', existing._id))
.collect()
for (const version of versions) {
await ctx.db.delete(version._id)
}
const embeddings = await ctx.db
.query('skillEmbeddings')
.withIndex('by_skill', (q) => q.eq('skillId', existing._id))
.collect()
for (const embedding of embeddings) {
await ctx.db.delete(embedding._id)
}
await ctx.db.delete(existing._id)
}
const now = Date.now()
const existingUsers = await ctx.db
.query('users')
.withIndex('handle', (q) => q.eq('handle', 'local'))
.collect()
const userId =
existingUsers[0]?._id ??
(await ctx.db.insert('users', {
handle: 'local',
displayName: 'Local Dev',
role: 'admin',
createdAt: now,
updatedAt: now,
}))
const skillId = await ctx.db.insert('skills', {
slug: args.slug,
displayName: args.displayName,
summary: args.summary,
ownerUserId: userId,
latestVersionId: undefined,
tags: {},
softDeletedAt: undefined,
badges: { redactionApproved: undefined },
stats: {
downloads: 0,
installsCurrent: 0,
installsAllTime: 0,
stars: 0,
versions: 0,
comments: 0,
},
createdAt: now,
updatedAt: now,
})
const versionId = await ctx.db.insert('skillVersions', {
skillId,
version: args.version,
changelog: 'Seeded local version for screenshots.',
files: [
{
path: 'SKILL.md',
size: args.skillMd.length,
storageId: args.storageId,
sha256: 'seeded',
contentType: 'text/markdown',
},
],
parsed: {
frontmatter: args.frontmatter,
metadata: args.metadata,
clawdis: args.clawdis,
},
createdBy: userId,
createdAt: now,
softDeletedAt: undefined,
})
const embeddingId = await ctx.db.insert('skillEmbeddings', {
skillId,
versionId,
ownerId: userId,
embedding: Array.from({ length: EMBEDDING_DIMENSIONS }, () => 0),
isLatest: true,
isApproved: true,
visibility: 'latest-approved',
updatedAt: now,
})
await ctx.db.patch(skillId, {
latestVersionId: versionId,
tags: { latest: versionId },
stats: {
downloads: 0,
installsCurrent: 0,
installsAllTime: 0,
stars: 0,
versions: 1,
comments: 0,
},
updatedAt: now,
})
return { ok: true, skillId, versionId, embeddingId }
},
})
+170
View File
@@ -0,0 +1,170 @@
import { v } from 'convex/values'
import { internal } from './_generated/api'
import type { Doc, Id } from './_generated/dataModel'
import { action, internalMutation, internalQuery } from './_generated/server'
import { assertRole, requireUserFromAction } from './lib/access'
const DEFAULT_BATCH_SIZE = 50
const MAX_BATCH_SIZE = 200
const SYNC_STATE_KEY = 'default'
type BackupPageItem =
| {
kind: 'ok'
skillId: Id<'skills'>
versionId: Id<'skillVersions'>
slug: string
displayName: string
version: string
ownerHandle: string
files: Doc<'skillVersions'>['files']
publishedAt: number
}
| { kind: 'missingLatestVersion'; skillId: Id<'skills'> }
| { kind: 'missingVersionDoc'; skillId: Id<'skills'>; versionId: Id<'skillVersions'> }
| { kind: 'missingOwner'; skillId: Id<'skills'>; ownerUserId: Id<'users'> }
type BackupPageResult = {
items: BackupPageItem[]
cursor: string | null
isDone: boolean
}
type BackupSyncState = {
cursor: string | null
}
export type SyncGitHubBackupsResult = {
stats: {
skillsScanned: number
skillsSkipped: number
skillsBackedUp: number
skillsMissingVersion: number
skillsMissingOwner: number
errors: number
}
cursor: string | null
isDone: boolean
}
export const getGitHubBackupPageInternal = internalQuery({
args: {
cursor: v.optional(v.string()),
batchSize: v.optional(v.number()),
},
handler: async (ctx, args): Promise<BackupPageResult> => {
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
const { page, isDone, continueCursor } = await ctx.db
.query('skills')
.order('asc')
.paginate({ cursor: args.cursor ?? null, numItems: batchSize })
const items: BackupPageItem[] = []
for (const skill of page) {
if (skill.softDeletedAt) continue
if (!skill.latestVersionId) {
items.push({ kind: 'missingLatestVersion', skillId: skill._id })
continue
}
const version = await ctx.db.get(skill.latestVersionId)
if (!version) {
items.push({
kind: 'missingVersionDoc',
skillId: skill._id,
versionId: skill.latestVersionId,
})
continue
}
const owner = await ctx.db.get(skill.ownerUserId)
if (!owner || owner.deletedAt) {
items.push({ kind: 'missingOwner', skillId: skill._id, ownerUserId: skill.ownerUserId })
continue
}
items.push({
kind: 'ok',
skillId: skill._id,
versionId: version._id,
slug: skill.slug,
displayName: skill.displayName,
version: version.version,
ownerHandle: owner.handle ?? owner._id,
files: version.files,
publishedAt: version.createdAt,
})
}
return { items, cursor: continueCursor, isDone }
},
})
export const getGitHubBackupSyncStateInternal = internalQuery({
args: {},
handler: async (ctx): Promise<BackupSyncState> => {
const state = await ctx.db
.query('githubBackupSyncState')
.withIndex('by_key', (q) => q.eq('key', SYNC_STATE_KEY))
.unique()
return { cursor: state?.cursor ?? null }
},
})
export const setGitHubBackupSyncStateInternal = internalMutation({
args: {
cursor: v.optional(v.string()),
},
handler: async (ctx, args) => {
const now = Date.now()
const state = await ctx.db
.query('githubBackupSyncState')
.withIndex('by_key', (q) => q.eq('key', SYNC_STATE_KEY))
.unique()
if (!state) {
await ctx.db.insert('githubBackupSyncState', {
key: SYNC_STATE_KEY,
cursor: args.cursor,
updatedAt: now,
})
return { ok: true as const }
}
await ctx.db.patch(state._id, {
cursor: args.cursor,
updatedAt: now,
})
return { ok: true as const }
},
})
export const syncGitHubBackups: ReturnType<typeof action> = action({
args: {
dryRun: v.optional(v.boolean()),
batchSize: v.optional(v.number()),
maxBatches: v.optional(v.number()),
resetCursor: v.optional(v.boolean()),
},
handler: async (ctx, args): Promise<SyncGitHubBackupsResult> => {
const { user } = await requireUserFromAction(ctx)
assertRole(user, ['admin'])
if (args.resetCursor && !args.dryRun) {
await ctx.runMutation(internal.githubBackups.setGitHubBackupSyncStateInternal, {
cursor: undefined,
})
}
return ctx.runAction(internal.githubBackupsNode.syncGitHubBackupsInternal, {
dryRun: args.dryRun,
batchSize: args.batchSize,
maxBatches: args.maxBatches,
}) as Promise<SyncGitHubBackupsResult>
},
})
function clampInt(value: number, min: number, max: number) {
return Math.max(min, Math.min(max, Math.floor(value)))
}
+183
View File
@@ -0,0 +1,183 @@
'use node'
import { v } from 'convex/values'
import { internal } from './_generated/api'
import type { Doc } from './_generated/dataModel'
import type { ActionCtx } from './_generated/server'
import { internalAction } from './_generated/server'
import {
backupSkillToGitHub,
fetchGitHubSkillMeta,
getGitHubBackupContext,
isGitHubBackupConfigured,
} from './lib/githubBackup'
const DEFAULT_BATCH_SIZE = 50
const MAX_BATCH_SIZE = 200
const DEFAULT_MAX_BATCHES = 5
const MAX_MAX_BATCHES = 200
type BackupPageItem =
| {
kind: 'ok'
slug: string
version: string
displayName: string
ownerHandle: string
files: Doc<'skillVersions'>['files']
publishedAt: number
}
| { kind: 'missingLatestVersion' }
| { kind: 'missingVersionDoc' }
| { kind: 'missingOwner' }
export type GitHubBackupSyncStats = {
skillsScanned: number
skillsSkipped: number
skillsBackedUp: number
skillsMissingVersion: number
skillsMissingOwner: number
errors: number
}
export type SyncGitHubBackupsInternalArgs = {
dryRun?: boolean
batchSize?: number
maxBatches?: number
}
export type SyncGitHubBackupsInternalResult = {
stats: GitHubBackupSyncStats
cursor: string | null
isDone: boolean
}
export const backupSkillForPublishInternal = internalAction({
args: {
slug: v.string(),
version: v.string(),
displayName: v.string(),
ownerHandle: v.string(),
files: v.array(
v.object({
path: v.string(),
size: v.number(),
storageId: v.id('_storage'),
sha256: v.string(),
contentType: v.optional(v.string()),
}),
),
publishedAt: v.number(),
},
handler: async (ctx, args) => {
if (!isGitHubBackupConfigured()) {
return { skipped: true as const }
}
await backupSkillToGitHub(ctx, args)
return { skipped: false as const }
},
})
export async function syncGitHubBackupsInternalHandler(
ctx: ActionCtx,
args: SyncGitHubBackupsInternalArgs,
): Promise<SyncGitHubBackupsInternalResult> {
const dryRun = Boolean(args.dryRun)
const stats: GitHubBackupSyncStats = {
skillsScanned: 0,
skillsSkipped: 0,
skillsBackedUp: 0,
skillsMissingVersion: 0,
skillsMissingOwner: 0,
errors: 0,
}
if (!isGitHubBackupConfigured()) {
return { stats, cursor: null, isDone: true }
}
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
const maxBatches = clampInt(args.maxBatches ?? DEFAULT_MAX_BATCHES, 1, MAX_MAX_BATCHES)
const context = await getGitHubBackupContext()
const state = dryRun
? { cursor: null as string | null }
: ((await ctx.runQuery(internal.githubBackups.getGitHubBackupSyncStateInternal, {})) as {
cursor: string | null
})
let cursor: string | null = state.cursor
let isDone = false
for (let batch = 0; batch < maxBatches; batch++) {
const page = (await ctx.runQuery(internal.githubBackups.getGitHubBackupPageInternal, {
cursor: cursor ?? undefined,
batchSize,
})) as { items: BackupPageItem[]; cursor: string | null; isDone: boolean }
cursor = page.cursor
isDone = page.isDone
for (const item of page.items) {
if (item.kind !== 'ok') {
if (item.kind === 'missingLatestVersion' || item.kind === 'missingVersionDoc') {
stats.skillsMissingVersion += 1
} else if (item.kind === 'missingOwner') {
stats.skillsMissingOwner += 1
}
continue
}
stats.skillsScanned += 1
try {
const meta = await fetchGitHubSkillMeta(context, item.ownerHandle, item.slug)
if (meta?.latest?.version === item.version) {
stats.skillsSkipped += 1
continue
}
if (!dryRun) {
await backupSkillToGitHub(
ctx,
{
slug: item.slug,
version: item.version,
displayName: item.displayName,
ownerHandle: item.ownerHandle,
files: item.files,
publishedAt: item.publishedAt,
},
context,
)
stats.skillsBackedUp += 1
}
} catch (error) {
console.error('GitHub backup sync failed', error)
stats.errors += 1
}
}
if (!dryRun) {
await ctx.runMutation(internal.githubBackups.setGitHubBackupSyncStateInternal, {
cursor: isDone ? undefined : (cursor ?? undefined),
})
}
if (isDone) break
}
return { stats, cursor, isDone }
}
export const syncGitHubBackupsInternal = internalAction({
args: {
dryRun: v.optional(v.boolean()),
batchSize: v.optional(v.number()),
maxBatches: v.optional(v.number()),
},
handler: syncGitHubBackupsInternalHandler,
})
function clampInt(value: number, min: number, max: number) {
return Math.max(min, Math.min(max, Math.floor(value)))
}
+317
View File
@@ -0,0 +1,317 @@
import { ConvexError, v } from 'convex/values'
import { unzipSync } from 'fflate'
import semver from 'semver'
import { api, internal } from './_generated/api'
import type { Id } from './_generated/dataModel'
import type { ActionCtx } from './_generated/server'
import { action } from './_generated/server'
import { requireUserFromAction } from './lib/access'
import {
buildGitHubImportFileList,
computeDefaultSelectedPaths,
detectGitHubImportCandidates,
fetchGitHubZipBytes,
listTextFilesUnderCandidate,
normalizeRepoPath,
parseGitHubImportUrl,
resolveGitHubCommit,
stripGitHubZipRoot,
suggestDisplayName,
suggestVersion,
} from './lib/githubImport'
import { publishVersionForUser } from './lib/skillPublish'
import { sanitizePath } from './lib/skills'
const MAX_SELECTED_BYTES = 50 * 1024 * 1024
const MAX_UNZIPPED_BYTES = 80 * 1024 * 1024
const MAX_FILE_COUNT = 7_500
const MAX_SINGLE_FILE_BYTES = 10 * 1024 * 1024
export const previewGitHubImport = action({
args: { url: v.string() },
handler: async (ctx, args) => {
await requireUserFromAction(ctx)
const parsed = parseGitHubImportUrl(args.url)
const resolved = await resolveGitHubCommit(parsed, fetch)
const zipBytes = await fetchGitHubZipBytes(resolved, fetch)
const entries = unzipToEntries(zipBytes)
const stripped = stripGitHubZipRoot(entries)
const candidates = detectGitHubImportCandidates(stripped).filter((candidate) =>
isCandidateUnderResolvedPath(candidate.path, resolved.path),
)
if (candidates.length === 0) throw new ConvexError('No SKILL.md found in this repo')
return {
resolved,
candidates: candidates.map((candidate) => ({
path: candidate.path,
readmePath: candidate.readmePath,
name: candidate.name ?? null,
description: candidate.description ?? null,
})),
}
},
})
export const previewGitHubImportCandidate = action({
args: { url: v.string(), candidatePath: v.string() },
handler: async (ctx, args) => {
const { userId } = await requireUserFromAction(ctx)
const parsed = parseGitHubImportUrl(args.url)
const resolved = await resolveGitHubCommit(parsed, fetch)
const zipBytes = await fetchGitHubZipBytes(resolved, fetch)
const entries = unzipToEntries(zipBytes)
const stripped = stripGitHubZipRoot(entries)
const normalizedCandidatePath = normalizeRepoPath(args.candidatePath)
if (!isCandidateUnderResolvedPath(normalizedCandidatePath, resolved.path)) {
throw new ConvexError('Candidate path is outside the requested import scope')
}
const candidates = detectGitHubImportCandidates(stripped).filter((candidate) =>
isCandidateUnderResolvedPath(candidate.path, resolved.path),
)
const candidate = candidates.find((item) => item.path === normalizedCandidatePath)
if (!candidate) throw new ConvexError('Candidate not found')
const files = listTextFilesUnderCandidate(stripped, candidate.path)
const defaultSelectedPaths = computeDefaultSelectedPaths({ candidate, files })
const fileList = buildGitHubImportFileList({
candidate,
files,
defaultSelectedPaths,
})
const baseForNaming = candidate.path ? (candidate.path.split('/').at(-1) ?? '') : resolved.repo
const suggestedDisplayName = suggestDisplayName(candidate, baseForNaming)
const rawSlugBase = sanitizeSlug(candidate.path ? baseForNaming : resolved.repo)
const suggestedSlug = await suggestAvailableSlug(ctx, userId, rawSlugBase)
const existing = await ctx.runQuery(api.skills.getBySlug, { slug: suggestedSlug })
const existingLatest =
existing?.skill && existing.skill.ownerUserId === userId
? (existing.latestVersion?.version ?? null)
: null
const suggestedVersion = suggestVersion(existingLatest)
return {
resolved,
candidate: {
path: candidate.path,
readmePath: candidate.readmePath,
name: candidate.name ?? null,
description: candidate.description ?? null,
},
defaults: {
selectedPaths: defaultSelectedPaths,
slug: suggestedSlug,
displayName: suggestedDisplayName,
version: suggestedVersion,
tags: ['latest'],
},
files: fileList,
}
},
})
export const importGitHubSkill = action({
args: {
url: v.string(),
commit: v.string(),
candidatePath: v.string(),
selectedPaths: v.array(v.string()),
slug: v.optional(v.string()),
displayName: v.optional(v.string()),
version: v.optional(v.string()),
tags: v.optional(v.array(v.string())),
},
handler: async (ctx, args) => {
const { userId } = await requireUserFromAction(ctx)
const parsed = parseGitHubImportUrl(args.url)
const resolved = await resolveGitHubCommit(parsed, fetch)
if (!/^[a-f0-9]{40}$/i.test(args.commit)) throw new ConvexError('Invalid commit')
if (args.commit.toLowerCase() !== resolved.commit.toLowerCase()) {
throw new ConvexError('Import is out of date. Re-run preview.')
}
const normalizedCandidatePath = normalizeRepoPath(args.candidatePath)
if (!isCandidateUnderResolvedPath(normalizedCandidatePath, resolved.path)) {
throw new ConvexError('Candidate path is outside the requested import scope')
}
const zipBytes = await fetchGitHubZipBytes(resolved, fetch)
const entries = stripGitHubZipRoot(unzipToEntries(zipBytes))
const candidates = detectGitHubImportCandidates(entries).filter((candidate) =>
isCandidateUnderResolvedPath(candidate.path, resolved.path),
)
const candidate = candidates.find((item) => item.path === normalizedCandidatePath)
if (!candidate) throw new ConvexError('Candidate not found')
const filesUnderCandidate = listTextFilesUnderCandidate(entries, candidate.path)
const byPath = new Map(filesUnderCandidate.map((file) => [file.path, file.bytes]))
const selected = Array.from(
new Set(args.selectedPaths.map((path) => normalizeRepoPath(path)).filter(Boolean)),
)
if (selected.length === 0) throw new ConvexError('No files selected')
const candidateRoot = candidate.path ? `${candidate.path}/` : ''
const normalizedReadmePath = normalizeRepoPath(candidate.readmePath)
if (!selected.includes(normalizedReadmePath)) {
throw new ConvexError('SKILL.md must be selected')
}
let totalBytes = 0
const storedFiles: Array<{
path: string
size: number
storageId: Id<'_storage'>
sha256: string
contentType?: string
}> = []
for (const path of selected.sort()) {
if (candidateRoot && !path.startsWith(candidateRoot)) {
throw new ConvexError('Selected file is outside the chosen skill folder')
}
const bytes = byPath.get(path)
if (!bytes) continue
totalBytes += bytes.byteLength
if (totalBytes > MAX_SELECTED_BYTES) throw new ConvexError('Selected files exceed 50MB limit')
const relPath = candidateRoot ? path.slice(candidateRoot.length) : path
const sanitized = sanitizePath(relPath)
if (!sanitized) throw new ConvexError('Invalid file paths')
const sha256 = await sha256Hex(bytes)
const safeBytes = new Uint8Array(bytes)
const storageId = await ctx.storage.store(new Blob([safeBytes], { type: 'text/plain' }))
storedFiles.push({
path: sanitized,
size: bytes.byteLength,
storageId,
sha256,
contentType: 'text/plain',
})
}
if (storedFiles.length === 0) throw new ConvexError('No files selected')
const slugBase = (args.slug ?? '').trim().toLowerCase()
const displayName = (args.displayName ?? '').trim()
const tags = (args.tags ?? ['latest']).map((tag) => tag.trim()).filter(Boolean)
const version = (args.version ?? '').trim()
if (!slugBase) throw new ConvexError('Slug required')
if (!displayName) throw new ConvexError('Display name required')
if (!version || !semver.valid(version)) throw new ConvexError('Version must be valid semver')
const result = await publishVersionForUser(ctx, userId, {
slug: slugBase,
displayName,
version,
changelog: '',
tags,
files: storedFiles,
source: {
kind: 'github',
url: resolved.originalUrl,
repo: `${resolved.owner}/${resolved.repo}`,
ref: resolved.ref,
commit: resolved.commit,
path: candidate.path,
importedAt: Date.now(),
},
})
return { ok: true, slug: slugBase, version, ...result }
},
})
function unzipToEntries(zipBytes: Uint8Array) {
const entries = unzipSync(zipBytes)
const out: Record<string, Uint8Array> = {}
const rawPaths = Object.keys(entries)
if (rawPaths.length > MAX_FILE_COUNT) throw new ConvexError('Repo archive has too many files')
let totalBytes = 0
for (const [rawPath, bytes] of Object.entries(entries)) {
const normalizedPath = normalizeZipPath(rawPath)
if (!normalizedPath) continue
if (isJunkPath(normalizedPath)) continue
if (!bytes) continue
if (bytes.byteLength > MAX_SINGLE_FILE_BYTES) continue
totalBytes += bytes.byteLength
if (totalBytes > MAX_UNZIPPED_BYTES) throw new ConvexError('Repo archive is too large')
out[normalizedPath] = bytes
}
return out
}
function isCandidateUnderResolvedPath(candidatePath: string, resolvedPath: string) {
const root = normalizeRepoPath(resolvedPath)
if (!root) return true
if (!candidatePath) return false
if (candidatePath === root) return true
return candidatePath.startsWith(`${root}/`)
}
function sanitizeSlug(value: string) {
return value
.trim()
.toLowerCase()
.replace(/[^a-z0-9-]+/g, '-')
.replace(/^-+/, '')
.replace(/-+$/, '')
.replace(/--+/g, '-')
}
async function suggestAvailableSlug(ctx: ActionCtx, userId: Id<'users'>, base: string) {
const cleaned = sanitizeSlug(base)
if (!cleaned) throw new ConvexError('Could not derive slug')
for (let i = 0; i < 50; i += 1) {
const candidate = i === 0 ? cleaned : `${cleaned}-${i + 1}`
const existing = await ctx.runQuery(internal.skills.getSkillBySlugInternal, { slug: candidate })
if (!existing) return candidate
if (existing.ownerUserId === userId) return candidate
}
throw new ConvexError('Could not find an available slug')
}
async function sha256Hex(bytes: Uint8Array) {
const normalized = new Uint8Array(bytes)
const digest = await crypto.subtle.digest('SHA-256', normalized.buffer)
return toHex(new Uint8Array(digest))
}
function toHex(bytes: Uint8Array) {
let out = ''
for (const byte of bytes) out += byte.toString(16).padStart(2, '0')
return out
}
function normalizeZipPath(path: string) {
const normalized = path
.replaceAll('\u0000', '')
.replaceAll('\\', '/')
.trim()
.replace(/^\.\/+/, '')
.replace(/^\/+/, '')
if (!normalized) return ''
if (normalized.includes('..')) return ''
return normalized
}
function isJunkPath(path: string) {
const normalized = path.toLowerCase()
if (normalized.startsWith('__macosx/')) return true
if (normalized.endsWith('/.ds_store')) return true
if (normalized === '.ds_store') return true
return false
}
+170
View File
@@ -0,0 +1,170 @@
import { v } from 'convex/values'
import { internal } from './_generated/api'
import type { Doc, Id } from './_generated/dataModel'
import { action, internalMutation, internalQuery } from './_generated/server'
import { assertRole, requireUserFromAction } from './lib/access'
const DEFAULT_BATCH_SIZE = 50
const MAX_BATCH_SIZE = 200
const SYNC_STATE_KEY = 'souls'
type BackupPageItem =
| {
kind: 'ok'
soulId: Id<'souls'>
versionId: Id<'soulVersions'>
slug: string
displayName: string
version: string
ownerHandle: string
files: Doc<'soulVersions'>['files']
publishedAt: number
}
| { kind: 'missingLatestVersion'; soulId: Id<'souls'> }
| { kind: 'missingVersionDoc'; soulId: Id<'souls'>; versionId: Id<'soulVersions'> }
| { kind: 'missingOwner'; soulId: Id<'souls'>; ownerUserId: Id<'users'> }
type BackupPageResult = {
items: BackupPageItem[]
cursor: string | null
isDone: boolean
}
type BackupSyncState = {
cursor: string | null
}
export type SyncGitHubSoulBackupsResult = {
stats: {
soulsScanned: number
soulsSkipped: number
soulsBackedUp: number
soulsMissingVersion: number
soulsMissingOwner: number
errors: number
}
cursor: string | null
isDone: boolean
}
export const getGitHubSoulBackupPageInternal = internalQuery({
args: {
cursor: v.optional(v.string()),
batchSize: v.optional(v.number()),
},
handler: async (ctx, args): Promise<BackupPageResult> => {
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
const { page, isDone, continueCursor } = await ctx.db
.query('souls')
.order('asc')
.paginate({ cursor: args.cursor ?? null, numItems: batchSize })
const items: BackupPageItem[] = []
for (const soul of page) {
if (soul.softDeletedAt) continue
if (!soul.latestVersionId) {
items.push({ kind: 'missingLatestVersion', soulId: soul._id })
continue
}
const version = await ctx.db.get(soul.latestVersionId)
if (!version) {
items.push({
kind: 'missingVersionDoc',
soulId: soul._id,
versionId: soul.latestVersionId,
})
continue
}
const owner = await ctx.db.get(soul.ownerUserId)
if (!owner || owner.deletedAt) {
items.push({ kind: 'missingOwner', soulId: soul._id, ownerUserId: soul.ownerUserId })
continue
}
items.push({
kind: 'ok',
soulId: soul._id,
versionId: version._id,
slug: soul.slug,
displayName: soul.displayName,
version: version.version,
ownerHandle: owner.handle ?? owner._id,
files: version.files,
publishedAt: version.createdAt,
})
}
return { items, cursor: continueCursor, isDone }
},
})
export const getGitHubSoulBackupSyncStateInternal = internalQuery({
args: {},
handler: async (ctx): Promise<BackupSyncState> => {
const state = await ctx.db
.query('githubBackupSyncState')
.withIndex('by_key', (q) => q.eq('key', SYNC_STATE_KEY))
.unique()
return { cursor: state?.cursor ?? null }
},
})
export const setGitHubSoulBackupSyncStateInternal = internalMutation({
args: {
cursor: v.optional(v.string()),
},
handler: async (ctx, args) => {
const now = Date.now()
const state = await ctx.db
.query('githubBackupSyncState')
.withIndex('by_key', (q) => q.eq('key', SYNC_STATE_KEY))
.unique()
if (!state) {
await ctx.db.insert('githubBackupSyncState', {
key: SYNC_STATE_KEY,
cursor: args.cursor,
updatedAt: now,
})
return { ok: true as const }
}
await ctx.db.patch(state._id, {
cursor: args.cursor,
updatedAt: now,
})
return { ok: true as const }
},
})
export const syncGitHubSoulBackups: ReturnType<typeof action> = action({
args: {
dryRun: v.optional(v.boolean()),
batchSize: v.optional(v.number()),
maxBatches: v.optional(v.number()),
resetCursor: v.optional(v.boolean()),
},
handler: async (ctx, args): Promise<SyncGitHubSoulBackupsResult> => {
const { user } = await requireUserFromAction(ctx)
assertRole(user, ['admin'])
if (args.resetCursor && !args.dryRun) {
await ctx.runMutation(internal.githubSoulBackups.setGitHubSoulBackupSyncStateInternal, {
cursor: undefined,
})
}
return ctx.runAction(internal.githubSoulBackupsNode.syncGitHubSoulBackupsInternal, {
dryRun: args.dryRun,
batchSize: args.batchSize,
maxBatches: args.maxBatches,
}) as Promise<SyncGitHubSoulBackupsResult>
},
})
function clampInt(value: number, min: number, max: number) {
return Math.max(min, Math.min(max, Math.floor(value)))
}
+186
View File
@@ -0,0 +1,186 @@
'use node'
import { v } from 'convex/values'
import { internal } from './_generated/api'
import type { Doc } from './_generated/dataModel'
import type { ActionCtx } from './_generated/server'
import { internalAction } from './_generated/server'
import {
backupSoulToGitHub,
fetchGitHubSoulMeta,
getGitHubSoulBackupContext,
isGitHubSoulBackupConfigured,
} from './lib/githubSoulBackup'
const DEFAULT_BATCH_SIZE = 50
const MAX_BATCH_SIZE = 200
const DEFAULT_MAX_BATCHES = 5
const MAX_MAX_BATCHES = 200
type BackupPageItem =
| {
kind: 'ok'
slug: string
version: string
displayName: string
ownerHandle: string
files: Doc<'soulVersions'>['files']
publishedAt: number
}
| { kind: 'missingLatestVersion' }
| { kind: 'missingVersionDoc' }
| { kind: 'missingOwner' }
export type GitHubSoulBackupSyncStats = {
soulsScanned: number
soulsSkipped: number
soulsBackedUp: number
soulsMissingVersion: number
soulsMissingOwner: number
errors: number
}
export type SyncGitHubSoulBackupsInternalArgs = {
dryRun?: boolean
batchSize?: number
maxBatches?: number
}
export type SyncGitHubSoulBackupsInternalResult = {
stats: GitHubSoulBackupSyncStats
cursor: string | null
isDone: boolean
}
export const backupSoulForPublishInternal = internalAction({
args: {
slug: v.string(),
version: v.string(),
displayName: v.string(),
ownerHandle: v.string(),
files: v.array(
v.object({
path: v.string(),
size: v.number(),
storageId: v.id('_storage'),
sha256: v.string(),
contentType: v.optional(v.string()),
}),
),
publishedAt: v.number(),
},
handler: async (ctx, args) => {
if (!isGitHubSoulBackupConfigured()) {
return { skipped: true as const }
}
await backupSoulToGitHub(ctx, args)
return { skipped: false as const }
},
})
export async function syncGitHubSoulBackupsInternalHandler(
ctx: ActionCtx,
args: SyncGitHubSoulBackupsInternalArgs,
): Promise<SyncGitHubSoulBackupsInternalResult> {
const dryRun = Boolean(args.dryRun)
const stats: GitHubSoulBackupSyncStats = {
soulsScanned: 0,
soulsSkipped: 0,
soulsBackedUp: 0,
soulsMissingVersion: 0,
soulsMissingOwner: 0,
errors: 0,
}
if (!isGitHubSoulBackupConfigured()) {
return { stats, cursor: null, isDone: true }
}
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
const maxBatches = clampInt(args.maxBatches ?? DEFAULT_MAX_BATCHES, 1, MAX_MAX_BATCHES)
const context = await getGitHubSoulBackupContext()
const state = dryRun
? { cursor: null as string | null }
: ((await ctx.runQuery(
internal.githubSoulBackups.getGitHubSoulBackupSyncStateInternal,
{},
)) as {
cursor: string | null
})
let cursor: string | null = state.cursor
let isDone = false
for (let batch = 0; batch < maxBatches; batch++) {
const page = (await ctx.runQuery(internal.githubSoulBackups.getGitHubSoulBackupPageInternal, {
cursor: cursor ?? undefined,
batchSize,
})) as { items: BackupPageItem[]; cursor: string | null; isDone: boolean }
cursor = page.cursor
isDone = page.isDone
for (const item of page.items) {
if (item.kind !== 'ok') {
if (item.kind === 'missingLatestVersion' || item.kind === 'missingVersionDoc') {
stats.soulsMissingVersion += 1
} else if (item.kind === 'missingOwner') {
stats.soulsMissingOwner += 1
}
continue
}
stats.soulsScanned += 1
try {
const meta = await fetchGitHubSoulMeta(context, item.ownerHandle, item.slug)
if (meta?.latest?.version === item.version) {
stats.soulsSkipped += 1
continue
}
if (!dryRun) {
await backupSoulToGitHub(
ctx,
{
slug: item.slug,
version: item.version,
displayName: item.displayName,
ownerHandle: item.ownerHandle,
files: item.files,
publishedAt: item.publishedAt,
},
context,
)
stats.soulsBackedUp += 1
}
} catch (error) {
console.error('GitHub soul backup sync failed', error)
stats.errors += 1
}
}
if (!dryRun) {
await ctx.runMutation(internal.githubSoulBackups.setGitHubSoulBackupSyncStateInternal, {
cursor: isDone ? undefined : (cursor ?? undefined),
})
}
if (isDone) break
}
return { stats, cursor, isDone }
}
export const syncGitHubSoulBackupsInternal = internalAction({
args: {
dryRun: v.optional(v.boolean()),
batchSize: v.optional(v.number()),
maxBatches: v.optional(v.number()),
},
handler: syncGitHubSoulBackupsInternalHandler,
})
function clampInt(value: number, min: number, max: number) {
return Math.max(min, Math.min(max, Math.floor(value)))
}
+108 -9
View File
@@ -1,4 +1,4 @@
import { ApiRoutes } from 'clawdhub-schema'
import { ApiRoutes, LegacyApiRoutes } from 'clawdhub-schema'
import { httpRouter } from 'convex/server'
import { auth } from './auth'
import { downloadZip } from './downloads'
@@ -13,6 +13,21 @@ import {
resolveSkillVersionHttp,
searchSkillsHttp,
} from './httpApi'
import {
listSkillsV1Http,
listSoulsV1Http,
publishSkillV1Http,
publishSoulV1Http,
resolveSkillVersionV1Http,
searchSkillsV1Http,
skillsDeleteRouterV1Http,
skillsGetRouterV1Http,
skillsPostRouterV1Http,
soulsDeleteRouterV1Http,
soulsGetRouterV1Http,
soulsPostRouterV1Http,
whoamiV1Http,
} from './httpApiV1'
const http = httpRouter()
@@ -27,53 +42,137 @@ http.route({
http.route({
path: ApiRoutes.search,
method: 'GET',
handler: searchSkillsV1Http,
})
http.route({
path: ApiRoutes.resolve,
method: 'GET',
handler: resolveSkillVersionV1Http,
})
http.route({
path: ApiRoutes.skills,
method: 'GET',
handler: listSkillsV1Http,
})
http.route({
pathPrefix: `${ApiRoutes.skills}/`,
method: 'GET',
handler: skillsGetRouterV1Http,
})
http.route({
path: ApiRoutes.skills,
method: 'POST',
handler: publishSkillV1Http,
})
http.route({
pathPrefix: `${ApiRoutes.skills}/`,
method: 'POST',
handler: skillsPostRouterV1Http,
})
http.route({
pathPrefix: `${ApiRoutes.skills}/`,
method: 'DELETE',
handler: skillsDeleteRouterV1Http,
})
http.route({
path: ApiRoutes.whoami,
method: 'GET',
handler: whoamiV1Http,
})
http.route({
path: ApiRoutes.souls,
method: 'GET',
handler: listSoulsV1Http,
})
http.route({
pathPrefix: `${ApiRoutes.souls}/`,
method: 'GET',
handler: soulsGetRouterV1Http,
})
http.route({
path: ApiRoutes.souls,
method: 'POST',
handler: publishSoulV1Http,
})
http.route({
pathPrefix: `${ApiRoutes.souls}/`,
method: 'POST',
handler: soulsPostRouterV1Http,
})
http.route({
pathPrefix: `${ApiRoutes.souls}/`,
method: 'DELETE',
handler: soulsDeleteRouterV1Http,
})
// TODO: remove legacy /api routes after deprecation window.
http.route({
path: LegacyApiRoutes.download,
method: 'GET',
handler: downloadZip,
})
http.route({
path: LegacyApiRoutes.search,
method: 'GET',
handler: searchSkillsHttp,
})
http.route({
path: ApiRoutes.skill,
path: LegacyApiRoutes.skill,
method: 'GET',
handler: getSkillHttp,
})
http.route({
path: ApiRoutes.skillResolve,
path: LegacyApiRoutes.skillResolve,
method: 'GET',
handler: resolveSkillVersionHttp,
})
http.route({
path: ApiRoutes.cliWhoami,
path: LegacyApiRoutes.cliWhoami,
method: 'GET',
handler: cliWhoamiHttp,
})
http.route({
path: ApiRoutes.cliUploadUrl,
path: LegacyApiRoutes.cliUploadUrl,
method: 'POST',
handler: cliUploadUrlHttp,
})
http.route({
path: ApiRoutes.cliPublish,
path: LegacyApiRoutes.cliPublish,
method: 'POST',
handler: cliPublishHttp,
})
http.route({
path: ApiRoutes.cliTelemetrySync,
path: LegacyApiRoutes.cliTelemetrySync,
method: 'POST',
handler: cliTelemetrySyncHttp,
})
http.route({
path: ApiRoutes.cliSkillDelete,
path: LegacyApiRoutes.cliSkillDelete,
method: 'POST',
handler: cliSkillDeleteHttp,
})
http.route({
path: ApiRoutes.cliSkillUndelete,
path: LegacyApiRoutes.cliSkillUndelete,
method: 'POST',
handler: cliSkillUndeleteHttp,
})
+143 -17
View File
@@ -11,7 +11,7 @@ vi.mock('./skills', () => ({
const { requireApiTokenUser } = await import('./lib/apiTokenAuth')
const { publishVersionForUser } = await import('./skills')
const { __handlers } = await import('./httpApi')
const { __handlers, cliSkillDeleteHttp, cliSkillUndeleteHttp } = await import('./httpApi')
const { hashSkillFiles } = await import('./lib/skills')
function makeCtx(partial: Record<string, unknown>) {
@@ -55,6 +55,19 @@ describe('httpApi handlers', () => {
expect(json.results[0].slug).toBe('a')
})
it('searchSkillsHttp omits approvedOnly when false', async () => {
const runAction = vi.fn().mockResolvedValue([])
await __handlers.searchSkillsHandler(
makeCtx({ runAction }),
new Request('https://example.com/api/search?q=test&approvedOnly=false'),
)
expect(runAction).toHaveBeenCalledWith(expect.anything(), {
query: 'test',
limit: undefined,
approvedOnly: undefined,
})
})
it('getSkillHttp validates slug', async () => {
const response = await __handlers.getSkillHandler(
makeCtx({ runQuery: vi.fn() }),
@@ -104,6 +117,30 @@ describe('httpApi handlers', () => {
expect(json.owner.handle).toBe('p')
})
it('getSkillHttp returns payload with null owner/latestVersion', async () => {
const runQuery = vi.fn().mockResolvedValue({
skill: {
slug: 'demo',
displayName: 'Demo',
summary: null,
tags: {},
stats: {},
createdAt: 1,
updatedAt: 2,
},
latestVersion: null,
owner: null,
})
const response = await __handlers.getSkillHandler(
makeCtx({ runQuery }),
new Request('https://example.com/api/skill?slug=demo'),
)
expect(response.status).toBe(200)
const json = await response.json()
expect(json.latestVersion).toBeNull()
expect(json.owner).toBeNull()
})
it('resolveSkillVersionHttp validates hash', async () => {
const response = await __handlers.resolveSkillVersionHandler(
makeCtx({ runQuery: vi.fn() }),
@@ -125,22 +162,10 @@ describe('httpApi handlers', () => {
it('resolveSkillVersionHttp returns match and latestVersion', async () => {
const matchHash = await hashSkillFiles([{ path: 'SKILL.md', sha256: 'abc' }])
const runQuery = vi
.fn()
.mockResolvedValueOnce({
skill: {
_id: 's',
slug: 'demo',
displayName: 'Demo',
tags: {},
stats: {},
createdAt: 1,
updatedAt: 2,
},
latestVersion: { version: '2.0.0', createdAt: 3, changelog: 'c' },
owner: null,
})
.mockResolvedValueOnce([{ version: '1.0.0', files: [{ path: 'SKILL.md', sha256: 'abc' }] }])
const runQuery = vi.fn().mockResolvedValueOnce({
match: { version: '1.0.0' },
latestVersion: { version: '2.0.0' },
})
const response = await __handlers.resolveSkillVersionHandler(
makeCtx({ runQuery }),
@@ -198,6 +223,65 @@ describe('httpApi handlers', () => {
expect(runMutation).toHaveBeenCalledTimes(1)
})
it('cliTelemetrySyncHttp returns 400 on invalid payload', async () => {
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: 'users:1' } as never)
const response = await __handlers.cliTelemetrySyncHandler(
makeCtx({ runMutation: vi.fn() }),
new Request('https://x/api/cli/telemetry/sync', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ roots: 'nope' }),
}),
)
expect(response.status).toBe(400)
})
it('cliTelemetrySyncHttp forwards skill versions when provided', async () => {
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: 'users:1' } as never)
const runMutation = vi.fn().mockResolvedValue(null)
await __handlers.cliTelemetrySyncHandler(
makeCtx({ runMutation }),
new Request('https://x/api/cli/telemetry/sync', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
roots: [
{
rootId: 'abc',
label: '~/skills',
skills: [{ slug: 'weather', version: '1.0.0' }],
},
],
}),
}),
)
expect(runMutation).toHaveBeenCalledWith(expect.anything(), {
userId: 'users:1',
roots: [
{ rootId: 'abc', label: '~/skills', skills: [{ slug: 'weather', version: '1.0.0' }] },
],
})
})
it('cliTelemetrySyncHttp returns 400 on invalid json', async () => {
const request = new Request('https://x/api/cli/telemetry/sync', { method: 'POST', body: '{' })
const response = await __handlers.cliTelemetrySyncHandler(makeCtx({}), request)
expect(response.status).toBe(400)
})
it('cliTelemetrySyncHttp returns 401 when unauthorized', async () => {
vi.mocked(requireApiTokenUser).mockRejectedValueOnce(new Error('Unauthorized'))
const response = await __handlers.cliTelemetrySyncHandler(
makeCtx({}),
new Request('https://x/api/cli/telemetry/sync', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ roots: [] }),
}),
)
expect(response.status).toBe(401)
})
it('cliUploadUrlHttp returns uploadUrl', async () => {
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: 'user1' } as never)
const runMutation = vi.fn().mockResolvedValue('https://upload.local')
@@ -328,6 +412,48 @@ describe('httpApi handlers', () => {
})
})
it('cliSkillUndeleteHttp calls delete handler with deleted=false', async () => {
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: 'user1' } as never)
const runMutation = vi.fn().mockResolvedValue({ ok: true })
const response = await cliSkillUndeleteHttp(
makeCtx({ runMutation }),
new Request('https://x/api/cli/skill/undelete', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ slug: 'demo' }),
}),
)
expect(response.status).toBe(200)
expect(runMutation).toHaveBeenCalledWith(expect.anything(), {
userId: 'user1',
slug: 'demo',
deleted: false,
})
warnSpy.mockRestore()
})
it('cliSkillDeleteHttp calls delete handler with deleted=true', async () => {
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: 'user1' } as never)
const runMutation = vi.fn().mockResolvedValue({ ok: true })
const response = await cliSkillDeleteHttp(
makeCtx({ runMutation }),
new Request('https://x/api/cli/skill/delete', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ slug: 'demo' }),
}),
)
expect(response.status).toBe(200)
expect(runMutation).toHaveBeenCalledWith(expect.anything(), {
userId: 'user1',
slug: 'demo',
deleted: true,
})
warnSpy.mockRestore()
})
it('cliSkillDeleteHandler returns 400 on invalid json', async () => {
const request = new Request('https://x/api/cli/skill/delete', { method: 'POST', body: '{' })
const response = await __handlers.cliSkillDeleteHandler(makeCtx({}), request, true)
+23
View File
@@ -25,6 +25,29 @@ describe('httpApi', () => {
expect(parsed.files[0]?.path).toBe('SKILL.md')
})
it('normalizes optional fields in publish payload', () => {
const parsed = __test.parsePublishBody({
slug: 'cool-skill',
displayName: 'Cool Skill',
version: '1.2.3',
changelog: '',
tags: [],
forkOf: { slug: 'base-skill' },
files: [
{
path: 'SKILL.md',
size: 5,
storageId: 'fakeStorageId',
sha256: 'abcd',
contentType: 'text/markdown',
},
],
})
expect(parsed.tags).toBeUndefined()
expect(parsed.source).toBeUndefined()
expect(parsed.forkOf).toEqual({ slug: 'base-skill', version: undefined })
})
it('rejects invalid publish payloads', () => {
expect(() => __test.parsePublishBody(null)).toThrow(/Publish payload/i)
expect(() =>
+10 -21
View File
@@ -11,7 +11,6 @@ import type { Id } from './_generated/dataModel'
import type { ActionCtx } from './_generated/server'
import { httpAction } from './_generated/server'
import { requireApiTokenUser } from './lib/apiTokenAuth'
import { hashSkillFiles } from './lib/skills'
import { publishVersionForUser } from './skills'
type SearchSkillEntry = {
@@ -112,27 +111,10 @@ async function resolveSkillVersionHandler(ctx: ActionCtx, request: Request) {
if (!slug || !hash) return text('Missing slug or hash', 400)
if (!/^[a-f0-9]{64}$/.test(hash)) return text('Invalid hash', 400)
const result = (await ctx.runQuery(api.skills.getBySlug, { slug })) as GetBySlugResult
if (!result?.skill) return text('Skill not found', 404)
const resolved = await ctx.runQuery(api.skills.resolveVersionByHash, { slug, hash })
if (!resolved) return text('Skill not found', 404)
const versions = (await ctx.runQuery(api.skills.listVersions, {
skillId: result.skill._id,
limit: 200,
})) as Array<{ version: string; files: Array<{ path: string; sha256: string }> }>
let match: { version: string } | null = null
for (const version of versions) {
const fingerprint = await hashSkillFiles(version.files)
if (fingerprint === hash) {
match = { version: version.version }
break
}
}
return json({
slug,
match,
latestVersion: result.latestVersion ? { version: result.latestVersion.version } : null,
})
return json({ slug, match: resolved.match, latestVersion: resolved.latestVersion })
}
export const resolveSkillVersionHttp = httpAction(resolveSkillVersionHandler)
@@ -291,6 +273,13 @@ function parsePublishBody(body: unknown) {
version: parsed.version,
changelog: parsed.changelog,
tags,
source: parsed.source ?? undefined,
forkOf: parsed.forkOf
? {
slug: parsed.forkOf.slug,
version: parsed.forkOf.version ?? undefined,
}
: undefined,
files: parsed.files.map((file) => ({
...file,
storageId: file.storageId as Id<'_storage'>,
+501
View File
@@ -0,0 +1,501 @@
/* @vitest-environment node */
import { beforeEach, describe, expect, it, vi } from 'vitest'
vi.mock('./lib/apiTokenAuth', () => ({
requireApiTokenUser: vi.fn(),
}))
vi.mock('./skills', () => ({
publishVersionForUser: vi.fn(),
}))
const { requireApiTokenUser } = await import('./lib/apiTokenAuth')
const { publishVersionForUser } = await import('./skills')
const { __handlers } = await import('./httpApiV1')
type ActionCtx = import('./_generated/server').ActionCtx
function makeCtx(partial: Record<string, unknown>) {
return partial as unknown as ActionCtx
}
const okRate = () => ({
allowed: true,
remaining: 10,
limit: 100,
resetAt: Date.now() + 60_000,
})
const blockedRate = () => ({
allowed: false,
remaining: 0,
limit: 100,
resetAt: Date.now() + 60_000,
})
beforeEach(() => {
vi.mocked(requireApiTokenUser).mockReset()
vi.mocked(publishVersionForUser).mockReset()
})
describe('httpApiV1 handlers', () => {
it('search returns empty results for blank query', async () => {
const runAction = vi.fn()
const runMutation = vi.fn().mockResolvedValue(okRate())
const response = await __handlers.searchSkillsV1Handler(
makeCtx({ runAction, runMutation }),
new Request('https://example.com/api/v1/search?q=%20%20'),
)
if (response.status !== 200) {
throw new Error(await response.text())
}
expect(await response.json()).toEqual({ results: [] })
expect(runAction).not.toHaveBeenCalled()
})
it('search forwards limit and highlightedOnly', async () => {
const runAction = vi.fn().mockResolvedValue([
{
score: 1,
skill: { slug: 'a', displayName: 'A', summary: null, updatedAt: 1 },
version: { version: '1.0.0' },
},
])
const runMutation = vi.fn().mockResolvedValue(okRate())
const response = await __handlers.searchSkillsV1Handler(
makeCtx({ runAction, runMutation }),
new Request('https://example.com/api/v1/search?q=test&limit=5&highlightedOnly=true'),
)
if (response.status !== 200) {
throw new Error(await response.text())
}
expect(runAction).toHaveBeenCalledWith(expect.anything(), {
query: 'test',
limit: 5,
highlightedOnly: true,
})
})
it('search rate limits', async () => {
const runMutation = vi.fn().mockResolvedValue(blockedRate())
const response = await __handlers.searchSkillsV1Handler(
makeCtx({ runAction: vi.fn(), runMutation }),
new Request('https://example.com/api/v1/search?q=test'),
)
expect(response.status).toBe(429)
})
it('resolve validates hash', async () => {
const runMutation = vi.fn().mockResolvedValue(okRate())
const response = await __handlers.resolveSkillVersionV1Handler(
makeCtx({ runQuery: vi.fn(), runMutation }),
new Request('https://example.com/api/v1/resolve?slug=demo&hash=bad'),
)
expect(response.status).toBe(400)
})
it('resolve returns 404 when missing', async () => {
const runQuery = vi.fn().mockResolvedValue(null)
const runMutation = vi.fn().mockResolvedValue(okRate())
const response = await __handlers.resolveSkillVersionV1Handler(
makeCtx({ runQuery, runMutation }),
new Request(
'https://example.com/api/v1/resolve?slug=demo&hash=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
),
)
expect(response.status).toBe(404)
})
it('resolve returns match and latestVersion', async () => {
const runQuery = vi.fn().mockResolvedValue({
match: { version: '1.0.0' },
latestVersion: { version: '2.0.0' },
})
const runMutation = vi.fn().mockResolvedValue(okRate())
const response = await __handlers.resolveSkillVersionV1Handler(
makeCtx({ runQuery, runMutation }),
new Request(
'https://example.com/api/v1/resolve?slug=demo&hash=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
),
)
expect(response.status).toBe(200)
const json = await response.json()
expect(json.match.version).toBe('1.0.0')
})
it('lists skills with resolved tags', async () => {
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
if ('cursor' in args || 'limit' in args) {
return {
items: [
{
skill: {
_id: 'skills:1',
slug: 'demo',
displayName: 'Demo',
summary: 's',
tags: { latest: 'versions:1' },
stats: { downloads: 0, stars: 0, versions: 1, comments: 0 },
createdAt: 1,
updatedAt: 2,
},
latestVersion: { version: '1.0.0', createdAt: 3, changelog: 'c' },
},
],
nextCursor: null,
}
}
if ('versionId' in args) return { version: '1.0.0' }
return null
})
const runMutation = vi.fn().mockResolvedValue(okRate())
const response = await __handlers.listSkillsV1Handler(
makeCtx({ runQuery, runMutation }),
new Request('https://example.com/api/v1/skills?limit=1'),
)
expect(response.status).toBe(200)
const json = await response.json()
expect(json.items[0].tags.latest).toBe('1.0.0')
})
it('get skill returns 404 when missing', async () => {
const runQuery = vi.fn().mockResolvedValue(null)
const runMutation = vi.fn().mockResolvedValue(okRate())
const response = await __handlers.skillsGetRouterV1Handler(
makeCtx({ runQuery, runMutation }),
new Request('https://example.com/api/v1/skills/missing'),
)
expect(response.status).toBe(404)
})
it('get skill returns payload', async () => {
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
if ('slug' in args) {
return {
skill: {
_id: 'skills:1',
slug: 'demo',
displayName: 'Demo',
summary: 's',
tags: { latest: 'versions:1' },
stats: { downloads: 0, stars: 0, versions: 1, comments: 0 },
createdAt: 1,
updatedAt: 2,
},
latestVersion: {
version: '1.0.0',
createdAt: 3,
changelog: 'c',
files: [],
},
owner: { handle: 'p', displayName: 'Peter', image: null },
}
}
if ('versionId' in args) return { version: '1.0.0' }
return null
})
const runMutation = vi.fn().mockResolvedValue(okRate())
const response = await __handlers.skillsGetRouterV1Handler(
makeCtx({ runQuery, runMutation }),
new Request('https://example.com/api/v1/skills/demo'),
)
expect(response.status).toBe(200)
const json = await response.json()
expect(json.skill.slug).toBe('demo')
expect(json.latestVersion.version).toBe('1.0.0')
})
it('lists versions', async () => {
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
if ('slug' in args) {
return { _id: 'skills:1', slug: 'demo', displayName: 'Demo' }
}
if ('skillId' in args && 'cursor' in args) {
return {
items: [
{
version: '1.0.0',
createdAt: 1,
changelog: 'c',
changelogSource: 'user',
files: [],
},
],
nextCursor: null,
}
}
return null
})
const runMutation = vi.fn().mockResolvedValue(okRate())
const response = await __handlers.skillsGetRouterV1Handler(
makeCtx({ runQuery, runMutation }),
new Request('https://example.com/api/v1/skills/demo/versions?limit=1'),
)
expect(response.status).toBe(200)
const json = await response.json()
expect(json.items[0].version).toBe('1.0.0')
})
it('returns version detail', async () => {
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
if ('slug' in args) {
return { _id: 'skills:1', slug: 'demo', displayName: 'Demo' }
}
if ('skillId' in args && 'version' in args) {
return {
version: '1.0.0',
createdAt: 1,
changelog: 'c',
changelogSource: 'auto',
files: [
{
path: 'SKILL.md',
size: 1,
storageId: 'storage:1',
sha256: 'abc',
contentType: 'text/plain',
},
],
}
}
return null
})
const runMutation = vi.fn().mockResolvedValue(okRate())
const response = await __handlers.skillsGetRouterV1Handler(
makeCtx({ runQuery, runMutation }),
new Request('https://example.com/api/v1/skills/demo/versions/1.0.0'),
)
expect(response.status).toBe(200)
const json = await response.json()
expect(json.version.files[0].path).toBe('SKILL.md')
})
it('returns raw file content', async () => {
const version = {
version: '1.0.0',
createdAt: 1,
changelog: 'c',
files: [
{
path: 'SKILL.md',
size: 5,
storageId: 'storage:1',
sha256: 'abcd',
contentType: 'text/plain',
},
],
softDeletedAt: undefined,
}
const runQuery = vi.fn().mockResolvedValue({
skill: {
_id: 'skills:1',
slug: 'demo',
displayName: 'Demo',
summary: 's',
tags: {},
stats: {},
createdAt: 1,
updatedAt: 2,
},
latestVersion: version,
owner: null,
})
const runMutation = vi.fn().mockResolvedValue(okRate())
const storage = {
get: vi.fn().mockResolvedValue(new Blob(['hello'], { type: 'text/plain' })),
}
const response = await __handlers.skillsGetRouterV1Handler(
makeCtx({ runQuery, runMutation, storage }),
new Request('https://example.com/api/v1/skills/demo/file?path=SKILL.md'),
)
expect(response.status).toBe(200)
expect(await response.text()).toBe('hello')
expect(response.headers.get('X-Content-SHA256')).toBe('abcd')
})
it('returns 413 when raw file too large', async () => {
const version = {
version: '1.0.0',
createdAt: 1,
changelog: 'c',
files: [
{
path: 'SKILL.md',
size: 210 * 1024,
storageId: 'storage:1',
sha256: 'abcd',
contentType: 'text/plain',
},
],
softDeletedAt: undefined,
}
const runQuery = vi.fn().mockResolvedValue({
skill: {
_id: 'skills:1',
slug: 'demo',
displayName: 'Demo',
summary: 's',
tags: {},
stats: {},
createdAt: 1,
updatedAt: 2,
},
latestVersion: version,
owner: null,
})
const runMutation = vi.fn().mockResolvedValue(okRate())
const response = await __handlers.skillsGetRouterV1Handler(
makeCtx({ runQuery, runMutation, storage: { get: vi.fn() } }),
new Request('https://example.com/api/v1/skills/demo/file?path=SKILL.md'),
)
expect(response.status).toBe(413)
})
it('publish json succeeds', async () => {
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({
userId: 'users:1',
user: { handle: 'p' },
} as never)
vi.mocked(publishVersionForUser).mockResolvedValueOnce({
skillId: 's',
versionId: 'v',
embeddingId: 'e',
} as never)
const runMutation = vi.fn().mockResolvedValue(okRate())
const body = JSON.stringify({
slug: 'demo',
displayName: 'Demo',
version: '1.0.0',
changelog: 'c',
files: [
{
path: 'SKILL.md',
size: 1,
storageId: 'storage:1',
sha256: 'abc',
contentType: 'text/plain',
},
],
})
const response = await __handlers.publishSkillV1Handler(
makeCtx({ runMutation }),
new Request('https://example.com/api/v1/skills', {
method: 'POST',
headers: { 'Content-Type': 'application/json', Authorization: 'Bearer clh_test' },
body,
}),
)
expect(response.status).toBe(200)
const json = await response.json()
expect(json.ok).toBe(true)
expect(publishVersionForUser).toHaveBeenCalled()
})
it('publish multipart succeeds', async () => {
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({
userId: 'users:1',
user: { handle: 'p' },
} as never)
vi.mocked(publishVersionForUser).mockResolvedValueOnce({
skillId: 's',
versionId: 'v',
embeddingId: 'e',
} as never)
const runMutation = vi.fn().mockResolvedValue(okRate())
const form = new FormData()
form.set(
'payload',
JSON.stringify({
slug: 'demo',
displayName: 'Demo',
version: '1.0.0',
changelog: '',
tags: ['latest'],
}),
)
form.append('files', new Blob(['hello'], { type: 'text/plain' }), 'SKILL.md')
const response = await __handlers.publishSkillV1Handler(
makeCtx({ runMutation, storage: { store: vi.fn().mockResolvedValue('storage:1') } }),
new Request('https://example.com/api/v1/skills', {
method: 'POST',
headers: { Authorization: 'Bearer clh_test' },
body: form,
}),
)
if (response.status !== 200) {
throw new Error(await response.text())
}
})
it('publish rejects missing token', async () => {
const runMutation = vi.fn().mockResolvedValue(okRate())
const response = await __handlers.publishSkillV1Handler(
makeCtx({ runMutation }),
new Request('https://example.com/api/v1/skills', { method: 'POST' }),
)
expect(response.status).toBe(401)
})
it('whoami returns user payload', async () => {
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({
userId: 'users:1',
user: { handle: 'p', displayName: 'Peter', image: null },
} as never)
const runMutation = vi.fn().mockResolvedValue(okRate())
const response = await __handlers.whoamiV1Handler(
makeCtx({ runMutation }),
new Request('https://example.com/api/v1/whoami', {
headers: { Authorization: 'Bearer clh_test' },
}),
)
expect(response.status).toBe(200)
const json = await response.json()
expect(json.user.handle).toBe('p')
})
it('delete and undelete require auth', async () => {
vi.mocked(requireApiTokenUser).mockRejectedValueOnce(new Error('Unauthorized'))
const runMutation = vi.fn().mockResolvedValue(okRate())
const response = await __handlers.skillsDeleteRouterV1Handler(
makeCtx({ runMutation }),
new Request('https://example.com/api/v1/skills/demo', { method: 'DELETE' }),
)
expect(response.status).toBe(401)
vi.mocked(requireApiTokenUser).mockRejectedValueOnce(new Error('Unauthorized'))
const response2 = await __handlers.skillsPostRouterV1Handler(
makeCtx({ runMutation }),
new Request('https://example.com/api/v1/skills/demo/undelete', { method: 'POST' }),
)
expect(response2.status).toBe(401)
})
it('delete and undelete succeed', async () => {
vi.mocked(requireApiTokenUser).mockResolvedValue({
userId: 'users:1',
user: { handle: 'p' },
} as never)
const runMutation = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
if ('key' in args) return okRate()
return { ok: true }
})
const response = await __handlers.skillsDeleteRouterV1Handler(
makeCtx({ runMutation }),
new Request('https://example.com/api/v1/skills/demo', {
method: 'DELETE',
headers: { Authorization: 'Bearer clh_test' },
}),
)
expect(response.status).toBe(200)
const response2 = await __handlers.skillsPostRouterV1Handler(
makeCtx({ runMutation }),
new Request('https://example.com/api/v1/skills/demo/undelete', {
method: 'POST',
headers: { Authorization: 'Bearer clh_test' },
}),
)
expect(response2.status).toBe(200)
})
})
+1064
View File
File diff suppressed because it is too large Load Diff
+8 -1
View File
@@ -1,9 +1,16 @@
export const EMBEDDING_MODEL = 'text-embedding-3-small'
export const EMBEDDING_DIMENSIONS = 1536
function emptyEmbedding() {
return Array.from({ length: EMBEDDING_DIMENSIONS }, () => 0)
}
export async function generateEmbedding(text: string) {
const apiKey = process.env.OPENAI_API_KEY
if (!apiKey) throw new Error('OPENAI_API_KEY is not configured')
if (!apiKey) {
console.warn('OPENAI_API_KEY is not configured; using zero embeddings')
return emptyEmbedding()
}
const response = await fetch('https://api.openai.com/v1/embeddings', {
method: 'POST',
+443
View File
@@ -0,0 +1,443 @@
'use node'
import { createPrivateKey, createSign } from 'node:crypto'
import type { Id } from '../_generated/dataModel'
import type { ActionCtx } from '../_generated/server'
const GITHUB_API = 'https://api.github.com'
const DEFAULT_REPO = 'clawdbot/skills'
const DEFAULT_ROOT = 'skills'
const META_FILENAME = '_meta.json'
const USER_AGENT = 'clawdhub/skills-backup'
type BackupFile = {
path: string
size: number
storageId: Id<'_storage'>
sha256: string
contentType?: string
}
type BackupParams = {
slug: string
version: string
displayName: string
ownerHandle: string
files: BackupFile[]
publishedAt: number
}
type RepoInfo = {
default_branch?: string
}
type GitRef = {
object: { sha: string }
}
type GitCommit = {
sha: string
tree: { sha: string }
}
type GitTreeEntry = {
path?: string
type?: string
}
type GitTree = {
tree?: GitTreeEntry[]
}
type MetaFile = {
owner: string
slug: string
displayName: string
latest: {
version: string
publishedAt: number
commit: string | null
}
history: Array<{
version: string
publishedAt: number
commit: string
}>
}
export type GitHubBackupContext = {
token: string
repo: string
repoOwner: string
repoName: string
branch: string
root: string
}
export function isGitHubBackupConfigured() {
return Boolean(
process.env.GITHUB_APP_ID &&
process.env.GITHUB_APP_PRIVATE_KEY &&
process.env.GITHUB_APP_INSTALLATION_ID,
)
}
export async function getGitHubBackupContext(): Promise<GitHubBackupContext> {
const repo = process.env.GITHUB_SKILLS_REPO ?? DEFAULT_REPO
const root = process.env.GITHUB_SKILLS_ROOT ?? DEFAULT_ROOT
const [repoOwner, repoName] = parseRepo(repo)
const token = await createInstallationToken()
const repoInfo = await githubGet<RepoInfo>(token, `/repos/${repoOwner}/${repoName}`)
const branch = repoInfo.default_branch ?? 'main'
return { token, repo, repoOwner, repoName, branch, root }
}
export async function fetchGitHubSkillMeta(
context: GitHubBackupContext,
ownerHandle: string,
slug: string,
): Promise<MetaFile | null> {
const skillRoot = buildSkillRoot(context.root, ownerHandle, slug)
return fetchMetaFile(
context.token,
context.repoOwner,
context.repoName,
`${skillRoot}/${META_FILENAME}`,
context.branch,
)
}
export async function backupSkillToGitHub(
ctx: ActionCtx,
params: BackupParams,
context?: GitHubBackupContext,
) {
if (!isGitHubBackupConfigured()) return
const resolved = context ?? (await getGitHubBackupContext())
const skillRoot = buildSkillRoot(resolved.root, params.ownerHandle, params.slug)
const ref = await githubGet<GitRef>(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/ref/heads/${resolved.branch}`,
)
const baseCommitSha = ref.object.sha
const baseCommit = await githubGet<GitCommit>(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/commits/${baseCommitSha}`,
)
const baseTreeSha = baseCommit.tree.sha
const existingTree = await githubGet<GitTree>(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/trees/${baseTreeSha}?recursive=1`,
)
const prefix = `${skillRoot}/`
const existingPaths = new Set(
(existingTree.tree ?? [])
.filter((entry) => entry.type === 'blob' && entry.path?.startsWith(prefix))
.map((entry) => entry.path ?? ''),
)
const newPaths = new Set<string>()
const treeEntries: Array<{
path: string
mode: '100644'
type: 'blob'
sha: string | null
}> = []
for (const file of params.files) {
const content = await fetchStorageBase64(ctx, file.storageId)
const blobSha = await createBlob(resolved.token, resolved.repoOwner, resolved.repoName, content)
const path = `${skillRoot}/${file.path}`
newPaths.add(path)
treeEntries.push({ path, mode: '100644', type: 'blob', sha: blobSha })
}
const existingMeta = await fetchMetaFile(
resolved.token,
resolved.repoOwner,
resolved.repoName,
`${skillRoot}/${META_FILENAME}`,
resolved.branch,
)
const metaPath = `${skillRoot}/${META_FILENAME}`
const metaDraft = buildMetaFile(params, existingMeta, resolved.repo, baseCommitSha, null)
const metaDraftContent = `${JSON.stringify(metaDraft, null, 2)}\n`
const metaDraftSha = await createBlob(
resolved.token,
resolved.repoOwner,
resolved.repoName,
toBase64(metaDraftContent),
)
newPaths.add(metaPath)
treeEntries.push({ path: metaPath, mode: '100644', type: 'blob', sha: metaDraftSha })
for (const path of existingPaths) {
if (newPaths.has(path)) continue
treeEntries.push({ path, mode: '100644', type: 'blob', sha: null })
}
const newTree = await githubPost<{ sha: string }>(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/trees`,
{
base_tree: baseTreeSha,
tree: treeEntries,
},
)
const commit = await githubPost<GitCommit>(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/commits`,
{
message: `skill: ${params.slug} v${params.version}`,
tree: newTree.sha,
parents: [baseCommitSha],
},
)
const metaFinal = buildMetaFile(params, existingMeta, resolved.repo, baseCommitSha, commit.sha)
const metaFinalContent = `${JSON.stringify(metaFinal, null, 2)}\n`
const metaFinalSha = await createBlob(
resolved.token,
resolved.repoOwner,
resolved.repoName,
toBase64(metaFinalContent),
)
const metaTree = await githubPost<{ sha: string }>(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/trees`,
{
base_tree: commit.tree.sha,
tree: [{ path: metaPath, mode: '100644', type: 'blob', sha: metaFinalSha }],
},
)
const metaCommit = await githubPost<GitCommit>(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/commits`,
{
message: `meta: ${params.slug} v${params.version}`,
tree: metaTree.sha,
parents: [commit.sha],
},
)
await githubPatch(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/refs/heads/${resolved.branch}`,
{
sha: metaCommit.sha,
},
)
}
function buildMetaFile(
params: BackupParams,
existing: MetaFile | null,
repo: string,
baseCommitSha: string,
latestCommitSha: string | null,
): MetaFile {
let history = [...(existing?.history ?? [])]
if (existing?.latest?.version) {
const previousCommit = existing.latest.commit ?? commitUrl(repo, baseCommitSha)
const previous = {
version: existing.latest.version,
publishedAt: existing.latest.publishedAt,
commit: previousCommit,
}
history = [previous, ...history.filter((entry) => entry.version !== previous.version)]
}
return {
owner: normalizeOwner(params.ownerHandle),
slug: params.slug,
displayName: params.displayName,
latest: {
version: params.version,
publishedAt: params.publishedAt,
commit: latestCommitSha ? commitUrl(repo, latestCommitSha) : null,
},
history: history.slice(0, 200),
}
}
async function fetchMetaFile(
token: string,
repoOwner: string,
repoName: string,
path: string,
branch: string,
): Promise<MetaFile | null> {
try {
const response = await githubGet<{ content?: string }>(
token,
`/repos/${repoOwner}/${repoName}/contents/${encodePath(path)}?ref=${branch}`,
)
if (!response.content) return null
const raw = fromBase64(response.content)
return JSON.parse(raw) as MetaFile
} catch (error) {
if (isNotFoundError(error)) return null
throw error
}
}
async function fetchStorageBase64(ctx: ActionCtx, storageId: Id<'_storage'>) {
const blob = await ctx.storage.get(storageId)
if (!blob) throw new Error('File missing in storage')
const buffer = Buffer.from(await blob.arrayBuffer())
return buffer.toString('base64')
}
async function createInstallationToken() {
const appId = process.env.GITHUB_APP_ID
const installationId = process.env.GITHUB_APP_INSTALLATION_ID
if (!appId || !installationId) {
throw new Error('GitHub App credentials missing')
}
const jwt = createAppJwt(appId)
const response = await fetch(`${GITHUB_API}/app/installations/${installationId}/access_tokens`, {
method: 'POST',
headers: buildHeaders(jwt, true),
})
if (!response.ok) {
const message = await response.text()
throw new Error(`GitHub App token failed: ${message}`)
}
const payload = (await response.json()) as { token?: string }
if (!payload.token) throw new Error('GitHub App token missing')
return payload.token
}
function createAppJwt(appId: string) {
const privateKey = loadPrivateKey()
const now = Math.floor(Date.now() / 1000)
const header = { alg: 'RS256', typ: 'JWT' }
const payload = { iat: now - 60, exp: now + 9 * 60, iss: appId }
const encodedHeader = base64Url(JSON.stringify(header))
const encodedPayload = base64Url(JSON.stringify(payload))
const signingInput = `${encodedHeader}.${encodedPayload}`
const sign = createSign('RSA-SHA256')
sign.update(signingInput)
sign.end()
const signature = sign.sign(privateKey)
return `${signingInput}.${base64Url(signature)}`
}
function loadPrivateKey() {
const raw = process.env.GITHUB_APP_PRIVATE_KEY
if (!raw) throw new Error('GITHUB_APP_PRIVATE_KEY is not configured')
const normalized = raw.replace(/\\n/g, '\n')
return createPrivateKey(normalized)
}
async function createBlob(token: string, repoOwner: string, repoName: string, content: string) {
const result = await githubPost<{ sha: string }>(
token,
`/repos/${repoOwner}/${repoName}/git/blobs`,
{
content,
encoding: 'base64',
},
)
if (!result.sha) throw new Error('GitHub blob missing sha')
return result.sha
}
async function githubGet<T>(token: string, path: string): Promise<T> {
const response = await fetch(`${GITHUB_API}${path}`, {
headers: buildHeaders(token),
})
if (!response.ok) {
const message = await response.text()
throw new Error(`GitHub GET ${path} failed: ${message}`)
}
return (await response.json()) as T
}
async function githubPost<T>(token: string, path: string, body: unknown): Promise<T> {
const response = await fetch(`${GITHUB_API}${path}`, {
method: 'POST',
headers: buildHeaders(token),
body: JSON.stringify(body),
})
if (!response.ok) {
const message = await response.text()
throw new Error(`GitHub POST ${path} failed: ${message}`)
}
return (await response.json()) as T
}
async function githubPatch(token: string, path: string, body: unknown) {
const response = await fetch(`${GITHUB_API}${path}`, {
method: 'PATCH',
headers: buildHeaders(token),
body: JSON.stringify(body),
})
if (!response.ok) {
const message = await response.text()
throw new Error(`GitHub PATCH ${path} failed: ${message}`)
}
}
function buildHeaders(token: string, isAppJwt = false) {
return {
Authorization: `${isAppJwt ? 'Bearer' : 'token'} ${token}`,
Accept: 'application/vnd.github+json',
'User-Agent': USER_AGENT,
}
}
function parseRepo(repo: string) {
const [owner, name] = repo.split('/')
if (!owner || !name) throw new Error('GITHUB_SKILLS_REPO must be owner/repo')
return [owner, name] as const
}
function normalizeOwner(value: string) {
const normalized = value
.trim()
.toLowerCase()
.replace(/[^a-z0-9-]/g, '-')
.replace(/-+/g, '-')
.replace(/^-+|-+$/g, '')
return normalized || 'unknown'
}
function commitUrl(repo: string, sha: string) {
return `https://github.com/${repo}/commit/${sha}`
}
function buildSkillRoot(root: string, ownerHandle: string, slug: string) {
const ownerSegment = normalizeOwner(ownerHandle)
return `${root}/${ownerSegment}/${slug}`
}
function encodePath(path: string) {
return path
.split('/')
.map((segment) => encodeURIComponent(segment))
.join('/')
}
function base64Url(value: string | Buffer) {
const buffer = typeof value === 'string' ? Buffer.from(value) : value
return buffer.toString('base64').replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/g, '')
}
function toBase64(value: string) {
return Buffer.from(value).toString('base64')
}
function fromBase64(value: string) {
return Buffer.from(value, 'base64').toString('utf8')
}
function isNotFoundError(error: unknown) {
return (
error instanceof Error && (error.message.includes('404') || error.message.includes('Not Found'))
)
}
+247
View File
@@ -0,0 +1,247 @@
/* @vitest-environment node */
import { unzipSync } from 'fflate'
import { describe, expect, it } from 'vitest'
import {
buildGitHubZipForTests,
computeDefaultSelectedPaths,
detectGitHubImportCandidates,
extractMarkdownRelativeTargets,
fetchGitHubZipBytes,
parseGitHubImportUrl,
resolveGitHubCommit,
resolveMarkdownTarget,
stripGitHubZipRoot,
} from './githubImport'
function requestInfoToUrlString(input: RequestInfo | URL): string {
if (typeof input === 'string') return input
if (input instanceof URL) return input.toString()
if (input instanceof Request) return input.url
throw new Error('Unexpected fetch input type')
}
describe('github import', () => {
it('parses repo root urls', () => {
expect(parseGitHubImportUrl('https://github.com/visionik/ouracli')).toEqual({
owner: 'visionik',
repo: 'ouracli',
originalUrl: 'https://github.com/visionik/ouracli',
})
})
it('rejects non-https and non-github urls', () => {
expect(() => parseGitHubImportUrl('http://github.com/a/b')).toThrow(/https/i)
expect(() => parseGitHubImportUrl('https://example.com/a/b')).toThrow(/github\.com/i)
expect(() => parseGitHubImportUrl('not-a-url')).toThrow(/Invalid URL/i)
})
it('rejects malformed tree/blob urls', () => {
expect(() => parseGitHubImportUrl('https://github.com/a/b/tree/')).toThrow(/Missing ref/i)
expect(() => parseGitHubImportUrl('https://github.com/a/b/blob/main')).toThrow(/Missing path/i)
expect(() => parseGitHubImportUrl('https://github.com/a/b/tree/main/bad%5cpath')).toThrow()
})
it('parses tree urls with ref and path', () => {
expect(parseGitHubImportUrl('https://github.com/a/b/tree/main/skills/foo')).toEqual({
owner: 'a',
repo: 'b',
ref: 'main',
path: 'skills/foo',
originalUrl: 'https://github.com/a/b/tree/main/skills/foo',
})
})
it('parses blob urls and derives folder path', () => {
expect(parseGitHubImportUrl('https://github.com/a/b/blob/main/skills/foo/SKILL.md')).toEqual({
owner: 'a',
repo: 'b',
ref: 'main',
path: 'skills/foo',
originalUrl: 'https://github.com/a/b/blob/main/skills/foo/SKILL.md',
})
})
it('strips single top-level folder from GitHub zip entries', () => {
const zip = buildGitHubZipForTests({
'repo-1/skill/SKILL.md': 'Body',
'repo-1/skill/a.txt': 'a',
})
const stripped = stripGitHubZipRoot(unzipSync(zip))
expect(Object.keys(stripped).sort()).toEqual(['skill/SKILL.md', 'skill/a.txt'])
})
it('keeps paths when zip has multiple top-level roots', () => {
const zip = buildGitHubZipForTests({
'a/SKILL.md': 'Body',
'b/SKILL.md': 'Body',
})
const stripped = stripGitHubZipRoot(unzipSync(zip))
expect(Object.keys(stripped).sort()).toEqual(['a/SKILL.md', 'b/SKILL.md'])
})
it('detects candidates in a GitHub zip and strips the root folder', () => {
const zip = buildGitHubZipForTests({
'ouracli-123/SKILL.md': `---\nname: demo\ndescription: Hello\n---\nBody`,
'ouracli-123/src/index.ts': 'export {}',
})
const stripped = stripGitHubZipRoot(unzipSync(zip))
const candidates = detectGitHubImportCandidates(stripped)
expect(candidates.map((c) => c.path)).toEqual([''])
expect(candidates[0]?.name).toBe('demo')
})
it('detects multiple candidates and supports skills.md', () => {
const zip = buildGitHubZipForTests({
'repo-1/alpha/SKILL.md': `---\nname: Alpha\n---\nBody`,
'repo-1/beta/skills.md': `---\nname: Beta\n---\nBody`,
'repo-1/readme.md': 'x',
})
const stripped = stripGitHubZipRoot(unzipSync(zip))
const candidates = detectGitHubImportCandidates(stripped)
expect(candidates.map((c) => c.path)).toEqual(['alpha', 'beta'])
expect(candidates.map((c) => c.name)).toEqual(['Alpha', 'Beta'])
})
it('computes default selection via markdown references', () => {
const entries = {
'skill/SKILL.md': `---\nname: demo\n---\nSee [usage](docs/usage.md) and ![logo](img/logo.svg).\nIgnore [web](https://example.com).`,
'skill/docs/usage.md': `See [more](more.md)`,
'skill/docs/more.md': `Ok`,
'skill/img/logo.svg': `<svg/>`,
'skill/extra.txt': 'not referenced',
}
const zip = buildGitHubZipForTests(
Object.fromEntries(Object.entries(entries).map(([k, v]) => [`repo-1/${k}`, v])),
)
const raw = unzipSync(zip)
const stripped = stripGitHubZipRoot(raw)
const candidates = detectGitHubImportCandidates(stripped)
const candidate = candidates.find((c) => c.path === 'skill')
expect(candidate).toBeTruthy()
if (!candidate) throw new Error('candidate not found')
const files = Object.entries(stripped)
.filter(([path]) => path.startsWith('skill/'))
.map(([path, bytes]) => ({ path, bytes }))
const selected = computeDefaultSelectedPaths({ candidate, files })
expect(selected).toContain('skill/SKILL.md')
expect(selected).toContain('skill/docs/usage.md')
expect(selected).toContain('skill/docs/more.md')
expect(selected).toContain('skill/img/logo.svg')
expect(selected).not.toContain('skill/extra.txt')
})
it('does not select files outside skill folder (even when referenced)', () => {
const entries = {
'skill/SKILL.md': `See [outside](../outside.md) and [abs](/abs.md) and [mail](mailto:test@example.com).`,
'outside.md': `secret`,
'skill/docs/usage.md': `Ok`,
}
const zip = buildGitHubZipForTests(
Object.fromEntries(Object.entries(entries).map(([k, v]) => [`repo-1/${k}`, v])),
)
const stripped = stripGitHubZipRoot(unzipSync(zip))
const candidate = detectGitHubImportCandidates(stripped).find((c) => c.path === 'skill')
expect(candidate).toBeTruthy()
if (!candidate) throw new Error('candidate not found')
const files = Object.entries(stripped).map(([path, bytes]) => ({ path, bytes }))
const selected = computeDefaultSelectedPaths({ candidate, files })
expect(selected).toContain('skill/SKILL.md')
expect(selected).not.toContain('outside.md')
})
it('extracts markdown targets with titles and angle brackets', () => {
const targets = extractMarkdownRelativeTargets(
`See [a](docs/usage.md "Title") and [b](<docs/my file.md>) and ![c](img/logo.svg)`,
)
expect(targets).toEqual(['docs/usage.md', 'docs/my file.md', 'img/logo.svg'])
})
it('resolves markdown targets safely', () => {
expect(resolveMarkdownTarget('a/SKILL.md', 'docs/usage.md')).toBe('a/docs/usage.md')
expect(resolveMarkdownTarget('a/SKILL.md', '../oops.md')).toBeNull()
expect(resolveMarkdownTarget('a/SKILL.md', '/abs.md')).toBeNull()
expect(resolveMarkdownTarget('a/SKILL.md', 'docs/usage.md#section')).toBe('a/docs/usage.md')
expect(resolveMarkdownTarget('a/SKILL.md', 'docs/usage.md?x=1')).toBe('a/docs/usage.md')
})
it('resolves HEAD commit via redirect chain and refuses unexpected redirect hosts', async () => {
const fetcher: typeof fetch = async (input) => {
const url = requestInfoToUrlString(input)
if (url.includes('/archive/HEAD.zip')) {
return new Response(null, {
status: 302,
headers: {
location:
'https://codeload.github.com/a/b/zip/0123456789012345678901234567890123456789',
},
})
}
if (url.startsWith('https://codeload.github.com/a/b/zip/')) {
return new Response(null, { status: 200 })
}
throw new Error(`Unexpected fetch: ${url}`)
}
const resolved = await resolveGitHubCommit(
{ owner: 'a', repo: 'b', originalUrl: 'https://github.com/a/b' },
fetcher,
)
expect(resolved.commit).toBe('0123456789012345678901234567890123456789')
const badFetcher: typeof fetch = async (input) => {
const url = requestInfoToUrlString(input)
if (url.includes('/archive/HEAD.zip')) {
return new Response(null, {
status: 302,
headers: { location: 'https://evil.example/zip/abc' },
})
}
throw new Error(`Unexpected fetch: ${url}`)
}
await expect(
resolveGitHubCommit(
{ owner: 'a', repo: 'b', originalUrl: 'https://github.com/a/b' },
badFetcher,
),
).rejects.toThrow(/redirect/i)
})
it('resolves explicit ref commit via GitHub API', async () => {
const fetcher: typeof fetch = async (input) => {
const url = requestInfoToUrlString(input)
if (url.startsWith('https://api.github.com/repos/a/b/commits/')) {
return new Response(JSON.stringify({ sha: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' }), {
status: 200,
})
}
throw new Error(`Unexpected fetch: ${url}`)
}
const resolved = await resolveGitHubCommit(
{ owner: 'a', repo: 'b', ref: 'main', originalUrl: 'https://github.com/a/b' },
fetcher,
)
expect(resolved.commit).toBe('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')
})
it('enforces zip byte cap when content-length is too large', async () => {
const resolved = {
owner: 'a',
repo: 'b',
ref: 'main',
commit: '0123456789012345678901234567890123456789',
path: '',
repoUrl: 'https://github.com/a/b',
originalUrl: 'https://github.com/a/b',
} as const
const fetcher: typeof fetch = async () =>
new Response(new Blob([new Uint8Array([1, 2, 3])]), {
status: 200,
headers: { 'content-length': String(999_999_999) },
})
await expect(fetchGitHubZipBytes(resolved, fetcher, { maxZipBytes: 10 })).rejects.toThrow(
/too large/i,
)
})
})
+425
View File
@@ -0,0 +1,425 @@
import { TEXT_FILE_EXTENSION_SET } from 'clawdhub-schema'
import { zipSync } from 'fflate'
import semver from 'semver'
import { parseFrontmatter } from './skills'
export type GitHubImportUrl = {
owner: string
repo: string
ref?: string
path?: string
originalUrl: string
}
export type GitHubImportResolved = {
owner: string
repo: string
ref: string
commit: string
path: string
repoUrl: string
originalUrl: string
}
export type GitHubImportCandidate = {
path: string
readmePath: string
name?: string
description?: string
}
export type GitHubImportFileEntry = {
path: string
size: number
defaultSelected: boolean
}
const MAX_REDIRECTS = 6
const GITHUB_HOST = 'github.com'
const CODELOAD_HOST = 'codeload.github.com'
const SKILL_FILENAMES = ['skill.md', 'skills.md']
export function parseGitHubImportUrl(input: string): GitHubImportUrl {
const originalUrl = input.trim()
let url: URL
try {
url = new URL(originalUrl)
} catch {
throw new Error('Invalid URL')
}
if (url.protocol !== 'https:') throw new Error('Only https:// URLs are supported')
if (url.hostname !== GITHUB_HOST) throw new Error('Only github.com URLs are supported')
const segments = url.pathname
.split('/')
.map((segment) => segment.trim())
.filter(Boolean)
.map((segment) => {
try {
return decodeURIComponent(segment)
} catch {
throw new Error('Invalid URL')
}
})
const owner = segments[0] ?? ''
const repo = (segments[1] ?? '').replace(/\.git$/, '')
if (!owner || !repo) throw new Error('GitHub URL must be /<owner>/<repo>')
const kind = segments[2] ?? ''
if (!kind) return { owner, repo, originalUrl }
if (kind !== 'tree' && kind !== 'blob') {
return { owner, repo, originalUrl }
}
const ref = segments[3] ?? ''
if (!ref) throw new Error('Missing ref in GitHub URL')
const rest = segments.slice(4).join('/')
const normalizedRest = normalizeRepoPath(rest)
if (kind === 'blob') {
if (!rest) throw new Error('Missing path in GitHub URL')
if (!normalizedRest) throw new Error('Invalid path in GitHub URL')
const dir = normalizedRest.split('/').slice(0, -1).join('/')
return { owner, repo, ref, path: dir || undefined, originalUrl }
}
if (rest && !normalizedRest) throw new Error('Invalid path in GitHub URL')
return { owner, repo, ref, path: normalizedRest || undefined, originalUrl }
}
export async function resolveGitHubCommit(
parsed: GitHubImportUrl,
fetcher: typeof fetch,
): Promise<GitHubImportResolved> {
const repoUrl = `https://${GITHUB_HOST}/${parsed.owner}/${parsed.repo}`
const ref = parsed.ref?.trim() || 'HEAD'
const path = normalizeRepoPath(parsed.path ?? '')
const commit =
ref === 'HEAD'
? await resolveHeadCommit(parsed, fetcher)
: await resolveRefCommit(parsed, ref, fetcher)
return {
owner: parsed.owner,
repo: parsed.repo,
ref,
commit,
path,
repoUrl,
originalUrl: parsed.originalUrl,
}
}
async function resolveRefCommit(parsed: GitHubImportUrl, ref: string, fetcher: typeof fetch) {
const apiUrl = `https://api.github.com/repos/${parsed.owner}/${parsed.repo}/commits/${encodeURIComponent(ref)}`
const response = await fetcher(apiUrl, {
headers: {
Accept: 'application/vnd.github+json',
'User-Agent': 'clawdhub/github-import',
},
})
if (!response.ok) throw new Error('GitHub ref not found')
const body = (await response.json()) as { sha?: unknown }
const sha = typeof body.sha === 'string' ? body.sha : ''
if (!/^[a-f0-9]{40}$/i.test(sha)) throw new Error('GitHub commit sha missing')
return sha.toLowerCase()
}
async function resolveHeadCommit(parsed: GitHubImportUrl, fetcher: typeof fetch) {
let url = `https://${GITHUB_HOST}/${parsed.owner}/${parsed.repo}/archive/HEAD.zip`
for (let i = 0; i < MAX_REDIRECTS; i += 1) {
const response = await fetcher(url, { redirect: 'manual' })
const location = response.headers.get('location')
if (!location) break
const next = new URL(location, url)
if (next.hostname !== GITHUB_HOST && next.hostname !== CODELOAD_HOST) {
throw new Error('Unexpected redirect host')
}
url = next.toString()
}
const maybe = url.split('/').at(-1) ?? ''
if (!/^[a-f0-9]{40}$/i.test(maybe)) {
throw new Error('Could not resolve commit for HEAD')
}
return maybe.toLowerCase()
}
export async function fetchGitHubZipBytes(
resolved: GitHubImportResolved,
fetcher: typeof fetch,
limits?: { maxZipBytes?: number },
): Promise<Uint8Array> {
const maxZipBytes = limits?.maxZipBytes ?? 25 * 1024 * 1024
const url = `https://${CODELOAD_HOST}/${resolved.owner}/${resolved.repo}/zip/${resolved.commit}`
const response = await fetcher(url, {
headers: { 'User-Agent': 'clawdhub/github-import' },
})
if (!response.ok) throw new Error('GitHub archive download failed')
const lengthHeader = response.headers.get('content-length')
if (lengthHeader) {
const contentLength = Number.parseInt(lengthHeader, 10)
if (Number.isFinite(contentLength) && contentLength > maxZipBytes) {
throw new Error('GitHub archive too large')
}
}
const reader = response.body?.getReader()
if (!reader) {
const buffer = new Uint8Array(await response.arrayBuffer())
if (buffer.byteLength > maxZipBytes) throw new Error('GitHub archive too large')
return buffer
}
const chunks: Uint8Array[] = []
let total = 0
while (true) {
const { done, value } = await reader.read()
if (done) break
if (!value) continue
total += value.byteLength
if (total > maxZipBytes) throw new Error('GitHub archive too large')
chunks.push(value)
}
const out = new Uint8Array(total)
let offset = 0
for (const chunk of chunks) {
out.set(chunk, offset)
offset += chunk.byteLength
}
return out
}
export type ZipEntryMap = Record<string, Uint8Array>
export function buildGitHubZipForTests(entries: Record<string, string>) {
const asBytes = Object.fromEntries(
Object.entries(entries).map(([path, text]) => [path, new TextEncoder().encode(text)]),
)
return Uint8Array.from(zipSync(asBytes, { level: 1 }))
}
export function stripGitHubZipRoot(entries: ZipEntryMap): ZipEntryMap {
const paths = Object.keys(entries)
if (paths.length === 0) return {}
const first = paths[0] ?? ''
const firstRoot = first.split('/')[0] ?? ''
if (!firstRoot) return entries
const prefix = `${firstRoot}/`
if (!paths.every((path) => path.startsWith(prefix))) return entries
const out: ZipEntryMap = {}
for (const [path, data] of Object.entries(entries)) {
const stripped = path.slice(prefix.length)
if (!stripped) continue
out[stripped] = data
}
return out
}
export function detectGitHubImportCandidates(entries: ZipEntryMap): GitHubImportCandidate[] {
const candidates: GitHubImportCandidate[] = []
for (const path of Object.keys(entries)) {
const normalized = normalizeRepoPath(path)
const lower = normalized.toLowerCase()
const isSkill = SKILL_FILENAMES.some((name) => lower === name || lower.endsWith(`/${name}`))
if (!isSkill) continue
const dir = normalized.split('/').slice(0, -1).join('/')
const readmePath = normalized
const raw = new TextDecoder().decode(entries[path] ?? new Uint8Array())
const frontmatter = parseFrontmatter(raw)
const name = typeof frontmatter.name === 'string' ? frontmatter.name : undefined
const description =
typeof frontmatter.description === 'string' ? frontmatter.description : undefined
candidates.push({
path: normalizeRepoPath(dir),
readmePath,
name: name?.trim() || undefined,
description: description?.trim() || undefined,
})
}
return uniqCandidates(candidates)
}
function uniqCandidates(candidates: GitHubImportCandidate[]) {
const seen = new Set<string>()
const out: GitHubImportCandidate[] = []
for (const candidate of candidates) {
const key = `${candidate.path}::${candidate.readmePath}`
if (seen.has(key)) continue
seen.add(key)
out.push(candidate)
}
return out.sort((a, b) => a.path.localeCompare(b.path))
}
export function listTextFilesUnderCandidate(
entries: ZipEntryMap,
candidatePath: string,
): Array<{ path: string; bytes: Uint8Array }> {
const root = normalizeCandidateRoot(candidatePath)
const out: Array<{ path: string; bytes: Uint8Array }> = []
for (const [path, bytes] of Object.entries(entries)) {
const normalized = normalizeRepoPath(path)
if (!isUnderRoot(normalized, root)) continue
if (!isTextPath(normalized)) continue
out.push({ path: normalized, bytes })
}
return out.sort((a, b) => a.path.localeCompare(b.path))
}
export function computeDefaultSelectedPaths(params: {
candidate: GitHubImportCandidate
files: Array<{ path: string; bytes: Uint8Array }>
maxDepth?: number
maxAdds?: number
}) {
const maxDepth = params.maxDepth ?? 4
const maxAdds = params.maxAdds ?? 200
const byPath = new Map(params.files.map((file) => [file.path, file.bytes]))
const candidateRoot = normalizeCandidateRoot(params.candidate.path)
const selected = new Set<string>()
let added = 0
const add = (path: string) => {
const normalized = normalizeRepoPath(path)
if (!isUnderRoot(normalized, candidateRoot)) return
if (!byPath.has(normalized)) return
if (!selected.has(normalized)) {
selected.add(normalized)
added += 1
}
}
add(params.candidate.readmePath)
const visited = new Set<string>()
const queue: Array<{ path: string; depth: number }> = [
{ path: params.candidate.readmePath, depth: 0 },
]
while (queue.length > 0) {
const item = queue.shift()
if (!item) break
if (item.depth >= maxDepth) continue
if (visited.has(item.path)) continue
visited.add(item.path)
const bytes = byPath.get(item.path)
if (!bytes) continue
if (!item.path.toLowerCase().endsWith('.md')) continue
const text = new TextDecoder().decode(bytes)
const refs = extractMarkdownRelativeTargets(text)
for (const ref of refs) {
if (added >= maxAdds) break
const resolved = resolveMarkdownTarget(item.path, ref)
if (!resolved) continue
add(resolved)
if (resolved.toLowerCase().endsWith('.md') && byPath.has(resolved)) {
queue.push({ path: resolved, depth: item.depth + 1 })
}
}
if (added >= maxAdds) break
}
return Array.from(selected).sort()
}
export function buildGitHubImportFileList(params: {
candidate: GitHubImportCandidate
files: Array<{ path: string; bytes: Uint8Array }>
defaultSelectedPaths: string[]
}): GitHubImportFileEntry[] {
const selected = new Set(params.defaultSelectedPaths)
return params.files.map((file) => ({
path: file.path,
size: file.bytes.byteLength,
defaultSelected: selected.has(file.path),
}))
}
export function normalizeRepoPath(path: string) {
const stripped = path.replace(/^\/+/, '').trim()
if (!stripped) return ''
const cleaned = stripped.split('/').filter(Boolean).join('/')
if (!cleaned || cleaned.includes('\\') || cleaned.includes('..')) return ''
return cleaned
}
export function normalizeCandidateRoot(candidatePath: string) {
const normalized = normalizeRepoPath(candidatePath)
return normalized ? `${normalized}/` : ''
}
function isUnderRoot(path: string, rootWithSlash: string) {
if (!rootWithSlash) return true
return path === rootWithSlash.slice(0, -1) || path.startsWith(rootWithSlash)
}
function isTextPath(path: string) {
const lower = path.toLowerCase()
const ext = lower.split('.').at(-1) ?? ''
if (!ext) return false
return TEXT_FILE_EXTENSION_SET.has(ext)
}
export function suggestDisplayName(candidate: GitHubImportCandidate, fallbackBase: string) {
const base = candidate.name?.trim() || fallbackBase.trim()
if (!base) return ''
return base
.replace(/[-_]+/g, ' ')
.replace(/\s+/g, ' ')
.replace(/\b\w/g, (char) => char.toUpperCase())
}
export function suggestVersion(latestVersion?: string | null) {
const latest = latestVersion?.trim() || ''
if (latest && semver.valid(latest)) {
return semver.inc(latest, 'patch') ?? '0.1.0'
}
return '0.1.0'
}
export function extractMarkdownRelativeTargets(markdown: string): string[] {
const out: string[] = []
const pattern = /!?\[[^\]]*]\(([^)]+)\)/g
for (const match of markdown.matchAll(pattern)) {
const raw = (match[1] ?? '').trim()
if (!raw) continue
const isAngleWrapped = raw.startsWith('<') && raw.endsWith('>')
const cleaned = raw.replace(/^<|>$/g, '').trim()
if (!cleaned) continue
const target = isAngleWrapped ? cleaned : (cleaned.split(/\s+/)[0] ?? '')
if (!target) continue
if (target.startsWith('#')) continue
const lower = target.toLowerCase()
if (lower.startsWith('http:') || lower.startsWith('https:')) continue
if (lower.startsWith('mailto:')) continue
out.push(target)
}
return out
}
export function resolveMarkdownTarget(fromPath: string, target: string) {
const withoutHash = target.split('#')[0] ?? ''
const withoutQuery = (withoutHash.split('?')[0] ?? '').trim()
if (!withoutQuery) return null
if (withoutQuery.startsWith('/')) return null
if (withoutQuery.includes('\\') || withoutQuery.includes('..')) return null
const fromDirParts = normalizeRepoPath(fromPath).split('/').slice(0, -1)
const targetParts = withoutQuery.split('/').filter(Boolean)
const combined = [...fromDirParts, ...targetParts]
const normalized: string[] = []
for (const part of combined) {
if (part === '.') continue
if (part === '..') return null
normalized.push(part)
}
return normalizeRepoPath(normalized.join('/')) || null
}
+443
View File
@@ -0,0 +1,443 @@
'use node'
import { createPrivateKey, createSign } from 'node:crypto'
import type { Id } from '../_generated/dataModel'
import type { ActionCtx } from '../_generated/server'
const GITHUB_API = 'https://api.github.com'
const DEFAULT_REPO = 'clawdbot/souls'
const DEFAULT_ROOT = 'souls'
const META_FILENAME = '_meta.json'
const USER_AGENT = 'clawdhub/souls-backup'
type BackupFile = {
path: string
size: number
storageId: Id<'_storage'>
sha256: string
contentType?: string
}
type BackupParams = {
slug: string
version: string
displayName: string
ownerHandle: string
files: BackupFile[]
publishedAt: number
}
type RepoInfo = {
default_branch?: string
}
type GitRef = {
object: { sha: string }
}
type GitCommit = {
sha: string
tree: { sha: string }
}
type GitTreeEntry = {
path?: string
type?: string
}
type GitTree = {
tree?: GitTreeEntry[]
}
type MetaFile = {
owner: string
slug: string
displayName: string
latest: {
version: string
publishedAt: number
commit: string | null
}
history: Array<{
version: string
publishedAt: number
commit: string
}>
}
export type GitHubBackupContext = {
token: string
repo: string
repoOwner: string
repoName: string
branch: string
root: string
}
export function isGitHubSoulBackupConfigured() {
return Boolean(
process.env.GITHUB_APP_ID &&
process.env.GITHUB_APP_PRIVATE_KEY &&
process.env.GITHUB_APP_INSTALLATION_ID,
)
}
export async function getGitHubSoulBackupContext(): Promise<GitHubBackupContext> {
const repo = process.env.GITHUB_SOULS_REPO ?? DEFAULT_REPO
const root = process.env.GITHUB_SOULS_ROOT ?? DEFAULT_ROOT
const [repoOwner, repoName] = parseRepo(repo)
const token = await createInstallationToken()
const repoInfo = await githubGet<RepoInfo>(token, `/repos/${repoOwner}/${repoName}`)
const branch = repoInfo.default_branch ?? 'main'
return { token, repo, repoOwner, repoName, branch, root }
}
export async function fetchGitHubSoulMeta(
context: GitHubBackupContext,
ownerHandle: string,
slug: string,
): Promise<MetaFile | null> {
const soulRoot = buildSoulRoot(context.root, ownerHandle, slug)
return fetchMetaFile(
context.token,
context.repoOwner,
context.repoName,
`${soulRoot}/${META_FILENAME}`,
context.branch,
)
}
export async function backupSoulToGitHub(
ctx: ActionCtx,
params: BackupParams,
context?: GitHubBackupContext,
) {
if (!isGitHubSoulBackupConfigured()) return
const resolved = context ?? (await getGitHubSoulBackupContext())
const soulRoot = buildSoulRoot(resolved.root, params.ownerHandle, params.slug)
const ref = await githubGet<GitRef>(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/ref/heads/${resolved.branch}`,
)
const baseCommitSha = ref.object.sha
const baseCommit = await githubGet<GitCommit>(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/commits/${baseCommitSha}`,
)
const baseTreeSha = baseCommit.tree.sha
const existingTree = await githubGet<GitTree>(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/trees/${baseTreeSha}?recursive=1`,
)
const prefix = `${soulRoot}/`
const existingPaths = new Set(
(existingTree.tree ?? [])
.filter((entry) => entry.type === 'blob' && entry.path?.startsWith(prefix))
.map((entry) => entry.path ?? ''),
)
const newPaths = new Set<string>()
const treeEntries: Array<{
path: string
mode: '100644'
type: 'blob'
sha: string | null
}> = []
for (const file of params.files) {
const content = await fetchStorageBase64(ctx, file.storageId)
const blobSha = await createBlob(resolved.token, resolved.repoOwner, resolved.repoName, content)
const path = `${soulRoot}/${file.path}`
newPaths.add(path)
treeEntries.push({ path, mode: '100644', type: 'blob', sha: blobSha })
}
const existingMeta = await fetchMetaFile(
resolved.token,
resolved.repoOwner,
resolved.repoName,
`${soulRoot}/${META_FILENAME}`,
resolved.branch,
)
const metaPath = `${soulRoot}/${META_FILENAME}`
const metaDraft = buildMetaFile(params, existingMeta, resolved.repo, baseCommitSha, null)
const metaDraftContent = `${JSON.stringify(metaDraft, null, 2)}\n`
const metaDraftSha = await createBlob(
resolved.token,
resolved.repoOwner,
resolved.repoName,
toBase64(metaDraftContent),
)
newPaths.add(metaPath)
treeEntries.push({ path: metaPath, mode: '100644', type: 'blob', sha: metaDraftSha })
for (const path of existingPaths) {
if (newPaths.has(path)) continue
treeEntries.push({ path, mode: '100644', type: 'blob', sha: null })
}
const newTree = await githubPost<{ sha: string }>(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/trees`,
{
base_tree: baseTreeSha,
tree: treeEntries,
},
)
const commit = await githubPost<GitCommit>(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/commits`,
{
message: `soul: ${params.slug} v${params.version}`,
tree: newTree.sha,
parents: [baseCommitSha],
},
)
const metaFinal = buildMetaFile(params, existingMeta, resolved.repo, baseCommitSha, commit.sha)
const metaFinalContent = `${JSON.stringify(metaFinal, null, 2)}\n`
const metaFinalSha = await createBlob(
resolved.token,
resolved.repoOwner,
resolved.repoName,
toBase64(metaFinalContent),
)
const metaTree = await githubPost<{ sha: string }>(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/trees`,
{
base_tree: commit.tree.sha,
tree: [{ path: metaPath, mode: '100644', type: 'blob', sha: metaFinalSha }],
},
)
const metaCommit = await githubPost<GitCommit>(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/commits`,
{
message: `meta: ${params.slug} v${params.version}`,
tree: metaTree.sha,
parents: [commit.sha],
},
)
await githubPatch(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/refs/heads/${resolved.branch}`,
{
sha: metaCommit.sha,
},
)
}
function buildMetaFile(
params: BackupParams,
existing: MetaFile | null,
repo: string,
baseCommitSha: string,
latestCommitSha: string | null,
): MetaFile {
let history = [...(existing?.history ?? [])]
if (existing?.latest?.version) {
const previousCommit = existing.latest.commit ?? commitUrl(repo, baseCommitSha)
const previous = {
version: existing.latest.version,
publishedAt: existing.latest.publishedAt,
commit: previousCommit,
}
history = [previous, ...history.filter((entry) => entry.version !== previous.version)]
}
return {
owner: normalizeOwner(params.ownerHandle),
slug: params.slug,
displayName: params.displayName,
latest: {
version: params.version,
publishedAt: params.publishedAt,
commit: latestCommitSha ? commitUrl(repo, latestCommitSha) : null,
},
history: history.slice(0, 200),
}
}
async function fetchMetaFile(
token: string,
repoOwner: string,
repoName: string,
path: string,
branch: string,
): Promise<MetaFile | null> {
try {
const response = await githubGet<{ content?: string }>(
token,
`/repos/${repoOwner}/${repoName}/contents/${encodePath(path)}?ref=${branch}`,
)
if (!response.content) return null
const raw = fromBase64(response.content)
return JSON.parse(raw) as MetaFile
} catch (error) {
if (isNotFoundError(error)) return null
throw error
}
}
async function fetchStorageBase64(ctx: ActionCtx, storageId: Id<'_storage'>) {
const blob = await ctx.storage.get(storageId)
if (!blob) throw new Error('File missing in storage')
const buffer = Buffer.from(await blob.arrayBuffer())
return buffer.toString('base64')
}
async function createInstallationToken() {
const appId = process.env.GITHUB_APP_ID
const installationId = process.env.GITHUB_APP_INSTALLATION_ID
if (!appId || !installationId) {
throw new Error('GitHub App credentials missing')
}
const jwt = createAppJwt(appId)
const response = await fetch(`${GITHUB_API}/app/installations/${installationId}/access_tokens`, {
method: 'POST',
headers: buildHeaders(jwt, true),
})
if (!response.ok) {
const message = await response.text()
throw new Error(`GitHub App token failed: ${message}`)
}
const payload = (await response.json()) as { token?: string }
if (!payload.token) throw new Error('GitHub App token missing')
return payload.token
}
function createAppJwt(appId: string) {
const privateKey = loadPrivateKey()
const now = Math.floor(Date.now() / 1000)
const header = { alg: 'RS256', typ: 'JWT' }
const payload = { iat: now - 60, exp: now + 9 * 60, iss: appId }
const encodedHeader = base64Url(JSON.stringify(header))
const encodedPayload = base64Url(JSON.stringify(payload))
const signingInput = `${encodedHeader}.${encodedPayload}`
const sign = createSign('RSA-SHA256')
sign.update(signingInput)
sign.end()
const signature = sign.sign(privateKey)
return `${signingInput}.${base64Url(signature)}`
}
function loadPrivateKey() {
const raw = process.env.GITHUB_APP_PRIVATE_KEY
if (!raw) throw new Error('GITHUB_APP_PRIVATE_KEY is not configured')
const normalized = raw.replace(/\\n/g, '\n')
return createPrivateKey(normalized)
}
async function createBlob(token: string, repoOwner: string, repoName: string, content: string) {
const result = await githubPost<{ sha: string }>(
token,
`/repos/${repoOwner}/${repoName}/git/blobs`,
{
content,
encoding: 'base64',
},
)
if (!result.sha) throw new Error('GitHub blob missing sha')
return result.sha
}
async function githubGet<T>(token: string, path: string): Promise<T> {
const response = await fetch(`${GITHUB_API}${path}`, {
headers: buildHeaders(token),
})
if (!response.ok) {
const message = await response.text()
throw new Error(`GitHub GET ${path} failed: ${message}`)
}
return (await response.json()) as T
}
async function githubPost<T>(token: string, path: string, body: unknown): Promise<T> {
const response = await fetch(`${GITHUB_API}${path}`, {
method: 'POST',
headers: buildHeaders(token),
body: JSON.stringify(body),
})
if (!response.ok) {
const message = await response.text()
throw new Error(`GitHub POST ${path} failed: ${message}`)
}
return (await response.json()) as T
}
async function githubPatch(token: string, path: string, body: unknown) {
const response = await fetch(`${GITHUB_API}${path}`, {
method: 'PATCH',
headers: buildHeaders(token),
body: JSON.stringify(body),
})
if (!response.ok) {
const message = await response.text()
throw new Error(`GitHub PATCH ${path} failed: ${message}`)
}
}
function buildHeaders(token: string, isAppJwt = false) {
return {
Authorization: `${isAppJwt ? 'Bearer' : 'token'} ${token}`,
Accept: 'application/vnd.github+json',
'User-Agent': USER_AGENT,
}
}
function parseRepo(repo: string) {
const [owner, name] = repo.split('/')
if (!owner || !name) throw new Error('GITHUB_SOULS_REPO must be owner/repo')
return [owner, name] as const
}
function normalizeOwner(value: string) {
const normalized = value
.trim()
.toLowerCase()
.replace(/[^a-z0-9-]/g, '-')
.replace(/-+/g, '-')
.replace(/^-+|-+$/g, '')
return normalized || 'unknown'
}
function commitUrl(repo: string, sha: string) {
return `https://github.com/${repo}/commit/${sha}`
}
function buildSoulRoot(root: string, ownerHandle: string, slug: string) {
const ownerSegment = normalizeOwner(ownerHandle)
return `${root}/${ownerSegment}/${slug}`
}
function encodePath(path: string) {
return path
.split('/')
.map((segment) => encodeURIComponent(segment))
.join('/')
}
function base64Url(value: string | Buffer) {
const buffer = typeof value === 'string' ? Buffer.from(value) : value
return buffer.toString('base64').replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/g, '')
}
function toBase64(value: string) {
return Buffer.from(value).toString('base64')
}
function fromBase64(value: string) {
return Buffer.from(value, 'base64').toString('utf8')
}
function isNotFoundError(error: unknown) {
return (
error instanceof Error && (error.message.includes('404') || error.message.includes('Not Found'))
)
}
+28
View File
@@ -0,0 +1,28 @@
import { describe, expect, it } from 'vitest'
import { __test } from './skillPublish'
describe('skillPublish', () => {
it('merges github source into metadata', () => {
const merged = __test.mergeSourceIntoMetadata(
{ clawdis: { emoji: 'x' } },
{
kind: 'github',
url: 'https://github.com/a/b',
repo: 'a/b',
ref: 'main',
commit: '0123456789012345678901234567890123456789',
path: 'skills/demo',
importedAt: 123,
},
)
expect((merged as Record<string, unknown>).clawdis).toEqual({ emoji: 'x' })
const source = (merged as Record<string, unknown>).source
expect(source).toEqual(
expect.objectContaining({
kind: 'github',
repo: 'a/b',
path: 'skills/demo',
}),
)
})
})
+62 -2
View File
@@ -8,6 +8,7 @@ import { generateEmbedding } from './embeddings'
import {
buildEmbeddingText,
getFrontmatterMetadata,
hashSkillFiles,
isTextFile,
parseClawdisMetadata,
parseFrontmatter,
@@ -30,6 +31,16 @@ export type PublishVersionArgs = {
version: string
changelog: string
tags?: string[]
forkOf?: { slug: string; version?: string }
source?: {
kind: 'github'
url: string
repo: string
ref: string
commit: string
path: string
importedAt: number
}
files: Array<{
path: string
size: number
@@ -81,7 +92,7 @@ export async function publishVersionForUser(
const readmeText = await fetchText(ctx, readmeFile.storageId)
const frontmatter = parseFrontmatter(readmeText)
const clawdis = parseClawdisMetadata(frontmatter)
const metadata = getFrontmatterMetadata(frontmatter)
const metadata = mergeSourceIntoMetadata(getFrontmatterMetadata(frontmatter), args.source)
const otherFiles = [] as Array<{ path: string; content: string }>
for (const file of sanitizedFiles) {
@@ -98,6 +109,10 @@ export async function publishVersionForUser(
otherFiles,
})
const fingerprintPromise = hashSkillFiles(
sanitizedFiles.map((file) => ({ path: file.path ?? '', sha256: file.sha256 })),
)
const changelogPromise =
changelogSource === 'user'
? Promise.resolve(suppliedChangelog)
@@ -110,7 +125,8 @@ export async function publishVersionForUser(
const embeddingPromise = generateEmbedding(embeddingText)
const [changelogText, embedding] = await Promise.all([
const [fingerprint, changelogText, embedding] = await Promise.all([
fingerprintPromise,
changelogPromise,
embeddingPromise.catch((error) => {
throw new ConvexError(formatEmbeddingError(error))
@@ -125,6 +141,13 @@ export async function publishVersionForUser(
changelog: changelogText,
changelogSource,
tags: args.tags?.map((tag) => tag.trim()).filter(Boolean),
fingerprint,
forkOf: args.forkOf
? {
slug: args.forkOf.slug.trim().toLowerCase(),
version: args.forkOf.version?.trim() || undefined,
}
: undefined,
files: sanitizedFiles.map((file) => ({
...file,
path: file.path ?? '',
@@ -137,6 +160,22 @@ export async function publishVersionForUser(
embedding,
})) as PublishResult
const owner = (await ctx.runQuery(api.users.getById, { userId })) as Doc<'users'> | null
const ownerHandle = owner?.handle ?? owner?.displayName ?? owner?.name ?? 'unknown'
void ctx.scheduler
.runAfter(0, internal.githubBackupsNode.backupSkillForPublishInternal, {
slug,
version,
displayName,
ownerHandle,
files: sanitizedFiles,
publishedAt: Date.now(),
})
.catch((error) => {
console.error('GitHub backup scheduling failed', error)
})
void schedulePublishWebhook(ctx, {
slug,
version,
@@ -146,6 +185,27 @@ export async function publishVersionForUser(
return publishResult
}
function mergeSourceIntoMetadata(metadata: unknown, source: PublishVersionArgs['source']) {
if (!source) return metadata === undefined ? undefined : metadata
const sourceValue = {
kind: source.kind,
url: source.url,
repo: source.repo,
ref: source.ref,
commit: source.commit,
path: source.path,
importedAt: source.importedAt,
}
if (!metadata) return { source: sourceValue }
if (typeof metadata !== 'object' || Array.isArray(metadata)) return { source: sourceValue }
return { ...(metadata as Record<string, unknown>), source: sourceValue }
}
export const __test = {
mergeSourceIntoMetadata,
}
export async function queueHighlightedWebhook(ctx: MutationCtx, skillId: Id<'skills'>) {
const skill = await ctx.db.get(skillId)
if (!skill) return
+29
View File
@@ -107,6 +107,35 @@ describe('skills utils', () => {
expect(clawdis?.requires?.anyBins).toEqual(['rg', 'fd'])
})
it('parses clawdbot metadata with nix plugin pointer', () => {
const frontmatter = parseFrontmatter(
`---\nmetadata: {"clawdbot":{"nix":{"plugin":"github:clawdbot/nix-steipete-tools?dir=tools/peekaboo","systems":["aarch64-darwin"]}}}\n---\nBody`,
)
const clawdis = parseClawdisMetadata(frontmatter)
expect(clawdis?.nix?.plugin).toBe('github:clawdbot/nix-steipete-tools?dir=tools/peekaboo')
expect(clawdis?.nix?.systems).toEqual(['aarch64-darwin'])
})
it('parses clawdbot config requirements with example', () => {
const frontmatter = parseFrontmatter(
`---\nmetadata: {"clawdbot":{"config":{"requiredEnv":["PADEL_AUTH_FILE"],"stateDirs":[".config/padel"],"example":"config = { env = { PADEL_AUTH_FILE = \\"/run/agenix/padel-auth\\"; }; };"}}}\n---\nBody`,
)
const clawdis = parseClawdisMetadata(frontmatter)
expect(clawdis?.config?.requiredEnv).toEqual(['PADEL_AUTH_FILE'])
expect(clawdis?.config?.stateDirs).toEqual(['.config/padel'])
expect(clawdis?.config?.example).toBe(
'config = { env = { PADEL_AUTH_FILE = "/run/agenix/padel-auth"; }; };',
)
})
it('parses cli help output', () => {
const frontmatter = parseFrontmatter(
`---\nmetadata: {"clawdbot":{"cliHelp":"padel --help\\nUsage: padel [command]\\n"}}\n---\nBody`,
)
const clawdis = parseClawdisMetadata(frontmatter)
expect(clawdis?.cliHelp).toBe('padel --help\nUsage: padel [command]')
})
it('sanitizes file paths', () => {
expect(sanitizePath('good/file.md')).toBe('good/file.md')
expect(sanitizePath('../bad/file.md')).toBeNull()
+43 -3
View File
@@ -1,7 +1,9 @@
import {
type ClawdbotConfigSpec,
type ClawdisSkillMetadata,
ClawdisSkillMetadataSchema,
isTextContentType,
type NixPluginSpec,
parseArk,
type SkillInstallSpec,
TEXT_FILE_EXTENSION_SET,
@@ -59,11 +61,19 @@ export function getFrontmatterMetadata(frontmatter: ParsedSkillFrontmatter) {
export function parseClawdisMetadata(frontmatter: ParsedSkillFrontmatter) {
const metadata = getFrontmatterMetadata(frontmatter)
const clawdisFromMetadata =
const metadataRecord =
metadata && typeof metadata === 'object' && !Array.isArray(metadata)
? (metadata as Record<string, unknown>).clawdis
? (metadata as Record<string, unknown>)
: undefined
const clawdisRaw = clawdisFromMetadata ?? frontmatter.clawdis
const clawdbotMeta = metadataRecord?.clawdbot
const clawdisMeta = metadataRecord?.clawdis
const metadataSource =
clawdbotMeta && typeof clawdbotMeta === 'object' && !Array.isArray(clawdbotMeta)
? (clawdbotMeta as Record<string, unknown>)
: clawdisMeta && typeof clawdisMeta === 'object' && !Array.isArray(clawdisMeta)
? (clawdisMeta as Record<string, unknown>)
: undefined
const clawdisRaw = metadataSource ?? frontmatter.clawdis
if (!clawdisRaw || typeof clawdisRaw !== 'object' || Array.isArray(clawdisRaw)) return undefined
try {
@@ -84,6 +94,7 @@ export function parseClawdisMetadata(frontmatter: ParsedSkillFrontmatter) {
if (typeof clawdisObj.homepage === 'string') metadata.homepage = clawdisObj.homepage
if (typeof clawdisObj.skillKey === 'string') metadata.skillKey = clawdisObj.skillKey
if (typeof clawdisObj.primaryEnv === 'string') metadata.primaryEnv = clawdisObj.primaryEnv
if (typeof clawdisObj.cliHelp === 'string') metadata.cliHelp = clawdisObj.cliHelp
if (osRaw.length > 0) metadata.os = osRaw
if (requiresRaw) {
@@ -101,6 +112,10 @@ export function parseClawdisMetadata(frontmatter: ParsedSkillFrontmatter) {
}
if (install.length > 0) metadata.install = install
const nix = parseNixPluginSpec(clawdisObj.nix)
if (nix) metadata.nix = nix
const config = parseClawdbotConfigSpec(clawdisObj.config)
if (config) metadata.config = config
return parseArk(ClawdisSkillMetadataSchema, metadata, 'Clawdis metadata')
} catch {
@@ -223,6 +238,31 @@ function parseInstallSpec(input: unknown): SkillInstallSpec | undefined {
return spec
}
function parseNixPluginSpec(input: unknown): NixPluginSpec | undefined {
if (!input || typeof input !== 'object') return undefined
const raw = input as Record<string, unknown>
if (typeof raw.plugin !== 'string') return undefined
const plugin = raw.plugin.trim()
if (!plugin) return undefined
const systems = normalizeStringList(raw.systems)
const spec: NixPluginSpec = { plugin }
if (systems.length > 0) spec.systems = systems
return spec
}
function parseClawdbotConfigSpec(input: unknown): ClawdbotConfigSpec | undefined {
if (!input || typeof input !== 'object') return undefined
const raw = input as Record<string, unknown>
const requiredEnv = normalizeStringList(raw.requiredEnv)
const stateDirs = normalizeStringList(raw.stateDirs)
const example = typeof raw.example === 'string' ? raw.example.trim() : ''
const spec: ClawdbotConfigSpec = {}
if (requiredEnv.length > 0) spec.requiredEnv = requiredEnv
if (stateDirs.length > 0) spec.stateDirs = stateDirs
if (example) spec.example = example
return Object.keys(spec).length > 0 ? spec : undefined
}
function toHex(bytes: Uint8Array) {
let out = ''
for (const byte of bytes) out += byte.toString(16).padStart(2, '0')
+273
View File
@@ -0,0 +1,273 @@
import { internal } from '../_generated/api'
import type { Doc } from '../_generated/dataModel'
import type { ActionCtx } from '../_generated/server'
const CHANGELOG_MODEL = process.env.OPENAI_CHANGELOG_MODEL ?? 'gpt-4.1'
const MAX_README_CHARS = 8_000
const MAX_PATHS_IN_PROMPT = 30
type FileMeta = { path: string; sha256?: string }
type FileDiffSummary = {
added: string[]
removed: string[]
changed: string[]
}
function clampText(value: string, maxChars: number) {
const trimmed = value.trim()
if (trimmed.length <= maxChars) return trimmed
return `${trimmed.slice(0, maxChars).trimEnd()}\n…`
}
function summarizeFileDiff(oldFiles: FileMeta[], nextFiles: FileMeta[]): FileDiffSummary {
const oldByPath = new Map(oldFiles.map((f) => [f.path, f] as const))
const nextByPath = new Map(nextFiles.map((f) => [f.path, f] as const))
const added: string[] = []
const removed: string[] = []
const changed: string[] = []
for (const [path, file] of nextByPath.entries()) {
const prev = oldByPath.get(path)
if (!prev) {
added.push(path)
continue
}
if (file.sha256 && prev.sha256 && file.sha256 !== prev.sha256) changed.push(path)
}
for (const path of oldByPath.keys()) {
if (!nextByPath.has(path)) removed.push(path)
}
added.sort()
removed.sort()
changed.sort()
return { added, removed, changed }
}
function formatDiffSummary(diff: FileDiffSummary) {
const parts: string[] = []
if (diff.added.length) parts.push(`${diff.added.length} added`)
if (diff.changed.length) parts.push(`${diff.changed.length} changed`)
if (diff.removed.length) parts.push(`${diff.removed.length} removed`)
return parts.join(', ') || 'no file changes detected'
}
function pickPaths(values: string[]) {
if (values.length <= MAX_PATHS_IN_PROMPT) return values
return values.slice(0, MAX_PATHS_IN_PROMPT)
}
function extractResponseText(payload: unknown) {
if (!payload || typeof payload !== 'object') return null
const output = (payload as { output?: unknown }).output
if (!Array.isArray(output)) return null
const chunks: string[] = []
for (const item of output) {
if (!item || typeof item !== 'object') continue
if ((item as { type?: unknown }).type !== 'message') continue
const content = (item as { content?: unknown }).content
if (!Array.isArray(content)) continue
for (const part of content) {
if (!part || typeof part !== 'object') continue
if ((part as { type?: unknown }).type !== 'output_text') continue
const text = (part as { text?: unknown }).text
if (typeof text === 'string' && text.trim()) chunks.push(text)
}
}
const joined = chunks.join('\n').trim()
return joined || null
}
async function generateWithOpenAI(args: {
slug: string
version: string
oldReadme: string | null
nextReadme: string
fileDiff: FileDiffSummary | null
}) {
const apiKey = process.env.OPENAI_API_KEY
if (!apiKey) return null
const oldReadme = args.oldReadme ? clampText(args.oldReadme, MAX_README_CHARS) : ''
const nextReadme = clampText(args.nextReadme, MAX_README_CHARS)
const fileDiff = args.fileDiff
const diffSummary = fileDiff ? formatDiffSummary(fileDiff) : 'unknown'
const changedPaths = fileDiff ? pickPaths(fileDiff.changed) : []
const addedPaths = fileDiff ? pickPaths(fileDiff.added) : []
const removedPaths = fileDiff ? pickPaths(fileDiff.removed) : []
const input = [
`Soul: ${args.slug}`,
`Version: ${args.version}`,
`File changes: ${diffSummary}`,
changedPaths.length ? `Changed files (sample): ${changedPaths.join(', ')}` : null,
addedPaths.length ? `Added files (sample): ${addedPaths.join(', ')}` : null,
removedPaths.length ? `Removed files (sample): ${removedPaths.join(', ')}` : null,
oldReadme ? `Previous SOUL.md:\n${oldReadme}` : null,
`New SOUL.md:\n${nextReadme}`,
]
.filter(Boolean)
.join('\n\n')
const response = await fetch('https://api.openai.com/v1/responses', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${apiKey}`,
},
body: JSON.stringify({
model: CHANGELOG_MODEL,
instructions:
'Write a concise changelog for this soul version. Audience: everyone. Output plain text. Prefer 26 bullet points. If it is a big change, include a short 1-line summary first, then bullets. Dont mention that you are AI. Dont invent details; only use the inputs.',
input,
max_output_tokens: 220,
}),
})
if (!response.ok) return null
const payload = (await response.json()) as unknown
return extractResponseText(payload)
}
function generateFallback(args: {
slug: string
version: string
oldReadme: string | null
nextReadme: string
fileDiff: FileDiffSummary | null
}) {
const lines: string[] = []
if (!args.oldReadme) {
lines.push(`- Initial release.`)
return lines.join('\n')
}
const diff = args.fileDiff
if (diff) {
const parts: string[] = []
if (diff.added.length) parts.push(`added ${diff.added.length}`)
if (diff.changed.length) parts.push(`updated ${diff.changed.length}`)
if (diff.removed.length) parts.push(`removed ${diff.removed.length}`)
if (parts.length) lines.push(`- ${parts.join(', ')} file(s).`)
}
lines.push(`- Updated SOUL.md.`)
return lines.join('\n')
}
export async function generateSoulChangelogForPublish(
ctx: ActionCtx,
args: { slug: string; version: string; readmeText: string; files: FileMeta[] },
): Promise<string> {
try {
const soul = (await ctx.runQuery(internal.souls.getSoulBySlugInternal, {
slug: args.slug,
})) as Doc<'souls'> | null
const previous: Doc<'soulVersions'> | null =
soul?.latestVersionId && !soul.softDeletedAt
? ((await ctx.runQuery(internal.souls.getVersionByIdInternal, {
versionId: soul.latestVersionId,
})) as Doc<'soulVersions'> | null)
: null
const oldReadmeText: string | null = previous
? await readReadmeFromVersion(ctx, previous)
: null
const oldFiles = previous
? previous.files.map((file) => ({ path: file.path, sha256: file.sha256 }))
: []
const fileDiff = previous ? summarizeFileDiff(oldFiles, args.files) : null
const ai = await generateWithOpenAI({
slug: args.slug,
version: args.version,
oldReadme: oldReadmeText,
nextReadme: args.readmeText,
fileDiff,
}).catch(() => null)
return (
ai ??
generateFallback({
slug: args.slug,
version: args.version,
oldReadme: oldReadmeText,
nextReadme: args.readmeText,
fileDiff,
})
)
} catch {
return '- Updated soul.'
}
}
export async function generateSoulChangelogPreview(
ctx: ActionCtx,
args: {
slug: string
version: string
readmeText: string
filePaths?: string[]
},
): Promise<string> {
try {
const soul = (await ctx.runQuery(internal.souls.getSoulBySlugInternal, {
slug: args.slug,
})) as Doc<'souls'> | null
const previous: Doc<'soulVersions'> | null =
soul?.latestVersionId && !soul.softDeletedAt
? ((await ctx.runQuery(internal.souls.getVersionByIdInternal, {
versionId: soul.latestVersionId,
})) as Doc<'soulVersions'> | null)
: null
const oldReadmeText: string | null = previous
? await readReadmeFromVersion(ctx, previous)
: null
const oldPaths = previous ? previous.files.map((file) => file.path) : []
const nextPaths = args.filePaths ?? []
const diff = previous ? summarizeFileDiffFromPaths(oldPaths, nextPaths) : null
const ai = await generateWithOpenAI({
slug: args.slug,
version: args.version,
oldReadme: oldReadmeText,
nextReadme: args.readmeText,
fileDiff: diff,
}).catch(() => null)
return (
ai ??
generateFallback({
slug: args.slug,
version: args.version,
oldReadme: oldReadmeText,
nextReadme: args.readmeText,
fileDiff: diff,
})
)
} catch {
return '- Updated soul.'
}
}
async function readReadmeFromVersion(ctx: ActionCtx, version: Doc<'soulVersions'>) {
const file = version.files.find((entry) => entry.path.toLowerCase() === 'soul.md')
if (!file) return null
const blob = await ctx.storage.get(file.storageId)
if (!blob) return null
return blob.text()
}
function summarizeFileDiffFromPaths(oldPaths: string[], nextPaths: string[]) {
const oldFiles = oldPaths.map((path) => ({ path }))
const nextFiles = nextPaths.map((path) => ({ path }))
return summarizeFileDiff(oldFiles, nextFiles)
}
export const __test = {
summarizeFileDiff,
}
+234
View File
@@ -0,0 +1,234 @@
import { ConvexError } from 'convex/values'
import semver from 'semver'
import { api, internal } from '../_generated/api'
import type { Doc, Id } from '../_generated/dataModel'
import type { ActionCtx } from '../_generated/server'
import { generateEmbedding } from './embeddings'
import {
buildEmbeddingText,
getFrontmatterMetadata,
getFrontmatterValue,
hashSkillFiles,
isTextFile,
parseFrontmatter,
sanitizePath,
} from './skills'
import { generateSoulChangelogForPublish } from './soulChangelog'
const MAX_TOTAL_BYTES = 50 * 1024 * 1024
const MAX_SUMMARY_LENGTH = 160
function deriveSoulSummary(readmeText: string) {
const lines = readmeText.split(/\r?\n/)
let inFrontmatter = false
for (const raw of lines) {
const trimmed = raw.trim()
if (!trimmed) continue
if (!inFrontmatter && trimmed === '---') {
inFrontmatter = true
continue
}
if (inFrontmatter) {
if (trimmed === '---') {
inFrontmatter = false
}
continue
}
const cleaned = trimmed.replace(/^#+\s*/, '')
if (!cleaned) continue
if (cleaned.length > MAX_SUMMARY_LENGTH) {
return `${cleaned.slice(0, MAX_SUMMARY_LENGTH - 3).trimEnd()}...`
}
return cleaned
}
return undefined
}
export type PublishResult = {
soulId: Id<'souls'>
versionId: Id<'soulVersions'>
embeddingId: Id<'soulEmbeddings'>
}
export type PublishVersionArgs = {
slug: string
displayName: string
version: string
changelog: string
tags?: string[]
source?: {
kind: 'github'
url: string
repo: string
ref: string
commit: string
path: string
importedAt: number
}
files: Array<{
path: string
size: number
storageId: Id<'_storage'>
sha256: string
contentType?: string
}>
}
export async function publishSoulVersionForUser(
ctx: ActionCtx,
userId: Id<'users'>,
args: PublishVersionArgs,
): Promise<PublishResult> {
const version = args.version.trim()
const slug = args.slug.trim().toLowerCase()
const displayName = args.displayName.trim()
if (!slug || !displayName) throw new ConvexError('Slug and display name required')
if (!/^[a-z0-9][a-z0-9-]*$/.test(slug)) {
throw new ConvexError('Slug must be lowercase and url-safe')
}
if (!semver.valid(version)) {
throw new ConvexError('Version must be valid semver')
}
const suppliedChangelog = args.changelog.trim()
const changelogSource = suppliedChangelog ? ('user' as const) : ('auto' as const)
const sanitizedFiles = args.files.map((file) => {
const path = sanitizePath(file.path)
if (!path) throw new ConvexError('Invalid file paths')
if (!isTextFile(path, file.contentType ?? undefined)) {
throw new ConvexError('Only text-based files are allowed')
}
return { ...file, path }
})
const totalBytes = sanitizedFiles.reduce((sum, file) => sum + file.size, 0)
if (totalBytes > MAX_TOTAL_BYTES) {
throw new ConvexError('Soul bundle exceeds 50MB limit')
}
const isSoulFile = (path: string) => path.toLowerCase() === 'soul.md'
const readmeFile = sanitizedFiles.find((file) => isSoulFile(file.path))
if (!readmeFile) throw new ConvexError('SOUL.md is required')
const nonSoulFiles = sanitizedFiles.filter((file) => !isSoulFile(file.path))
if (nonSoulFiles.length > 0) {
throw new ConvexError('Only SOUL.md is allowed for soul bundles')
}
const readmeText = await fetchText(ctx, readmeFile.storageId)
const frontmatter = parseFrontmatter(readmeText)
const summary = getFrontmatterValue(frontmatter, 'description') ?? deriveSoulSummary(readmeText)
const metadata = mergeSourceIntoMetadata(getFrontmatterMetadata(frontmatter), args.source)
const embeddingText = buildEmbeddingText({
frontmatter,
readme: readmeText,
otherFiles: [],
})
const fingerprint = await hashSkillFiles(
sanitizedFiles.map((file) => ({
path: file.path ?? '',
sha256: file.sha256,
})),
)
const changelogPromise =
changelogSource === 'user'
? Promise.resolve(suppliedChangelog)
: generateSoulChangelogForPublish(ctx, {
slug,
version,
readmeText,
files: sanitizedFiles.map((file) => ({ path: file.path ?? '', sha256: file.sha256 })),
})
const embeddingPromise = generateEmbedding(embeddingText)
const [changelogText, embedding] = await Promise.all([
changelogPromise,
embeddingPromise.catch((error) => {
throw new ConvexError(formatEmbeddingError(error))
}),
])
const publishResult = (await ctx.runMutation(internal.souls.insertVersion, {
userId,
slug,
displayName,
version,
changelog: changelogText,
changelogSource,
tags: args.tags?.map((tag) => tag.trim()).filter(Boolean),
fingerprint,
files: sanitizedFiles,
parsed: {
frontmatter,
metadata,
},
summary,
embedding,
})) as PublishResult
const owner = (await ctx.runQuery(api.users.getById, { userId })) as Doc<'users'> | null
const ownerHandle = owner?.handle ?? owner?.name ?? userId
void ctx.scheduler
.runAfter(0, internal.githubSoulBackupsNode.backupSoulForPublishInternal, {
slug,
version,
displayName,
ownerHandle,
files: sanitizedFiles,
publishedAt: Date.now(),
})
.catch((error) => {
console.error('GitHub soul backup scheduling failed', error)
})
return publishResult
}
function mergeSourceIntoMetadata(metadata: unknown, source: PublishVersionArgs['source']) {
if (!source) return metadata === undefined ? undefined : metadata
const sourceValue = {
kind: source.kind,
url: source.url,
repo: source.repo,
ref: source.ref,
commit: source.commit,
path: source.path,
importedAt: source.importedAt,
}
if (!metadata) return { source: sourceValue }
if (typeof metadata !== 'object' || Array.isArray(metadata)) return { source: sourceValue }
return { ...(metadata as Record<string, unknown>), source: sourceValue }
}
export async function fetchText(
ctx: { storage: { get: (id: Id<'_storage'>) => Promise<Blob | null> } },
storageId: Id<'_storage'>,
) {
const blob = await ctx.storage.get(storageId)
if (!blob) throw new Error('File missing in storage')
return blob.text()
}
function formatEmbeddingError(error: unknown) {
if (error instanceof Error) {
if (error.message.includes('OPENAI_API_KEY')) {
return 'OPENAI_API_KEY is not configured.'
}
if (error.message.startsWith('Embedding failed')) {
return error.message
}
}
return 'Embedding failed. Please try again.'
}
export const __test = {
getSummary: (frontmatter: Record<string, unknown>) =>
getFrontmatterValue(frontmatter, 'description'),
}
+26 -13
View File
@@ -1,20 +1,33 @@
/* @vitest-environment node */
import { describe, expect, it } from 'vitest'
import { API_TOKEN_PREFIX, generateToken, hashToken } from './tokens'
import { __test, generateToken, hashToken } from './tokens'
describe('tokens', () => {
it('generates token with prefix and url-safe chars', () => {
const { token, prefix } = generateToken()
expect(token.startsWith(API_TOKEN_PREFIX)).toBe(true)
expect(prefix).toBe(token.slice(0, 12))
expect(token).toMatch(/^[a-z0-9_-]+$/i)
it('hashToken returns sha256 hex', async () => {
await expect(hashToken('test')).resolves.toBe(
'9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08',
)
})
it('hashes tokens deterministically', async () => {
const a = await hashToken('clh_test')
const b = await hashToken('clh_test')
const c = await hashToken('clh_other')
expect(a).toBe(b)
expect(a).not.toBe(c)
expect(a).toMatch(/^[a-f0-9]{64}$/)
it('generateToken returns token + prefix', () => {
const { token, prefix } = generateToken()
expect(token).toMatch(/^clh_[A-Za-z0-9_-]+$/)
expect(prefix).toBe(token.slice(0, 12))
})
it('toHex encodes bytes', () => {
expect(__test.toHex(new Uint8Array([0, 15, 255]))).toBe('000fff')
})
it('toBase64 encodes 1/2/3-byte tails', () => {
expect(__test.toBase64(new Uint8Array([0xff]))).toBe('/w==')
expect(__test.toBase64(new Uint8Array([0xff, 0xee]))).toBe('/+4=')
expect(__test.toBase64(new Uint8Array([0xff, 0xee, 0xdd]))).toBe('/+7d')
})
it('toBase64Url replaces alphabet and strips padding', () => {
expect(__test.toBase64Url(new Uint8Array([0xff]))).toBe('_w')
expect(__test.toBase64Url(new Uint8Array([0xfa, 0x00, 0x00]))).toBe('-gAA')
})
})
+6
View File
@@ -43,3 +43,9 @@ function toBase64(bytes: Uint8Array) {
}
return output
}
export const __test = {
toHex,
toBase64,
toBase64Url,
}
+151 -1
View File
@@ -7,11 +7,17 @@ vi.mock('./_generated/api', () => ({
getSkillBackfillPageInternal: Symbol('getSkillBackfillPageInternal'),
applySkillBackfillPatchInternal: Symbol('applySkillBackfillPatchInternal'),
backfillSkillSummariesInternal: Symbol('backfillSkillSummariesInternal'),
getSkillFingerprintBackfillPageInternal: Symbol('getSkillFingerprintBackfillPageInternal'),
applySkillFingerprintBackfillPatchInternal: Symbol(
'applySkillFingerprintBackfillPatchInternal',
),
backfillSkillFingerprintsInternal: Symbol('backfillSkillFingerprintsInternal'),
},
},
}))
const { backfillSkillSummariesInternalHandler } = await import('./maintenance')
const { backfillSkillFingerprintsInternalHandler, backfillSkillSummariesInternalHandler } =
await import('./maintenance')
function makeBlob(text: string) {
return { text: () => Promise.resolve(text) } as unknown as Blob
@@ -118,3 +124,147 @@ describe('maintenance backfill', () => {
expect(runMutation).not.toHaveBeenCalled()
})
})
describe('maintenance fingerprint backfill', () => {
it('backfills fingerprint field and inserts index entry', async () => {
const { hashSkillFiles } = await import('./lib/skills')
const expected = await hashSkillFiles([{ path: 'SKILL.md', sha256: 'abc' }])
const runQuery = vi.fn().mockResolvedValue({
items: [
{
skillId: 'skills:1',
versionId: 'skillVersions:1',
versionFingerprint: undefined,
files: [{ path: 'SKILL.md', sha256: 'abc' }],
existingEntries: [],
},
],
cursor: null,
isDone: true,
})
const runMutation = vi.fn().mockResolvedValue({ ok: true })
const result = await backfillSkillFingerprintsInternalHandler(
{ runQuery, runMutation } as never,
{ dryRun: false, batchSize: 10, maxBatches: 1 },
)
expect(result.ok).toBe(true)
expect(result.stats.versionsScanned).toBe(1)
expect(result.stats.versionsPatched).toBe(1)
expect(result.stats.fingerprintsInserted).toBe(1)
expect(result.stats.fingerprintMismatches).toBe(0)
expect(runMutation).toHaveBeenCalledTimes(1)
expect(runMutation).toHaveBeenCalledWith(expect.anything(), {
versionId: 'skillVersions:1',
fingerprint: expected,
patchVersion: true,
replaceEntries: true,
existingEntryIds: [],
})
})
it('dryRun does not patch', async () => {
const runQuery = vi.fn().mockResolvedValue({
items: [
{
skillId: 'skills:1',
versionId: 'skillVersions:1',
versionFingerprint: undefined,
files: [{ path: 'SKILL.md', sha256: 'abc' }],
existingEntries: [],
},
],
cursor: null,
isDone: true,
})
const runMutation = vi.fn()
const result = await backfillSkillFingerprintsInternalHandler(
{ runQuery, runMutation } as never,
{ dryRun: true, batchSize: 10, maxBatches: 1 },
)
expect(result.ok).toBe(true)
expect(result.stats.versionsPatched).toBe(1)
expect(result.stats.fingerprintsInserted).toBe(1)
expect(runMutation).not.toHaveBeenCalled()
})
it('patches missing version fingerprint without touching correct entries', async () => {
const { hashSkillFiles } = await import('./lib/skills')
const expected = await hashSkillFiles([{ path: 'SKILL.md', sha256: 'abc' }])
const runQuery = vi.fn().mockResolvedValue({
items: [
{
skillId: 'skills:1',
versionId: 'skillVersions:1',
versionFingerprint: undefined,
files: [{ path: 'SKILL.md', sha256: 'abc' }],
existingEntries: [{ id: 'skillVersionFingerprints:1', fingerprint: expected }],
},
],
cursor: null,
isDone: true,
})
const runMutation = vi.fn().mockResolvedValue({ ok: true })
const result = await backfillSkillFingerprintsInternalHandler(
{ runQuery, runMutation } as never,
{ dryRun: false, batchSize: 10, maxBatches: 1 },
)
expect(result.ok).toBe(true)
expect(result.stats.versionsPatched).toBe(1)
expect(result.stats.fingerprintsInserted).toBe(0)
expect(result.stats.fingerprintMismatches).toBe(0)
expect(runMutation).toHaveBeenCalledWith(expect.anything(), {
versionId: 'skillVersions:1',
fingerprint: expected,
patchVersion: true,
replaceEntries: false,
existingEntryIds: [],
})
})
it('replaces mismatched fingerprint entries', async () => {
const { hashSkillFiles } = await import('./lib/skills')
const expected = await hashSkillFiles([{ path: 'SKILL.md', sha256: 'abc' }])
const runQuery = vi.fn().mockResolvedValue({
items: [
{
skillId: 'skills:1',
versionId: 'skillVersions:1',
versionFingerprint: 'wrong',
files: [{ path: 'SKILL.md', sha256: 'abc' }],
existingEntries: [{ id: 'skillVersionFingerprints:1', fingerprint: 'wrong' }],
},
],
cursor: null,
isDone: true,
})
const runMutation = vi.fn().mockResolvedValue({ ok: true })
const result = await backfillSkillFingerprintsInternalHandler(
{ runQuery, runMutation } as never,
{ dryRun: false, batchSize: 10, maxBatches: 1 },
)
expect(result.ok).toBe(true)
expect(result.stats.fingerprintMismatches).toBe(1)
expect(runMutation).toHaveBeenCalledWith(expect.anything(), {
versionId: 'skillVersions:1',
fingerprint: expected,
patchVersion: true,
replaceEntries: true,
existingEntryIds: ['skillVersionFingerprints:1'],
})
})
})
+224
View File
@@ -5,6 +5,7 @@ import type { ActionCtx } from './_generated/server'
import { action, internalAction, internalMutation, internalQuery } from './_generated/server'
import { assertRole, requireUserFromAction } from './lib/access'
import { buildSkillSummaryBackfillPatch, type ParsedSkillData } from './lib/skillBackfill'
import { hashSkillFiles } from './lib/skills'
const DEFAULT_BATCH_SIZE = 50
const MAX_BATCH_SIZE = 200
@@ -243,6 +244,229 @@ export const scheduleBackfillSkillSummaries: ReturnType<typeof action> = action(
},
})
type FingerprintBackfillStats = {
versionsScanned: number
versionsPatched: number
fingerprintsInserted: number
fingerprintMismatches: number
}
type FingerprintBackfillPageItem = {
skillId: Id<'skills'>
versionId: Id<'skillVersions'>
versionFingerprint?: string
files: Array<{ path: string; sha256: string }>
existingEntries: Array<{ id: Id<'skillVersionFingerprints'>; fingerprint: string }>
}
type FingerprintBackfillPageResult = {
items: FingerprintBackfillPageItem[]
cursor: string | null
isDone: boolean
}
export const getSkillFingerprintBackfillPageInternal = internalQuery({
args: {
cursor: v.optional(v.string()),
batchSize: v.optional(v.number()),
},
handler: async (ctx, args): Promise<FingerprintBackfillPageResult> => {
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
const { page, isDone, continueCursor } = await ctx.db
.query('skillVersions')
.order('asc')
.paginate({ cursor: args.cursor ?? null, numItems: batchSize })
const items: FingerprintBackfillPageItem[] = []
for (const version of page) {
const existingEntries = await ctx.db
.query('skillVersionFingerprints')
.withIndex('by_version', (q) => q.eq('versionId', version._id))
.take(20)
const normalizedFiles = version.files.map((file) => ({
path: file.path,
sha256: file.sha256,
}))
const hasAnyEntry = existingEntries.length > 0
const entryFingerprints = new Set(existingEntries.map((entry) => entry.fingerprint))
const hasFingerprintMismatch =
typeof version.fingerprint === 'string' &&
hasAnyEntry &&
(entryFingerprints.size !== 1 || !entryFingerprints.has(version.fingerprint))
const needsFingerprintField = !version.fingerprint
const needsFingerprintEntry = !hasAnyEntry
if (!needsFingerprintField && !needsFingerprintEntry && !hasFingerprintMismatch) continue
items.push({
skillId: version.skillId,
versionId: version._id,
versionFingerprint: version.fingerprint ?? undefined,
files: normalizedFiles,
existingEntries: existingEntries.map((entry) => ({
id: entry._id,
fingerprint: entry.fingerprint,
})),
})
}
return { items, cursor: continueCursor, isDone }
},
})
export const applySkillFingerprintBackfillPatchInternal = internalMutation({
args: {
versionId: v.id('skillVersions'),
fingerprint: v.string(),
patchVersion: v.boolean(),
replaceEntries: v.boolean(),
existingEntryIds: v.optional(v.array(v.id('skillVersionFingerprints'))),
},
handler: async (ctx, args) => {
const version = await ctx.db.get(args.versionId)
if (!version) return { ok: false as const, reason: 'missingVersion' as const }
const now = Date.now()
if (args.patchVersion) {
await ctx.db.patch(version._id, { fingerprint: args.fingerprint })
}
if (args.replaceEntries) {
const existing = args.existingEntryIds ?? []
for (const id of existing) {
await ctx.db.delete(id)
}
await ctx.db.insert('skillVersionFingerprints', {
skillId: version.skillId,
versionId: version._id,
fingerprint: args.fingerprint,
createdAt: now,
})
}
return { ok: true as const }
},
})
export type FingerprintBackfillActionArgs = {
dryRun?: boolean
batchSize?: number
maxBatches?: number
}
export type FingerprintBackfillActionResult = { ok: true; stats: FingerprintBackfillStats }
export async function backfillSkillFingerprintsInternalHandler(
ctx: ActionCtx,
args: FingerprintBackfillActionArgs,
): Promise<FingerprintBackfillActionResult> {
const dryRun = Boolean(args.dryRun)
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
const maxBatches = clampInt(args.maxBatches ?? DEFAULT_MAX_BATCHES, 1, MAX_MAX_BATCHES)
const totals: FingerprintBackfillStats = {
versionsScanned: 0,
versionsPatched: 0,
fingerprintsInserted: 0,
fingerprintMismatches: 0,
}
let cursor: string | null = null
let isDone = false
for (let i = 0; i < maxBatches; i++) {
const page = (await ctx.runQuery(internal.maintenance.getSkillFingerprintBackfillPageInternal, {
cursor: cursor ?? undefined,
batchSize,
})) as FingerprintBackfillPageResult
cursor = page.cursor
isDone = page.isDone
for (const item of page.items) {
totals.versionsScanned++
const fingerprint = await hashSkillFiles(item.files)
const existingFingerprints = new Set(item.existingEntries.map((entry) => entry.fingerprint))
const hasAnyEntry = item.existingEntries.length > 0
const entryIsCorrect =
hasAnyEntry && existingFingerprints.size === 1 && existingFingerprints.has(fingerprint)
const versionFingerprintIsCorrect = item.versionFingerprint === fingerprint
if (hasAnyEntry && !entryIsCorrect) totals.fingerprintMismatches++
const shouldPatchVersion = !versionFingerprintIsCorrect
const shouldReplaceEntries = !entryIsCorrect
if (!shouldPatchVersion && !shouldReplaceEntries) continue
if (shouldPatchVersion) totals.versionsPatched++
if (shouldReplaceEntries) totals.fingerprintsInserted++
if (dryRun) continue
await ctx.runMutation(internal.maintenance.applySkillFingerprintBackfillPatchInternal, {
versionId: item.versionId,
fingerprint,
patchVersion: shouldPatchVersion,
replaceEntries: shouldReplaceEntries,
existingEntryIds: shouldReplaceEntries ? item.existingEntries.map((entry) => entry.id) : [],
})
}
if (isDone) break
}
if (!isDone) {
throw new ConvexError('Backfill incomplete (maxBatches reached)')
}
return { ok: true as const, stats: totals }
}
export const backfillSkillFingerprintsInternal = internalAction({
args: {
dryRun: v.optional(v.boolean()),
batchSize: v.optional(v.number()),
maxBatches: v.optional(v.number()),
},
handler: backfillSkillFingerprintsInternalHandler,
})
export const backfillSkillFingerprints: ReturnType<typeof action> = action({
args: {
dryRun: v.optional(v.boolean()),
batchSize: v.optional(v.number()),
maxBatches: v.optional(v.number()),
},
handler: async (ctx, args): Promise<FingerprintBackfillActionResult> => {
const { user } = await requireUserFromAction(ctx)
assertRole(user, ['admin'])
return ctx.runAction(
internal.maintenance.backfillSkillFingerprintsInternal,
args,
) as Promise<FingerprintBackfillActionResult>
},
})
export const scheduleBackfillSkillFingerprints: ReturnType<typeof action> = action({
args: { dryRun: v.optional(v.boolean()) },
handler: async (ctx, args) => {
const { user } = await requireUserFromAction(ctx)
assertRole(user, ['admin'])
await ctx.scheduler.runAfter(0, internal.maintenance.backfillSkillFingerprintsInternal, {
dryRun: Boolean(args.dryRun),
batchSize: DEFAULT_BATCH_SIZE,
maxBatches: DEFAULT_MAX_BATCHES,
})
return { ok: true as const }
},
})
function clampInt(value: number, min: number, max: number) {
const rounded = Math.trunc(value)
if (!Number.isFinite(rounded)) return min
+50
View File
@@ -0,0 +1,50 @@
import { v } from 'convex/values'
import { internalMutation } from './_generated/server'
export const checkRateLimitInternal = internalMutation({
args: {
key: v.string(),
limit: v.number(),
windowMs: v.number(),
},
handler: async (ctx, args) => {
const now = Date.now()
const windowStart = Math.floor(now / args.windowMs) * args.windowMs
const resetAt = windowStart + args.windowMs
if (args.limit <= 0) {
return { allowed: false, remaining: 0, limit: args.limit, resetAt }
}
const existing = await ctx.db
.query('rateLimits')
.withIndex('by_key_window', (q) => q.eq('key', args.key).eq('windowStart', windowStart))
.unique()
if (!existing) {
await ctx.db.insert('rateLimits', {
key: args.key,
windowStart,
count: 1,
limit: args.limit,
updatedAt: now,
})
return { allowed: true, remaining: Math.max(0, args.limit - 1), limit: args.limit, resetAt }
}
if (existing.count >= args.limit) {
return { allowed: false, remaining: 0, limit: args.limit, resetAt }
}
await ctx.db.patch(existing._id, {
count: existing.count + 1,
limit: args.limit,
updatedAt: now,
})
return {
allowed: true,
remaining: Math.max(0, args.limit - existing.count - 1),
limit: args.limit,
resetAt,
}
},
})
+140
View File
@@ -28,6 +28,15 @@ const skills = defineTable({
displayName: v.string(),
summary: v.optional(v.string()),
ownerUserId: v.id('users'),
canonicalSkillId: v.optional(v.id('skills')),
forkOf: v.optional(
v.object({
skillId: v.id('skills'),
kind: v.union(v.literal('fork'), v.literal('duplicate')),
version: v.optional(v.string()),
at: v.number(),
}),
),
latestVersionId: v.optional(v.id('skillVersions')),
tags: v.record(v.string(), v.id('skillVersions')),
softDeletedAt: v.optional(v.number()),
@@ -56,9 +65,31 @@ const skills = defineTable({
.index('by_updated', ['updatedAt'])
.index('by_batch', ['batch'])
const souls = defineTable({
slug: v.string(),
displayName: v.string(),
summary: v.optional(v.string()),
ownerUserId: v.id('users'),
latestVersionId: v.optional(v.id('soulVersions')),
tags: v.record(v.string(), v.id('soulVersions')),
softDeletedAt: v.optional(v.number()),
stats: v.object({
downloads: v.number(),
stars: v.number(),
versions: v.number(),
comments: v.number(),
}),
createdAt: v.number(),
updatedAt: v.number(),
})
.index('by_slug', ['slug'])
.index('by_owner', ['ownerUserId'])
.index('by_updated', ['updatedAt'])
const skillVersions = defineTable({
skillId: v.id('skills'),
version: v.string(),
fingerprint: v.optional(v.string()),
changelog: v.string(),
changelogSource: v.optional(v.union(v.literal('auto'), v.literal('user'))),
files: v.array(
@@ -82,6 +113,52 @@ const skillVersions = defineTable({
.index('by_skill', ['skillId'])
.index('by_skill_version', ['skillId', 'version'])
const soulVersions = defineTable({
soulId: v.id('souls'),
version: v.string(),
fingerprint: v.optional(v.string()),
changelog: v.string(),
changelogSource: v.optional(v.union(v.literal('auto'), v.literal('user'))),
files: v.array(
v.object({
path: v.string(),
size: v.number(),
storageId: v.id('_storage'),
sha256: v.string(),
contentType: v.optional(v.string()),
}),
),
parsed: v.object({
frontmatter: v.record(v.string(), v.any()),
metadata: v.optional(v.any()),
}),
createdBy: v.id('users'),
createdAt: v.number(),
softDeletedAt: v.optional(v.number()),
})
.index('by_soul', ['soulId'])
.index('by_soul_version', ['soulId', 'version'])
const skillVersionFingerprints = defineTable({
skillId: v.id('skills'),
versionId: v.id('skillVersions'),
fingerprint: v.string(),
createdAt: v.number(),
})
.index('by_version', ['versionId'])
.index('by_fingerprint', ['fingerprint'])
.index('by_skill_fingerprint', ['skillId', 'fingerprint'])
const soulVersionFingerprints = defineTable({
soulId: v.id('souls'),
versionId: v.id('soulVersions'),
fingerprint: v.string(),
createdAt: v.number(),
})
.index('by_version', ['versionId'])
.index('by_fingerprint', ['fingerprint'])
.index('by_soul_fingerprint', ['soulId', 'fingerprint'])
const skillEmbeddings = defineTable({
skillId: v.id('skills'),
versionId: v.id('skillVersions'),
@@ -100,6 +177,24 @@ const skillEmbeddings = defineTable({
filterFields: ['visibility'],
})
const soulEmbeddings = defineTable({
soulId: v.id('souls'),
versionId: v.id('soulVersions'),
ownerId: v.id('users'),
embedding: v.array(v.number()),
isLatest: v.boolean(),
isApproved: v.boolean(),
visibility: v.string(),
updatedAt: v.number(),
})
.index('by_soul', ['soulId'])
.index('by_version', ['versionId'])
.vectorIndex('by_embedding', {
vectorField: 'embedding',
dimensions: EMBEDDING_DIMENSIONS,
filterFields: ['visibility'],
})
const comments = defineTable({
skillId: v.id('skills'),
userId: v.id('users'),
@@ -111,6 +206,17 @@ const comments = defineTable({
.index('by_skill', ['skillId'])
.index('by_user', ['userId'])
const soulComments = defineTable({
soulId: v.id('souls'),
userId: v.id('users'),
body: v.string(),
createdAt: v.number(),
softDeletedAt: v.optional(v.number()),
deletedBy: v.optional(v.id('users')),
})
.index('by_soul', ['soulId'])
.index('by_user', ['userId'])
const stars = defineTable({
skillId: v.id('skills'),
userId: v.id('users'),
@@ -120,6 +226,15 @@ const stars = defineTable({
.index('by_user', ['userId'])
.index('by_skill_user', ['skillId', 'userId'])
const soulStars = defineTable({
soulId: v.id('souls'),
userId: v.id('users'),
createdAt: v.number(),
})
.index('by_soul', ['soulId'])
.index('by_user', ['userId'])
.index('by_soul_user', ['soulId', 'userId'])
const auditLogs = defineTable({
actorUserId: v.id('users'),
action: v.string(),
@@ -143,6 +258,22 @@ const apiTokens = defineTable({
.index('by_user', ['userId'])
.index('by_hash', ['tokenHash'])
const rateLimits = defineTable({
key: v.string(),
windowStart: v.number(),
count: v.number(),
limit: v.number(),
updatedAt: v.number(),
})
.index('by_key_window', ['key', 'windowStart'])
.index('by_key', ['key'])
const githubBackupSyncState = defineTable({
key: v.string(),
cursor: v.optional(v.string()),
updatedAt: v.number(),
}).index('by_key', ['key'])
const userSyncRoots = defineTable({
userId: v.id('users'),
rootId: v.string(),
@@ -185,12 +316,21 @@ export default defineSchema({
...authTables,
users,
skills,
souls,
skillVersions,
soulVersions,
skillVersionFingerprints,
soulVersionFingerprints,
skillEmbeddings,
soulEmbeddings,
comments,
soulComments,
stars,
soulStars,
auditLogs,
apiTokens,
rateLimits,
githubBackupSyncState,
userSyncRoots,
userSkillInstalls,
userSkillRootInstalls,
+58
View File
@@ -66,3 +66,61 @@ export const hydrateResults = internalQuery({
return entries
},
})
type HydratedSoulEntry = {
embeddingId: Id<'soulEmbeddings'>
soul: Doc<'souls'> | null
version: Doc<'soulVersions'> | null
}
type SoulSearchResult = HydratedSoulEntry & { score: number }
export const searchSouls: ReturnType<typeof action> = action({
args: {
query: v.string(),
limit: v.optional(v.number()),
},
handler: async (ctx, args): Promise<SoulSearchResult[]> => {
const query = args.query.trim()
if (!query) return []
const vector = await generateEmbedding(query)
const results = await ctx.vectorSearch('soulEmbeddings', 'by_embedding', {
vector,
limit: args.limit ?? 10,
filter: (q) => q.or(q.eq('visibility', 'latest'), q.eq('visibility', 'latest-approved')),
})
const hydrated = (await ctx.runQuery(internal.search.hydrateSoulResults, {
embeddingIds: results.map((result) => result._id),
})) as HydratedSoulEntry[]
const scoreById = new Map<Id<'soulEmbeddings'>, number>(
results.map((result) => [result._id, result._score]),
)
return hydrated
.map((entry) => ({
...entry,
score: scoreById.get(entry.embeddingId) ?? 0,
}))
.filter((entry) => entry.soul)
},
})
export const hydrateSoulResults = internalQuery({
args: { embeddingIds: v.array(v.id('soulEmbeddings')) },
handler: async (ctx, args): Promise<HydratedSoulEntry[]> => {
const entries: HydratedSoulEntry[] = []
for (const embeddingId of args.embeddingIds) {
const embedding = await ctx.db.get(embeddingId)
if (!embedding) continue
const soul = await ctx.db.get(embedding.soulId)
if (soul?.softDeletedAt) continue
const version = await ctx.db.get(embedding.versionId)
entries.push({ embeddingId, soul, version })
}
return entries
},
})
+37
View File
@@ -0,0 +1,37 @@
import { describe, expect, it } from 'vitest'
import type { Doc } from './_generated/dataModel'
import { decideSeedStart } from './seed'
function seedState(cursor: string, updatedAt: number) {
return { cursor, updatedAt } as unknown as Doc<'githubBackupSyncState'>
}
describe('decideSeedStart', () => {
it('returns done when done', () => {
expect(decideSeedStart(seedState('done', Date.now()), Date.now())).toEqual({
started: false,
reason: 'done',
})
})
it('returns running when lock fresh', () => {
const now = Date.now()
expect(decideSeedStart(seedState('running', now), now + 1000)).toEqual({
started: false,
reason: 'running',
})
})
it('starts when lock stale', () => {
const now = Date.now()
const stale = now - 10 * 60 * 1000 - 1
expect(decideSeedStart(seedState('running', stale), now)).toEqual({
started: true,
reason: 'patched',
})
})
it('starts when missing', () => {
expect(decideSeedStart(null, Date.now())).toEqual({ started: true, reason: 'inserted' })
})
})
+253
View File
@@ -0,0 +1,253 @@
import { v } from 'convex/values'
import { internal } from './_generated/api'
import type { Doc, Id } from './_generated/dataModel'
import type { ActionCtx, DatabaseReader, DatabaseWriter } from './_generated/server'
import { action, internalMutation, internalQuery } from './_generated/server'
import { publishSoulVersionForUser } from './lib/soulPublish'
import { SOUL_SEED_DISPLAY_NAME, SOUL_SEED_HANDLE, SOUL_SEED_KEY, SOUL_SEEDS } from './seedSouls'
const SEED_LOCK_STALE_MS = 10 * 60 * 1000
type SeedStateDoc = Doc<'githubBackupSyncState'>
type SeedStartDecision = {
started: boolean
reason: 'done' | 'running' | 'patched' | 'inserted'
}
async function getSeedState(ctx: { db: DatabaseReader }): Promise<SeedStateDoc | null> {
const entries = (await ctx.db
.query('githubBackupSyncState')
.withIndex('by_key', (q) => q.eq('key', SOUL_SEED_KEY))
.order('desc')
.take(2)) as SeedStateDoc[]
return entries[0] ?? null
}
async function cleanupSeedState(ctx: { db: DatabaseWriter }, keepId: Id<'githubBackupSyncState'>) {
const entries = (await ctx.db
.query('githubBackupSyncState')
.withIndex('by_key', (q) => q.eq('key', SOUL_SEED_KEY))
.order('desc')
.take(50)) as SeedStateDoc[]
for (const entry of entries) {
if (entry._id === keepId) continue
await ctx.db.delete(entry._id)
}
}
export function decideSeedStart(existing: SeedStateDoc | null, now: number): SeedStartDecision {
const cursor = existing?.cursor ?? null
if (cursor === 'done') return { started: false, reason: 'done' }
if (cursor === 'running' && existing && now - existing.updatedAt < SEED_LOCK_STALE_MS) {
return { started: false, reason: 'running' }
}
return existing ? { started: true, reason: 'patched' } : { started: true, reason: 'inserted' }
}
export const getSoulSeedStateInternal = internalQuery({
args: {},
handler: async (ctx) => getSeedState(ctx),
})
export const setSoulSeedStateInternal = internalMutation({
args: { status: v.string() },
handler: async (ctx, args) => {
const existing = await getSeedState(ctx)
const now = Date.now()
if (existing) {
await ctx.db.patch(existing._id, { cursor: args.status, updatedAt: now })
await cleanupSeedState(ctx, existing._id)
return existing._id
}
const id = await ctx.db.insert('githubBackupSyncState', {
key: SOUL_SEED_KEY,
cursor: args.status,
updatedAt: now,
})
await cleanupSeedState(ctx, id)
return id
},
})
export const tryStartSoulSeedInternal = internalMutation({
args: {},
handler: async (ctx) => {
const now = Date.now()
const existing = await getSeedState(ctx)
const decision = decideSeedStart(existing, now)
if (!decision.started) return decision
if (existing) {
await ctx.db.patch(existing._id, { cursor: 'running', updatedAt: now })
await cleanupSeedState(ctx, existing._id)
return { started: true, reason: 'patched' as const }
}
const id = await ctx.db.insert('githubBackupSyncState', {
key: SOUL_SEED_KEY,
cursor: 'running',
updatedAt: now,
})
await cleanupSeedState(ctx, id)
return { started: true, reason: 'inserted' as const }
},
})
export const hasAnySoulsInternal = internalQuery({
args: {},
handler: async (ctx) => {
const entry = await ctx.db.query('souls').take(1)
return entry.length > 0
},
})
export const ensureSoulSeeds = action({
args: {},
handler: async (ctx) => {
const started = (await ctx.runMutation(internal.seed.tryStartSoulSeedInternal, {})) as {
started: boolean
reason: 'done' | 'running' | 'patched' | 'inserted'
}
if (!started.started) {
if (started.reason === 'done') return { seeded: false, reason: 'already-seeded' as const }
return { seeded: false, reason: 'in-progress' as const }
}
const hasSouls = (await ctx.runQuery(internal.seed.hasAnySoulsInternal, {})) as boolean
if (hasSouls) {
await ctx.runMutation(internal.seed.setSoulSeedStateInternal, { status: 'done' })
return { seeded: false, reason: 'souls-exist' as const }
}
try {
const result = await runSeed(ctx)
await ctx.runMutation(internal.seed.setSoulSeedStateInternal, { status: 'done' })
return { seeded: true, reason: 'seeded' as const, ...result }
} catch (error) {
await ctx.runMutation(internal.seed.setSoulSeedStateInternal, { status: 'error' })
throw error
}
},
})
export const seed = action({
args: {},
handler: async (ctx) => runSeed(ctx),
})
async function runSeed(ctx: ActionCtx) {
const userId = (await ctx.runMutation(internal.seed.ensureSeedUserInternal, {
handle: SOUL_SEED_HANDLE,
displayName: SOUL_SEED_DISPLAY_NAME,
})) as Id<'users'>
const created: string[] = []
const skipped: string[] = []
for (const seedEntry of SOUL_SEEDS) {
const existing = (await ctx.runQuery(internal.souls.getSoulBySlugInternal, {
slug: seedEntry.slug,
})) as Doc<'souls'> | null
if (existing) {
if (existing.softDeletedAt && existing.ownerUserId === userId) {
await ctx.runMutation(internal.souls.setSoulSoftDeletedInternal, {
userId,
slug: seedEntry.slug,
deleted: false,
})
}
skipped.push(seedEntry.slug)
continue
}
const body = seedEntry.readme
if (!body) {
skipped.push(seedEntry.slug)
continue
}
const bytes = new TextEncoder().encode(body)
const sha256 = await sha256Hex(bytes)
const storageId = await ctx.storage.store(new Blob([bytes], { type: 'text/markdown' }))
try {
await publishSoulVersionForUser(ctx, userId, {
slug: seedEntry.slug,
displayName: seedEntry.displayName,
version: seedEntry.version,
changelog: '',
tags: seedEntry.tags,
files: [
{
path: 'SOUL.md',
size: bytes.byteLength,
storageId,
sha256,
contentType: 'text/markdown',
},
],
})
created.push(seedEntry.slug)
} catch (error) {
if (!isExpectedSeedSkipError(error)) throw error
skipped.push(seedEntry.slug)
}
}
return { created, skipped }
}
function isExpectedSeedSkipError(error: unknown) {
const message = error instanceof Error ? error.message : String(error)
return (
message.includes('Version already exists') || message.includes('Only the owner can publish')
)
}
export const ensureSeedUserInternal = internalMutation({
args: {
handle: v.string(),
displayName: v.string(),
},
handler: async (ctx, args) => {
const baseHandle = args.handle.trim()
const displayName = args.displayName.trim()
const candidates = [baseHandle, `${baseHandle}-bot`]
for (let i = 2; i <= 6; i += 1) candidates.push(`${baseHandle}-bot-${i}`)
for (const candidate of candidates) {
const existing = await ctx.db
.query('users')
.withIndex('handle', (q) => q.eq('handle', candidate))
.take(2)
const user = (existing[0] ?? null) as Doc<'users'> | null
if (user) {
if ((user.displayName ?? user.name) === displayName) return user._id
continue
}
return ctx.db.insert('users', {
handle: candidate,
displayName,
createdAt: Date.now(),
updatedAt: Date.now(),
})
}
throw new Error('Unable to allocate seed user handle')
},
})
async function sha256Hex(bytes: Uint8Array) {
const digest = await crypto.subtle.digest('SHA-256', bytes)
return toHex(new Uint8Array(digest))
}
function toHex(bytes: Uint8Array) {
let out = ''
for (const byte of bytes) out += byte.toString(16).padStart(2, '0')
return out
}
+111
View File
File diff suppressed because one or more lines are too long
+303 -10
View File
@@ -1,20 +1,27 @@
import { ConvexError, v } from 'convex/values'
import { internal } from './_generated/api'
import type { Doc, Id } from './_generated/dataModel'
import type { MutationCtx } from './_generated/server'
import { action, internalMutation, internalQuery, mutation, query } from './_generated/server'
import { assertRole, requireUser, requireUserFromAction } from './lib/access'
import { generateChangelogPreview as buildChangelogPreview } from './lib/changelog'
import { getFrontmatterValue } from './lib/skills'
import {
fetchText,
type PublishResult,
publishVersionForUser,
queueHighlightedWebhook,
type PublishResult,
} from './lib/skillPublish'
import { getFrontmatterValue, hashSkillFiles } from './lib/skills'
export { publishVersionForUser } from './lib/skillPublish'
type ReadmeResult = { path: string; text: string }
type FileTextResult = { path: string; text: string; size: number; sha256: string }
const MAX_DIFF_FILE_BYTES = 200 * 1024
const MAX_LIST_LIMIT = 50
const MAX_LIST_BULK_LIMIT = 200
const MAX_LIST_TAKE = 1000
export const getBySlug = query({
args: { slug: v.string() },
@@ -26,7 +33,42 @@ export const getBySlug = query({
if (!skill || skill.softDeletedAt) return null
const latestVersion = skill.latestVersionId ? await ctx.db.get(skill.latestVersionId) : null
const owner = await ctx.db.get(skill.ownerUserId)
return { skill, latestVersion, owner }
const forkOfSkill = skill.forkOf?.skillId ? await ctx.db.get(skill.forkOf.skillId) : null
const forkOfOwner = forkOfSkill ? await ctx.db.get(forkOfSkill.ownerUserId) : null
const canonicalSkill = skill.canonicalSkillId ? await ctx.db.get(skill.canonicalSkillId) : null
const canonicalOwner = canonicalSkill ? await ctx.db.get(canonicalSkill.ownerUserId) : null
return {
skill,
latestVersion,
owner,
forkOf: forkOfSkill
? {
kind: skill.forkOf?.kind ?? 'fork',
version: skill.forkOf?.version ?? null,
skill: {
slug: forkOfSkill.slug,
displayName: forkOfSkill.displayName,
},
owner: {
handle: forkOfOwner?.handle ?? forkOfOwner?.name ?? null,
},
}
: null,
canonical: canonicalSkill
? {
skill: {
slug: canonicalSkill.slug,
displayName: canonicalSkill.displayName,
},
owner: {
handle: canonicalOwner?.handle ?? canonicalOwner?.name ?? null,
},
}
: null,
}
},
})
@@ -47,13 +89,14 @@ export const list = query({
limit: v.optional(v.number()),
},
handler: async (ctx, args) => {
const limit = args.limit ?? 24
const limit = clampInt(args.limit ?? 24, 1, MAX_LIST_BULK_LIMIT)
const takeLimit = Math.min(limit * 5, MAX_LIST_TAKE)
if (args.batch) {
const entries = await ctx.db
.query('skills')
.withIndex('by_batch', (q) => q.eq('batch', args.batch))
.order('desc')
.take(limit * 5)
.take(takeLimit)
return entries.filter((skill) => !skill.softDeletedAt).slice(0, limit)
}
const ownerUserId = args.ownerUserId
@@ -62,17 +105,79 @@ export const list = query({
.query('skills')
.withIndex('by_owner', (q) => q.eq('ownerUserId', ownerUserId))
.order('desc')
.take(limit * 5)
.take(takeLimit)
return entries.filter((skill) => !skill.softDeletedAt).slice(0, limit)
}
const entries = await ctx.db
.query('skills')
.order('desc')
.take(limit * 5)
const entries = await ctx.db.query('skills').order('desc').take(takeLimit)
return entries.filter((skill) => !skill.softDeletedAt).slice(0, limit)
},
})
export const listWithLatest = query({
args: {
batch: v.optional(v.string()),
ownerUserId: v.optional(v.id('users')),
limit: v.optional(v.number()),
},
handler: async (ctx, args) => {
const limit = clampInt(args.limit ?? 24, 1, MAX_LIST_BULK_LIMIT)
const takeLimit = Math.min(limit * 5, MAX_LIST_TAKE)
let entries: Doc<'skills'>[] = []
if (args.batch) {
entries = await ctx.db
.query('skills')
.withIndex('by_batch', (q) => q.eq('batch', args.batch))
.order('desc')
.take(takeLimit)
} else if (args.ownerUserId) {
entries = await ctx.db
.query('skills')
.withIndex('by_owner', (q) => q.eq('ownerUserId', args.ownerUserId))
.order('desc')
.take(takeLimit)
} else {
entries = await ctx.db.query('skills').order('desc').take(takeLimit)
}
const filtered = entries.filter((skill) => !skill.softDeletedAt).slice(0, limit)
const items = await Promise.all(
filtered.map(async (skill) => ({
skill,
latestVersion: skill.latestVersionId ? await ctx.db.get(skill.latestVersionId) : null,
})),
)
return items
},
})
export const listPublicPage = query({
args: {
cursor: v.optional(v.string()),
limit: v.optional(v.number()),
},
handler: async (ctx, args) => {
const limit = clampInt(args.limit ?? 24, 1, MAX_LIST_LIMIT)
const { page, isDone, continueCursor } = await ctx.db
.query('skills')
.withIndex('by_updated', (q) => q)
.order('desc')
.paginate({ cursor: args.cursor ?? null, numItems: limit })
const items: Array<{
skill: Doc<'skills'>
latestVersion: Doc<'skillVersions'> | null
}> = []
for (const skill of page) {
if (skill.softDeletedAt) continue
const latestVersion = skill.latestVersionId ? await ctx.db.get(skill.latestVersionId) : null
items.push({ skill, latestVersion })
}
return { items, nextCursor: isDone ? null : continueCursor }
},
})
export const listVersions = query({
args: { skillId: v.id('skills'), limit: v.optional(v.number()) },
handler: async (ctx, args) => {
@@ -85,6 +190,24 @@ export const listVersions = query({
},
})
export const listVersionsPage = query({
args: {
skillId: v.id('skills'),
cursor: v.optional(v.string()),
limit: v.optional(v.number()),
},
handler: async (ctx, args) => {
const limit = clampInt(args.limit ?? 20, 1, MAX_LIST_LIMIT)
const { page, isDone, continueCursor } = await ctx.db
.query('skillVersions')
.withIndex('by_skill', (q) => q.eq('skillId', args.skillId))
.order('desc')
.paginate({ cursor: args.cursor ?? null, numItems: limit })
const items = page.filter((version) => !version.softDeletedAt)
return { items, nextCursor: isDone ? null : continueCursor }
},
})
export const getVersionById = query({
args: { versionId: v.id('skillVersions') },
handler: async (ctx, args) => ctx.db.get(args.versionId),
@@ -114,6 +237,12 @@ export const publishVersion: ReturnType<typeof action> = action({
version: v.string(),
changelog: v.string(),
tags: v.optional(v.array(v.string())),
forkOf: v.optional(
v.object({
slug: v.string(),
version: v.optional(v.string()),
}),
),
files: v.array(
v.object({
path: v.string(),
@@ -165,6 +294,92 @@ export const getReadme: ReturnType<typeof action> = action({
},
})
export const getFileText: ReturnType<typeof action> = action({
args: { versionId: v.id('skillVersions'), path: v.string() },
handler: async (ctx, args): Promise<FileTextResult> => {
const version = (await ctx.runQuery(internal.skills.getVersionByIdInternal, {
versionId: args.versionId,
})) as Doc<'skillVersions'> | null
if (!version) throw new ConvexError('Version not found')
const normalizedPath = args.path.trim()
const normalizedLower = normalizedPath.toLowerCase()
const file =
version.files.find((entry) => entry.path === normalizedPath) ??
version.files.find((entry) => entry.path.toLowerCase() === normalizedLower)
if (!file) throw new ConvexError('File not found')
if (file.size > MAX_DIFF_FILE_BYTES) {
throw new ConvexError('File exceeds 200KB limit')
}
const text = await fetchText(ctx, file.storageId)
return { path: file.path, text, size: file.size, sha256: file.sha256 }
},
})
export const resolveVersionByHash = query({
args: { slug: v.string(), hash: v.string() },
handler: async (ctx, args) => {
const slug = args.slug.trim().toLowerCase()
const hash = args.hash.trim().toLowerCase()
if (!slug || !/^[a-f0-9]{64}$/.test(hash)) return null
const skill = await ctx.db
.query('skills')
.withIndex('by_slug', (q) => q.eq('slug', slug))
.unique()
if (!skill || skill.softDeletedAt) return null
const latestVersion = skill.latestVersionId ? await ctx.db.get(skill.latestVersionId) : null
const fingerprintMatches = await ctx.db
.query('skillVersionFingerprints')
.withIndex('by_skill_fingerprint', (q) => q.eq('skillId', skill._id).eq('fingerprint', hash))
.take(25)
let match: { version: string } | null = null
if (fingerprintMatches.length > 0) {
const newest = fingerprintMatches.reduce(
(best, entry) => (entry.createdAt > best.createdAt ? entry : best),
fingerprintMatches[0] as (typeof fingerprintMatches)[number],
)
const version = await ctx.db.get(newest.versionId)
if (version && !version.softDeletedAt) {
match = { version: version.version }
}
}
if (!match) {
const versions = await ctx.db
.query('skillVersions')
.withIndex('by_skill', (q) => q.eq('skillId', skill._id))
.order('desc')
.take(200)
for (const version of versions) {
if (version.softDeletedAt) continue
if (typeof version.fingerprint === 'string' && version.fingerprint === hash) {
match = { version: version.version }
break
}
const fingerprint = await hashSkillFiles(
version.files.map((file) => ({ path: file.path, sha256: file.sha256 })),
)
if (fingerprint === hash) {
match = { version: version.version }
break
}
}
}
return {
match,
latestVersion: latestVersion ? { version: latestVersion.version } : null,
}
},
})
export const updateTags = mutation({
args: {
skillId: v.id('skills'),
@@ -283,6 +498,13 @@ export const insertVersion = internalMutation({
changelog: v.string(),
changelogSource: v.optional(v.union(v.literal('auto'), v.literal('user'))),
tags: v.optional(v.array(v.string())),
fingerprint: v.string(),
forkOf: v.optional(
v.object({
slug: v.string(),
version: v.optional(v.string()),
}),
),
files: v.array(
v.object({
path: v.string(),
@@ -315,12 +537,52 @@ export const insertVersion = internalMutation({
const now = Date.now()
if (!skill) {
const forkOfSlug = args.forkOf?.slug.trim().toLowerCase() || ''
const forkOfVersion = args.forkOf?.version?.trim() || undefined
let canonicalSkillId: Id<'skills'> | undefined
let forkOf:
| {
skillId: Id<'skills'>
kind: 'fork' | 'duplicate'
version?: string
at: number
}
| undefined
if (forkOfSlug) {
const upstream = await ctx.db
.query('skills')
.withIndex('by_slug', (q) => q.eq('slug', forkOfSlug))
.unique()
if (!upstream || upstream.softDeletedAt) throw new Error('Upstream skill not found')
canonicalSkillId = upstream.canonicalSkillId ?? upstream._id
forkOf = {
skillId: upstream._id,
kind: 'fork',
version: forkOfVersion,
at: now,
}
} else {
const match = await findCanonicalSkillForFingerprint(ctx, args.fingerprint)
if (match) {
canonicalSkillId = match.canonicalSkillId ?? match._id
forkOf = {
skillId: match._id,
kind: 'duplicate',
at: now,
}
}
}
const summary = getFrontmatterValue(args.parsed.frontmatter, 'description')
const skillId = await ctx.db.insert('skills', {
slug: args.slug,
displayName: args.displayName,
summary: summary ?? undefined,
ownerUserId: userId,
canonicalSkillId,
forkOf,
latestVersionId: undefined,
tags: {},
softDeletedAt: undefined,
@@ -352,6 +614,7 @@ export const insertVersion = internalMutation({
const versionId = await ctx.db.insert('skillVersions', {
skillId: skill._id,
version: args.version,
fingerprint: args.fingerprint,
changelog: args.changelog,
changelogSource: args.changelogSource,
files: args.files,
@@ -404,6 +667,13 @@ export const insertVersion = internalMutation({
}
}
await ctx.db.insert('skillVersionFingerprints', {
skillId: skill._id,
versionId,
fingerprint: args.fingerprint,
createdAt: now,
})
return { skillId: skill._id, versionId, embeddingId }
},
})
@@ -469,3 +739,26 @@ function visibilityFor(isLatest: boolean, isApproved: boolean) {
if (isApproved) return 'archived-approved'
return 'archived'
}
function clampInt(value: number, min: number, max: number) {
const rounded = Number.isFinite(value) ? Math.round(value) : min
return Math.min(max, Math.max(min, rounded))
}
async function findCanonicalSkillForFingerprint(
ctx: { db: MutationCtx['db'] },
fingerprint: string,
) {
const matches = await ctx.db
.query('skillVersionFingerprints')
.withIndex('by_fingerprint', (q) => q.eq('fingerprint', fingerprint))
.take(25)
for (const entry of matches) {
const skill = await ctx.db.get(entry.skillId)
if (!skill || skill.softDeletedAt) continue
return skill
}
return null
}
+87
View File
@@ -0,0 +1,87 @@
import { v } from 'convex/values'
import type { Doc } from './_generated/dataModel'
import { mutation, query } from './_generated/server'
import { assertRole, requireUser } from './lib/access'
export const listBySoul = query({
args: { soulId: v.id('souls'), limit: v.optional(v.number()) },
handler: async (ctx, args) => {
const limit = args.limit ?? 50
const comments = await ctx.db
.query('soulComments')
.withIndex('by_soul', (q) => q.eq('soulId', args.soulId))
.order('desc')
.take(limit)
const results: Array<{ comment: Doc<'soulComments'>; user: Doc<'users'> | null }> = []
for (const comment of comments) {
if (comment.softDeletedAt) continue
const user = await ctx.db.get(comment.userId)
results.push({ comment, user })
}
return results
},
})
export const add = mutation({
args: { soulId: v.id('souls'), body: v.string() },
handler: async (ctx, args) => {
const { userId } = await requireUser(ctx)
const body = args.body.trim()
if (!body) throw new Error('Comment body required')
const soul = await ctx.db.get(args.soulId)
if (!soul) throw new Error('Soul not found')
await ctx.db.insert('soulComments', {
soulId: args.soulId,
userId,
body,
createdAt: Date.now(),
softDeletedAt: undefined,
deletedBy: undefined,
})
await ctx.db.patch(soul._id, {
stats: { ...soul.stats, comments: soul.stats.comments + 1 },
updatedAt: Date.now(),
})
},
})
export const remove = mutation({
args: { commentId: v.id('soulComments') },
handler: async (ctx, args) => {
const { user } = await requireUser(ctx)
const comment = await ctx.db.get(args.commentId)
if (!comment) throw new Error('Comment not found')
if (comment.softDeletedAt) return
const isOwner = comment.userId === user._id
if (!isOwner) {
assertRole(user, ['admin', 'moderator'])
}
await ctx.db.patch(comment._id, {
softDeletedAt: Date.now(),
deletedBy: user._id,
})
const soul = await ctx.db.get(comment.soulId)
if (soul) {
await ctx.db.patch(soul._id, {
stats: { ...soul.stats, comments: Math.max(0, soul.stats.comments - 1) },
updatedAt: Date.now(),
})
}
await ctx.db.insert('auditLogs', {
actorUserId: user._id,
action: 'soul.comment.delete',
targetType: 'soulComment',
targetId: comment._id,
metadata: { soulId: comment.soulId },
createdAt: Date.now(),
})
},
})
+14
View File
@@ -0,0 +1,14 @@
import { v } from 'convex/values'
import { mutation } from './_generated/server'
export const increment = mutation({
args: { soulId: v.id('souls') },
handler: async (ctx, args) => {
const soul = await ctx.db.get(args.soulId)
if (!soul) return
await ctx.db.patch(soul._id, {
stats: { ...soul.stats, downloads: soul.stats.downloads + 1 },
updatedAt: Date.now(),
})
},
})
+69
View File
@@ -0,0 +1,69 @@
import { v } from 'convex/values'
import type { Doc } from './_generated/dataModel'
import { mutation, query } from './_generated/server'
import { requireUser } from './lib/access'
export const isStarred = query({
args: { soulId: v.id('souls') },
handler: async (ctx, args) => {
const { userId } = await requireUser(ctx)
const existing = await ctx.db
.query('soulStars')
.withIndex('by_soul_user', (q) => q.eq('soulId', args.soulId).eq('userId', userId))
.unique()
return Boolean(existing)
},
})
export const toggle = mutation({
args: { soulId: v.id('souls') },
handler: async (ctx, args) => {
const { userId } = await requireUser(ctx)
const soul = await ctx.db.get(args.soulId)
if (!soul) throw new Error('Soul not found')
const existing = await ctx.db
.query('soulStars')
.withIndex('by_soul_user', (q) => q.eq('soulId', args.soulId).eq('userId', userId))
.unique()
if (existing) {
await ctx.db.delete(existing._id)
await ctx.db.patch(soul._id, {
stats: { ...soul.stats, stars: Math.max(0, soul.stats.stars - 1) },
updatedAt: Date.now(),
})
return { starred: false }
}
await ctx.db.insert('soulStars', {
soulId: args.soulId,
userId,
createdAt: Date.now(),
})
await ctx.db.patch(soul._id, {
stats: { ...soul.stats, stars: soul.stats.stars + 1 },
updatedAt: Date.now(),
})
return { starred: true }
},
})
export const listByUser = query({
args: { userId: v.id('users'), limit: v.optional(v.number()) },
handler: async (ctx, args) => {
const limit = args.limit ?? 50
const stars = await ctx.db
.query('soulStars')
.withIndex('by_user', (q) => q.eq('userId', args.userId))
.order('desc')
.take(limit)
const souls: Doc<'souls'>[] = []
for (const star of stars) {
const soul = await ctx.db.get(star.soulId)
if (soul) souls.push(soul)
}
return souls
},
})
+554
View File
@@ -0,0 +1,554 @@
import { ConvexError, v } from 'convex/values'
import { internal } from './_generated/api'
import type { Doc, Id } from './_generated/dataModel'
import { action, internalMutation, internalQuery, mutation, query } from './_generated/server'
import { assertRole, requireUser, requireUserFromAction } from './lib/access'
import { getFrontmatterValue, hashSkillFiles } from './lib/skills'
import { generateSoulChangelogPreview } from './lib/soulChangelog'
import { fetchText, type PublishResult, publishSoulVersionForUser } from './lib/soulPublish'
export { publishSoulVersionForUser } from './lib/soulPublish'
type ReadmeResult = { path: string; text: string }
type FileTextResult = { path: string; text: string; size: number; sha256: string }
const MAX_DIFF_FILE_BYTES = 200 * 1024
const MAX_LIST_LIMIT = 50
export const getBySlug = query({
args: { slug: v.string() },
handler: async (ctx, args) => {
const matches = await ctx.db
.query('souls')
.withIndex('by_slug', (q) => q.eq('slug', args.slug))
.order('desc')
.take(2)
const soul = matches[0] ?? null
if (!soul || soul.softDeletedAt) return null
const latestVersion = soul.latestVersionId ? await ctx.db.get(soul.latestVersionId) : null
const owner = await ctx.db.get(soul.ownerUserId)
return { soul, latestVersion, owner }
},
})
export const getSoulBySlugInternal = internalQuery({
args: { slug: v.string() },
handler: async (ctx, args) => {
const matches = await ctx.db
.query('souls')
.withIndex('by_slug', (q) => q.eq('slug', args.slug))
.order('desc')
.take(2)
return matches[0] ?? null
},
})
export const list = query({
args: {
ownerUserId: v.optional(v.id('users')),
limit: v.optional(v.number()),
},
handler: async (ctx, args) => {
const limit = args.limit ?? 24
const ownerUserId = args.ownerUserId
if (ownerUserId) {
const entries = await ctx.db
.query('souls')
.withIndex('by_owner', (q) => q.eq('ownerUserId', ownerUserId))
.order('desc')
.take(limit * 5)
return entries.filter((soul) => !soul.softDeletedAt).slice(0, limit)
}
const entries = await ctx.db
.query('souls')
.order('desc')
.take(limit * 5)
return entries.filter((soul) => !soul.softDeletedAt).slice(0, limit)
},
})
export const listPublicPage = query({
args: {
cursor: v.optional(v.string()),
limit: v.optional(v.number()),
},
handler: async (ctx, args) => {
const limit = clampInt(args.limit ?? 24, 1, MAX_LIST_LIMIT)
const { page, isDone, continueCursor } = await ctx.db
.query('souls')
.withIndex('by_updated', (q) => q)
.order('desc')
.paginate({ cursor: args.cursor ?? null, numItems: limit })
const items: Array<{ soul: Doc<'souls'>; latestVersion: Doc<'soulVersions'> | null }> = []
for (const soul of page) {
if (soul.softDeletedAt) continue
const latestVersion = soul.latestVersionId ? await ctx.db.get(soul.latestVersionId) : null
items.push({ soul, latestVersion })
}
return { items, nextCursor: isDone ? null : continueCursor }
},
})
export const listVersions = query({
args: { soulId: v.id('souls'), limit: v.optional(v.number()) },
handler: async (ctx, args) => {
const limit = args.limit ?? 20
return ctx.db
.query('soulVersions')
.withIndex('by_soul', (q) => q.eq('soulId', args.soulId))
.order('desc')
.take(limit)
},
})
export const listVersionsPage = query({
args: {
soulId: v.id('souls'),
cursor: v.optional(v.string()),
limit: v.optional(v.number()),
},
handler: async (ctx, args) => {
const limit = clampInt(args.limit ?? 20, 1, MAX_LIST_LIMIT)
const { page, isDone, continueCursor } = await ctx.db
.query('soulVersions')
.withIndex('by_soul', (q) => q.eq('soulId', args.soulId))
.order('desc')
.paginate({ cursor: args.cursor ?? null, numItems: limit })
const items = page.filter((version) => !version.softDeletedAt)
return { items, nextCursor: isDone ? null : continueCursor }
},
})
export const getVersionById = query({
args: { versionId: v.id('soulVersions') },
handler: async (ctx, args) => ctx.db.get(args.versionId),
})
export const getVersionByIdInternal = internalQuery({
args: { versionId: v.id('soulVersions') },
handler: async (ctx, args) => ctx.db.get(args.versionId),
})
export const getVersionBySoulAndVersion = query({
args: { soulId: v.id('souls'), version: v.string() },
handler: async (ctx, args) => {
return ctx.db
.query('soulVersions')
.withIndex('by_soul_version', (q) => q.eq('soulId', args.soulId).eq('version', args.version))
.unique()
},
})
export const publishVersion: ReturnType<typeof action> = action({
args: {
slug: v.string(),
displayName: v.string(),
version: v.string(),
changelog: v.string(),
tags: v.optional(v.array(v.string())),
source: v.optional(
v.object({
kind: v.literal('github'),
url: v.string(),
repo: v.string(),
ref: v.string(),
commit: v.string(),
path: v.string(),
importedAt: v.number(),
}),
),
files: v.array(
v.object({
path: v.string(),
size: v.number(),
storageId: v.id('_storage'),
sha256: v.string(),
contentType: v.optional(v.string()),
}),
),
},
handler: async (ctx, args): Promise<PublishResult> => {
const { userId } = await requireUserFromAction(ctx)
return publishSoulVersionForUser(ctx, userId, args)
},
})
export const generateChangelogPreview = action({
args: {
slug: v.string(),
version: v.string(),
readmeText: v.string(),
filePaths: v.optional(v.array(v.string())),
},
handler: async (ctx, args) => {
await requireUserFromAction(ctx)
const changelog = await generateSoulChangelogPreview(ctx, {
slug: args.slug.trim().toLowerCase(),
version: args.version.trim(),
readmeText: args.readmeText,
filePaths: args.filePaths?.map((value) => value.trim()).filter(Boolean),
})
return { changelog, source: 'auto' as const }
},
})
export const getReadme: ReturnType<typeof action> = action({
args: { versionId: v.id('soulVersions') },
handler: async (ctx, args): Promise<ReadmeResult> => {
const version = (await ctx.runQuery(internal.souls.getVersionByIdInternal, {
versionId: args.versionId,
})) as Doc<'soulVersions'> | null
if (!version) throw new ConvexError('Version not found')
const readmeFile = version.files.find((file) => file.path.toLowerCase() === 'soul.md')
if (!readmeFile) throw new ConvexError('SOUL.md not found')
const text = await fetchText(ctx, readmeFile.storageId)
return { path: readmeFile.path, text }
},
})
export const getFileText: ReturnType<typeof action> = action({
args: { versionId: v.id('soulVersions'), path: v.string() },
handler: async (ctx, args): Promise<FileTextResult> => {
const version = (await ctx.runQuery(internal.souls.getVersionByIdInternal, {
versionId: args.versionId,
})) as Doc<'soulVersions'> | null
if (!version) throw new ConvexError('Version not found')
const normalizedPath = args.path.trim()
const normalizedLower = normalizedPath.toLowerCase()
const file =
version.files.find((entry) => entry.path === normalizedPath) ??
version.files.find((entry) => entry.path.toLowerCase() === normalizedLower)
if (!file) throw new ConvexError('File not found')
if (file.size > MAX_DIFF_FILE_BYTES) {
throw new ConvexError('File exceeds 200KB limit')
}
const text = await fetchText(ctx, file.storageId)
return { path: file.path, text, size: file.size, sha256: file.sha256 }
},
})
export const resolveVersionByHash = query({
args: { slug: v.string(), hash: v.string() },
handler: async (ctx, args) => {
const slug = args.slug.trim().toLowerCase()
const hash = args.hash.trim().toLowerCase()
if (!slug || !/^[a-f0-9]{64}$/.test(hash)) return null
const soulMatches = await ctx.db
.query('souls')
.withIndex('by_slug', (q) => q.eq('slug', slug))
.order('desc')
.take(2)
const soul = soulMatches[0] ?? null
if (!soul || soul.softDeletedAt) return null
const latestVersion = soul.latestVersionId ? await ctx.db.get(soul.latestVersionId) : null
const fingerprintMatches = await ctx.db
.query('soulVersionFingerprints')
.withIndex('by_soul_fingerprint', (q) => q.eq('soulId', soul._id).eq('fingerprint', hash))
.take(25)
let match: { version: string } | null = null
if (fingerprintMatches.length > 0) {
const newest = fingerprintMatches.reduce(
(best, entry) => (entry.createdAt > best.createdAt ? entry : best),
fingerprintMatches[0] as (typeof fingerprintMatches)[number],
)
const version = await ctx.db.get(newest.versionId)
if (version && !version.softDeletedAt) {
match = { version: version.version }
}
}
if (!match) {
const versions = await ctx.db
.query('soulVersions')
.withIndex('by_soul', (q) => q.eq('soulId', soul._id))
.order('desc')
.take(200)
for (const version of versions) {
if (version.softDeletedAt) continue
if (typeof version.fingerprint === 'string' && version.fingerprint === hash) {
match = { version: version.version }
break
}
const fingerprint = await hashSkillFiles(
version.files.map((file) => ({ path: file.path, sha256: file.sha256 })),
)
if (fingerprint === hash) {
match = { version: version.version }
break
}
}
}
return {
match,
latestVersion: latestVersion ? { version: latestVersion.version } : null,
}
},
})
export const updateTags = mutation({
args: {
soulId: v.id('souls'),
tags: v.array(v.object({ tag: v.string(), versionId: v.id('soulVersions') })),
},
handler: async (ctx, args) => {
const { user } = await requireUser(ctx)
const soul = await ctx.db.get(args.soulId)
if (!soul) throw new Error('Soul not found')
if (soul.ownerUserId !== user._id) {
assertRole(user, ['admin', 'moderator'])
}
const nextTags = { ...soul.tags }
for (const entry of args.tags) {
nextTags[entry.tag] = entry.versionId
}
const latestEntry = args.tags.find((entry) => entry.tag === 'latest')
await ctx.db.patch(soul._id, {
tags: nextTags,
latestVersionId: latestEntry ? latestEntry.versionId : soul.latestVersionId,
updatedAt: Date.now(),
})
if (latestEntry) {
const embeddings = await ctx.db
.query('soulEmbeddings')
.withIndex('by_soul', (q) => q.eq('soulId', soul._id))
.collect()
for (const embedding of embeddings) {
const isLatest = embedding.versionId === latestEntry.versionId
await ctx.db.patch(embedding._id, {
isLatest,
visibility: visibilityFor(isLatest, embedding.isApproved),
updatedAt: Date.now(),
})
}
}
},
})
export const insertVersion = internalMutation({
args: {
userId: v.id('users'),
slug: v.string(),
displayName: v.string(),
version: v.string(),
changelog: v.string(),
changelogSource: v.optional(v.union(v.literal('auto'), v.literal('user'))),
tags: v.optional(v.array(v.string())),
fingerprint: v.string(),
summary: v.optional(v.string()),
files: v.array(
v.object({
path: v.string(),
size: v.number(),
storageId: v.id('_storage'),
sha256: v.string(),
contentType: v.optional(v.string()),
}),
),
parsed: v.object({
frontmatter: v.record(v.string(), v.any()),
metadata: v.optional(v.any()),
}),
embedding: v.array(v.number()),
},
handler: async (ctx, args) => {
const userId = args.userId
const user = await ctx.db.get(userId)
if (!user || user.deletedAt) throw new Error('User not found')
const soulMatches = await ctx.db
.query('souls')
.withIndex('by_slug', (q) => q.eq('slug', args.slug))
.order('desc')
.take(2)
let soul = soulMatches[0] ?? null
if (soul && soul.ownerUserId !== userId) {
throw new Error('Only the owner can publish updates')
}
const now = Date.now()
if (!soul) {
const summary = args.summary ?? getFrontmatterValue(args.parsed.frontmatter, 'description')
const soulId = await ctx.db.insert('souls', {
slug: args.slug,
displayName: args.displayName,
summary: summary ?? undefined,
ownerUserId: userId,
latestVersionId: undefined,
tags: {},
softDeletedAt: undefined,
stats: {
downloads: 0,
stars: 0,
versions: 0,
comments: 0,
},
createdAt: now,
updatedAt: now,
})
soul = await ctx.db.get(soulId)
}
if (!soul) throw new Error('Soul creation failed')
const existingVersion = await ctx.db
.query('soulVersions')
.withIndex('by_soul_version', (q) => q.eq('soulId', soul._id).eq('version', args.version))
.unique()
if (existingVersion) {
throw new Error('Version already exists')
}
const versionId = await ctx.db.insert('soulVersions', {
soulId: soul._id,
version: args.version,
fingerprint: args.fingerprint,
changelog: args.changelog,
changelogSource: args.changelogSource,
files: args.files,
parsed: args.parsed,
createdBy: userId,
createdAt: now,
softDeletedAt: undefined,
})
const nextTags: Record<string, Id<'soulVersions'>> = { ...soul.tags }
nextTags.latest = versionId
for (const tag of args.tags ?? []) {
nextTags[tag] = versionId
}
const latestBefore = soul.latestVersionId
await ctx.db.patch(soul._id, {
displayName: args.displayName,
summary:
args.summary ?? getFrontmatterValue(args.parsed.frontmatter, 'description') ?? soul.summary,
latestVersionId: versionId,
tags: nextTags,
stats: { ...soul.stats, versions: soul.stats.versions + 1 },
softDeletedAt: undefined,
updatedAt: now,
})
const embeddingId = await ctx.db.insert('soulEmbeddings', {
soulId: soul._id,
versionId,
ownerId: userId,
embedding: args.embedding,
isLatest: true,
isApproved: true,
visibility: visibilityFor(true, true),
updatedAt: now,
})
if (latestBefore) {
const previousEmbedding = await ctx.db
.query('soulEmbeddings')
.withIndex('by_version', (q) => q.eq('versionId', latestBefore))
.unique()
if (previousEmbedding) {
await ctx.db.patch(previousEmbedding._id, {
isLatest: false,
visibility: visibilityFor(false, previousEmbedding.isApproved),
updatedAt: now,
})
}
}
await ctx.db.insert('soulVersionFingerprints', {
soulId: soul._id,
versionId,
fingerprint: args.fingerprint,
createdAt: now,
})
return { soulId: soul._id, versionId, embeddingId }
},
})
export const setSoulSoftDeletedInternal = internalMutation({
args: {
userId: v.id('users'),
slug: v.string(),
deleted: v.boolean(),
},
handler: async (ctx, args) => {
const user = await ctx.db.get(args.userId)
if (!user || user.deletedAt) throw new Error('User not found')
const slug = args.slug.trim().toLowerCase()
if (!slug) throw new Error('Slug required')
const soulMatches = await ctx.db
.query('souls')
.withIndex('by_slug', (q) => q.eq('slug', slug))
.order('desc')
.take(2)
const soul = soulMatches[0] ?? null
if (!soul) throw new Error('Soul not found')
if (soul.ownerUserId !== args.userId) {
assertRole(user, ['admin', 'moderator'])
}
const now = Date.now()
await ctx.db.patch(soul._id, {
softDeletedAt: args.deleted ? now : undefined,
updatedAt: now,
})
const embeddings = await ctx.db
.query('soulEmbeddings')
.withIndex('by_soul', (q) => q.eq('soulId', soul._id))
.collect()
for (const embedding of embeddings) {
await ctx.db.patch(embedding._id, {
visibility: args.deleted
? 'deleted'
: visibilityFor(embedding.isLatest, embedding.isApproved),
updatedAt: now,
})
}
await ctx.db.insert('auditLogs', {
actorUserId: args.userId,
action: args.deleted ? 'soul.delete' : 'soul.undelete',
targetType: 'soul',
targetId: soul._id,
metadata: { slug, softDeletedAt: args.deleted ? now : null },
createdAt: now,
})
return { ok: true as const }
},
})
function visibilityFor(isLatest: boolean, isApproved: boolean) {
if (isLatest && isApproved) return 'latest-approved'
if (isLatest) return 'latest'
if (isApproved) return 'archived-approved'
return 'archived'
}
function clampInt(value: number, min: number, max: number) {
const rounded = Number.isFinite(value) ? Math.round(value) : min
return Math.min(max, Math.max(min, rounded))
}
+32
View File
@@ -0,0 +1,32 @@
---
summary: 'Documentation index + reading order.'
read_when:
- New contributor onboarding
- Looking for the right doc
---
# Docs
Reading order (new contributor):
1. `README.md` (repo root): run locally.
2. `docs/quickstart.md`: end-to-end: search → install → publish → sync.
3. `docs/architecture.md`: how the pieces fit (TanStack Start + Convex + CLI).
4. `docs/skill-format.md`: what a “skill” is on disk + on the registry.
5. `docs/cli.md`: CLI reference (flags, config, lockfiles, sync rules).
6. `docs/http-api.md`: HTTP endpoints used by the CLI + public API.
7. `docs/auth.md`: GitHub OAuth + API tokens + CLI loopback login.
8. `docs/deploy.md`: Convex + Vercel deployment + rewrites.
9. `docs/troubleshooting.md`: common failure modes.
Feature/ops docs (already present):
- `docs/spec.md`: product + implementation spec (data model + flows).
- `docs/telemetry.md`: what `clawdhub sync` reports; opt-out.
- `docs/webhook.md`: Discord webhook events/payload.
- `docs/diffing.md`: version-to-version diff UI spec.
- `docs/manual-testing.md`: CLI smoke scripts.
Docs tooling:
- `docs/mintlify.md`: publish these docs with Mintlify.
+50
View File
@@ -0,0 +1,50 @@
---
summary: 'Public REST API (v1) overview and conventions.'
read_when:
- Building API clients
- Adding endpoints or schemas
---
# API v1
Base: `https://clawdhub.com`
OpenAPI: `/api/v1/openapi.json`
## Auth
- Public read: no token required.
- Write + account: `Authorization: Bearer clh_...`.
## Rate limits
Per IP + per API key:
- Read: 120/min per IP, 600/min per key
- Write: 30/min per IP, 120/min per key
Headers: `X-RateLimit-Limit`, `X-RateLimit-Remaining`, `X-RateLimit-Reset`, `Retry-After` (on 429).
## Endpoints
Public read:
- `GET /api/v1/search?q=...`
- `GET /api/v1/skills?limit=&cursor=`
- `GET /api/v1/skills/{slug}`
- `GET /api/v1/skills/{slug}/versions?limit=&cursor=`
- `GET /api/v1/skills/{slug}/versions/{version}`
- `GET /api/v1/skills/{slug}/file?path=&version=&tag=`
- `GET /api/v1/resolve?slug=&hash=`
- `GET /api/v1/download?slug=&version=&tag=`
Auth required:
- `POST /api/v1/skills` (publish, multipart preferred)
- `DELETE /api/v1/skills/{slug}`
- `POST /api/v1/skills/{slug}/undelete`
- `GET /api/v1/whoami`
## Legacy
Legacy `/api/*` and `/api/cli/*` still available. See `DEPRECATIONS.md`.
+61
View File
@@ -0,0 +1,61 @@
---
summary: 'System overview: web app + Convex backend + CLI + shared schema.'
read_when:
- Orienting in codebase
- Tracing a user flow across layers
---
# Architecture
## Pieces
- Web app: TanStack Start (React) under `src/`.
- Backend: Convex under `convex/` (DB, storage, actions, HTTP routes).
- CLI: `packages/clawdhub/` (published as `clawdhub`).
- Shared schemas/routes: `packages/schema/` (`clawdhub-schema`).
## Data + storage
- Skill “bundle” = versioned set of text files stored in Convex `_storage`.
- Metadata extracted from `SKILL.md` frontmatter.
- Stats stored on `skills` (downloads, installs, stars, comments, …).
## Main flows
### Browse (web)
- UI reads skill metadata + latest version from Convex queries/actions.
- `SKILL.md` rendered as Markdown.
### Search (HTTP)
- `/api/v1/search?q=...` routes to Convex action for vector search.
- Embeddings currently generated during publish.
### Install (CLI)
- Resolve latest version via `/api/v1/skills/<slug>`.
- Download zip via `/api/v1/download?slug=...&version=...`.
- Extract into `./skills/<slug>` (default).
- Persist install state:
- `./.clawdhub/lock.json` (per workdir)
- `./skills/<slug>/.clawdhub/origin.json` (per skill folder)
### Update (CLI)
- Hash local files, call `/api/v1/resolve?slug=...&hash=<sha256>`.
- If local matches a known version → use that for “current”.
- If local doesnt match:
- refuse by default
- or overwrite with `--force`
### Publish (CLI)
- Publish via `POST /api/v1/skills` (multipart; requires Bearer token).
### Sync (CLI)
- Scan roots for skill folders (contain `SKILL.md`).
- Compute fingerprint; compare to registry state.
- Optionally reports telemetry (see `docs/telemetry.md`).
- Publishes new/changed skills (skips modified installed skills inside install root).
+54
View File
@@ -0,0 +1,54 @@
---
summary: 'Auth overview: GitHub OAuth (web) + API tokens (CLI).'
read_when:
- Working on login/token flows
- Debugging 401s
---
# Auth
## Web auth (GitHub OAuth)
- Convex Auth + GitHub OAuth App.
- Env vars:
- `AUTH_GITHUB_ID`
- `AUTH_GITHUB_SECRET`
- `CONVEX_SITE_URL` (used by auth config)
Local setup steps are in the repo root `README.md`.
## API tokens (CLI)
The CLI uses a long-lived API token (Bearer token) for publish/sync/delete.
### Browser flow (default)
`clawdhub login` does:
1. Starts a loopback HTTP server on `127.0.0.1` (random port).
2. Opens `<site>/cli/auth?redirect_uri=http://127.0.0.1:<port>/callback&state=...`.
3. Web UI requires GitHub login, then creates a token and redirects back to the loopback server.
4. CLI stores the token in the global config file.
### Headless flow
Create a token in the web UI (Settings → API tokens) and paste it:
```bash
clawdhub login --token clh_...
```
### Token storage
Default global config path:
- macOS: `~/Library/Application Support/clawdhub/config.json`
Override:
- `CLAWDHUB_CONFIG_PATH=/path/to/config.json`
### Revocation
- Tokens can be revoked in the web UI.
- Revoked tokens return `401 Unauthorized` on CLI endpoints.
+102
View File
@@ -0,0 +1,102 @@
---
summary: 'CLI reference: commands, flags, config, lockfile, sync behavior.'
read_when:
- Working on CLI behavior
- Debugging install/update/sync
---
# CLI
CLI package: `packages/clawdhub/` (bin: `clawdhub`).
From this repo you can run it via the wrapper script:
```bash
bun clawdhub --help
```
## Global flags
- `--workdir <dir>`: working directory (default: cwd; falls back to Clawdbot workspace if configured)
- `--dir <dir>`: install dir under workdir (default: `skills`)
- `--site <url>`: base URL for browser login (default: `https://clawdhub.com`)
- `--registry <url>`: API base URL (default: discovered, else `https://clawdhub.com`)
- `--no-input`: disable prompts
Env equivalents:
- `CLAWDHUB_SITE`
- `CLAWDHUB_REGISTRY`
- `CLAWDHUB_WORKDIR`
## Config file
Stores your API token + cached registry URL.
- macOS: `~/Library/Application Support/clawdhub/config.json`
- override: `CLAWDHUB_CONFIG_PATH`
## Commands
### `login` / `auth login`
- Default: opens browser to `<site>/cli/auth` and completes via loopback callback.
- Headless: `clawdhub login --token clh_...`
### `whoami`
- Verifies the stored token via `/api/v1/whoami`.
### `search <query...>`
- Calls `/api/v1/search?q=...`.
### `install <slug>`
- Resolves latest version via `/api/v1/skills/<slug>`.
- Downloads zip via `/api/v1/download`.
- Extracts into `<workdir>/<dir>/<slug>`.
- Writes:
- `<workdir>/.clawdhub/lock.json`
- `<skill>/.clawdhub/origin.json`
### `list`
- Reads `<workdir>/.clawdhub/lock.json`.
### `update [slug]` / `update --all`
- Computes fingerprint from local files.
- If fingerprint matches a known version: no prompt.
- If fingerprint does not match:
- refuses by default
- overwrites with `--force` (or prompt, if interactive)
### `publish <path>`
- Publishes via `POST /api/v1/skills` (multipart).
- Requires semver: `--version 1.2.3`.
### `sync`
- Scans for local skill folders and publishes new/changed ones.
- Roots can be any folder: a skills directory or a single skill folder with `SKILL.md`.
- Auto-adds Clawdbot skill roots when `~/.clawdbot/clawdbot.json` is present:
- `agent.workspace/skills` (main agent)
- `routing.agents.*.workspace/skills` (per-agent)
- `~/.clawdbot/skills` (shared)
- `skills.load.extraDirs` (shared packs)
- Respects `CLAWDBOT_CONFIG_PATH` and `CLAWDBOT_STATE_DIR`.
- Flags:
- `--root <dir...>` extra scan roots
- `--all` upload without prompting
- `--dry-run` show plan only
- `--bump patch|minor|major` (default: patch)
- `--changelog <text>` (non-interactive)
- `--tags a,b,c` (default: latest)
- `--concurrency <n>` (default: 4)
Telemetry:
- Sent during `sync` when logged in, unless `CLAWDHUB_DISABLE_TELEMETRY=1`.
- Details: `docs/telemetry.md`.
+78
View File
@@ -0,0 +1,78 @@
---
summary: 'Deploy checklist: Convex backend + Vercel web app + /api rewrites.'
read_when:
- Shipping to production
- Debugging /api routing
---
# Deploy
ClawdHub is two deployables:
- Web app (TanStack Start) → typically Vercel.
- Convex backend → Convex deployment (serves `/api/...` routes).
## 1) Deploy Convex
From your local machine:
```bash
bunx convex deploy
```
Ensure Convex env is set (auth + embeddings):
- `AUTH_GITHUB_ID`
- `AUTH_GITHUB_SECRET`
- `CONVEX_SITE_URL`
- `JWT_PRIVATE_KEY`
- `JWKS`
- `OPENAI_API_KEY`
- `SITE_URL` (your web app URL)
- Optional webhook env (see `docs/webhook.md`)
## 2) Deploy web app (Vercel)
Set env vars:
- `VITE_CONVEX_URL`
- `VITE_CONVEX_SITE_URL` (Convex “site” URL)
- `CONVEX_SITE_URL` (same value; used by auth provider config)
- `SITE_URL` (web app URL)
## 3) Route `/api/*` to Convex
This repo currently uses `vercel.json` rewrites:
- `source: /api/:path*`
- `destination: https://<deployment>.convex.site/api/:path*`
For self-host:
- update `vercel.json` to your deployments Convex site URL.
## 4) Registry discovery
The CLI can discover the API base from:
- `/.well-known/clawdhub.json`
If you dont serve that file, users must set:
```bash
export CLAWDHUB_REGISTRY=https://your-site.example
```
## 5) Post-deploy checks
```bash
curl -i "https://<site>/api/v1/search?q=test"
curl -i "https://<site>/api/v1/skills/gifgrep"
```
Then:
```bash
clawdhub login --site https://<site>
clawdhub whoami
```
+84
View File
@@ -0,0 +1,84 @@
---
summary: "Skill version diffing mode (Monaco-backed)"
read_when:
- Implementing skill diff UI
- Adding version comparisons
---
# Diffing mode
## Goals
- Compare any file between two versions.
- Default compare: `latest` vs `previous` (SemVer precedence).
- UX feels native to ClawdHub (theme + typography + motion).
- Inline or side-by-side toggle.
- Public access.
## UX
- Diff card on skill detail page.
- Two selectors: Left/Right.
- Items: version strings, plus tags (e.g. `latest`), plus `previous`.
- Default: Left = `previous`, Right = `latest`.
- File list with status: added / removed / changed / same.
- Default file: `SKILL.md` if present; else first changed file.
- Toggle: Inline vs Side-by-side.
- Show size guard message when file > 200KB.
## SemVer ordering
- Use SemVer precedence to sort versions.
- `previous` = immediate predecessor of `latest` by SemVer.
- If `latest` missing or only one version:
- Disable `previous` and show empty-state copy.
## Data sources
- Versions: `api.skills.listVersions` (all, not just latest 10).
- Tags: `skill.tags` map.
- File list: `version.files` with `path`, `sha256`, `size`.
## API
Add action:
- `skills.getFileText({ versionId, path }) -> { text, size, sha256 }`
- Validate version exists + file path exists in version.
- Enforce size <= 200KB (both in action and client).
- Use `fetchText` from `convex/lib/skillPublish.ts`.
Optional helper action:
- `skills.getVersionFiles({ versionId }) -> files[]`
- If we want lightweight fetch without full version object.
## Client flow
1. Fetch versions + tags.
2. Resolve default compare pair:
- Right = tag `latest` if present else highest SemVer.
- Left = `previous` (SemVer predecessor).
3. Build file union by path.
4. For selected file:
- Fetch left/right text (guard by size).
- Feed into Monaco diff editor.
## Monaco theming
- Define `clawdhub-light` / `clawdhub-dark` via `monaco.editor.defineTheme`.
- Derive colors from CSS variables on `document.documentElement`:
- `--surface`, `--surface-muted`, `--ink`, `--ink-soft`, `--line`, `--accent`.
- Apply theme on load + when theme changes (`data-theme`).
- Match font: `var(--font-mono)`.
- Set diff options:
- `renderSideBySide` toggle
- `diffAlgorithm: 'advanced'`
- `renderSideBySideInlineBreakpoint` for mobile
- `wordWrap: 'on'`
## Edge cases
- File removed/added: show empty buffer on missing side + label.
- Non-text file should not exist (upload rejects), but still guard.
- Large file: show size warning + disable fetch.
- Missing version: show error state.
## Perf
- Cache file text per version+path in client state.
- Debounce selector changes (100-200ms).
- Limit concurrent fetches to 2.
## Tests
- Unit: SemVer ordering + `previous` selection.
- Component: default selectors, tag inclusion, size guard.
+171
View File
@@ -0,0 +1,171 @@
---
summary: 'Feature spec: import a skill from a public GitHub URL (auto-detect SKILL.md, selective file upload, provenance).'
read_when:
- Adding GitHub import (web + API)
- Reviewing safety limits (SSRF/zip-bombs)
- Implementing provenance + canonical-claim flows
---
# GitHub import (public repos)
Goal: paste a GitHub URL → auto-detect skill → preview files → publish (selective) → persist provenance.
Non-goal (v1): private repos (no OAuth/PAT support).
Related:
- `docs/skill-format.md` (what counts as a skill; text-only limits)
- `docs/api.md` / `docs/http-api.md` (REST patterns + auth)
## UX
Upload page: “Import from GitHub” mode.
Flow:
1) URL input
2) Detect skill candidates (SKILL.md)
3) If multiple candidates: choose one
4) File picker: check/uncheck; smart-select referenced files
5) Confirm slug/name/version/tags
6) Import → publish
## Accepted URLs
Allowlist: `https://github.com/...` only.
Supported shapes:
- Repo root: `https://github.com/<owner>/<repo>`
- Tree path: `https://github.com/<owner>/<repo>/tree/<ref>/<path>`
- Blob path (file): `https://github.com/<owner>/<repo>/blob/<ref>/<path>`
Normalization:
- Strip query/hash for fetch.
- From `blob/.../SKILL.md` derive `path` as parent folder.
- If `ref` missing: use `HEAD`.
Reject:
- Non-GitHub hosts.
- Unknown URL patterns.
- Paths containing `..` after normalization.
## Fetch strategy (public)
Download archive:
- `https://github.com/<owner>/<repo>/archive/<ref>.zip`
- Follow redirects. Final redirect usually pins a commit via `codeload.github.com/.../zip/<sha-or-branch>`.
Unzip server-side (Node or Convex node action). Scan for skill candidates.
Skill candidate definition:
- Any folder containing `SKILL.md` or `skill.md` (also accept `skills.md` for compatibility).
- Treat repo root as a folder too.
Multiple skills:
- Return candidate list: `{ path, frontmatter.name, frontmatter.description }`.
- User chooses one.
## Smart file selection
Defaults:
- Always select `SKILL.md` (or chosen readme file).
- Prefer selecting only within chosen skill folder; allow “include out-of-folder refs” if explicitly toggled.
Referenced file expansion:
- Parse Markdown links/images from selected `.md` files:
- `[](<rel>)`, `![](<rel>)`, `<rel>` only when relative.
- Ignore `http(s):`, `mailto:`, `#anchors`.
- Strip query/hash from relative targets.
- Resolve against the current files directory.
- Normalize, reject escapes (`..`).
- Add referenced file if present in archive and is text-allowed.
- Recurse for newly added `.md` files.
Hard caps:
- Max recursion depth (e.g. 4).
- Max referenced additions (e.g. 200).
UI affordances:
- “Select referenced”
- “Select all text”
- “Clear”
- Search/filter by path
## Publish behavior
Server publishes using existing pipeline:
- Text-only enforced (see `docs/skill-format.md`).
- Total ≤ 50MB (selected set).
- Must include `SKILL.md` (or accepted variant).
Suggested defaults (UI):
- `displayName`: frontmatter `name` else folder basename → title case.
- `slug`: sanitize folder basename; if collision, suffix (`-2`, `-3`, …).
- `version`: if new skill → `0.1.0`; if updating own existing skill → bump patch.
- `tags`: default `latest`.
## Provenance (persist source)
Persist on each published version (server-side injection; no mutation of imported files):
- Store in `skillVersions.parsed.metadata.source`:
Example:
```json
{
"kind": "github",
"url": "https://github.com/visionik/ouracli",
"repo": "visionik/ouracli",
"ref": "HEAD",
"commit": "66ac8fb266b7c5ff6519431862be6a375bbfb883",
"path": "",
"importedAt": 1767930000000
}
```
Why `parsed.metadata`:
- Already optional and stored with each version.
- No schema churn for v1.
Future: canonical-claim
- “claim canonical” can key off `{ kind:'github', repo, path }`.
- Prefer commit-pinned provenance for auditability; allow UI to show “Imported from …”.
## API sketch (internal actions)
Two-step (recommended):
- `previewGitHubImport(url)``{ commit, candidates:[...], files:[...], defaults:{...} }`
- `importGitHubSkill({ url, commit, candidatePath, selectedPaths, slug, displayName, version, tags })`
Notes:
- `importGitHubSkill` should re-fetch by pinned `commit` (not floating branch), to avoid TOCTOU.
- Validate `selectedPaths` subset of fetched archive manifest.
## Security / abuse controls
SSRF:
- Only `github.com` (+ `codeload.github.com` during redirect follow).
- No arbitrary redirects to other hosts.
Zip safety:
- Max compressed bytes (from `Content-Length` if present; else streaming cap).
- Max uncompressed total bytes.
- Max file count.
- Max single file size.
- Reject symlinks; reject absolute paths; reject `..` segments.
Rate limits:
- Tie to existing write limits (import == publish).
- Cache preview results briefly (e.g. 60s) keyed by `{repo, commit}`.
Error UX:
- “No SKILL.md found.”
- “Multiple skills found; pick one.”
- “Repo too large / too many files.”
- “Selected files exceed 50MB.”
## Manual test checklist
- Repo root skill (`SKILL.md` at root).
- Nested skill (`skills/foo/SKILL.md`).
- Multi-skill repo (two SKILL.md).
- SKILL.md references `docs/usage.md` + images; smart-select picks `.md` and referenced text files; ignores external links.
- Huge repo → clean “too large” error.
- Redirect pinning → import stores commit sha in provenance.
+168
View File
@@ -0,0 +1,168 @@
---
summary: 'HTTP API reference (public + CLI endpoints + auth).'
read_when:
- Adding/changing endpoints
- Debugging CLI ↔ registry requests
---
# HTTP API
Base URL: `https://clawdhub.com` (default).
All v1 paths are under `/api/v1/...` and implemented by Convex HTTP routes (`convex/http.ts`).
Legacy `/api/...` and `/api/cli/...` remain for compatibility (see `DEPRECATIONS.md`).
OpenAPI: `/api/v1/openapi.json`.
## Rate limits
Enforced per IP + per API key:
- Read: 120/min per IP, 600/min per key
- Write: 30/min per IP, 120/min per key
Headers:
- `X-RateLimit-Limit`, `X-RateLimit-Remaining`, `X-RateLimit-Reset`, `Retry-After` (when limited)
## Public endpoints (no auth)
### `GET /api/v1/search`
Query params:
- `q` (required): query string
- `limit` (optional): integer
- `highlightedOnly` (optional): `true` to filter to highlighted skills
Response:
```json
{ "results": [{ "score": 0.123, "slug": "gifgrep", "displayName": "GifGrep", "summary": "…", "version": "1.2.3", "updatedAt": 1730000000000 }] }
```
### `GET /api/v1/skills`
Query params:
- `limit` (optional): integer
- `cursor` (optional): pagination cursor
Response:
```json
{ "items": [{ "slug": "gifgrep", "displayName": "GifGrep", "summary": "…", "tags": { "latest": "1.2.3" }, "stats": {}, "createdAt": 0, "updatedAt": 0, "latestVersion": { "version": "1.2.3", "createdAt": 0, "changelog": "…" } }], "nextCursor": null }
```
### `GET /api/v1/skills/{slug}`
Response:
```json
{ "skill": { "slug": "gifgrep", "displayName": "GifGrep", "summary": "…", "tags": { "latest": "1.2.3" }, "stats": {}, "createdAt": 0, "updatedAt": 0 }, "latestVersion": { "version": "1.2.3", "createdAt": 0, "changelog": "…" }, "owner": { "handle": "steipete", "displayName": "Peter", "image": null } }
```
### `GET /api/v1/skills/{slug}/versions`
Query params:
- `limit` (optional): integer
- `cursor` (optional): pagination cursor
### `GET /api/v1/skills/{slug}/versions/{version}`
Returns version metadata + files list.
### `GET /api/v1/skills/{slug}/file`
Returns raw text content.
Query params:
- `path` (required)
- `version` (optional)
- `tag` (optional)
Notes:
- Defaults to latest version.
- File size limit: 200KB.
### `GET /api/v1/resolve`
Used by the CLI to map a local fingerprint to a known version.
Query params:
- `slug` (required)
- `hash` (required): 64-char hex sha256 of the bundle fingerprint
Response:
```json
{ "slug": "gifgrep", "match": { "version": "1.2.2" }, "latestVersion": { "version": "1.2.3" } }
```
### `GET /api/v1/download`
Downloads a zip of a skill version.
Query params:
- `slug` (required)
- `version` (optional): semver string
- `tag` (optional): tag name (e.g. `latest`)
Notes:
- If neither `version` nor `tag` is provided, the latest version is used.
- Soft-deleted versions return `410`.
## Auth endpoints (Bearer token)
All endpoints require:
```
Authorization: Bearer clh_...
```
### `GET /api/v1/whoami`
Validates token and returns the user handle.
### `POST /api/v1/skills`
Publishes a new version.
- Preferred: `multipart/form-data` with `payload` JSON + `files[]` blobs.
- JSON body with `files` (storageId-based) is also accepted.
### `DELETE /api/v1/skills/{slug}` / `POST /api/v1/skills/{slug}/undelete`
Soft-delete / restore a skill (owner/admin only).
## Legacy CLI endpoints (deprecated)
Still supported for older CLI versions:
- `GET /api/cli/whoami`
- `POST /api/cli/upload-url`
- `POST /api/cli/publish`
- `POST /api/cli/telemetry/sync`
- `POST /api/cli/skill/delete`
- `POST /api/cli/skill/undelete`
See `DEPRECATIONS.md` for removal plan.
## Registry discovery (`/.well-known/clawdhub.json`)
The CLI can discover registry/auth settings from the site:
- `/.well-known/clawdhub.json` (JSON)
Schema:
```json
{ "apiBase": "https://clawdhub.com", "authBase": "https://clawdhub.com", "minCliVersion": "0.0.5" }
```
If you self-host, serve this file (or set `CLAWDHUB_REGISTRY` explicitly).
+22 -1
View File
@@ -1,3 +1,10 @@
---
summary: 'Copy/paste CLI smoke checklist for local verification.'
read_when:
- Pre-merge validation
- Reproducing a reported CLI bug
---
# Manual testing (CLI)
## Setup
@@ -33,7 +40,7 @@
## Delete / undelete (owner/admin)
- `bun clawdhub delete clawdhub-manual-<ts> --yes`
- Verify hidden:
- `curl -i "https://clawdhub.com/api/skill?slug=clawdhub-manual-<ts>"`
- `curl -i "https://clawdhub.com/api/v1/skills/clawdhub-manual-<ts>"`
- Restore:
- `bun clawdhub undelete clawdhub-manual-<ts> --yes`
- Cleanup:
@@ -41,3 +48,17 @@
## Sync
- `bun clawdhub sync --dry-run --all`
## Playwright (menu smoke)
Run against prod:
```
PLAYWRIGHT_BASE_URL=https://clawdhub.com bun run test:pw
```
Run against a local preview server:
```
bun run test:e2e:local
```
+43
View File
@@ -0,0 +1,43 @@
---
summary: 'Mintlify setup notes for publishing docs/.'
read_when:
- Setting up docs site
---
# Mintlify
Goal: publish `docs/` as a browsable docs site (nice UX for OSS users).
This repo does **not** include Mintlify config yet (`mint.json` missing).
## Minimal setup
1) Install Mintlify CLI (per Mintlify docs).
2) Add a `mint.json` at repo root that points to `docs/` pages.
Example (starter):
```json
{
"name": "ClawdHub",
"logo": "public/logo.svg",
"navigation": [
{ "group": "Start", "pages": ["docs/README", "docs/quickstart"] },
{ "group": "Concepts", "pages": ["docs/architecture", "docs/skill-format", "docs/telemetry"] },
{ "group": "Reference", "pages": ["docs/cli", "docs/http-api", "docs/auth", "docs/deploy"] }
]
}
```
Notes:
- Mintlify usually wants page paths without extension; keep files as `.md`.
- If you prefer Mintlify conventions, rename to `.mdx` later (optional).
## Recommended “docs UX” additions
- Add an “Overview” page (use `docs/README.md`).
- Keep “Quickstart” copy/paste friendly.
- Provide CLI + HTTP API reference pages (done here).
- Add a Troubleshooting page for common setup failures.
+120
View File
@@ -0,0 +1,120 @@
---
summary: 'Local setup + CLI smoke: login, search, install, publish, sync.'
read_when:
- First run / local dev setup
- Verifying end-to-end flows
---
# Quickstart
## 0) Prereqs
- Bun
- Convex CLI (`bunx convex ...`)
- GitHub OAuth App (for login)
- OpenAI key (for embeddings/search)
## 1) Local dev (web + Convex)
```bash
bun install
cp .env.local.example .env.local
# terminal A
bun run dev
# terminal B
bunx convex dev
```
## 2) Auth setup (GitHub OAuth + Convex Auth keys)
Fill in `.env.local`:
- `AUTH_GITHUB_ID`
- `AUTH_GITHUB_SECRET`
- `VITE_CONVEX_URL`
- `VITE_CONVEX_SITE_URL`
- `CONVEX_SITE_URL` (same as `VITE_CONVEX_SITE_URL`)
- `OPENAI_API_KEY`
Generate Convex Auth keys for your deployment:
```bash
bunx auth --deployment-name <deployment> --web-server-url http://localhost:3000
```
Then paste the printed `JWT_PRIVATE_KEY` + `JWKS` into `.env.local` (and ensure the deployment got them too).
## 3) CLI: login + basic commands
From this repo:
```bash
bun clawdhub --help
bun clawdhub login
bun clawdhub whoami
bun clawdhub search gif --limit 5
```
Install a skill into `./skills/<slug>` (if Clawdbot is configured, installs into that workspace instead):
```bash
bun clawdhub install <slug>
bun clawdhub list
```
You can also install into any folder:
```bash
bun clawdhub install <slug> --workdir /tmp/clawdhub-demo --dir skills
```
Update:
```bash
bun clawdhub update --all
```
## 4) Publish a skill
Create a folder containing `SKILL.md` (required) plus any supporting text files:
```bash
mkdir -p /tmp/clawdhub-skill-demo && cd /tmp/clawdhub-skill-demo
cat > SKILL.md <<'EOF'
---
name: Demo Skill
description: Demo skill for local testing
---
# Demo Skill
Hello.
EOF
```
Publish:
```bash
bun clawdhub publish . \
--slug clawdhub-demo-$(date +%s) \
--name "Demo $(date +%s)" \
--version 1.0.0 \
--tags latest \
--changelog "Initial release"
```
## 5) Sync local skills (auto-publish new/changed)
`sync` scans for local skill folders and publishes the ones that arent “synced” yet.
```bash
bun clawdhub sync
```
Dry run + non-interactive:
```bash
bun clawdhub sync --all --dry-run --no-input
```
+58
View File
@@ -0,0 +1,58 @@
---
summary: 'Skill folder format, required files, allowed file types, limits.'
read_when:
- Publishing skills
- Debugging publish/sync failures
---
# Skill format
## On disk
A skill is a folder.
Required:
- `SKILL.md` (or `skill.md`)
Optional:
- any supporting *text-based* files (see “Allowed files”)
- `.clawdhubignore` (ignore patterns for publish/sync)
- `.gitignore` (also honored)
Local install metadata (written by the CLI):
- `<skill>/.clawdhub/origin.json`
Workdir install state (written by the CLI):
- `<workdir>/.clawdhub/lock.json`
## `SKILL.md`
- Markdown with optional YAML frontmatter.
- The server extracts metadata from frontmatter during publish.
- `description` is used as the skill summary in the UI/search.
## Allowed files
Only “text-based” files are accepted by publish.
- Extension allowlist is in `packages/schema/src/textFiles.ts` (`TEXT_FILE_EXTENSIONS`).
- Content types starting with `text/` are treated as text; plus a small allowlist (JSON/YAML/TOML/JS/TS/Markdown/SVG).
Limits (server-side):
- Total bundle size: 50MB.
- Embedding text includes `SKILL.md` + up to ~40 non-`.md` files (best-effort cap).
## Slugs
- Derived from folder name by default.
- Must be lowercase and URL-safe: `^[a-z0-9][a-z0-9-]*$`.
## Versioning + tags
- Each publish creates a new version (semver).
- Tags are string pointers to a version; `latest` is commonly used.
+37
View File
@@ -0,0 +1,37 @@
---
summary: 'Soul bundle format, required files, limits.'
read_when:
- Publishing souls
- Debugging soul publish failures
---
# Soul format
## On disk
A soul is a single file:
- `SOUL.md` (or `soul.md`)
For now, onlycrabs.ai rejects any extra files.
## `SOUL.md`
- Markdown with optional YAML frontmatter.
- The server extracts metadata from frontmatter during publish.
- `description` is used as the soul summary in the UI/search.
## Limits
- Total bundle size: 50MB.
- Embedding text includes `SOUL.md` only.
## Slugs
- Derived from folder name by default.
- Must be lowercase and URL-safe: `^[a-z0-9][a-z0-9-]*$`.
## Versioning + tags
- Each publish creates a new version (semver).
- Tags are string pointers to a version; `latest` is commonly used.
+45 -5
View File
@@ -9,9 +9,10 @@ read_when:
# ClawdHub — product + implementation spec (v1)
## Goals
- onlycrabs.ai mode for sharing `SOUL.md` bundles (host-based entry point).
- Minimal, fast SPA for browsing and publishing agent skills.
- Skills stored in Convex (files + metadata + versions + stats).
- GitHub OAuth login; optional GitHub App repo sync later.
- GitHub OAuth login; GitHub App backs up skills to `clawdbot/skills`.
- Vector-based search over skill text + metadata.
- Versioning, tags (`latest` + user tags), changelog, rollback (tag movement).
- Public read access; upload requires auth.
@@ -19,7 +20,7 @@ read_when:
## Non-goals (v1)
- Paid features, private skills, or binary assets.
- GitHub App sync (future phase).
- GitHub App sync beyond backups (future phase).
## Core objects
@@ -60,9 +61,45 @@ read_when:
From SKILL.md frontmatter + AgentSkills + Clawdis extensions:
- `name`, `description`, `homepage`, `website`, `url`, `emoji`
- `metadata.clawdis`: `always`, `skillKey`, `primaryEnv`, `emoji`, `homepage`, `os`,
`requires` (`bins`, `anyBins`, `env`, `config`), `install[]`
`requires` (`bins`, `anyBins`, `env`, `config`), `install[]`, `nix` (`plugin`, `systems`),
`config` (`requiredEnv`, `stateDirs`, `example`), `cliHelp` (string; `cli --help` output)
- `metadata.clawdbot`: alias of `metadata.clawdis` (preferred for nix-clawdbot plugin pointers)
- Nix plugins are different from regular skills; they bundle the skill pack, the CLI binary, and config flags/requirements together.
- `metadata` in frontmatter is YAML (object) preferred; legacy JSON-string accepted.
### Soul
- `slug` (unique)
- `displayName`
- `ownerUserId`
- `summary` (from SOUL.md frontmatter `description`)
- `latestVersionId`
- `tags` map: `{ tag -> versionId }`
- `stats`: `{ downloads, stars, versions, comments }`
- `status`: `active` only (soft-delete on version/comment only)
- `createdAt`, `updatedAt`
### SoulVersion
- `soulId`
- `version` (semver string)
- `tag` (string, optional; `latest` always maintained separately)
- `changelog` (required)
- `files`: list of file metadata (SOUL.md only)
- `path`, `size`, `storageId`, `sha256`
- `parsed` (metadata extracted from SOUL.md)
- `vectorDocId` (if using RAG component) OR `embeddingId`
- `createdBy`, `createdAt`
- `softDeletedAt` (nullable)
### SoulComment
- `soulId`, `userId`, `body`
- `softDeletedAt`, `deletedBy`
- `createdAt`
### SoulStar
- `soulId`, `userId`, `createdAt`
### Comment
- `skillId`, `userId`, `body`
- `softDeletedAt`, `deletedBy`
@@ -94,6 +131,9 @@ From SKILL.md frontmatter + AgentSkills + Clawdis extensions:
- version uniqueness
5) Server stores files + metadata, sets `latest` tag, updates stats.
Soul upload flow: same as skills, but only `SOUL.md` is allowed in the bundle.
Seed data lives in `convex/seed.ts` for local dev.
## Versioning + tags
- Each upload is a new `SkillVersion`.
- `latest` tag always points to most recent version unless user re-tags.
@@ -101,7 +141,7 @@ From SKILL.md frontmatter + AgentSkills + Clawdis extensions:
- Changelog is optional.
## Search
- Vector search over: SKILL.md + other text files + metadata summary.
- Vector search over: SKILL.md + other text files + metadata summary (souls index SOUL.md).
- Convex embeddings + vector index.
- Filters: tag, owner, `redactionApproved` only, min stars, updatedAt.
@@ -122,7 +162,7 @@ From SKILL.md frontmatter + AgentSkills + Clawdis extensions:
- Lint: Biome + Oxlint (type-aware).
## Vercel
- Env vars: Convex deployment URLs + GitHub OAuth client + OpenAI key (if used).
- Env vars: Convex deployment URLs + GitHub OAuth client + OpenAI key (if used) + GitHub App backup credentials.
- SPA feel: client-side transitions, prefetching, optimistic UI.
## Open questions (carry forward)
+7 -1
View File
@@ -1,3 +1,10 @@
---
summary: 'Install telemetry collected via `clawdhub sync` + opt-out.'
read_when:
- Working on telemetry / privacy controls
- Questions about what data is collected
---
# Telemetry
ClawdHub uses **minimal telemetry** to compute **install counts** (whats actually in use) and to power better sorting/filtering.
@@ -82,4 +89,3 @@ export CLAWDHUB_DISABLE_TELEMETRY=1
```
With this set, the CLI will not send telemetry during `clawdhub sync`.
+49
View File
@@ -0,0 +1,49 @@
---
summary: 'Common setup/runtime issues (CLI + backend) and fixes.'
read_when:
- Something is broken and you need a fix-fast checklist
---
# Troubleshooting
## `clawdhub login` opens browser but never completes
- Ensure your browser can reach `http://127.0.0.1:<port>/callback` (local firewalls/VPNs can interfere).
- Use headless mode:
- create a token in the web UI (Settings → API tokens)
- `clawdhub login --token clh_...`
## `whoami` / `publish` returns `Unauthorized` (401)
- Token missing or revoked: check your config file (`CLAWDHUB_CONFIG_PATH` override?).
- Ensure requests include `Authorization: Bearer ...` (CLI does this automatically).
## `publish` fails with `OPENAI_API_KEY is not configured`
- Set `OPENAI_API_KEY` in the Convex environment (not only locally).
- Re-run `bunx convex dev` / `bunx convex deploy` after setting env.
## `sync` says “No skills found”
- `sync` looks for folders containing `SKILL.md` (or `skill.md`).
- It scans:
- workdir first
- then fallback roots (legacy `~/clawdis/skills`, `~/clawdbot/skills`, etc.)
- Provide explicit roots:
```bash
clawdhub sync --root /path/to/skills
```
## `update` refuses due to “local changes (no match)”
- Your local files dont match any published fingerprint.
- Options:
- keep local edits; skip updating
- overwrite: `clawdhub update <slug> --force`
- publish as fork: copy to new folder/slug then `clawdhub publish ... --fork-of upstream@version`
## `GET /api/*` works locally but not on Vercel
- Check `vercel.json` rewrite destination points at your Convex site URL.
- Ensure `VITE_CONVEX_SITE_URL` and `CONVEX_SITE_URL` match your deployment.
+6
View File
@@ -1,3 +1,9 @@
---
summary: 'Discord webhook events/payloads for skill publish + highlight.'
read_when:
- Working on webhooks/integrations
---
# Webhooks (Discord)
ClawdHub can post Discord embeds when skills are published or highlighted.
+63 -8
View File
@@ -5,9 +5,9 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import {
ApiCliWhoamiResponseSchema,
ApiRoutes,
ApiSearchResponseSchema,
ApiV1SearchResponseSchema,
ApiV1WhoamiResponseSchema,
parseArk,
} from 'clawdhub-schema'
import { unzipSync } from 'fflate'
@@ -50,7 +50,7 @@ describe('clawdhub e2e', () => {
const response = await fetch(url.toString(), { headers: { Accept: 'application/json' } })
expect(response.ok).toBe(true)
const json = (await response.json()) as unknown
const parsed = parseArk(ApiSearchResponseSchema, json, 'API response')
const parsed = parseArk(ApiV1SearchResponseSchema, json, 'API response')
expect(Array.isArray(parsed.results)).toBe(true)
})
@@ -102,13 +102,13 @@ describe('clawdhub e2e', () => {
const cfg = await makeTempConfig(registry, token)
try {
const whoamiUrl = new URL(ApiRoutes.cliWhoami, registry)
const whoamiUrl = new URL(ApiRoutes.whoami, registry)
const whoamiRes = await fetch(whoamiUrl.toString(), {
headers: { Accept: 'application/json', Authorization: `Bearer ${token}` },
})
expect(whoamiRes.ok).toBe(true)
const whoami = parseArk(
ApiCliWhoamiResponseSchema,
ApiV1WhoamiResponseSchema,
(await whoamiRes.json()) as unknown,
'Whoami',
)
@@ -174,6 +174,61 @@ describe('clawdhub e2e', () => {
}
})
it('sync dry-run finds skills from clawdbot.json roots', async () => {
const registry = process.env.CLAWDHUB_REGISTRY?.trim() || 'https://clawdhub.com'
const site = process.env.CLAWDHUB_SITE?.trim() || 'https://clawdhub.com'
const token = mustGetToken() ?? (await readGlobalConfig())?.token ?? null
if (!token) {
throw new Error('Missing token. Set CLAWDHUB_E2E_TOKEN or run: bun clawdhub auth login')
}
const cfg = await makeTempConfig(registry, token)
const root = await mkdtemp(join(tmpdir(), 'clawdhub-e2e-clawdbot-'))
const stateDir = join(root, 'state')
const configPath = join(root, 'clawdbot.json')
const workspace = join(root, 'clawd-work')
const skillsRoot = join(workspace, 'skills')
const skillDir = join(skillsRoot, 'auto-skill')
try {
await mkdir(skillDir, { recursive: true })
await writeFile(join(skillDir, 'SKILL.md'), '# Skill\n', 'utf8')
const config = `{
// JSON5-style comments + trailing commas
routing: {
agents: {
work: { name: 'Work', workspace: '${workspace}', },
},
},
}`
await writeFile(configPath, config, 'utf8')
const result = spawnSync(
'bun',
['clawdhub', 'sync', '--dry-run', '--all', '--site', site, '--registry', registry],
{
cwd: process.cwd(),
env: {
...process.env,
CLAWDHUB_CONFIG_PATH: cfg.path,
CLAWDHUB_DISABLE_TELEMETRY: '1',
CLAWDBOT_CONFIG_PATH: configPath,
CLAWDBOT_STATE_DIR: stateDir,
},
encoding: 'utf8',
},
)
expect(result.status).toBe(0)
expect(result.stderr).not.toMatch(/error:/i)
expect(result.stdout).toMatch(/Dry run/i)
expect(result.stdout).toMatch(/auto-skill/i)
} finally {
await rm(root, { recursive: true, force: true })
await rm(cfg.dir, { recursive: true, force: true })
}
})
it('publishes, deletes, and undeletes a skill (logged-in)', async () => {
const registry = process.env.CLAWDHUB_REGISTRY?.trim() || 'https://clawdhub.com'
const site = process.env.CLAWDHUB_SITE?.trim() || 'https://clawdhub.com'
@@ -319,9 +374,9 @@ describe('clawdhub e2e', () => {
)
expect(update.status).toBe(0)
const metaUrl = new URL(ApiRoutes.skill, registry)
metaUrl.searchParams.set('slug', slug)
const metaRes = await fetch(metaUrl.toString(), { headers: { Accept: 'application/json' } })
const metaRes = await fetch(`${registry}${ApiRoutes.skills}/${slug}`, {
headers: { Accept: 'application/json' },
})
expect(metaRes.status).toBe(200)
const del = spawnSync(
+49
View File
@@ -0,0 +1,49 @@
import { expect, test } from '@playwright/test'
const navLabels = ['Skills', 'Upload', 'Import', 'Search']
test('skills loads without error', async ({ page }) => {
await page.goto('/skills', { waitUntil: 'domcontentloaded' })
await expect(page.locator('text=Something went wrong!')).toHaveCount(0)
await expect(page.locator('h1', { hasText: 'Skills' })).toBeVisible()
})
test('souls loads without error', async ({ page }) => {
await page.goto('/souls', { waitUntil: 'domcontentloaded' })
await expect(page.locator('text=Something went wrong!')).toHaveCount(0)
await expect(page.locator('h1', { hasText: 'Souls' })).toBeVisible()
})
test('header menu routes render', async ({ page }) => {
await page.goto('/', { waitUntil: 'domcontentloaded' })
for (const label of navLabels) {
const link = page.getByRole('link', { name: label }).first()
await expect(link).toBeVisible()
await link.click()
if (label === 'Skills') {
await expect(page).toHaveURL(/\/skills/)
await expect(page.locator('h1', { hasText: 'Skills' })).toBeVisible()
}
if (label === 'Upload') {
await expect(page).toHaveURL(/\/upload/)
const heading = page.locator('h1.section-title', { hasText: /^Publish a /i })
const signInCard = page.locator('text=Sign in to upload')
await expect(heading.or(signInCard)).toBeVisible()
}
if (label === 'Import') {
await expect(page).toHaveURL(/\/import/)
const heading = page.getByRole('heading', { name: 'Import from GitHub' })
const signInCard = page.locator('text=Sign in to import and publish skills.')
await expect(heading.or(signInCard)).toBeVisible()
}
if (label === 'Search') {
await expect(page).toHaveURL(/\/?(\?|$)/)
await expect(page.locator('h1', { hasText: 'ClawdHub' })).toBeVisible()
}
}
})
+9
View File
@@ -9,9 +9,12 @@
"dev": "bun --bun vite dev --port 3000",
"build": "bun --bun vite build",
"preview": "bun --bun vite preview",
"docs:list": "bun scripts/docs-list.ts",
"test": "vitest run",
"test:watch": "vitest",
"test:e2e": "vitest run -c vitest.e2e.config.ts",
"test:e2e:local": "bash scripts/run-playwright-local.sh",
"test:pw": "playwright test",
"coverage": "vitest run --coverage",
"convex:deploy": "bunx convex deploy --typecheck=disable --yes",
"lint": "bun run lint:biome && bun run lint:oxlint",
@@ -25,19 +28,24 @@
"@fontsource/bricolage-grotesque": "^5.2.10",
"@fontsource/ibm-plex-mono": "^5.2.7",
"@fontsource/manrope": "^5.2.8",
"@monaco-editor/react": "^4.7.0",
"@radix-ui/react-dropdown-menu": "^2.1.16",
"@radix-ui/react-toggle-group": "^1.1.11",
"@resvg/resvg-wasm": "^2.6.2",
"@tailwindcss/vite": "^4.1.18",
"@tanstack/react-devtools": "^0.9.0",
"@tanstack/react-router": "^1.144.0",
"@tanstack/react-router-devtools": "^1.144.0",
"@tanstack/react-start": "^1.145.3",
"@tanstack/router-plugin": "^1.145.2",
"@vercel/analytics": "^1.6.1",
"clawdhub-schema": "^0.0.2",
"clsx": "^2.1.1",
"convex": "^1.31.2",
"fflate": "^0.8.2",
"h3": "2.0.1-rc.5",
"lucide-react": "^0.562.0",
"monaco-editor": "^0.55.1",
"nitro": "^3.0.1-alpha.1",
"react": "^19.2.3",
"react-dom": "^19.2.3",
@@ -51,6 +59,7 @@
},
"devDependencies": {
"@biomejs/biome": "^2.3.11",
"@playwright/test": "^1.57.0",
"@tanstack/devtools-vite": "^0.4.0",
"@testing-library/dom": "^10.4.1",
"@testing-library/react": "^16.3.1",
+1 -1
View File
@@ -53,5 +53,5 @@ clawdhub sync --root ../clawdis/skills --all --dry-run
- Site: `https://clawdhub.com` (override via `--site` or `CLAWDHUB_SITE`)
- Registry: discovered from `/.well-known/clawdhub.json` on the site (override via `--registry` or `CLAWDHUB_REGISTRY`)
- Workdir: current directory (override via `--workdir`)
- Workdir: current directory (falls back to Clawdbot workspace if configured; override via `--workdir` or `CLAWDHUB_WORKDIR`)
- Install dir: `./skills` under workdir (override via `--dir`)
+3 -2
View File
@@ -1,7 +1,7 @@
{
"name": "clawdhub",
"version": "0.0.5",
"description": "ClawdHub CLI install, update, search, and publish agent skills.",
"version": "0.2.0",
"description": "ClawdHub CLI \u2014 install, update, search, and publish agent skills.",
"license": "MIT",
"type": "module",
"bin": {
@@ -24,6 +24,7 @@
"commander": "^14.0.2",
"fflate": "^0.8.2",
"ignore": "^7.0.5",
"json5": "^2.2.3",
"mime": "^4.1.0",
"ora": "^9.0.0",
"p-retry": "^7.1.1",
+50
View File
@@ -21,12 +21,23 @@ describe('browserAuth', () => {
expect(url).toContain('state=')
})
it('builds auth url without label', () => {
const url = buildCliAuthUrl({
siteUrl: 'https://example.com',
redirectUri: 'http://127.0.0.1:1234/callback',
state: 'state123',
})
expect(url).toContain('https://example.com/cli/auth?')
expect(url).not.toContain('label_b64=')
})
it('accepts only loopback http redirect uris', () => {
expect(isAllowedLoopbackRedirectUri('http://127.0.0.1:1234/callback')).toBe(true)
expect(isAllowedLoopbackRedirectUri('http://localhost:1234/callback')).toBe(true)
expect(isAllowedLoopbackRedirectUri('http://[::1]:1234/callback')).toBe(true)
expect(isAllowedLoopbackRedirectUri('https://127.0.0.1:1234/callback')).toBe(false)
expect(isAllowedLoopbackRedirectUri('http://evil.com/callback')).toBe(false)
expect(isAllowedLoopbackRedirectUri('not a url')).toBe(false)
})
it('receives token via loopback server', async () => {
@@ -43,4 +54,43 @@ describe('browserAuth', () => {
})
await expect(server.waitForResult()).resolves.toEqual(payload)
})
it('serves callback html', async () => {
const server = await startLoopbackAuthServer({ timeoutMs: 2000 })
const response = await fetch(server.redirectUri)
expect(response.status).toBe(200)
const text = await response.text()
expect(text).toContain('ClawdHub CLI Login')
server.close()
})
it('returns 404 for unknown routes', async () => {
const server = await startLoopbackAuthServer({ timeoutMs: 2000 })
const response = await fetch(server.redirectUri.replace('/callback', '/nope'))
expect(response.status).toBe(404)
server.close()
})
it('rejects invalid json payloads', async () => {
const server = await startLoopbackAuthServer({ timeoutMs: 2000 })
const tokenUrl = server.redirectUri.replace('/callback', '/token')
const response = await fetch(tokenUrl, { method: 'POST', body: '{' })
expect(response.status).toBe(400)
await expect(server.waitForResult()).rejects.toThrow()
})
it('rejects state mismatches', async () => {
const server = await startLoopbackAuthServer({ timeoutMs: 2000 })
await fetch(server.redirectUri.replace('/callback', '/token'), {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ token: 'clh_test', registry: 'https://example.com', state: 'nope' }),
})
await expect(server.waitForResult()).rejects.toThrow(/state mismatch/i)
})
it('times out waiting for login', async () => {
const server = await startLoopbackAuthServer({ timeoutMs: 25 })
await expect(server.waitForResult()).rejects.toThrow(/timed out waiting for browser login/i)
})
})
+54 -19
View File
@@ -1,7 +1,9 @@
#!/usr/bin/env node
import { resolve } from 'node:path'
import { stat } from 'node:fs/promises'
import { join, resolve } from 'node:path'
import { Command } from 'commander'
import { getCliBuildLabel, getCliVersion } from './cli/buildInfo.js'
import { resolveClawdbotDefaultWorkspace } from './cli/clawdbotConfig.js'
import { cmdLoginFlow, cmdLogout, cmdWhoami } from './cli/commands/auth.js'
import { cmdDeleteSkill, cmdUndeleteSkill } from './cli/commands/delete.js'
import { cmdPublish } from './cli/commands/publish.js'
@@ -28,13 +30,16 @@ const program = new Command()
.option('--no-input', 'Disable prompts')
.showHelpAfterError()
.showSuggestionAfterError()
.addHelpText('after', styleEnvBlock('\nEnv:\n CLAWDHUB_SITE\n CLAWDHUB_REGISTRY\n'))
.addHelpText(
'after',
styleEnvBlock('\nEnv:\n CLAWDHUB_SITE\n CLAWDHUB_REGISTRY\n CLAWDHUB_WORKDIR\n'),
)
configureCommanderHelp(program)
function resolveGlobalOpts(): GlobalOpts {
async function resolveGlobalOpts(): Promise<GlobalOpts> {
const raw = program.opts<{ workdir?: string; dir?: string; site?: string; registry?: string }>()
const workdir = resolve(raw.workdir ?? process.cwd())
const workdir = await resolveWorkdir(raw.workdir)
const dir = resolve(workdir, raw.dir ?? 'skills')
const site = raw.site ?? process.env.CLAWDHUB_SITE ?? DEFAULT_SITE
const registrySource = raw.registry ? 'cli' : process.env.CLAWDHUB_REGISTRY ? 'env' : 'default'
@@ -47,6 +52,35 @@ function isInputAllowed() {
return globalFlags.input !== false
}
async function resolveWorkdir(explicit?: string) {
if (explicit?.trim()) return resolve(explicit.trim())
const envWorkdir = process.env.CLAWDHUB_WORKDIR?.trim()
if (envWorkdir) return resolve(envWorkdir)
const cwd = resolve(process.cwd())
const hasMarker = await hasClawdhubMarker(cwd)
if (hasMarker) return cwd
const clawdbotWorkspace = await resolveClawdbotDefaultWorkspace()
return clawdbotWorkspace ? resolve(clawdbotWorkspace) : cwd
}
async function hasClawdhubMarker(workdir: string) {
const lockfile = join(workdir, '.clawdhub', 'lock.json')
if (await pathExists(lockfile)) return true
const markerDir = join(workdir, '.clawdhub')
return pathExists(markerDir)
}
async function pathExists(path: string) {
try {
await stat(path)
return true
} catch {
return false
}
}
program
.command('login')
.description('Log in (opens browser or stores token)')
@@ -54,7 +88,7 @@ program
.option('--label <label>', 'Token label (browser flow only)', 'CLI token')
.option('--no-browser', 'Do not open browser (requires --token)')
.action(async (options) => {
const opts = resolveGlobalOpts()
const opts = await resolveGlobalOpts()
await cmdLoginFlow(opts, options, isInputAllowed())
})
@@ -62,7 +96,7 @@ program
.command('logout')
.description('Remove stored token')
.action(async () => {
const opts = resolveGlobalOpts()
const opts = await resolveGlobalOpts()
await cmdLogout(opts)
})
@@ -70,7 +104,7 @@ program
.command('whoami')
.description('Validate token')
.action(async () => {
const opts = resolveGlobalOpts()
const opts = await resolveGlobalOpts()
await cmdWhoami(opts)
})
@@ -87,7 +121,7 @@ auth
.option('--label <label>', 'Token label (browser flow only)', 'CLI token')
.option('--no-browser', 'Do not open browser (requires --token)')
.action(async (options) => {
const opts = resolveGlobalOpts()
const opts = await resolveGlobalOpts()
await cmdLoginFlow(opts, options, isInputAllowed())
})
@@ -95,7 +129,7 @@ auth
.command('logout')
.description('Remove stored token')
.action(async () => {
const opts = resolveGlobalOpts()
const opts = await resolveGlobalOpts()
await cmdLogout(opts)
})
@@ -103,7 +137,7 @@ auth
.command('whoami')
.description('Validate token')
.action(async () => {
const opts = resolveGlobalOpts()
const opts = await resolveGlobalOpts()
await cmdWhoami(opts)
})
@@ -113,7 +147,7 @@ program
.argument('<query...>', 'Query string')
.option('--limit <n>', 'Max results', (value) => Number.parseInt(value, 10))
.action(async (queryParts, options) => {
const opts = resolveGlobalOpts()
const opts = await resolveGlobalOpts()
const query = queryParts.join(' ').trim()
await cmdSearch(opts, query, options.limit)
})
@@ -125,7 +159,7 @@ program
.option('--version <version>', 'Version to install')
.option('--force', 'Overwrite existing folder')
.action(async (slug, options) => {
const opts = resolveGlobalOpts()
const opts = await resolveGlobalOpts()
await cmdInstall(opts, slug, options.version, options.force)
})
@@ -137,7 +171,7 @@ program
.option('--version <version>', 'Update to specific version (single slug only)')
.option('--force', 'Overwrite when local files do not match any version')
.action(async (slug, options) => {
const opts = resolveGlobalOpts()
const opts = await resolveGlobalOpts()
await cmdUpdate(opts, slug, options, isInputAllowed())
})
@@ -145,7 +179,7 @@ program
.command('list')
.description('List installed skills (from lockfile)')
.action(async () => {
const opts = resolveGlobalOpts()
const opts = await resolveGlobalOpts()
await cmdList(opts)
})
@@ -156,10 +190,11 @@ program
.option('--slug <slug>', 'Skill slug')
.option('--name <name>', 'Display name')
.option('--version <version>', 'Version (semver)')
.option('--fork-of <slug[@version]>', 'Mark as a fork of an existing skill')
.option('--changelog <text>', 'Changelog text')
.option('--tags <tags>', 'Comma-separated tags', 'latest')
.action(async (folder, options) => {
const opts = resolveGlobalOpts()
const opts = await resolveGlobalOpts()
await cmdPublish(opts, folder, options)
})
@@ -169,7 +204,7 @@ program
.argument('<slug>', 'Skill slug')
.option('--yes', 'Skip confirmation')
.action(async (slug, options) => {
const opts = resolveGlobalOpts()
const opts = await resolveGlobalOpts()
await cmdDeleteSkill(opts, slug, options, isInputAllowed())
})
@@ -179,7 +214,7 @@ program
.argument('<slug>', 'Skill slug')
.option('--yes', 'Skip confirmation')
.action(async (slug, options) => {
const opts = resolveGlobalOpts()
const opts = await resolveGlobalOpts()
await cmdUndeleteSkill(opts, slug, options, isInputAllowed())
})
@@ -194,7 +229,7 @@ program
.option('--tags <tags>', 'Comma-separated tags', 'latest')
.option('--concurrency <n>', 'Concurrent registry checks (default: 4)', '4')
.action(async (options) => {
const opts = resolveGlobalOpts()
const opts = await resolveGlobalOpts()
const bump = String(options.bump ?? 'patch') as 'patch' | 'minor' | 'major'
if (!['patch', 'minor', 'major'].includes(bump)) fail('--bump must be patch|minor|major')
const concurrencyRaw = Number(options.concurrency ?? 4)
@@ -216,7 +251,7 @@ program
})
program.action(async () => {
const opts = resolveGlobalOpts()
const opts = await resolveGlobalOpts()
const cfg = await readGlobalConfig()
if (cfg?.token) {
await cmdSync(opts, {}, isInputAllowed())
@@ -0,0 +1,159 @@
/* @vitest-environment node */
import { mkdir, mkdtemp, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join, resolve } from 'node:path'
import { afterEach, describe, expect, it } from 'vitest'
import { resolveClawdbotDefaultWorkspace, resolveClawdbotSkillRoots } from './clawdbotConfig.js'
const originalEnv = { ...process.env }
afterEach(() => {
process.env = { ...originalEnv }
})
describe('resolveClawdbotSkillRoots', () => {
it('reads JSON5 config and resolves per-agent + shared skill roots', async () => {
const base = await mkdtemp(join(tmpdir(), 'clawdhub-clawdbot-'))
const home = join(base, 'home')
const stateDir = join(base, 'state')
const configPath = join(base, 'clawdbot.json')
process.env.HOME = home
process.env.CLAWDBOT_STATE_DIR = stateDir
process.env.CLAWDBOT_CONFIG_PATH = configPath
const config = `{
// JSON5 comments + trailing commas supported
agents: {
defaults: { workspace: '~/clawd-main', },
list: [
{ id: 'work', name: 'Work Bot', workspace: '~/clawd-work', },
{ id: 'family', workspace: '~/clawd-family', },
],
},
// legacy entries still supported
agent: { workspace: '~/clawd-legacy', },
routing: {
agents: {
work: { name: 'Work Bot', workspace: '~/clawd-work', },
family: { workspace: '~/clawd-family' },
},
},
skills: {
load: { extraDirs: ['~/shared/skills', '/opt/skills',], },
},
}`
await writeFile(configPath, config, 'utf8')
const { roots, labels } = await resolveClawdbotSkillRoots()
const expectedRoots = [
resolve(stateDir, 'skills'),
resolve(home, 'clawd-main', 'skills'),
resolve(home, 'clawd-work', 'skills'),
resolve(home, 'clawd-family', 'skills'),
resolve(home, 'shared', 'skills'),
resolve('/opt/skills'),
]
expect(roots).toEqual(expect.arrayContaining(expectedRoots))
expect(labels[resolve(stateDir, 'skills')]).toBe('Shared skills')
expect(labels[resolve(home, 'clawd-main', 'skills')]).toBe('Agent: main')
expect(labels[resolve(home, 'clawd-work', 'skills')]).toBe('Agent: Work Bot')
expect(labels[resolve(home, 'clawd-family', 'skills')]).toBe('Agent: family')
expect(labels[resolve(home, 'shared', 'skills')]).toBe('Extra: skills')
expect(labels[resolve('/opt/skills')]).toBe('Extra: skills')
})
it('resolves default workspace from agents.defaults and agents.list', async () => {
const base = await mkdtemp(join(tmpdir(), 'clawdhub-clawdbot-default-'))
const home = join(base, 'home')
const stateDir = join(base, 'state')
const configPath = join(base, 'clawdbot.json')
const workspaceMain = join(base, 'workspace-main')
const workspaceList = join(base, 'workspace-list')
process.env.HOME = home
process.env.CLAWDBOT_STATE_DIR = stateDir
process.env.CLAWDBOT_CONFIG_PATH = configPath
const config = `{
agents: {
defaults: { workspace: "${workspaceMain}", },
list: [
{ id: 'main', workspace: "${workspaceList}", default: true },
],
},
}`
await writeFile(configPath, config, 'utf8')
const workspace = await resolveClawdbotDefaultWorkspace()
expect(workspace).toBe(resolve(workspaceMain))
})
it('falls back to default agent in agents.list when defaults missing', async () => {
const base = await mkdtemp(join(tmpdir(), 'clawdhub-clawdbot-list-'))
const home = join(base, 'home')
const configPath = join(base, 'clawdbot.json')
const workspaceMain = join(base, 'workspace-main')
const workspaceWork = join(base, 'workspace-work')
process.env.HOME = home
process.env.CLAWDBOT_CONFIG_PATH = configPath
const config = `{
agents: {
list: [
{ id: 'main', workspace: "${workspaceMain}", default: true },
{ id: 'work', workspace: "${workspaceWork}" },
],
},
}`
await writeFile(configPath, config, 'utf8')
const workspace = await resolveClawdbotDefaultWorkspace()
expect(workspace).toBe(resolve(workspaceMain))
})
it('respects CLAWDBOT_STATE_DIR and CLAWDBOT_CONFIG_PATH overrides', async () => {
const base = await mkdtemp(join(tmpdir(), 'clawdhub-clawdbot-override-'))
const home = join(base, 'home')
const stateDir = join(base, 'custom-state')
const configPath = join(base, 'config', 'clawdbot.json')
process.env.HOME = home
process.env.CLAWDBOT_STATE_DIR = stateDir
process.env.CLAWDBOT_CONFIG_PATH = configPath
const config = `{
agent: { workspace: "${join(base, 'workspace-main')}" },
}`
await mkdir(join(base, 'config'), { recursive: true })
await writeFile(configPath, config, 'utf8')
const { roots, labels } = await resolveClawdbotSkillRoots()
expect(roots).toEqual(
expect.arrayContaining([
resolve(stateDir, 'skills'),
resolve(join(base, 'workspace-main'), 'skills'),
]),
)
expect(labels[resolve(stateDir, 'skills')]).toBe('Shared skills')
expect(labels[resolve(join(base, 'workspace-main'), 'skills')]).toBe('Agent: main')
})
it('returns shared skills root when config is missing', async () => {
const base = await mkdtemp(join(tmpdir(), 'clawdhub-clawdbot-missing-'))
const stateDir = join(base, 'state')
const configPath = join(base, 'missing', 'clawdbot.json')
process.env.CLAWDBOT_STATE_DIR = stateDir
process.env.CLAWDBOT_CONFIG_PATH = configPath
const { roots, labels } = await resolveClawdbotSkillRoots()
expect(roots).toEqual([resolve(stateDir, 'skills')])
expect(labels[resolve(stateDir, 'skills')]).toBe('Shared skills')
})
})
+147
View File
@@ -0,0 +1,147 @@
import { readFile } from 'node:fs/promises'
import { homedir } from 'node:os'
import { basename, join, resolve } from 'node:path'
import JSON5 from 'json5'
type ClawdbotConfig = {
agent?: { workspace?: string }
agents?: {
defaults?: { workspace?: string }
list?: Array<{
id?: string
name?: string
workspace?: string
default?: boolean
}>
}
routing?: {
agents?: Record<
string,
{
name?: string
workspace?: string
}
>
}
skills?: {
load?: {
extraDirs?: string[]
}
}
}
export type ClawdbotSkillRoots = {
roots: string[]
labels: Record<string, string>
}
export async function resolveClawdbotSkillRoots(): Promise<ClawdbotSkillRoots> {
const roots: string[] = []
const labels: Record<string, string> = {}
const stateDir = resolveClawdbotStateDir()
const sharedSkills = resolveUserPath(join(stateDir, 'skills'))
pushRoot(roots, labels, sharedSkills, 'Shared skills')
const config = await readClawdbotConfig()
if (!config) return { roots, labels }
const mainWorkspace = resolveUserPath(
config.agents?.defaults?.workspace ?? config.agent?.workspace ?? '',
)
if (mainWorkspace) {
pushRoot(roots, labels, join(mainWorkspace, 'skills'), 'Agent: main')
}
const listedAgents = config.agents?.list ?? []
for (const entry of listedAgents) {
const workspace = resolveUserPath(entry?.workspace ?? '')
if (!workspace) continue
const name = entry?.name?.trim() || entry?.id?.trim() || 'agent'
pushRoot(roots, labels, join(workspace, 'skills'), `Agent: ${name}`)
}
const agents = config.routing?.agents ?? {}
for (const [agentId, entry] of Object.entries(agents)) {
const workspace = resolveUserPath(entry?.workspace ?? '')
if (!workspace) continue
const name = entry?.name?.trim() || agentId
pushRoot(roots, labels, join(workspace, 'skills'), `Agent: ${name}`)
}
const extraDirs = config.skills?.load?.extraDirs ?? []
for (const dir of extraDirs) {
const resolved = resolveUserPath(String(dir))
if (!resolved) continue
const label = `Extra: ${basename(resolved) || resolved}`
pushRoot(roots, labels, resolved, label)
}
return { roots, labels }
}
export async function resolveClawdbotDefaultWorkspace(): Promise<string | null> {
const config = await readClawdbotConfig()
if (!config) return null
const defaultsWorkspace = resolveUserPath(
config.agents?.defaults?.workspace ?? config.agent?.workspace ?? '',
)
if (defaultsWorkspace) return defaultsWorkspace
const listedAgents = config.agents?.list ?? []
const defaultAgent =
listedAgents.find((entry) => entry.default) ?? listedAgents.find((entry) => entry.id === 'main')
const listWorkspace = resolveUserPath(defaultAgent?.workspace ?? '')
return listWorkspace || null
}
function resolveClawdbotStateDir() {
const override = process.env.CLAWDBOT_STATE_DIR?.trim()
if (override) return resolveUserPath(override)
return join(homedir(), '.clawdbot')
}
function resolveClawdbotConfigPath() {
const override = process.env.CLAWDBOT_CONFIG_PATH?.trim()
if (override) return resolveUserPath(override)
return join(resolveClawdbotStateDir(), 'clawdbot.json')
}
function resolveUserPath(input: string) {
const trimmed = input.trim()
if (!trimmed) return ''
if (trimmed.startsWith('~')) {
return resolve(trimmed.replace(/^~(?=$|[\\/])/, homedir()))
}
return resolve(trimmed)
}
async function readClawdbotConfig(): Promise<ClawdbotConfig | null> {
try {
const raw = await readFile(resolveClawdbotConfigPath(), 'utf8')
const parsed = JSON5.parse(raw)
if (!parsed || typeof parsed !== 'object') return null
return parsed as ClawdbotConfig
} catch {
return null
}
}
function pushRoot(roots: string[], labels: Record<string, string>, root: string, label?: string) {
const resolved = resolveUserPath(root)
if (!resolved) return
if (!roots.includes(resolved)) roots.push(resolved)
if (!label) return
const existing = labels[resolved]
if (!existing) {
labels[resolved] = label
return
}
const parts = existing
.split(', ')
.map((part) => part.trim())
.filter(Boolean)
if (parts.includes(label)) return
labels[resolved] = `${existing}, ${label}`
}
+5 -5
View File
@@ -2,7 +2,7 @@ import { buildCliAuthUrl, startLoopbackAuthServer } from '../../browserAuth.js'
import { readGlobalConfig, writeGlobalConfig } from '../../config.js'
import { discoverRegistryFromSite } from '../../discovery.js'
import { apiRequest } from '../../http.js'
import { ApiCliWhoamiResponseSchema, ApiRoutes } from '../../schema/index.js'
import { ApiRoutes, ApiV1WhoamiResponseSchema } from '../../schema/index.js'
import { getRegistry } from '../registry.js'
import type { GlobalOpts } from '../types.js'
import { createSpinner, fail, formatError, openInBrowser, promptHidden } from '../ui.js'
@@ -55,8 +55,8 @@ export async function cmdLogin(
try {
const whoami = await apiRequest(
registry,
{ method: 'GET', path: ApiRoutes.cliWhoami, token },
ApiCliWhoamiResponseSchema,
{ method: 'GET', path: ApiRoutes.whoami, token },
ApiV1WhoamiResponseSchema,
)
if (!whoami.user) fail('Login failed')
@@ -86,8 +86,8 @@ export async function cmdWhoami(opts: GlobalOpts) {
try {
const whoami = await apiRequest(
registry,
{ method: 'GET', path: ApiRoutes.cliWhoami, token },
ApiCliWhoamiResponseSchema,
{ method: 'GET', path: ApiRoutes.whoami, token },
ApiV1WhoamiResponseSchema,
)
spinner.succeed(whoami.user.handle ?? 'unknown')
} catch (error) {
@@ -56,7 +56,7 @@ describe('delete/undelete', () => {
await cmdDeleteSkill(makeOpts(), 'demo', { yes: true }, false)
expect(mockApiRequest).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({ method: 'POST', path: '/api/cli/skill/delete' }),
expect.objectContaining({ method: 'DELETE', path: '/api/v1/skills/demo' }),
expect.anything(),
)
})
@@ -66,7 +66,7 @@ describe('delete/undelete', () => {
await cmdUndeleteSkill(makeOpts(), 'demo', { yes: true }, false)
expect(mockApiRequest).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({ method: 'POST', path: '/api/cli/skill/undelete' }),
expect.objectContaining({ method: 'POST', path: '/api/v1/skills/demo/undelete' }),
expect.anything(),
)
})
+11 -9
View File
@@ -1,6 +1,6 @@
import { readGlobalConfig } from '../../config.js'
import { apiRequest } from '../../http.js'
import { ApiCliSkillDeleteResponseSchema, ApiRoutes, parseArk } from '../../schema/index.js'
import { ApiRoutes, ApiV1DeleteResponseSchema, parseArk } from '../../schema/index.js'
import { getRegistry } from '../registry.js'
import type { GlobalOpts } from '../types.js'
import { createSpinner, fail, formatError, isInteractive, promptConfirm } from '../ui.js'
@@ -32,14 +32,13 @@ export async function cmdDeleteSkill(
const registry = await getRegistry(opts, { cache: true })
const spinner = createSpinner(`Deleting ${slug}`)
try {
const body = { slug }
const result = await apiRequest(
registry,
{ method: 'POST', path: ApiRoutes.cliSkillDelete, token, body },
ApiCliSkillDeleteResponseSchema,
{ method: 'DELETE', path: `${ApiRoutes.skills}/${encodeURIComponent(slug)}`, token },
ApiV1DeleteResponseSchema,
)
spinner.succeed(`OK. Deleted ${slug}`)
return parseArk(ApiCliSkillDeleteResponseSchema, result, 'Delete response')
return parseArk(ApiV1DeleteResponseSchema, result, 'Delete response')
} catch (error) {
spinner.fail(formatError(error))
throw error
@@ -66,14 +65,17 @@ export async function cmdUndeleteSkill(
const registry = await getRegistry(opts, { cache: true })
const spinner = createSpinner(`Undeleting ${slug}`)
try {
const body = { slug }
const result = await apiRequest(
registry,
{ method: 'POST', path: ApiRoutes.cliSkillUndelete, token, body },
ApiCliSkillDeleteResponseSchema,
{
method: 'POST',
path: `${ApiRoutes.skills}/${encodeURIComponent(slug)}/undelete`,
token,
},
ApiV1DeleteResponseSchema,
)
spinner.succeed(`OK. Undeleted ${slug}`)
return parseArk(ApiCliSkillDeleteResponseSchema, result, 'Undelete response')
return parseArk(ApiV1DeleteResponseSchema, result, 'Undelete response')
} catch (error) {
spinner.fail(formatError(error))
throw error
@@ -4,7 +4,6 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, describe, expect, it, vi } from 'vitest'
import { sha256Hex } from '../../skills'
import type { GlobalOpts } from '../types'
vi.mock('../../config.js', () => ({
@@ -16,10 +15,10 @@ vi.mock('../registry.js', () => ({
getRegistry: (opts: unknown, params?: unknown) => mockGetRegistry(opts, params),
}))
const mockApiRequest = vi.fn()
const mockApiRequestForm = vi.fn()
vi.mock('../../http.js', () => ({
apiRequest: (registry: unknown, args: unknown, schema?: unknown) =>
mockApiRequest(registry, args, schema),
apiRequestForm: (registry: unknown, args: unknown, schema?: unknown) =>
mockApiRequestForm(registry, args, schema),
}))
const mockFail = vi.fn((message: string) => {
@@ -65,37 +64,7 @@ describe('cmdPublish', () => {
await writeFile(join(folder, 'SKILL.md'), skillContent, 'utf8')
await writeFile(join(folder, 'notes.md'), notesContent, 'utf8')
let uploadIndex = 0
mockApiRequest.mockImplementation(
async (_registry: string, args: { method: string; path: string }) => {
if (args.method === 'GET' && args.path.startsWith('/api/skill?slug=')) {
return { skill: null, latestVersion: { version: '9.9.9' } }
}
if (args.method === 'POST' && args.path === '/api/cli/upload-url') {
uploadIndex += 1
return { uploadUrl: `https://upload.example/${uploadIndex}` }
}
if (args.method === 'POST' && args.path === '/api/cli/publish') {
return { ok: true, skillId: 'skill_1', versionId: 'ver_1' }
}
throw new Error(`Unexpected apiRequest: ${args.method} ${args.path}`)
},
)
vi.stubGlobal(
'fetch',
vi.fn(async (url: string, init?: RequestInit) => {
expect(url).toMatch(/^https:\/\/upload\.example\/\d+$/)
expect(init?.method).toBe('POST')
expect((init?.headers as Record<string, string>)?.['Content-Type']).toMatch(
/text\/(markdown|plain)/,
)
return new Response(JSON.stringify({ storageId: `st_${String(url).split('/').pop()}` }), {
status: 200,
headers: { 'Content-Type': 'application/json' },
})
}) as unknown as typeof fetch,
)
mockApiRequestForm.mockResolvedValueOnce({ ok: true, skillId: 'skill_1', versionId: 'ver_1' })
await cmdPublish(makeOpts(workdir), 'my-skill', {
slug: 'my-skill',
@@ -105,30 +74,22 @@ describe('cmdPublish', () => {
tags: 'latest',
})
const publishCall = mockApiRequest.mock.calls.find((call) => {
const publishCall = mockApiRequestForm.mock.calls.find((call) => {
const req = call[1] as { path?: string } | undefined
return req?.path === '/api/cli/publish'
return req?.path === '/api/v1/skills'
})
if (!publishCall) throw new Error('Missing publish call')
const publishBody = (publishCall[1] as { body?: unknown }).body as {
slug: string
displayName: string
version: string
changelog: string
tags: string[]
files: Array<{ path: string; sha256: string; storageId: string }>
}
expect(publishBody.slug).toBe('my-skill')
expect(publishBody.displayName).toBe('My Skill')
expect(publishBody.version).toBe('1.0.0')
expect(publishBody.changelog).toBe('')
expect(publishBody.tags).toEqual(['latest'])
const byPath = Object.fromEntries(publishBody.files.map((f) => [f.path, f]))
expect(Object.keys(byPath).sort()).toEqual(['SKILL.md', 'notes.md'])
expect(byPath['SKILL.md']?.sha256).toBe(sha256Hex(new TextEncoder().encode(skillContent)))
expect(byPath['notes.md']?.sha256).toBe(sha256Hex(new TextEncoder().encode(notesContent)))
const publishForm = (publishCall[1] as { form?: FormData }).form as FormData
const payloadEntry = publishForm.get('payload')
if (typeof payloadEntry !== 'string') throw new Error('Missing publish payload')
const payload = JSON.parse(payloadEntry)
expect(payload.slug).toBe('my-skill')
expect(payload.displayName).toBe('My Skill')
expect(payload.version).toBe('1.0.0')
expect(payload.changelog).toBe('')
expect(payload.tags).toEqual(['latest'])
const files = publishForm.getAll('files') as Array<Blob & { name?: string }>
expect(files.map((file) => String(file.name ?? '')).sort()).toEqual(['SKILL.md', 'notes.md'])
} finally {
await rm(workdir, { recursive: true, force: true })
}
@@ -141,28 +102,7 @@ describe('cmdPublish', () => {
await mkdir(folder, { recursive: true })
await writeFile(join(folder, 'SKILL.md'), '# Skill\n', 'utf8')
let uploadIndex = 0
mockApiRequest.mockImplementation(
async (_registry: string, args: { method: string; path: string }) => {
if (args.method === 'GET' && args.path.startsWith('/api/skill?slug=')) {
return { skill: { slug: 'existing-skill' }, latestVersion: { version: '1.0.0' } }
}
if (args.method === 'POST' && args.path === '/api/cli/upload-url') {
uploadIndex += 1
return { uploadUrl: `https://upload.example/${uploadIndex}` }
}
if (args.method === 'POST' && args.path === '/api/cli/publish') {
return { ok: true, skillId: 'skill_1', versionId: 'ver_2' }
}
throw new Error(`Unexpected apiRequest: ${args.method} ${args.path}`)
},
)
vi.stubGlobal(
'fetch',
vi.fn(
async () => new Response(JSON.stringify({ storageId: 'st_1' }), { status: 200 }),
) as unknown as typeof fetch,
)
mockApiRequestForm.mockResolvedValueOnce({ ok: true, skillId: 'skill_1', versionId: 'ver_2' })
await cmdPublish(makeOpts(workdir), 'existing-skill', {
version: '1.0.1',
@@ -170,9 +110,9 @@ describe('cmdPublish', () => {
tags: 'latest',
})
expect(mockApiRequest).toHaveBeenCalledWith(
expect(mockApiRequestForm).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({ path: '/api/cli/publish', method: 'POST' }),
expect.objectContaining({ path: '/api/v1/skills', method: 'POST' }),
expect.anything(),
)
} finally {
+39 -56
View File
@@ -2,16 +2,9 @@ import { stat } from 'node:fs/promises'
import { basename, resolve } from 'node:path'
import semver from 'semver'
import { readGlobalConfig } from '../../config.js'
import { apiRequest } from '../../http.js'
import {
ApiCliPublishResponseSchema,
ApiCliUploadUrlResponseSchema,
ApiRoutes,
ApiUploadFileResponseSchema,
CliPublishRequestSchema,
parseArk,
} from '../../schema/index.js'
import { listTextFiles, sha256Hex } from '../../skills.js'
import { apiRequestForm } from '../../http.js'
import { ApiRoutes, ApiV1PublishResponseSchema } from '../../schema/index.js'
import { listTextFiles } from '../../skills.js'
import { getRegistry } from '../registry.js'
import { sanitizeSlug, titleCase } from '../slug.js'
import type { GlobalOpts } from '../types.js'
@@ -20,7 +13,14 @@ import { createSpinner, fail, formatError } from '../ui.js'
export async function cmdPublish(
opts: GlobalOpts,
folderArg: string,
options: { slug?: string; name?: string; version?: string; changelog?: string; tags?: string },
options: {
slug?: string
name?: string
version?: string
changelog?: string
tags?: string
forkOf?: string
},
) {
const folder = folderArg ? resolve(opts.workdir, folderArg) : null
if (!folder) fail('Path required')
@@ -42,6 +42,9 @@ export async function cmdPublish(
.map((tag) => tag.trim())
.filter(Boolean)
const forkOfRaw = options.forkOf?.trim()
const forkOf = forkOfRaw ? parseForkOf(forkOfRaw) : undefined
if (!slug) fail('--slug required')
if (!displayName) fail('--name required')
if (!version || !semver.valid(version)) fail('--version must be valid semver')
@@ -59,45 +62,32 @@ export async function cmdPublish(
fail('SKILL.md required')
}
const uploaded: Array<{
path: string
size: number
storageId: string
sha256: string
contentType?: string
}> = []
const form = new FormData()
form.set(
'payload',
JSON.stringify({
slug,
displayName,
version,
changelog,
tags,
...(forkOf ? { forkOf } : {}),
}),
)
let index = 0
for (const file of filesOnDisk) {
index += 1
spinner.text = `Uploading ${file.relPath} (${index}/${filesOnDisk.length})`
const { uploadUrl } = await apiRequest(
registry,
{ method: 'POST', path: ApiRoutes.cliUploadUrl, token },
ApiCliUploadUrlResponseSchema,
)
const storageId = await uploadFile(uploadUrl, file.bytes, file.contentType ?? 'text/plain')
const sha256 = sha256Hex(file.bytes)
uploaded.push({
path: file.relPath,
size: file.bytes.byteLength,
storageId,
sha256,
contentType: file.contentType ?? undefined,
})
const blob = new Blob([Buffer.from(file.bytes)], { type: file.contentType ?? 'text/plain' })
form.append('files', blob, file.relPath)
}
spinner.text = `Publishing ${slug}@${version}`
const body = parseArk(
CliPublishRequestSchema,
{ slug, displayName, version, changelog, tags, files: uploaded },
'Publish payload',
)
const result = await apiRequest(
const result = await apiRequestForm(
registry,
{ method: 'POST', path: ApiRoutes.cliPublish, token, body },
ApiCliPublishResponseSchema,
{ method: 'POST', path: ApiRoutes.skills, token, form },
ApiV1PublishResponseSchema,
)
spinner.succeed(`OK. Published ${slug}@${version} (${result.versionId})`)
@@ -107,19 +97,12 @@ export async function cmdPublish(
}
}
async function uploadFile(uploadUrl: string, bytes: Uint8Array, contentType: string) {
const response = await fetch(uploadUrl, {
method: 'POST',
headers: { 'Content-Type': contentType || 'application/octet-stream' },
body: Buffer.from(bytes),
})
if (!response.ok) {
throw new Error(`Upload failed: ${await response.text()}`)
}
const payload = parseArk(
ApiUploadFileResponseSchema,
(await response.json()) as unknown,
'Upload response',
)
return payload.storageId
function parseForkOf(value: string) {
const trimmed = value.trim()
const [slugRaw, versionRaw] = trimmed.split('@')
const slug = (slugRaw ?? '').trim().toLowerCase()
if (!slug) fail('--fork-of must be <slug> or <slug@version>')
const version = (versionRaw ?? '').trim()
if (version && !semver.valid(version)) fail('--fork-of version must be valid semver')
return { slug, version: version || undefined }
}
+34 -12
View File
@@ -4,16 +4,18 @@ import semver from 'semver'
import { apiRequest, downloadZip } from '../../http.js'
import {
ApiRoutes,
ApiSearchResponseSchema,
ApiSkillMetaResponseSchema,
ApiSkillResolveResponseSchema,
ApiV1SearchResponseSchema,
ApiV1SkillResolveResponseSchema,
ApiV1SkillResponseSchema,
} from '../../schema/index.js'
import {
extractZipToDir,
hashSkillFiles,
listTextFiles,
readLockfile,
readSkillOrigin,
writeLockfile,
writeSkillOrigin,
} from '../../skills.js'
import { getRegistry } from '../registry.js'
import type { GlobalOpts, ResolveResult } from '../types.js'
@@ -33,7 +35,7 @@ export async function cmdSearch(opts: GlobalOpts, query: string, limit?: number)
const result = await apiRequest(
registry,
{ method: 'GET', url: url.toString() },
ApiSearchResponseSchema,
ApiV1SearchResponseSchema,
)
spinner.stop()
@@ -75,8 +77,8 @@ export async function cmdInstall(
(
await apiRequest(
registry,
{ method: 'GET', path: `/api/skill?slug=${encodeURIComponent(trimmed)}` },
ApiSkillMetaResponseSchema,
{ method: 'GET', path: `${ApiRoutes.skills}/${encodeURIComponent(trimmed)}` },
ApiV1SkillResponseSchema,
)
).latestVersion?.version ??
null
@@ -86,6 +88,14 @@ export async function cmdInstall(
const zip = await downloadZip(registry, { slug: trimmed, version: resolvedVersion })
await extractZipToDir(zip, target)
await writeSkillOrigin(target, {
version: 1,
registry,
slug: trimmed,
installedVersion: resolvedVersion,
installedAt: Date.now(),
})
const lock = await readLockfile(opts.workdir)
lock.skills[trimmed] = {
version: resolvedVersion,
@@ -140,12 +150,10 @@ export async function cmdUpdate(
if (localFingerprint) {
resolveResult = await resolveSkillVersion(registry, entry, localFingerprint)
} else {
const url = new URL(ApiRoutes.skill, registry)
url.searchParams.set('slug', entry)
const meta = await apiRequest(
registry,
{ method: 'GET', url: url.toString() },
ApiSkillMetaResponseSchema,
{ method: 'GET', url: `${ApiRoutes.skills}/${encodeURIComponent(entry)}` },
ApiV1SkillResponseSchema,
)
resolveResult = { match: null, latestVersion: meta.latestVersion ?? null }
}
@@ -200,6 +208,16 @@ export async function cmdUpdate(
await rm(target, { recursive: true, force: true })
const zip = await downloadZip(registry, { slug: entry, version: targetVersion })
await extractZipToDir(zip, target)
const existingOrigin = await readSkillOrigin(target)
await writeSkillOrigin(target, {
version: 1,
registry: existingOrigin?.registry ?? registry,
slug: existingOrigin?.slug ?? entry,
installedVersion: targetVersion,
installedAt: existingOrigin?.installedAt ?? Date.now(),
})
lock.skills[entry] = { version: targetVersion, installedAt: Date.now() }
spinner.succeed(`${entry}: updated -> ${targetVersion}`)
} catch (error) {
@@ -224,10 +242,14 @@ export async function cmdList(opts: GlobalOpts) {
}
async function resolveSkillVersion(registry: string, slug: string, hash: string) {
const url = new URL(ApiRoutes.skillResolve, registry)
const url = new URL(ApiRoutes.resolve, registry)
url.searchParams.set('slug', slug)
url.searchParams.set('hash', hash)
return apiRequest(registry, { method: 'GET', url: url.toString() }, ApiSkillResolveResponseSchema)
return apiRequest(
registry,
{ method: 'GET', url: url.toString() },
ApiV1SkillResolveResponseSchema,
)
}
async function fileExists(path: string) {
+52 -12
View File
@@ -57,6 +57,17 @@ vi.mock('../scanSkills.js', () => ({
getFallbackSkillRoots: vi.fn(() => []),
}))
const mockResolveClawdbotSkillRoots = vi.fn(
async () =>
({
roots: [] as string[],
labels: {} as Record<string, string>,
}) as const,
)
vi.mock('../clawdbotConfig.js', () => ({
resolveClawdbotSkillRoots: () => mockResolveClawdbotSkillRoots(),
}))
vi.mock('../../skills.js', async () => {
const actual = await vi.importActual<typeof import('../../skills.js')>('../../skills.js')
return {
@@ -98,9 +109,9 @@ describe('cmdSync', () => {
it('classifies skills as new/update/synced (dry-run, mocked HTTP)', async () => {
interactive = false
mockApiRequest.mockImplementation(async (_registry: string, args: { path: string }) => {
if (args.path === '/api/cli/whoami') return { user: { handle: 'steipete' } }
if (args.path === '/api/v1/whoami') return { user: { handle: 'steipete' } }
if (args.path === '/api/cli/telemetry/sync') return { ok: true }
if (args.path.startsWith('/api/skill/resolve?')) {
if (args.path.startsWith('/api/v1/resolve?')) {
const u = new URL(`https://x.test${args.path}`)
const slug = u.searchParams.get('slug')
if (slug === 'new-skill') {
@@ -135,9 +146,9 @@ describe('cmdSync', () => {
return initialValues
})
mockApiRequest.mockImplementation(async (_registry: string, args: { path: string }) => {
if (args.path === '/api/cli/whoami') return { user: { handle: 'steipete' } }
if (args.path === '/api/v1/whoami') return { user: { handle: 'steipete' } }
if (args.path === '/api/cli/telemetry/sync') return { ok: true }
if (args.path.startsWith('/api/skill/resolve?')) {
if (args.path.startsWith('/api/v1/resolve?')) {
const u = new URL(`https://x.test${args.path}`)
const slug = u.searchParams.get('slug')
if (slug === 'new-skill') {
@@ -171,9 +182,9 @@ describe('cmdSync', () => {
it('shows condensed synced list when nothing to sync', async () => {
interactive = false
mockApiRequest.mockImplementation(async (_registry: string, args: { path: string }) => {
if (args.path === '/api/cli/whoami') return { user: { handle: 'steipete' } }
if (args.path === '/api/v1/whoami') return { user: { handle: 'steipete' } }
if (args.path === '/api/cli/telemetry/sync') return { ok: true }
if (args.path.startsWith('/api/skill/resolve?')) {
if (args.path.startsWith('/api/v1/resolve?')) {
return { match: { version: '1.0.0' }, latestVersion: { version: '1.0.0' } }
}
throw new Error(`Unexpected apiRequest: ${args.path}`)
@@ -203,9 +214,9 @@ describe('cmdSync', () => {
})
mockApiRequest.mockImplementation(async (_registry: string, args: { path: string }) => {
if (args.path === '/api/cli/whoami') return { user: { handle: 'steipete' } }
if (args.path === '/api/v1/whoami') return { user: { handle: 'steipete' } }
if (args.path === '/api/cli/telemetry/sync') return { ok: true }
if (args.path.startsWith('/api/skill/resolve?')) {
if (args.path.startsWith('/api/v1/resolve?')) {
return { match: null, latestVersion: null }
}
throw new Error(`Unexpected apiRequest: ${args.path}`)
@@ -219,12 +230,41 @@ describe('cmdSync', () => {
expect(output).toMatch(/dup-skill/)
})
it('prints labeled roots when clawdbot roots are detected', async () => {
interactive = false
mockResolveClawdbotSkillRoots.mockResolvedValueOnce({
roots: ['/auto'],
labels: { '/auto': 'Agent: Work' },
})
const { findSkillFolders } = await import('../scanSkills.js')
vi.mocked(findSkillFolders).mockImplementation(async (root: string) => {
if (root === '/auto') {
return [{ folder: '/auto/alpha', slug: 'alpha', displayName: 'Alpha' }]
}
return []
})
mockApiRequest.mockImplementation(async (_registry: string, args: { path: string }) => {
if (args.path === '/api/v1/whoami') return { user: { handle: 'steipete' } }
if (args.path === '/api/cli/telemetry/sync') return { ok: true }
if (args.path.startsWith('/api/v1/resolve?')) {
throw new Error('Skill not found')
}
throw new Error(`Unexpected apiRequest: ${args.path}`)
})
await cmdSync(makeOpts(), { all: true, dryRun: true }, true)
const output = mockLog.mock.calls.map((call) => String(call[0])).join('\n')
expect(output).toMatch(/Roots with skills/)
expect(output).toMatch(/Agent: Work/)
})
it('allows empty changelog for updates (interactive)', async () => {
interactive = true
mockApiRequest.mockImplementation(async (_registry: string, args: { path: string }) => {
if (args.path === '/api/cli/whoami') return { user: { handle: 'steipete' } }
if (args.path === '/api/v1/whoami') return { user: { handle: 'steipete' } }
if (args.path === '/api/cli/telemetry/sync') return { ok: true }
if (args.path.startsWith('/api/skill/resolve?')) {
if (args.path.startsWith('/api/v1/resolve?')) {
const u = new URL(`https://x.test${args.path}`)
const slug = u.searchParams.get('slug')
if (slug === 'new-skill') {
@@ -254,8 +294,8 @@ describe('cmdSync', () => {
interactive = false
process.env.CLAWDHUB_DISABLE_TELEMETRY = '1'
mockApiRequest.mockImplementation(async (_registry: string, args: { path: string }) => {
if (args.path === '/api/cli/whoami') return { user: { handle: 'steipete' } }
if (args.path.startsWith('/api/skill/resolve?')) {
if (args.path === '/api/v1/whoami') return { user: { handle: 'steipete' } }
if (args.path.startsWith('/api/v1/resolve?')) {
return { match: { version: '1.0.0' }, latestVersion: { version: '1.0.0' } }
}
throw new Error(`Unexpected apiRequest: ${args.path}`)
+32 -5
View File
@@ -1,11 +1,11 @@
import { intro, outro } from '@clack/prompts'
import { readGlobalConfig } from '../../config.js'
import { hashSkillFiles, listTextFiles } from '../../skills.js'
import { hashSkillFiles, listTextFiles, readSkillOrigin } from '../../skills.js'
import { resolveClawdbotSkillRoots } from '../clawdbotConfig.js'
import { getFallbackSkillRoots } from '../scanSkills.js'
import type { GlobalOpts } from '../types.js'
import { createSpinner, fail, formatError, isInteractive } from '../ui.js'
import { cmdPublish } from './publish.js'
import type { Candidate, LocalSkill, SyncOptions } from './syncTypes.js'
import {
buildScanRoots,
checkRegistrySyncState,
@@ -23,9 +23,10 @@ import {
printSection,
reportTelemetryIfEnabled,
resolvePublishMeta,
scanRoots,
scanRootsWithLabels,
selectToUpload,
} from './syncHelpers.js'
import type { Candidate, LocalSkill, SyncOptions } from './syncTypes.js'
export async function cmdSync(opts: GlobalOpts, options: SyncOptions, inputAllowed: boolean) {
const allowPrompt = isInteractive() && inputAllowed !== false
@@ -37,15 +38,19 @@ export async function cmdSync(opts: GlobalOpts, options: SyncOptions, inputAllow
const registry = await getRegistryWithAuth(opts, token)
const selectedRoots = buildScanRoots(opts, options.root)
const clawdbotRoots = await resolveClawdbotSkillRoots()
const combinedRoots = Array.from(
new Set([...selectedRoots, ...clawdbotRoots.roots].map((root) => root.trim()).filter(Boolean)),
)
const concurrency = normalizeConcurrency(options.concurrency)
const spinner = createSpinner('Scanning for local skills')
const primaryScan = await scanRoots(selectedRoots)
const primaryScan = await scanRootsWithLabels(combinedRoots, clawdbotRoots.labels)
let scan = primaryScan
let telemetryScan = primaryScan
if (primaryScan.skills.length === 0) {
const fallback = getFallbackSkillRoots(opts.workdir)
const fallbackScan = await scanRoots(fallback)
const fallbackScan = await scanRootsWithLabels(fallback)
spinner.stop()
telemetryScan = mergeScan(primaryScan, fallbackScan)
scan = fallbackScan
@@ -57,6 +62,15 @@ export async function cmdSync(opts: GlobalOpts, options: SyncOptions, inputAllow
)
} else {
spinner.stop()
const labeledRoots = primaryScan.rootsWithSkills
.map((root) => {
const label = primaryScan.rootLabels?.[root]
return label ? `${label} (${root})` : root
})
.filter(Boolean)
if (labeledRoots.length > 0) {
printSection('Roots with skills', formatList(labeledRoots, 10))
}
}
const deduped = dedupeSkillsBySlug(scan.skills)
const skills = deduped.skills
@@ -70,12 +84,14 @@ export async function cmdSync(opts: GlobalOpts, options: SyncOptions, inputAllow
const parsed = await mapWithConcurrency(skills, Math.min(concurrency, 12), async (skill) => {
const filesOnDisk = await listTextFiles(skill.folder)
const hashed = hashSkillFiles(filesOnDisk)
const origin = await readSkillOrigin(skill.folder)
done += 1
parsingSpinner.text = `Parsing local skills ${done}/${skills.length}`
return {
...skill,
fingerprint: hashed.fingerprint,
fileCount: filesOnDisk.length,
origin,
}
})
locals.push(...parsed)
@@ -160,14 +176,25 @@ export async function cmdSync(opts: GlobalOpts, options: SyncOptions, inputAllow
allowPrompt,
changelogFlag: options.changelog,
})
const forkOf =
skill.origin && normalizeRegistry(skill.origin.registry) === normalizeRegistry(registry)
? skill.origin.slug !== skill.slug
? `${skill.origin.slug}@${skill.origin.installedVersion}`
: undefined
: undefined
await cmdPublish(opts, skill.folder, {
slug: skill.slug,
name: skill.displayName,
version: publishVersion,
changelog,
tags,
forkOf,
})
}
outro(`Uploaded ${selected.length} skill(s).`)
}
function normalizeRegistry(value: string) {
return value.trim().replace(/\/+$/, '').toLowerCase()
}
@@ -0,0 +1,26 @@
/* @vitest-environment node */
import { describe, expect, it, vi } from 'vitest'
vi.mock('../scanSkills.js', () => ({
findSkillFolders: vi.fn(async (root: string) => {
if (root.endsWith('/with-skill')) {
return [{ folder: `${root}/demo`, slug: 'demo', displayName: 'Demo' }]
}
return []
}),
}))
const { scanRootsWithLabels } = await import('./syncHelpers.js')
describe('scanRootsWithLabels', () => {
it('attaches labels to roots with skills', async () => {
const roots = ['/tmp/with-skill', '/tmp/empty', '/tmp/with-skill']
const labels = { '/tmp/with-skill': 'Agent: Work' }
const result = await scanRootsWithLabels(roots, labels)
expect(result.rootsWithSkills).toEqual(['/tmp/with-skill'])
expect(result.rootLabels).toEqual({ '/tmp/with-skill': 'Agent: Work' })
expect(result.skills.map((skill) => skill.slug)).toEqual(['demo'])
})
})
@@ -7,16 +7,17 @@ import semver from 'semver'
import { apiRequest, downloadZip } from '../../http.js'
import {
ApiCliTelemetrySyncResponseSchema,
ApiCliWhoamiResponseSchema,
ApiRoutes,
ApiSkillMetaResponseSchema,
ApiSkillResolveResponseSchema,
ApiV1SkillResolveResponseSchema,
ApiV1SkillResponseSchema,
ApiV1WhoamiResponseSchema,
LegacyApiRoutes,
} from '../../schema/index.js'
import { hashSkillZip } from '../../skills.js'
import { getRegistry } from '../registry.js'
import { findSkillFolders, type SkillFolder } from '../scanSkills.js'
import type { GlobalOpts } from '../types.js'
import { fail, formatError, isInteractive } from '../ui.js'
import { fail, formatError } from '../ui.js'
import type { Candidate, LocalSkill } from './syncTypes.js'
export async function reportTelemetryIfEnabled(params: {
@@ -45,7 +46,7 @@ export async function reportTelemetryIfEnabled(params: {
params.registry,
{
method: 'POST',
path: ApiRoutes.cliTelemetrySync,
path: LegacyApiRoutes.cliTelemetrySync,
token: params.token,
body: { roots },
},
@@ -73,7 +74,11 @@ export function normalizeConcurrency(value: number | undefined) {
return Math.min(32, Math.max(1, rounded))
}
export async function mapWithConcurrency<T, R>(items: T[], limit: number, fn: (item: T) => Promise<R>) {
export async function mapWithConcurrency<T, R>(
items: T[],
limit: number,
fn: (item: T) => Promise<R>,
) {
const results = Array.from({ length: items.length }) as R[]
let nextIndex = 0
const workerCount = Math.min(Math.max(1, limit), items.length || 1)
@@ -102,9 +107,9 @@ export async function checkRegistrySyncState(
registry,
{
method: 'GET',
path: `${ApiRoutes.skillResolve}?slug=${encodeURIComponent(skill.slug)}&hash=${encodeURIComponent(skill.fingerprint)}`,
path: `${ApiRoutes.resolve}?slug=${encodeURIComponent(skill.slug)}&hash=${encodeURIComponent(skill.fingerprint)}`,
},
ApiSkillResolveResponseSchema,
ApiV1SkillResolveResponseSchema,
)
resolveSupport.value = true
const latestVersion = resolved.latestVersion?.version ?? null
@@ -144,8 +149,8 @@ export async function checkRegistrySyncState(
const meta = await apiRequest(
registry,
{ method: 'GET', path: `${ApiRoutes.skill}?slug=${encodeURIComponent(skill.slug)}` },
ApiSkillMetaResponseSchema,
{ method: 'GET', path: `${ApiRoutes.skills}/${encodeURIComponent(skill.slug)}` },
ApiV1SkillResponseSchema,
).catch(() => null)
const latestVersion = meta?.latestVersion?.version ?? null
@@ -171,15 +176,27 @@ export async function checkRegistrySyncState(
}
export async function scanRoots(roots: string[]) {
const result = await scanRootsWithLabels(roots)
return {
roots: result.roots,
skillsByRoot: result.skillsByRoot,
skills: result.skills,
rootsWithSkills: result.rootsWithSkills,
}
}
export async function scanRootsWithLabels(roots: string[], labels?: Record<string, string>) {
const all: SkillFolder[] = []
const rootsWithSkills: string[] = []
const uniqueRoots = await dedupeRoots(roots)
const skillsByRoot: Record<string, SkillFolder[]> = {}
const rootLabels: Record<string, string> = {}
for (const root of uniqueRoots) {
const found = await findSkillFolders(root)
skillsByRoot[root] = found
if (found.length > 0) rootsWithSkills.push(root)
all.push(...found)
if (labels?.[root]) rootLabels[root] = labels[root] as string
}
const byFolder = new Map<string, SkillFolder>()
for (const folder of all) {
@@ -190,6 +207,7 @@ export async function scanRoots(roots: string[]) {
skillsByRoot,
skills: Array.from(byFolder.values()),
rootsWithSkills,
rootLabels,
}
}
@@ -199,12 +217,14 @@ export function mergeScan(
skillsByRoot: Record<string, SkillFolder[]>
skills: SkillFolder[]
rootsWithSkills: string[]
rootLabels: Record<string, string>
},
right: {
roots: string[]
skillsByRoot: Record<string, SkillFolder[]>
skills: SkillFolder[]
rootsWithSkills: string[]
rootLabels: Record<string, string>
},
) {
const mergedRoots = Array.from(new Set([...left.roots, ...right.roots]))
@@ -212,13 +232,14 @@ export function mergeScan(
for (const root of mergedRoots) {
skillsByRoot[root] = right.skillsByRoot[root] ?? left.skillsByRoot[root] ?? []
}
const rootLabels: Record<string, string> = { ...left.rootLabels, ...right.rootLabels }
const byFolder = new Map<string, SkillFolder>()
for (const entry of [...left.skills, ...right.skills]) {
byFolder.set(entry.folder, entry)
}
const skills = Array.from(byFolder.values())
const rootsWithSkills = mergedRoots.filter((root) => (skillsByRoot[root]?.length ?? 0) > 0)
return { roots: mergedRoots, skillsByRoot, skills, rootsWithSkills }
return { roots: mergedRoots, skillsByRoot, skills, rootsWithSkills, rootLabels }
}
async function dedupeRoots(roots: string[]) {
@@ -286,8 +307,8 @@ export async function getRegistryWithAuth(opts: GlobalOpts, token: string) {
const registry = await getRegistry(opts, { cache: true })
await apiRequest(
registry,
{ method: 'GET', path: ApiRoutes.cliWhoami, token },
ApiCliWhoamiResponseSchema,
{ method: 'GET', path: ApiRoutes.whoami, token },
ApiV1WhoamiResponseSchema,
)
return registry
}
@@ -356,7 +377,10 @@ export function dedupeSkillsBySlug(skills: SkillFolder[]) {
return { skills: unique, duplicates }
}
export function formatActionableStatus(candidate: Candidate, bump: 'patch' | 'minor' | 'major'): string {
export function formatActionableStatus(
candidate: Candidate,
bump: 'patch' | 'minor' | 'major',
): string {
if (candidate.status === 'new') return 'NEW'
const latest = candidate.latestVersion
const next = latest ? semver.inc(latest, bump) : null
@@ -364,7 +388,10 @@ export function formatActionableStatus(candidate: Candidate, bump: 'patch' | 'mi
return 'UPDATE'
}
export function formatActionableLine(candidate: Candidate, bump: 'patch' | 'minor' | 'major'): string {
export function formatActionableLine(
candidate: Candidate,
bump: 'patch' | 'minor' | 'major',
): string {
return `${candidate.slug} ${formatActionableStatus(candidate, bump)} (${candidate.fileCount} files)`
}
@@ -1,3 +1,4 @@
import type { SkillOrigin } from '../../skills.js'
import type { SkillFolder } from '../scanSkills.js'
export type SyncOptions = {
@@ -13,6 +14,7 @@ export type SyncOptions = {
export type Candidate = SkillFolder & {
fingerprint: string
fileCount: number
origin: SkillOrigin | null
status: 'synced' | 'new' | 'update'
matchVersion: string | null
latestVersion: string | null
@@ -21,4 +23,5 @@ export type Candidate = SkillFolder & {
export type LocalSkill = SkillFolder & {
fingerprint: string
fileCount: number
origin: SkillOrigin | null
}
+39
View File
@@ -33,4 +33,43 @@ describe('discovery', () => {
minCliVersion: undefined,
})
})
it('parses apiBase config', async () => {
vi.stubGlobal(
'fetch',
vi.fn(
async () =>
new Response(
JSON.stringify({
apiBase: 'https://api.example.com',
authBase: 'https://auth.example.com',
minCliVersion: '1.2.3',
}),
{
status: 200,
headers: { 'Content-Type': 'application/json' },
},
),
) as unknown as typeof fetch,
)
await expect(discoverRegistryFromSite('https://example.com')).resolves.toEqual({
apiBase: 'https://api.example.com',
authBase: 'https://auth.example.com',
minCliVersion: '1.2.3',
})
})
it('returns null when apiBase is empty', async () => {
vi.stubGlobal(
'fetch',
vi.fn(
async () =>
new Response(JSON.stringify({ apiBase: '' }), {
status: 200,
headers: { 'Content-Type': 'application/json' },
}),
) as unknown as typeof fetch,
)
await expect(discoverRegistryFromSite('https://example.com')).resolves.toBeNull()
})
})
+76 -3
View File
@@ -1,8 +1,8 @@
/* @vitest-environment node */
import { describe, expect, it, vi } from 'vitest'
import { apiRequest, downloadZip } from './http'
import { ApiCliWhoamiResponseSchema } from './schema/index.js'
import { apiRequest, apiRequestForm, downloadZip } from './http'
import { ApiV1WhoamiResponseSchema } from './schema/index.js'
describe('apiRequest', () => {
it('adds bearer token and parses json', async () => {
@@ -14,7 +14,7 @@ describe('apiRequest', () => {
const result = await apiRequest(
'https://example.com',
{ method: 'GET', path: '/x', token: 'clh_token' },
ApiCliWhoamiResponseSchema,
ApiV1WhoamiResponseSchema,
)
expect(result.user.handle).toBeNull()
expect(fetchMock).toHaveBeenCalledTimes(1)
@@ -80,4 +80,77 @@ describe('apiRequest', () => {
expect(url).toContain('version=1.0.0')
vi.unstubAllGlobals()
})
it('does not retry on non-retryable errors', async () => {
const fetchMock = vi.fn().mockResolvedValue({
ok: false,
status: 404,
text: async () => 'nope',
})
vi.stubGlobal('fetch', fetchMock)
await expect(downloadZip('https://example.com', { slug: 'demo' })).rejects.toThrow('nope')
expect(fetchMock).toHaveBeenCalledTimes(1)
vi.unstubAllGlobals()
})
})
describe('apiRequestForm', () => {
it('posts form data and returns json', async () => {
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({ ok: true }),
})
vi.stubGlobal('fetch', fetchMock)
const form = new FormData()
form.append('x', '1')
const result = await apiRequestForm('https://example.com', {
method: 'POST',
path: '/upload',
token: 'clh_token',
form,
})
expect(result).toEqual({ ok: true })
const [, init] = fetchMock.mock.calls[0] as [string, RequestInit]
expect(init.body).toBe(form)
expect((init.headers as Record<string, string>).Authorization).toBe('Bearer clh_token')
vi.unstubAllGlobals()
})
it('retries on 429', async () => {
const fetchMock = vi.fn().mockResolvedValue({
ok: false,
status: 429,
text: async () => 'rate limited',
})
vi.stubGlobal('fetch', fetchMock)
await expect(
apiRequestForm('https://example.com', {
method: 'POST',
path: '/upload',
form: new FormData(),
}),
).rejects.toThrow('rate limited')
expect(fetchMock).toHaveBeenCalledTimes(3)
vi.unstubAllGlobals()
})
it('falls back to HTTP status when body cannot be read', async () => {
const fetchMock = vi.fn().mockResolvedValue({
ok: false,
status: 400,
text: async () => {
throw new Error('boom')
},
})
vi.stubGlobal('fetch', fetchMock)
await expect(
apiRequestForm('https://example.com', {
method: 'POST',
path: '/upload',
form: new FormData(),
}),
).rejects.toThrow('HTTP 400')
expect(fetchMock).toHaveBeenCalledTimes(1)
vi.unstubAllGlobals()
})
})
+39 -2
View File
@@ -3,8 +3,8 @@ import type { ArkValidator } from './schema/index.js'
import { ApiRoutes, parseArk } from './schema/index.js'
type RequestArgs =
| { method: 'GET' | 'POST'; path: string; token?: string; body?: unknown }
| { method: 'GET' | 'POST'; url: string; token?: string; body?: unknown }
| { method: 'GET' | 'POST' | 'DELETE'; path: string; token?: string; body?: unknown }
| { method: 'GET' | 'POST' | 'DELETE'; url: string; token?: string; body?: unknown }
export async function apiRequest<T>(registry: string, args: RequestArgs): Promise<T>
export async function apiRequest<T>(
@@ -44,6 +44,43 @@ export async function apiRequest<T>(
return json as T
}
type FormRequestArgs =
| { method: 'POST'; path: string; token?: string; form: FormData }
| { method: 'POST'; url: string; token?: string; form: FormData }
export async function apiRequestForm<T>(registry: string, args: FormRequestArgs): Promise<T>
export async function apiRequestForm<T>(
registry: string,
args: FormRequestArgs,
schema: ArkValidator<T>,
): Promise<T>
export async function apiRequestForm<T>(
registry: string,
args: FormRequestArgs,
schema?: ArkValidator<T>,
): Promise<T> {
const url = 'url' in args ? args.url : new URL(args.path, registry).toString()
const json = await pRetry(
async () => {
const headers: Record<string, string> = { Accept: 'application/json' }
if (args.token) headers.Authorization = `Bearer ${args.token}`
const response = await fetch(url, { method: args.method, headers, body: args.form })
if (!response.ok) {
const text = await response.text().catch(() => '')
const message = text || `HTTP ${response.status}`
if (response.status === 429 || response.status >= 500) {
throw new Error(message)
}
throw new AbortError(message)
}
return (await response.json()) as unknown
},
{ retries: 2 },
)
if (schema) return parseArk(schema, json, 'API response')
return json as T
}
export async function downloadZip(registry: string, args: { slug: string; version?: string }) {
const url = new URL(ApiRoutes.download, registry)
url.searchParams.set('slug', args.slug)
+1 -1
View File
@@ -1,5 +1,5 @@
export type { ArkValidator } from './ark.js'
export { formatArkErrors, parseArk } from './ark.js'
export { ApiRoutes } from './routes.js'
export { ApiRoutes, LegacyApiRoutes } from './routes.js'
export * from './schemas.js'
export * from './textFiles.js'
+10 -1
View File
@@ -1,4 +1,4 @@
export const ApiRoutes = {
export const LegacyApiRoutes = {
download: '/api/download',
search: '/api/search',
skill: '/api/skill',
@@ -10,3 +10,12 @@ export const ApiRoutes = {
cliSkillDelete: '/api/cli/skill/delete',
cliSkillUndelete: '/api/cli/skill/undelete',
} as const
export const ApiRoutes = {
search: '/api/v1/search',
resolve: '/api/v1/resolve',
download: '/api/v1/download',
skills: '/api/v1/skills',
souls: '/api/v1/souls',
whoami: '/api/v1/whoami',
} as const
+102
View File
@@ -73,6 +73,10 @@ export const CliPublishRequestSchema = type({
version: 'string',
changelog: 'string',
tags: 'string[]?',
forkOf: type({
slug: 'string',
version: 'string?',
}).optional(),
files: CliPublishFileSchema.array(),
})
export type CliPublishRequest = (typeof CliPublishRequestSchema)[inferred]
@@ -113,6 +117,104 @@ export const ApiCliTelemetrySyncResponseSchema = type({
ok: 'true',
})
export const ApiV1WhoamiResponseSchema = type({
user: {
handle: 'string|null',
displayName: 'string|null?',
image: 'string|null?',
},
})
export const ApiV1SearchResponseSchema = type({
results: type({
slug: 'string?',
displayName: 'string?',
summary: 'string|null?',
version: 'string|null?',
score: 'number',
updatedAt: 'number?',
}).array(),
})
export const ApiV1SkillListResponseSchema = type({
items: type({
slug: 'string',
displayName: 'string',
summary: 'string|null?',
tags: 'unknown',
stats: 'unknown',
createdAt: 'number',
updatedAt: 'number',
latestVersion: type({
version: 'string',
createdAt: 'number',
changelog: 'string',
}).optional(),
}).array(),
nextCursor: 'string|null',
})
export const ApiV1SkillResponseSchema = type({
skill: type({
slug: 'string',
displayName: 'string',
summary: 'string|null?',
tags: 'unknown',
stats: 'unknown',
createdAt: 'number',
updatedAt: 'number',
}).or('null'),
latestVersion: type({
version: 'string',
createdAt: 'number',
changelog: 'string',
}).or('null'),
owner: type({
handle: 'string|null',
displayName: 'string|null?',
image: 'string|null?',
}).or('null'),
})
export const ApiV1SkillVersionListResponseSchema = type({
items: type({
version: 'string',
createdAt: 'number',
changelog: 'string',
changelogSource: '"auto"|"user"|null?',
}).array(),
nextCursor: 'string|null',
})
export const ApiV1SkillVersionResponseSchema = type({
version: type({
version: 'string',
createdAt: 'number',
changelog: 'string',
changelogSource: '"auto"|"user"|null?',
files: 'unknown?',
}).or('null'),
skill: type({
slug: 'string',
displayName: 'string',
}).or('null'),
})
export const ApiV1SkillResolveResponseSchema = type({
match: type({ version: 'string' }).or('null'),
latestVersion: type({ version: 'string' }).or('null'),
})
export const ApiV1PublishResponseSchema = type({
ok: 'true',
skillId: 'string',
versionId: 'string',
})
export const ApiV1DeleteResponseSchema = type({
ok: 'true',
})
export const SkillInstallSpecSchema = type({
id: 'string?',
kind: '"brew"|"node"|"go"|"uv"',
@@ -0,0 +1,23 @@
/* @vitest-environment node */
import { describe, expect, it } from 'vitest'
import * as schema from '.'
import { isTextContentType, TEXT_FILE_EXTENSION_SET } from './textFiles'
describe('packages/clawdhub schema textFiles', () => {
it('exports text-file extension set', () => {
expect(TEXT_FILE_EXTENSION_SET.has('md')).toBe(true)
expect(TEXT_FILE_EXTENSION_SET.has('exe')).toBe(false)
})
it('detects text content types with parameters', () => {
expect(isTextContentType('text/plain; charset=utf-8')).toBe(true)
expect(isTextContentType('application/json; charset=utf-8')).toBe(true)
expect(isTextContentType('application/octet-stream')).toBe(false)
})
it('re-exports helpers from index', () => {
expect(typeof schema.isTextContentType).toBe('function')
expect(schema.isTextContentType('application/markdown')).toBe(true)
})
})
+89
View File
@@ -4,6 +4,7 @@ import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { strToU8, zipSync } from 'fflate'
import { describe, expect, it } from 'vitest'
import type { SkillOrigin } from './skills'
import {
buildSkillFingerprint,
extractZipToDir,
@@ -11,8 +12,10 @@ import {
hashSkillZip,
listTextFiles,
readLockfile,
readSkillOrigin,
sha256Hex,
writeLockfile,
writeSkillOrigin,
} from './skills'
describe('skills', () => {
@@ -46,6 +49,18 @@ describe('skills', () => {
expect(read).toEqual({ version: 1, skills: {} })
})
it('returns empty lockfile on schema mismatch', async () => {
const workdir = await mkdtemp(join(tmpdir(), 'clawdhub-work-schema-'))
await mkdir(join(workdir, '.clawdhub'), { recursive: true })
await writeFile(
join(workdir, '.clawdhub', 'lock.json'),
JSON.stringify({ version: 1, skills: 'nope' }),
'utf8',
)
const read = await readLockfile(workdir)
expect(read).toEqual({ version: 1, skills: {} })
})
it('skips dotfiles and node_modules when listing text files', async () => {
const workdir = await mkdtemp(join(tmpdir(), 'clawdhub-files-'))
await writeFile(join(workdir, 'SKILL.md'), 'hi', 'utf8')
@@ -74,6 +89,14 @@ describe('skills', () => {
)
})
it('falls back to text/plain for unknown text extensions', async () => {
const workdir = await mkdtemp(join(tmpdir(), 'clawdhub-env-'))
await writeFile(join(workdir, 'SKILL.md'), 'hi', 'utf8')
await writeFile(join(workdir, 'config.env'), 'TOKEN=demo', 'utf8')
const files = await listTextFiles(workdir)
expect(files.find((file) => file.relPath === 'config.env')?.contentType).toBe('text/plain')
})
it('hashes skill files deterministically', async () => {
const { fingerprint } = hashSkillFiles([
{ relPath: 'b.txt', bytes: strToU8('b') },
@@ -99,4 +122,70 @@ describe('skills', () => {
])
expect(fingerprint).toBe(expected)
})
it('ignores unsafe or non-text entries when hashing zips', () => {
const zip = zipSync({
'SKILL.md': strToU8('hello'),
'folder/': strToU8(''),
'../evil.txt': strToU8('nope'),
'bad\\path.txt': strToU8('nope'),
'image.png': strToU8('nope'),
})
const { files } = hashSkillZip(new Uint8Array(zip))
expect(files).toEqual([{ path: 'SKILL.md', sha256: sha256Hex(strToU8('hello')), size: 5 }])
})
it('builds fingerprints from valid entries only', () => {
const fingerprint = buildSkillFingerprint([
{ path: '', sha256: '' },
{ path: 'valid.txt', sha256: sha256Hex(strToU8('ok')) },
])
const expected = buildSkillFingerprint([
{ path: 'valid.txt', sha256: sha256Hex(strToU8('ok')) },
])
expect(fingerprint).toBe(expected)
})
it('returns null for invalid skill origin metadata', async () => {
const workdir = await mkdtemp(join(tmpdir(), 'clawdhub-origin-'))
expect(await readSkillOrigin(workdir)).toBeNull()
await mkdir(join(workdir, '.clawdhub'), { recursive: true })
await writeFile(
join(workdir, '.clawdhub', 'origin.json'),
JSON.stringify({ version: 2 }),
'utf8',
)
expect(await readSkillOrigin(workdir)).toBeNull()
await writeFile(
join(workdir, '.clawdhub', 'origin.json'),
JSON.stringify({ version: 1, registry: 'demo', slug: 'x', installedAt: 1 }),
'utf8',
)
expect(await readSkillOrigin(workdir)).toBeNull()
await writeFile(
join(workdir, '.clawdhub', 'origin.json'),
JSON.stringify({
version: 1,
registry: 'demo',
slug: 'x',
installedVersion: '0.1.0',
installedAt: 'nope',
}),
'utf8',
)
expect(await readSkillOrigin(workdir)).toBeNull()
const origin: SkillOrigin = {
version: 1,
registry: 'https://example.com',
slug: 'demo',
installedVersion: '1.2.3',
installedAt: 123,
}
await writeSkillOrigin(workdir, origin)
expect(await readSkillOrigin(workdir)).toEqual(origin)
})
})
+34
View File
@@ -6,6 +6,14 @@ import ignore from 'ignore'
import mime from 'mime'
import { type Lockfile, LockfileSchema, parseArk, TEXT_FILE_EXTENSION_SET } from './schema/index.js'
export type SkillOrigin = {
version: 1
registry: string
slug: string
installedVersion: string
installedAt: number
}
export async function extractZipToDir(zipBytes: Uint8Array, targetDir: string) {
const entries = unzipSync(zipBytes)
await mkdir(targetDir, { recursive: true })
@@ -95,6 +103,32 @@ export async function writeLockfile(workdir: string, lock: Lockfile) {
await writeFile(path, `${JSON.stringify(lock, null, 2)}\n`, 'utf8')
}
export async function readSkillOrigin(skillFolder: string): Promise<SkillOrigin | null> {
const path = join(skillFolder, '.clawdhub', 'origin.json')
try {
const raw = await readFile(path, 'utf8')
const parsed = JSON.parse(raw) as Partial<SkillOrigin>
if (parsed.version !== 1) return null
if (!parsed.registry || !parsed.slug || !parsed.installedVersion) return null
if (typeof parsed.installedAt !== 'number' || !Number.isFinite(parsed.installedAt)) return null
return {
version: 1,
registry: String(parsed.registry),
slug: String(parsed.slug),
installedVersion: String(parsed.installedVersion),
installedAt: parsed.installedAt,
}
} catch {
return null
}
}
export async function writeSkillOrigin(skillFolder: string, origin: SkillOrigin) {
const path = join(skillFolder, '.clawdhub', 'origin.json')
await mkdir(dirname(path), { recursive: true })
await writeFile(path, `${JSON.stringify(origin, null, 2)}\n`, 'utf8')
}
function normalizePath(path: string) {
return path
.split(sep)
+1 -1
View File
@@ -1,5 +1,5 @@
export type { ArkValidator } from './ark.js';
export { formatArkErrors, parseArk } from './ark.js';
export { ApiRoutes } from './routes.js';
export { ApiRoutes, LegacyApiRoutes } from './routes.js';
export * from './schemas.js';
export * from './textFiles.js';
+1 -1
View File
@@ -1,5 +1,5 @@
export { formatArkErrors, parseArk } from './ark.js';
export { ApiRoutes } from './routes.js';
export { ApiRoutes, LegacyApiRoutes } from './routes.js';
export * from './schemas.js';
export * from './textFiles.js';
//# sourceMappingURL=index.js.map
+1 -1
View File
@@ -1 +1 @@
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AACvC,cAAc,cAAc,CAAA;AAC5B,cAAc,gBAAgB,CAAA"}
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AACpD,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AACxD,cAAc,cAAc,CAAA;AAC5B,cAAc,gBAAgB,CAAA"}
+9 -1
View File
@@ -1,4 +1,4 @@
export declare const ApiRoutes: {
export declare const LegacyApiRoutes: {
readonly download: "/api/download";
readonly search: "/api/search";
readonly skill: "/api/skill";
@@ -10,3 +10,11 @@ export declare const ApiRoutes: {
readonly cliSkillDelete: "/api/cli/skill/delete";
readonly cliSkillUndelete: "/api/cli/skill/undelete";
};
export declare const ApiRoutes: {
readonly search: "/api/v1/search";
readonly resolve: "/api/v1/resolve";
readonly download: "/api/v1/download";
readonly skills: "/api/v1/skills";
readonly souls: "/api/v1/souls";
readonly whoami: "/api/v1/whoami";
};
+9 -1
View File
@@ -1,4 +1,4 @@
export const ApiRoutes = {
export const LegacyApiRoutes = {
download: '/api/download',
search: '/api/search',
skill: '/api/skill',
@@ -10,4 +10,12 @@ export const ApiRoutes = {
cliSkillDelete: '/api/cli/skill/delete',
cliSkillUndelete: '/api/cli/skill/undelete',
};
export const ApiRoutes = {
search: '/api/v1/search',
resolve: '/api/v1/resolve',
download: '/api/v1/download',
skills: '/api/v1/skills',
souls: '/api/v1/souls',
whoami: '/api/v1/whoami',
};
//# sourceMappingURL=routes.js.map

Some files were not shown because too many files have changed in this diff Show More