mirror of
https://github.com/openclaw/clawhub.git
synced 2026-08-15 01:12:11 +00:00
Compare commits
101
Commits
v0.1.0
...
shadow/rewrite
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ffa25f47da | ||
|
|
f866dc05d1 | ||
|
|
69436fd79b | ||
|
|
f185ca6f55 | ||
|
|
f3fc8d62b6 | ||
|
|
71f94a774f | ||
|
|
07349e6107 | ||
|
|
5ef00e8c6a | ||
|
|
c5e5e657dd | ||
|
|
69e1e5c507 | ||
|
|
481f1b9188 | ||
|
|
d4f5832554 | ||
|
|
ca3275ba92 | ||
|
|
d577add8a0 | ||
|
|
90bd065f7e | ||
|
|
219f05b160 | ||
|
|
c8091ee8a8 | ||
|
|
63f367e1d6 | ||
|
|
e701d7b713 | ||
|
|
79bd1f1d6c | ||
|
|
ac51cc0236 | ||
|
|
b9c23dc00e | ||
|
|
f4e96995bc | ||
|
|
33165db598 | ||
|
|
4c10e4847b | ||
|
|
8d68b55333 | ||
|
|
251de1f540 | ||
|
|
123f60fa93 | ||
|
|
643faf71c8 | ||
|
|
a2c46fbb5d | ||
|
|
f51e0a087d | ||
|
|
d9108b0948 | ||
|
|
d7a017e1c3 | ||
|
|
fffdf82540 | ||
|
|
a16e624766 | ||
|
|
decce1d35c | ||
|
|
468832af3f | ||
|
|
54c793a660 | ||
|
|
5d9a89a885 | ||
|
|
31e9a57678 | ||
|
|
7680cc4ce8 | ||
|
|
de56255b95 | ||
|
|
eaaa5e4423 | ||
|
|
49e9c3c071 | ||
|
|
ef96fbee84 | ||
|
|
9fc032216e | ||
|
|
236058b1b5 | ||
|
|
595bd2cf05 | ||
|
|
918b5528df | ||
|
|
2f6f11af3f | ||
|
|
c90f92dbc7 | ||
|
|
8db8c89206 | ||
|
|
00c31dcdd4 | ||
|
|
47646937b6 | ||
|
|
10d250ea32 | ||
|
|
e7fa7afdf7 | ||
|
|
aa97727be8 | ||
|
|
d375496c17 | ||
|
|
9a912ee5eb | ||
|
|
11b257a062 | ||
|
|
02e509404a | ||
|
|
2cf6182991 | ||
|
|
e108789ab1 | ||
|
|
18bfea5035 | ||
|
|
5f93ddb390 | ||
|
|
2b552c4803 | ||
|
|
ece0b830fb | ||
|
|
57afea9de5 | ||
|
|
1e788b2e0c | ||
|
|
f025721261 | ||
|
|
c2cc61df24 | ||
|
|
7a8a3f75ce | ||
|
|
ac0bdf0375 | ||
|
|
6c7dd2ec6d | ||
|
|
c72e3007a1 | ||
|
|
b66d28a184 | ||
|
|
dddee828af | ||
|
|
106cb1896a | ||
|
|
e29ec88afd | ||
|
|
0eb3047ca6 | ||
|
|
b34c7261bd | ||
|
|
e0d553602a | ||
|
|
f80fa90e01 | ||
|
|
0cc0bdcd50 | ||
|
|
cc0027a094 | ||
|
|
dddbd3a78e | ||
|
|
f350442002 | ||
|
|
826c60f4da | ||
|
|
a679c3a999 | ||
|
|
d5d8e6ae5b | ||
|
|
f0772e7215 | ||
|
|
770bb3aeb8 | ||
|
|
6811691055 | ||
|
|
26b46d9f6e | ||
|
|
d145c186a7 | ||
|
|
57af81d054 | ||
|
|
0131229843 | ||
|
|
d7650583dc | ||
|
|
cf2ad58e86 | ||
|
|
153c3f5b9e | ||
|
|
243ca9ca2b |
@@ -1,6 +1,9 @@
|
||||
# Frontend
|
||||
VITE_CONVEX_URL=
|
||||
VITE_CONVEX_SITE_URL=
|
||||
VITE_SOULHUB_SITE_URL=
|
||||
VITE_SOULHUB_HOST=
|
||||
VITE_SITE_MODE=
|
||||
SITE_URL=http://localhost:3000
|
||||
CONVEX_SITE_URL=
|
||||
|
||||
|
||||
@@ -15,10 +15,12 @@ jobs:
|
||||
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
with:
|
||||
bun-version: 1.3.5
|
||||
bun-version: 1.3.6
|
||||
|
||||
- name: Install
|
||||
run: bun install --frozen-lockfile
|
||||
- name: Peer deps
|
||||
run: bun run check:peers
|
||||
|
||||
- name: Lint
|
||||
run: bun run lint
|
||||
@@ -36,4 +38,3 @@ jobs:
|
||||
|
||||
- name: Build
|
||||
run: bun run build
|
||||
|
||||
|
||||
+3
-1
@@ -16,9 +16,11 @@ count.txt
|
||||
.wrangler
|
||||
.output
|
||||
.vinxi
|
||||
*.bun-build
|
||||
todos.json
|
||||
.cta.json
|
||||
.vscode
|
||||
.env*.local
|
||||
coverage
|
||||
playwright-report
|
||||
test-results
|
||||
.playwright
|
||||
|
||||
@@ -40,5 +40,6 @@
|
||||
- OAuth: GitHub OAuth App credentials required for login.
|
||||
|
||||
## Convex Ops (Gotchas)
|
||||
- New Convex functions must be pushed before `convex run`: use `bunx convex dev --once` (dev) or `bunx convex deploy --prod` (prod).
|
||||
- New Convex functions must be pushed before `convex run`: use `bunx convex dev --once` (dev) or `bunx convex deploy` (prod).
|
||||
- For non-interactive prod deploys, use `bunx convex deploy -y` to skip confirmation.
|
||||
- If `bunx convex run --env-file .env.local ...` returns `401 MissingAccessToken` despite `bunx convex login`, workaround: omit `--env-file` and use `--deployment-name <name>` / `--prod`.
|
||||
|
||||
+59
-6
@@ -1,11 +1,64 @@
|
||||
# Changelog
|
||||
|
||||
## 0.1.0 - 2026-01-07
|
||||
## Unreleased
|
||||
|
||||
### Added
|
||||
|
||||
### Changed
|
||||
|
||||
### Fixed
|
||||
|
||||
## 0.4.0 - 2026-01-30
|
||||
|
||||
### Added
|
||||
- Web: show published skills on user profiles (thanks @njoylab, #20).
|
||||
- CLI: include OpenClaw + Moltbot fallback skill roots for sync scans.
|
||||
- CLI: support OpenClaw configuration files (`OPENCLAW_CONFIG_PATH` / `OPENCLAW_STATE_DIR`).
|
||||
|
||||
### Changed
|
||||
- Brand: rebrand to OpenClaw and publish CLI as `clawhub` (legacy `clawdhub` supported).
|
||||
- Domain: default site/registry now `https://clawhub.ai`; `.well-known/clawhub.json` preferred.
|
||||
- Theme: persist theme under `clawhub-theme` (legacy key still read).
|
||||
|
||||
### Fixed
|
||||
- Registry: drop missing skills during search hydration (thanks @aaronn, #28).
|
||||
- CLI: use path-based skill metadata lookup for updates (thanks @daveonkels, #22).
|
||||
- Search: keep highlighted-only filtering and clamp vector candidates to Convex limits (thanks @aaronn, #30).
|
||||
|
||||
## 0.3.0 - 2026-01-19
|
||||
|
||||
### Added
|
||||
- CLI: add `explore` command for latest updates, with limit clamping + tests/docs (thanks @jdrhyne, #14).
|
||||
- CLI: `explore --json` output + new sorts (`installs`, `installsAllTime`, `trending`) and limit up to 200.
|
||||
- API: `/api/v1/skills` supports installs + trending sorts (7-day installs).
|
||||
- API: idempotent `POST/DELETE /api/v1/stars/{slug}` endpoints.
|
||||
- Registry: trending leaderboard + daily stats backfill for installs-based sorts.
|
||||
|
||||
### Fixed
|
||||
- Web: keep search mode navigation and state in sync (thanks @NACC96, #12).
|
||||
|
||||
## 0.2.0 - 2026-01-13
|
||||
|
||||
### Added
|
||||
- Web: dynamic OG image cards for skills (name, description, version).
|
||||
- CLI: auto-scan Clawdbot skill roots (per-agent workspaces, shared skills, extraDirs).
|
||||
- Web: import skills from public GitHub URLs (auto-detect `SKILL.md`, smart file selection, provenance).
|
||||
- Web/API: SoulHub (SOUL.md registry) with v1 endpoints and first-run auto-seed.
|
||||
|
||||
### Fixed
|
||||
- Web: stabilize skill OG image generation on server runtimes.
|
||||
- Web: prevent skill OG text overflow outside the card.
|
||||
- Registry: make SoulHub auto-seed idempotent and non-user-owned.
|
||||
- Registry: keep GitHub backup state + publish backups intact (thanks @joshp123, #1).
|
||||
- CLI/Registry: restore fork lineage on sync + clamp bulk list queries (thanks @joshp123, #1).
|
||||
- CLI: default workdir falls back to Clawdbot workspace (override with `--workdir` / `CLAWHUB_WORKDIR`).
|
||||
|
||||
## 0.0.6 - 2026-01-07
|
||||
|
||||
|
||||
### Added
|
||||
- API: v1 public REST endpoints with rate limits, raw file fetch, and OpenAPI spec.
|
||||
- Docs: `docs/api.md` and `DEPRECATIONS.md` for the v1 cutover plan.
|
||||
- Registry: GitHub App backs up published skills to `clawdbot/skills` (thanks @thewilloftheshadow, #5).
|
||||
|
||||
### Changed
|
||||
- CLI: publish now uses single multipart `POST /api/v1/skills`.
|
||||
@@ -14,7 +67,7 @@
|
||||
## 0.0.5 - 2026-01-06
|
||||
|
||||
### Added
|
||||
- Telemetry: track installs via `clawdhub sync` (logged-in only), per root, with 120-day staleness.
|
||||
- Telemetry: track installs via `clawhub sync` (logged-in only), per root, with 120-day staleness.
|
||||
- Skills: show current + all-time installs; sort by installs.
|
||||
- Profile: private "Installed" tab with JSON export + delete telemetry controls.
|
||||
- Docs: add `docs/telemetry.md` (what we track + how to opt out).
|
||||
@@ -22,7 +75,7 @@
|
||||
- Web: dashboard for managing your published skills (thanks @dbhurley!).
|
||||
|
||||
### Changed
|
||||
- CLI: telemetry opt-out via `CLAWDHUB_DISABLE_TELEMETRY=1`.
|
||||
- CLI: telemetry opt-out via `CLAWHUB_DISABLE_TELEMETRY=1`.
|
||||
- Web: move theme picker into mobile menu.
|
||||
|
||||
### Fixed
|
||||
@@ -70,8 +123,8 @@
|
||||
### Fixed
|
||||
- CLI sync: wrap note output to avoid terminal overflow; cap list lengths.
|
||||
- CLI sync: label fallback scans as fallback locations.
|
||||
- CLI package: bundle schema internally (no external `clawdhub-schema` publish).
|
||||
- Repo: mark `clawdhub-schema` as private to prevent publishing.
|
||||
- CLI package: bundle schema internally (no external `clawhub-schema` publish).
|
||||
- Repo: mark `clawhub-schema` as private to prevent publishing.
|
||||
|
||||
## 0.0.2 - 2026-01-04
|
||||
|
||||
|
||||
@@ -1,37 +1,49 @@
|
||||
# ClawdHub
|
||||
# OpenClaw
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/clawdbot/clawdhub/actions/workflows/ci.yml?branch=main"><img src="https://img.shields.io/github/actions/workflow/status/clawdbot/clawdhub/ci.yml?branch=main&style=for-the-badge" alt="CI status"></a>
|
||||
<a href="https://github.com/openclaw/clawhub/actions/workflows/ci.yml?branch=main"><img src="https://img.shields.io/github/actions/workflow/status/openclaw/clawhub/ci.yml?branch=main&style=for-the-badge" alt="CI status"></a>
|
||||
<a href="https://discord.gg/clawd"><img src="https://img.shields.io/discord/1456350064065904867?label=Discord&logo=discord&logoColor=white&color=5865F2&style=for-the-badge" alt="Discord"></a>
|
||||
<a href="LICENSE"><img src="https://img.shields.io/badge/License-MIT-blue.svg?style=for-the-badge" alt="MIT License"></a>
|
||||
</p>
|
||||
|
||||
ClawdHub is the **public skill registry for Clawdbot**: publish, version, and search text-based agent skills (a `SKILL.md` plus supporting files).
|
||||
OpenClaw is the **public skill registry for Clawdbot**: publish, version, and search text-based agent skills (a `SKILL.md` plus supporting files).
|
||||
It’s designed for fast browsing + a CLI-friendly API, with moderation hooks and vector search.
|
||||
|
||||
Live: `https://clawdhub.com`
|
||||
onlycrabs.ai is the **SOUL.md registry**: publish and share system lore the same way you publish skills.
|
||||
|
||||
Live: `https://clawhub.ai`
|
||||
onlycrabs.ai: `https://onlycrabs.ai`
|
||||
|
||||
## What you can do
|
||||
|
||||
- Browse skills + render their `SKILL.md`.
|
||||
- Publish new versions with changelogs + tags (including `latest`).
|
||||
- Publish new skill versions with changelogs + tags (including `latest`).
|
||||
- Browse souls + render their `SOUL.md`.
|
||||
- Publish new soul versions with changelogs + tags.
|
||||
- Search via embeddings (vector index) instead of brittle keywords.
|
||||
- Star + comment; admins/mods can curate and approve.
|
||||
- Star + comment; admins/mods can curate and approve skills.
|
||||
|
||||
## onlycrabs.ai (SOUL.md registry)
|
||||
|
||||
- Entry point is host-based: `onlycrabs.ai`.
|
||||
- On the onlycrabs.ai host, the home page and nav default to souls.
|
||||
- On OpenClaw, souls live under `/souls`.
|
||||
- Soul bundles only accept `SOUL.md` for now (no extra files).
|
||||
|
||||
## How it works (high level)
|
||||
|
||||
- Web app: TanStack Start (React, Vite/Nitro).
|
||||
- Backend: Convex (DB + file storage + HTTP actions) + Convex Auth (GitHub OAuth).
|
||||
- Search: OpenAI embeddings (`text-embedding-3-small`) + Convex vector search.
|
||||
- API schema + routes: `packages/schema` (`clawdhub-schema`).
|
||||
- API schema + routes: `packages/schema` (`clawhub-schema`).
|
||||
|
||||
## Telemetry
|
||||
|
||||
ClawdHub tracks minimal **install telemetry** (to compute install counts) when you run `clawdhub sync` while logged in.
|
||||
OpenClaw tracks minimal **install telemetry** (to compute install counts) when you run `clawhub sync` while logged in.
|
||||
Disable via:
|
||||
|
||||
```bash
|
||||
export CLAWDHUB_DISABLE_TELEMETRY=1
|
||||
export CLAWHUB_DISABLE_TELEMETRY=1
|
||||
```
|
||||
|
||||
Details: `docs/telemetry.md`.
|
||||
@@ -72,12 +84,61 @@ This writes `JWT_PRIVATE_KEY` + `JWKS` to the deployment and prints values for y
|
||||
|
||||
- `VITE_CONVEX_URL`: Convex deployment URL (`https://<deployment>.convex.cloud`).
|
||||
- `VITE_CONVEX_SITE_URL`: Convex site URL (`https://<deployment>.convex.site`).
|
||||
- `VITE_SOULHUB_SITE_URL`: onlycrabs.ai site URL (`https://onlycrabs.ai`).
|
||||
- `VITE_SOULHUB_HOST`: onlycrabs.ai host match (`onlycrabs.ai`).
|
||||
- `VITE_SITE_MODE`: Optional override (`skills` or `souls`) for SSR builds.
|
||||
- `CONVEX_SITE_URL`: same as `VITE_CONVEX_SITE_URL` (auth + cookies).
|
||||
- `SITE_URL`: App URL (local: `http://localhost:3000`).
|
||||
- `AUTH_GITHUB_ID` / `AUTH_GITHUB_SECRET`: GitHub OAuth App.
|
||||
- `JWT_PRIVATE_KEY` / `JWKS`: Convex Auth keys.
|
||||
- `OPENAI_API_KEY`: embeddings for search + indexing.
|
||||
|
||||
## Nix plugins (nixmode skills)
|
||||
|
||||
OpenClaw can store a nix-clawdbot plugin pointer in SKILL frontmatter so the registry knows which
|
||||
Nix package bundle to install. A nix plugin is different from a regular skill pack: it bundles the
|
||||
skill pack, the CLI binary, and its config flags/requirements together.
|
||||
|
||||
Add this to `SKILL.md`:
|
||||
|
||||
```yaml
|
||||
---
|
||||
name: peekaboo
|
||||
description: Capture and automate macOS UI with the Peekaboo CLI.
|
||||
metadata: {"clawdbot":{"nix":{"plugin":"github:clawdbot/nix-steipete-tools?dir=tools/peekaboo","systems":["aarch64-darwin"]}}}
|
||||
---
|
||||
```
|
||||
|
||||
Install via nix-clawdbot:
|
||||
|
||||
```nix
|
||||
programs.clawdbot.plugins = [
|
||||
{ source = "github:clawdbot/nix-steipete-tools?dir=tools/peekaboo"; }
|
||||
];
|
||||
```
|
||||
|
||||
You can also declare config requirements + an example snippet:
|
||||
|
||||
```yaml
|
||||
---
|
||||
name: padel
|
||||
description: Check padel court availability and manage bookings via Playtomic.
|
||||
metadata: {"clawdbot":{"config":{"requiredEnv":["PADEL_AUTH_FILE"],"stateDirs":[".config/padel"],"example":"config = { env = { PADEL_AUTH_FILE = \\\"/run/agenix/padel-auth\\\"; }; };"}}}
|
||||
---
|
||||
```
|
||||
|
||||
To show CLI help (recommended for nix plugins), include the `cli --help` output:
|
||||
|
||||
```yaml
|
||||
---
|
||||
name: padel
|
||||
description: Check padel court availability and manage bookings via Playtomic.
|
||||
metadata: {"clawdbot":{"cliHelp":"padel --help\\nUsage: padel [command]\\n"}}
|
||||
---
|
||||
```
|
||||
|
||||
`metadata.clawdbot` is preferred, but `metadata.clawdis` is accepted as an alias for compatibility.
|
||||
|
||||
## Scripts
|
||||
|
||||
```bash
|
||||
|
||||
+5
-2
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"$schema": "https://biomejs.dev/schemas/2.3.11/schema.json",
|
||||
"$schema": "https://biomejs.dev/schemas/2.3.13/schema.json",
|
||||
"files": {
|
||||
"includes": [
|
||||
"**",
|
||||
@@ -10,9 +10,12 @@
|
||||
"!**/.output",
|
||||
"!**/coverage",
|
||||
"!**/convex/_generated",
|
||||
"!**/test-results",
|
||||
"!**/src/routeTree.gen.ts",
|
||||
"!**/.tanstack",
|
||||
"!**/public"
|
||||
"!**/public",
|
||||
"!**/.devenv",
|
||||
"!**/.devenv"
|
||||
]
|
||||
},
|
||||
"assist": { "actions": { "source": { "organizeImports": "on" } } },
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env bun
|
||||
import { existsSync } from 'node:fs'
|
||||
import { stat } from 'node:fs/promises'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
const distCliUrl = new URL('./packages/clawdhub/dist/cli.js', import.meta.url)
|
||||
const distCliPath = fileURLToPath(distCliUrl)
|
||||
const srcRootPath = fileURLToPath(new URL('./packages/clawdhub/src/', import.meta.url))
|
||||
|
||||
const shouldBuild = await (async () => {
|
||||
if (!existsSync(distCliPath)) return true
|
||||
try {
|
||||
const dist = await stat(distCliPath)
|
||||
const latestSrcMtime = await getLatestMtime(srcRootPath)
|
||||
return latestSrcMtime > dist.mtimeMs
|
||||
} catch {
|
||||
return true
|
||||
}
|
||||
})()
|
||||
|
||||
if (shouldBuild) {
|
||||
const proc = Bun.spawn(['bunx', 'tsc', '-p', 'packages/clawdhub/tsconfig.json'], {
|
||||
stdin: 'inherit',
|
||||
stdout: 'inherit',
|
||||
stderr: 'inherit',
|
||||
})
|
||||
const code = await proc.exited
|
||||
if (code !== 0) process.exit(code)
|
||||
}
|
||||
|
||||
await import(distCliUrl.href)
|
||||
|
||||
async function getLatestMtime(root: string) {
|
||||
let latest = 0
|
||||
const glob = new Bun.Glob('**/*.ts')
|
||||
for await (const rel of glob.scan({ cwd: root, onlyFiles: true })) {
|
||||
const path = `${root}${root.endsWith('/') ? '' : '/'}${rel}`
|
||||
try {
|
||||
const entry = await stat(path)
|
||||
latest = Math.max(latest, entry.mtimeMs)
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
return latest
|
||||
}
|
||||
Vendored
+48
@@ -11,27 +11,51 @@
|
||||
import type * as auth from "../auth.js";
|
||||
import type * as comments from "../comments.js";
|
||||
import type * as crons from "../crons.js";
|
||||
import type * as devSeed from "../devSeed.js";
|
||||
import type * as devSeedExtra from "../devSeedExtra.js";
|
||||
import type * as downloads from "../downloads.js";
|
||||
import type * as githubBackups from "../githubBackups.js";
|
||||
import type * as githubBackupsNode from "../githubBackupsNode.js";
|
||||
import type * as githubImport from "../githubImport.js";
|
||||
import type * as githubSoulBackups from "../githubSoulBackups.js";
|
||||
import type * as githubSoulBackupsNode from "../githubSoulBackupsNode.js";
|
||||
import type * as http from "../http.js";
|
||||
import type * as httpApi from "../httpApi.js";
|
||||
import type * as httpApiV1 from "../httpApiV1.js";
|
||||
import type * as leaderboards from "../leaderboards.js";
|
||||
import type * as lib_access from "../lib/access.js";
|
||||
import type * as lib_apiTokenAuth from "../lib/apiTokenAuth.js";
|
||||
import type * as lib_badges from "../lib/badges.js";
|
||||
import type * as lib_changelog from "../lib/changelog.js";
|
||||
import type * as lib_embeddings from "../lib/embeddings.js";
|
||||
import type * as lib_githubBackup from "../lib/githubBackup.js";
|
||||
import type * as lib_githubImport from "../lib/githubImport.js";
|
||||
import type * as lib_githubSoulBackup from "../lib/githubSoulBackup.js";
|
||||
import type * as lib_leaderboards from "../lib/leaderboards.js";
|
||||
import type * as lib_moderation from "../lib/moderation.js";
|
||||
import type * as lib_public from "../lib/public.js";
|
||||
import type * as lib_searchText from "../lib/searchText.js";
|
||||
import type * as lib_skillBackfill from "../lib/skillBackfill.js";
|
||||
import type * as lib_skillPublish from "../lib/skillPublish.js";
|
||||
import type * as lib_skillStats from "../lib/skillStats.js";
|
||||
import type * as lib_skills from "../lib/skills.js";
|
||||
import type * as lib_soulChangelog from "../lib/soulChangelog.js";
|
||||
import type * as lib_soulPublish from "../lib/soulPublish.js";
|
||||
import type * as lib_tokens from "../lib/tokens.js";
|
||||
import type * as lib_webhooks from "../lib/webhooks.js";
|
||||
import type * as maintenance from "../maintenance.js";
|
||||
import type * as rateLimits from "../rateLimits.js";
|
||||
import type * as search from "../search.js";
|
||||
import type * as seed from "../seed.js";
|
||||
import type * as seedSouls from "../seedSouls.js";
|
||||
import type * as skillStatEvents from "../skillStatEvents.js";
|
||||
import type * as skills from "../skills.js";
|
||||
import type * as soulComments from "../soulComments.js";
|
||||
import type * as soulDownloads from "../soulDownloads.js";
|
||||
import type * as soulStars from "../soulStars.js";
|
||||
import type * as souls from "../souls.js";
|
||||
import type * as stars from "../stars.js";
|
||||
import type * as statsMaintenance from "../statsMaintenance.js";
|
||||
import type * as telemetry from "../telemetry.js";
|
||||
import type * as tokens from "../tokens.js";
|
||||
import type * as uploads from "../uploads.js";
|
||||
@@ -48,27 +72,51 @@ declare const fullApi: ApiFromModules<{
|
||||
auth: typeof auth;
|
||||
comments: typeof comments;
|
||||
crons: typeof crons;
|
||||
devSeed: typeof devSeed;
|
||||
devSeedExtra: typeof devSeedExtra;
|
||||
downloads: typeof downloads;
|
||||
githubBackups: typeof githubBackups;
|
||||
githubBackupsNode: typeof githubBackupsNode;
|
||||
githubImport: typeof githubImport;
|
||||
githubSoulBackups: typeof githubSoulBackups;
|
||||
githubSoulBackupsNode: typeof githubSoulBackupsNode;
|
||||
http: typeof http;
|
||||
httpApi: typeof httpApi;
|
||||
httpApiV1: typeof httpApiV1;
|
||||
leaderboards: typeof leaderboards;
|
||||
"lib/access": typeof lib_access;
|
||||
"lib/apiTokenAuth": typeof lib_apiTokenAuth;
|
||||
"lib/badges": typeof lib_badges;
|
||||
"lib/changelog": typeof lib_changelog;
|
||||
"lib/embeddings": typeof lib_embeddings;
|
||||
"lib/githubBackup": typeof lib_githubBackup;
|
||||
"lib/githubImport": typeof lib_githubImport;
|
||||
"lib/githubSoulBackup": typeof lib_githubSoulBackup;
|
||||
"lib/leaderboards": typeof lib_leaderboards;
|
||||
"lib/moderation": typeof lib_moderation;
|
||||
"lib/public": typeof lib_public;
|
||||
"lib/searchText": typeof lib_searchText;
|
||||
"lib/skillBackfill": typeof lib_skillBackfill;
|
||||
"lib/skillPublish": typeof lib_skillPublish;
|
||||
"lib/skillStats": typeof lib_skillStats;
|
||||
"lib/skills": typeof lib_skills;
|
||||
"lib/soulChangelog": typeof lib_soulChangelog;
|
||||
"lib/soulPublish": typeof lib_soulPublish;
|
||||
"lib/tokens": typeof lib_tokens;
|
||||
"lib/webhooks": typeof lib_webhooks;
|
||||
maintenance: typeof maintenance;
|
||||
rateLimits: typeof rateLimits;
|
||||
search: typeof search;
|
||||
seed: typeof seed;
|
||||
seedSouls: typeof seedSouls;
|
||||
skillStatEvents: typeof skillStatEvents;
|
||||
skills: typeof skills;
|
||||
soulComments: typeof soulComments;
|
||||
soulDownloads: typeof soulDownloads;
|
||||
soulStars: typeof soulStars;
|
||||
souls: typeof souls;
|
||||
stars: typeof stars;
|
||||
statsMaintenance: typeof statsMaintenance;
|
||||
telemetry: typeof telemetry;
|
||||
tokens: typeof tokens;
|
||||
uploads: typeof uploads;
|
||||
|
||||
+5
-4
@@ -1,7 +1,8 @@
|
||||
import { v } from 'convex/values'
|
||||
import type { Doc } from './_generated/dataModel'
|
||||
import { mutation, query } from './_generated/server'
|
||||
import { assertRole, requireUser } from './lib/access'
|
||||
import { assertModerator, requireUser } from './lib/access'
|
||||
import { type PublicUser, toPublicUser } from './lib/public'
|
||||
|
||||
export const listBySkill = query({
|
||||
args: { skillId: v.id('skills'), limit: v.optional(v.number()) },
|
||||
@@ -13,10 +14,10 @@ export const listBySkill = query({
|
||||
.order('desc')
|
||||
.take(limit)
|
||||
|
||||
const results: Array<{ comment: Doc<'comments'>; user: Doc<'users'> | null }> = []
|
||||
const results: Array<{ comment: Doc<'comments'>; user: PublicUser | null }> = []
|
||||
for (const comment of comments) {
|
||||
if (comment.softDeletedAt) continue
|
||||
const user = await ctx.db.get(comment.userId)
|
||||
const user = toPublicUser(await ctx.db.get(comment.userId))
|
||||
results.push({ comment, user })
|
||||
}
|
||||
return results
|
||||
@@ -59,7 +60,7 @@ export const remove = mutation({
|
||||
|
||||
const isOwner = comment.userId === user._id
|
||||
if (!isOwner) {
|
||||
assertRole(user, ['admin', 'moderator'])
|
||||
assertModerator(user)
|
||||
}
|
||||
|
||||
await ctx.db.patch(comment._id, {
|
||||
|
||||
@@ -10,4 +10,25 @@ crons.interval(
|
||||
{ batchSize: 50, maxBatches: 5 },
|
||||
)
|
||||
|
||||
crons.interval(
|
||||
'trending-leaderboard',
|
||||
{ minutes: 60 },
|
||||
internal.leaderboards.rebuildTrendingLeaderboardInternal,
|
||||
{ limit: 200 },
|
||||
)
|
||||
|
||||
crons.interval(
|
||||
'skill-stats-backfill',
|
||||
{ minutes: 10 },
|
||||
internal.statsMaintenance.runSkillStatBackfillInternal,
|
||||
{ batchSize: 200, maxBatches: 5 },
|
||||
)
|
||||
|
||||
crons.interval(
|
||||
'skill-stat-events',
|
||||
{ minutes: 15 },
|
||||
internal.skillStatEvents.processSkillStatEventsAction,
|
||||
{},
|
||||
)
|
||||
|
||||
export default crons
|
||||
|
||||
@@ -0,0 +1,459 @@
|
||||
import { v } from 'convex/values'
|
||||
import { internal } from './_generated/api'
|
||||
import type { ActionCtx } from './_generated/server'
|
||||
import { internalAction, internalMutation } from './_generated/server'
|
||||
import { EMBEDDING_DIMENSIONS } from './lib/embeddings'
|
||||
import { parseClawdisMetadata, parseFrontmatter } from './lib/skills'
|
||||
|
||||
type SeedSkillSpec = {
|
||||
slug: string
|
||||
displayName: string
|
||||
summary: string
|
||||
version: string
|
||||
metadata: Record<string, unknown>
|
||||
rawSkillMd: string
|
||||
}
|
||||
|
||||
type SeedActionArgs = {
|
||||
reset?: boolean
|
||||
}
|
||||
|
||||
type SeedActionResult = {
|
||||
ok: true
|
||||
results: Array<Record<string, unknown> & { slug: string }>
|
||||
}
|
||||
|
||||
type SeedMutationResult = Record<string, unknown>
|
||||
|
||||
const SEED_SKILLS: SeedSkillSpec[] = [
|
||||
{
|
||||
slug: 'padel',
|
||||
displayName: 'Padel',
|
||||
summary: 'Check padel court availability and manage bookings via Playtomic.',
|
||||
version: '0.1.0',
|
||||
metadata: {
|
||||
clawdbot: {
|
||||
nix: {
|
||||
plugin: 'github:joshp123/padel-cli',
|
||||
systems: ['aarch64-darwin', 'x86_64-linux'],
|
||||
},
|
||||
config: {
|
||||
requiredEnv: ['PADEL_AUTH_FILE'],
|
||||
stateDirs: ['.config/padel'],
|
||||
example:
|
||||
'config = { env = { PADEL_AUTH_FILE = "/run/agenix/padel-auth"; }; stateDirs = [ ".config/padel" ]; };',
|
||||
},
|
||||
cliHelp: `Padel CLI for availability
|
||||
|
||||
Usage:
|
||||
padel [command]
|
||||
|
||||
Available Commands:
|
||||
auth Manage authentication
|
||||
availability Show availability for a club on a date
|
||||
book Book a court
|
||||
bookings Manage bookings history
|
||||
search Search for available courts
|
||||
venues Manage saved venues
|
||||
|
||||
Flags:
|
||||
-h, --help help for padel
|
||||
--json Output JSON
|
||||
|
||||
Use "padel [command] --help" for more information about a command.
|
||||
`,
|
||||
},
|
||||
},
|
||||
rawSkillMd: `---
|
||||
name: padel
|
||||
description: Check padel court availability and manage bookings via the padel CLI.
|
||||
---
|
||||
|
||||
# Padel Booking Skill
|
||||
|
||||
## CLI
|
||||
|
||||
\`\`\`bash
|
||||
padel # On PATH (clawdbot plugin bundle)
|
||||
\`\`\`
|
||||
|
||||
## Venues
|
||||
|
||||
Use the configured venue list in order of preference. If no venues are configured, ask for a venue name or location.
|
||||
|
||||
## Commands
|
||||
|
||||
### Check next booking
|
||||
\`\`\`bash
|
||||
padel bookings list 2>&1 | head -3
|
||||
\`\`\`
|
||||
|
||||
### Search availability
|
||||
\`\`\`bash
|
||||
padel search --venues VENUE1,VENUE2 --date YYYY-MM-DD --time 09:00-12:00
|
||||
\`\`\`
|
||||
|
||||
## Response guidelines
|
||||
|
||||
- Keep responses concise.
|
||||
- Use 🎾 emoji.
|
||||
- End with a call to action.
|
||||
|
||||
## Authorization
|
||||
|
||||
Only the authorized booker can confirm bookings. If the requester is not authorized, ask the authorized user to confirm.
|
||||
`,
|
||||
},
|
||||
{
|
||||
slug: 'gohome',
|
||||
displayName: 'GoHome',
|
||||
summary: 'Operate GoHome via gRPC discovery, metrics, and Grafana dashboards.',
|
||||
version: '0.1.0',
|
||||
metadata: {
|
||||
clawdbot: {
|
||||
nix: {
|
||||
plugin: 'github:joshp123/gohome',
|
||||
systems: ['x86_64-linux', 'aarch64-linux'],
|
||||
},
|
||||
config: {
|
||||
requiredEnv: ['GOHOME_GRPC_ADDR', 'GOHOME_HTTP_BASE'],
|
||||
example:
|
||||
'config = { env = { GOHOME_GRPC_ADDR = "gohome:9000"; GOHOME_HTTP_BASE = "http://gohome:8080"; }; };',
|
||||
},
|
||||
cliHelp: `GoHome CLI
|
||||
|
||||
Usage:
|
||||
gohome-cli [command]
|
||||
|
||||
Available Commands:
|
||||
services List registered services
|
||||
plugins Inspect loaded plugins
|
||||
methods List RPC methods
|
||||
call Call an RPC method
|
||||
roborock Manage roborock devices
|
||||
tado Manage tado zones
|
||||
|
||||
Flags:
|
||||
--grpc-addr string gRPC endpoint (host:port)
|
||||
-h, --help help for gohome-cli
|
||||
`,
|
||||
},
|
||||
},
|
||||
rawSkillMd: `---
|
||||
name: gohome
|
||||
description: Use when Clawdbot needs to test or operate GoHome via gRPC discovery, metrics, and Grafana.
|
||||
---
|
||||
|
||||
# GoHome Skill
|
||||
|
||||
## Quick start
|
||||
|
||||
\`\`\`bash
|
||||
export GOHOME_HTTP_BASE="http://gohome:8080"
|
||||
export GOHOME_GRPC_ADDR="gohome:9000"
|
||||
\`\`\`
|
||||
|
||||
## CLI
|
||||
|
||||
\`\`\`bash
|
||||
gohome-cli services
|
||||
\`\`\`
|
||||
|
||||
## Discovery flow (read-only)
|
||||
|
||||
1) List plugins.
|
||||
2) Describe a plugin.
|
||||
3) List RPC methods.
|
||||
4) Call a read-only RPC.
|
||||
|
||||
## Metrics validation
|
||||
|
||||
\`\`\`bash
|
||||
curl -s "\${GOHOME_HTTP_BASE}/gohome/metrics" | rg -n "gohome_"
|
||||
\`\`\`
|
||||
|
||||
## Stateful actions
|
||||
|
||||
Only call write RPCs after explicit user approval.
|
||||
`,
|
||||
},
|
||||
{
|
||||
slug: 'xuezh',
|
||||
displayName: 'Xuezh',
|
||||
summary: 'Teach Mandarin with the xuezh engine for review, speaking, and audits.',
|
||||
version: '0.1.0',
|
||||
metadata: {
|
||||
clawdbot: {
|
||||
nix: {
|
||||
plugin: 'github:joshp123/xuezh',
|
||||
systems: ['aarch64-darwin', 'x86_64-linux'],
|
||||
},
|
||||
config: {
|
||||
requiredEnv: ['XUEZH_AZURE_SPEECH_KEY_FILE', 'XUEZH_AZURE_SPEECH_REGION'],
|
||||
stateDirs: ['.config/xuezh'],
|
||||
example:
|
||||
'config = { env = { XUEZH_AZURE_SPEECH_KEY_FILE = "/run/agenix/xuezh-azure-speech-key"; XUEZH_AZURE_SPEECH_REGION = "westeurope"; }; stateDirs = [ ".config/xuezh" ]; };',
|
||||
},
|
||||
cliHelp: `xuezh - Chinese learning engine
|
||||
|
||||
Usage:
|
||||
xuezh [command]
|
||||
|
||||
Available Commands:
|
||||
snapshot Fetch learner state snapshot
|
||||
review Review due items
|
||||
audio Process speech audio
|
||||
items Manage learning items
|
||||
events Log learning events
|
||||
|
||||
Flags:
|
||||
-h, --help help for xuezh
|
||||
--json Output JSON
|
||||
`,
|
||||
},
|
||||
},
|
||||
rawSkillMd: `---
|
||||
name: xuezh
|
||||
description: Teach Mandarin using the xuezh engine for review, speaking, and audits.
|
||||
---
|
||||
|
||||
# Xuezh Skill
|
||||
|
||||
## Contract
|
||||
|
||||
Use the xuezh CLI exactly as specified. If a command is missing, ask for implementation instead of guessing.
|
||||
|
||||
## Default loop
|
||||
|
||||
1) Call \`xuezh snapshot\`.
|
||||
2) Pick a tiny plan (1-2 bullets).
|
||||
3) Run a short activity.
|
||||
4) Log outcomes.
|
||||
|
||||
## CLI examples
|
||||
|
||||
\`\`\`bash
|
||||
xuezh snapshot --profile default
|
||||
xuezh review next --limit 10
|
||||
xuezh audio process-voice --file ./utterance.wav
|
||||
\`\`\`
|
||||
`,
|
||||
},
|
||||
]
|
||||
|
||||
function injectMetadata(rawSkillMd: string, metadata: Record<string, unknown>) {
|
||||
const frontmatterEnd = rawSkillMd.indexOf('\n---', 3)
|
||||
if (frontmatterEnd === -1) return rawSkillMd
|
||||
return `${rawSkillMd.slice(0, frontmatterEnd)}\nmetadata: ${JSON.stringify(
|
||||
metadata,
|
||||
)}${rawSkillMd.slice(frontmatterEnd)}`
|
||||
}
|
||||
|
||||
async function seedNixSkillsHandler(
|
||||
ctx: ActionCtx,
|
||||
args: SeedActionArgs,
|
||||
): Promise<SeedActionResult> {
|
||||
const results: Array<Record<string, unknown> & { slug: string }> = []
|
||||
|
||||
for (const spec of SEED_SKILLS) {
|
||||
const skillMd = injectMetadata(spec.rawSkillMd, spec.metadata)
|
||||
const frontmatter = parseFrontmatter(skillMd)
|
||||
const clawdis = parseClawdisMetadata(frontmatter)
|
||||
const storageId = await ctx.storage.store(new Blob([skillMd], { type: 'text/markdown' }))
|
||||
|
||||
const result: SeedMutationResult = await ctx.runMutation(internal.devSeed.seedSkillMutation, {
|
||||
reset: args.reset,
|
||||
storageId,
|
||||
metadata: spec.metadata,
|
||||
frontmatter,
|
||||
clawdis,
|
||||
skillMd,
|
||||
slug: spec.slug,
|
||||
displayName: spec.displayName,
|
||||
summary: spec.summary,
|
||||
version: spec.version,
|
||||
})
|
||||
|
||||
results.push({ slug: spec.slug, ...result })
|
||||
}
|
||||
|
||||
return { ok: true, results }
|
||||
}
|
||||
|
||||
export const seedNixSkills: ReturnType<typeof internalAction> = internalAction({
|
||||
args: {
|
||||
reset: v.optional(v.boolean()),
|
||||
},
|
||||
handler: seedNixSkillsHandler,
|
||||
})
|
||||
|
||||
async function seedPadelSkillHandler(
|
||||
ctx: ActionCtx,
|
||||
args: SeedActionArgs,
|
||||
): Promise<SeedMutationResult> {
|
||||
const spec = SEED_SKILLS.find((entry) => entry.slug === 'padel')
|
||||
if (!spec) throw new Error('padel seed spec missing')
|
||||
|
||||
const skillMd = injectMetadata(spec.rawSkillMd, spec.metadata)
|
||||
const frontmatter = parseFrontmatter(skillMd)
|
||||
const clawdis = parseClawdisMetadata(frontmatter)
|
||||
const storageId = await ctx.storage.store(new Blob([skillMd], { type: 'text/markdown' }))
|
||||
|
||||
return (await ctx.runMutation(internal.devSeed.seedSkillMutation, {
|
||||
reset: args.reset,
|
||||
storageId,
|
||||
metadata: spec.metadata,
|
||||
frontmatter,
|
||||
clawdis,
|
||||
skillMd,
|
||||
slug: spec.slug,
|
||||
displayName: spec.displayName,
|
||||
summary: spec.summary,
|
||||
version: spec.version,
|
||||
})) as SeedMutationResult
|
||||
}
|
||||
|
||||
export const seedPadelSkill: ReturnType<typeof internalAction> = internalAction({
|
||||
args: {
|
||||
reset: v.optional(v.boolean()),
|
||||
},
|
||||
handler: seedPadelSkillHandler,
|
||||
})
|
||||
|
||||
export const seedSkillMutation = internalMutation({
|
||||
args: {
|
||||
reset: v.optional(v.boolean()),
|
||||
storageId: v.id('_storage'),
|
||||
metadata: v.any(),
|
||||
frontmatter: v.any(),
|
||||
clawdis: v.any(),
|
||||
skillMd: v.string(),
|
||||
slug: v.string(),
|
||||
displayName: v.string(),
|
||||
summary: v.optional(v.string()),
|
||||
version: v.string(),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const existing = await ctx.db
|
||||
.query('skills')
|
||||
.withIndex('by_slug', (q) => q.eq('slug', args.slug))
|
||||
.unique()
|
||||
|
||||
if (existing && !args.reset) {
|
||||
return { ok: true, skipped: true, skillId: existing._id }
|
||||
}
|
||||
|
||||
if (existing && args.reset) {
|
||||
const versions = await ctx.db
|
||||
.query('skillVersions')
|
||||
.withIndex('by_skill', (q) => q.eq('skillId', existing._id))
|
||||
.collect()
|
||||
for (const version of versions) {
|
||||
await ctx.db.delete(version._id)
|
||||
}
|
||||
const embeddings = await ctx.db
|
||||
.query('skillEmbeddings')
|
||||
.withIndex('by_skill', (q) => q.eq('skillId', existing._id))
|
||||
.collect()
|
||||
for (const embedding of embeddings) {
|
||||
await ctx.db.delete(embedding._id)
|
||||
}
|
||||
await ctx.db.delete(existing._id)
|
||||
}
|
||||
|
||||
const now = Date.now()
|
||||
const existingUsers = await ctx.db
|
||||
.query('users')
|
||||
.withIndex('handle', (q) => q.eq('handle', 'local'))
|
||||
.collect()
|
||||
|
||||
const userId =
|
||||
existingUsers[0]?._id ??
|
||||
(await ctx.db.insert('users', {
|
||||
handle: 'local',
|
||||
displayName: 'Local Dev',
|
||||
role: 'admin',
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
}))
|
||||
|
||||
const skillId = await ctx.db.insert('skills', {
|
||||
slug: args.slug,
|
||||
displayName: args.displayName,
|
||||
summary: args.summary,
|
||||
ownerUserId: userId,
|
||||
latestVersionId: undefined,
|
||||
tags: {},
|
||||
softDeletedAt: undefined,
|
||||
badges: { redactionApproved: undefined },
|
||||
statsDownloads: 0,
|
||||
statsStars: 0,
|
||||
statsInstallsCurrent: 0,
|
||||
statsInstallsAllTime: 0,
|
||||
stats: {
|
||||
downloads: 0,
|
||||
installsCurrent: 0,
|
||||
installsAllTime: 0,
|
||||
stars: 0,
|
||||
versions: 0,
|
||||
comments: 0,
|
||||
},
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
})
|
||||
|
||||
const versionId = await ctx.db.insert('skillVersions', {
|
||||
skillId,
|
||||
version: args.version,
|
||||
changelog: 'Seeded local version for screenshots.',
|
||||
files: [
|
||||
{
|
||||
path: 'SKILL.md',
|
||||
size: args.skillMd.length,
|
||||
storageId: args.storageId,
|
||||
sha256: 'seeded',
|
||||
contentType: 'text/markdown',
|
||||
},
|
||||
],
|
||||
parsed: {
|
||||
frontmatter: args.frontmatter,
|
||||
metadata: args.metadata,
|
||||
clawdis: args.clawdis,
|
||||
},
|
||||
createdBy: userId,
|
||||
createdAt: now,
|
||||
softDeletedAt: undefined,
|
||||
})
|
||||
|
||||
const embeddingId = await ctx.db.insert('skillEmbeddings', {
|
||||
skillId,
|
||||
versionId,
|
||||
ownerId: userId,
|
||||
embedding: Array.from({ length: EMBEDDING_DIMENSIONS }, () => 0),
|
||||
isLatest: true,
|
||||
isApproved: true,
|
||||
visibility: 'latest-approved',
|
||||
updatedAt: now,
|
||||
})
|
||||
|
||||
await ctx.db.patch(skillId, {
|
||||
latestVersionId: versionId,
|
||||
tags: { latest: versionId },
|
||||
statsDownloads: 0,
|
||||
statsStars: 0,
|
||||
statsInstallsCurrent: 0,
|
||||
statsInstallsAllTime: 0,
|
||||
stats: {
|
||||
downloads: 0,
|
||||
installsCurrent: 0,
|
||||
installsAllTime: 0,
|
||||
stars: 0,
|
||||
versions: 1,
|
||||
comments: 0,
|
||||
},
|
||||
updatedAt: now,
|
||||
})
|
||||
|
||||
return { ok: true, skillId, versionId, embeddingId }
|
||||
},
|
||||
})
|
||||
@@ -0,0 +1,541 @@
|
||||
/**
|
||||
* Extra seed skills for pagination testing.
|
||||
*
|
||||
* This file contains 50 placeholder skills to test pagination behavior.
|
||||
* Run with: bunx convex run internal.devSeedExtra.seedExtraSkillsInternal
|
||||
* Or with reset: bunx convex run internal.devSeedExtra.seedExtraSkillsInternal '{"reset": true}'
|
||||
*/
|
||||
|
||||
import { v } from 'convex/values'
|
||||
import { internal } from './_generated/api'
|
||||
import type { Id } from './_generated/dataModel'
|
||||
import type { ActionCtx } from './_generated/server'
|
||||
import { internalAction, internalMutation } from './_generated/server'
|
||||
import { parseClawdisMetadata, parseFrontmatter } from './lib/skills'
|
||||
|
||||
type SeedSkillSpec = {
|
||||
slug: string
|
||||
displayName: string
|
||||
summary: string
|
||||
version: string
|
||||
metadata: Record<string, unknown>
|
||||
rawSkillMd: string
|
||||
}
|
||||
|
||||
function makeSkill(
|
||||
slug: string,
|
||||
displayName: string,
|
||||
summary: string,
|
||||
envVars: string[] = [],
|
||||
commands: string[] = ['help', 'status', 'run'],
|
||||
): SeedSkillSpec {
|
||||
const cliHelp = `${slug} - ${summary}
|
||||
|
||||
Usage:
|
||||
${slug} [command]
|
||||
|
||||
Commands:
|
||||
${commands.map((cmd) => ` ${cmd.padEnd(12)} Run ${cmd} operation`).join('\n')}
|
||||
|
||||
Flags:
|
||||
-h, --help Show help
|
||||
--json Output as JSON
|
||||
`
|
||||
|
||||
const rawSkillMd = `---
|
||||
name: ${slug}
|
||||
description: ${summary}
|
||||
---
|
||||
|
||||
# ${displayName}
|
||||
|
||||
## CLI
|
||||
|
||||
\`\`\`bash
|
||||
${commands.map((cmd) => `${slug} ${cmd}`).join('\n')}
|
||||
\`\`\`
|
||||
|
||||
## Usage
|
||||
|
||||
Use this skill to ${summary.toLowerCase()}.
|
||||
`
|
||||
|
||||
return {
|
||||
slug,
|
||||
displayName,
|
||||
summary,
|
||||
version: '0.1.0',
|
||||
metadata: {
|
||||
clawdbot: {
|
||||
nix: {
|
||||
plugin: `github:example/${slug}`,
|
||||
systems: ['aarch64-darwin', 'x86_64-linux'],
|
||||
},
|
||||
config: {
|
||||
requiredEnv: envVars,
|
||||
},
|
||||
cliHelp,
|
||||
},
|
||||
},
|
||||
rawSkillMd,
|
||||
}
|
||||
}
|
||||
|
||||
// 50 placeholder skills for pagination testing
|
||||
const EXTRA_SEED_SKILLS: SeedSkillSpec[] = [
|
||||
// DevOps & Infrastructure (10)
|
||||
makeSkill(
|
||||
'kubectl-helper',
|
||||
'Kubectl Helper',
|
||||
'Simplified kubectl commands for common Kubernetes operations.',
|
||||
['KUBECONFIG'],
|
||||
['pods', 'logs', 'exec', 'describe', 'apply'],
|
||||
),
|
||||
makeSkill(
|
||||
'terraform-runner',
|
||||
'Terraform Runner',
|
||||
'Execute Terraform plans and applies with safety checks.',
|
||||
['TF_VAR_region', 'AWS_PROFILE'],
|
||||
['plan', 'apply', 'destroy', 'output', 'state'],
|
||||
),
|
||||
makeSkill(
|
||||
'ansible-exec',
|
||||
'Ansible Exec',
|
||||
'Run Ansible playbooks and ad-hoc commands.',
|
||||
['ANSIBLE_INVENTORY'],
|
||||
['playbook', 'adhoc', 'inventory', 'facts', 'vault'],
|
||||
),
|
||||
makeSkill(
|
||||
'docker-compose-mgr',
|
||||
'Docker Compose Manager',
|
||||
'Manage Docker Compose stacks and services.',
|
||||
['DOCKER_HOST'],
|
||||
['up', 'down', 'logs', 'ps', 'restart'],
|
||||
),
|
||||
makeSkill(
|
||||
'k9s-wrapper',
|
||||
'K9s Wrapper',
|
||||
'Interactive Kubernetes cluster management via K9s.',
|
||||
['KUBECONFIG'],
|
||||
['launch', 'contexts', 'namespaces', 'pods', 'logs'],
|
||||
),
|
||||
makeSkill(
|
||||
'helm-charts',
|
||||
'Helm Charts',
|
||||
'Manage Helm chart deployments and releases.',
|
||||
['KUBECONFIG', 'HELM_REPO'],
|
||||
['install', 'upgrade', 'rollback', 'list', 'search'],
|
||||
),
|
||||
makeSkill(
|
||||
'prometheus-alerts',
|
||||
'Prometheus Alerts',
|
||||
'Query Prometheus metrics and manage alerting rules.',
|
||||
['PROMETHEUS_URL'],
|
||||
['query', 'alerts', 'rules', 'targets', 'status'],
|
||||
),
|
||||
makeSkill(
|
||||
'grafana-dash',
|
||||
'Grafana Dashboards',
|
||||
'Create and manage Grafana dashboards programmatically.',
|
||||
['GRAFANA_URL', 'GRAFANA_API_KEY'],
|
||||
['list', 'export', 'import', 'create', 'delete'],
|
||||
),
|
||||
makeSkill(
|
||||
'nginx-config',
|
||||
'Nginx Config',
|
||||
'Generate and validate Nginx configuration files.',
|
||||
['NGINX_CONF_DIR'],
|
||||
['generate', 'validate', 'reload', 'test', 'sites'],
|
||||
),
|
||||
makeSkill(
|
||||
'jenkins-jobs',
|
||||
'Jenkins Jobs',
|
||||
'Manage Jenkins jobs and pipelines.',
|
||||
['JENKINS_URL', 'JENKINS_TOKEN'],
|
||||
['list', 'build', 'status', 'logs', 'config'],
|
||||
),
|
||||
|
||||
// Productivity (8)
|
||||
makeSkill(
|
||||
'todoist-sync',
|
||||
'Todoist Sync',
|
||||
'Sync and manage Todoist tasks from the command line.',
|
||||
['TODOIST_API_TOKEN'],
|
||||
['list', 'add', 'complete', 'projects', 'labels'],
|
||||
),
|
||||
makeSkill(
|
||||
'notion-backup',
|
||||
'Notion Backup',
|
||||
'Export and backup Notion workspaces.',
|
||||
['NOTION_TOKEN'],
|
||||
['export', 'backup', 'restore', 'pages', 'databases'],
|
||||
),
|
||||
makeSkill(
|
||||
'gcal-manager',
|
||||
'Google Calendar Manager',
|
||||
'Manage Google Calendar events and schedules.',
|
||||
['GOOGLE_CREDENTIALS_FILE'],
|
||||
['events', 'create', 'delete', 'calendars', 'reminders'],
|
||||
),
|
||||
makeSkill(
|
||||
'time-tracker',
|
||||
'Time Tracker',
|
||||
'Track time spent on projects and tasks.',
|
||||
['TIMETRACK_DB'],
|
||||
['start', 'stop', 'status', 'report', 'projects'],
|
||||
),
|
||||
makeSkill(
|
||||
'email-digest',
|
||||
'Email Digest',
|
||||
'Generate email digests and summaries.',
|
||||
['IMAP_SERVER', 'IMAP_USER'],
|
||||
['fetch', 'digest', 'search', 'folders', 'unread'],
|
||||
),
|
||||
makeSkill(
|
||||
'habit-tracker',
|
||||
'Habit Tracker',
|
||||
'Track daily habits and streaks.',
|
||||
['HABITS_DB'],
|
||||
['log', 'streak', 'stats', 'habits', 'remind'],
|
||||
),
|
||||
makeSkill(
|
||||
'bookmark-sync',
|
||||
'Bookmark Sync',
|
||||
'Sync bookmarks across browsers and devices.',
|
||||
['BOOKMARKS_DIR'],
|
||||
['sync', 'export', 'import', 'search', 'tags'],
|
||||
),
|
||||
makeSkill(
|
||||
'notes-export',
|
||||
'Notes Export',
|
||||
'Export notes to various formats.',
|
||||
['NOTES_DIR'],
|
||||
['export', 'convert', 'search', 'list', 'tags'],
|
||||
),
|
||||
|
||||
// Media & Entertainment (6)
|
||||
makeSkill(
|
||||
'spotify-ctl',
|
||||
'Spotify Control',
|
||||
'Control Spotify playback from the terminal.',
|
||||
['SPOTIFY_CLIENT_ID', 'SPOTIFY_CLIENT_SECRET'],
|
||||
['play', 'pause', 'next', 'prev', 'search'],
|
||||
),
|
||||
makeSkill(
|
||||
'plex-manager',
|
||||
'Plex Manager',
|
||||
'Manage Plex media libraries and playback.',
|
||||
['PLEX_URL', 'PLEX_TOKEN'],
|
||||
['libraries', 'scan', 'search', 'play', 'sessions'],
|
||||
),
|
||||
makeSkill(
|
||||
'ytdl-wrapper',
|
||||
'YouTube Downloader',
|
||||
'Download videos from YouTube and other platforms.',
|
||||
['YTDL_OUTPUT_DIR'],
|
||||
['download', 'info', 'playlist', 'audio', 'formats'],
|
||||
),
|
||||
makeSkill(
|
||||
'podcast-dl',
|
||||
'Podcast Downloader',
|
||||
'Download and manage podcast episodes.',
|
||||
['PODCAST_DIR'],
|
||||
['subscribe', 'download', 'list', 'play', 'search'],
|
||||
),
|
||||
makeSkill(
|
||||
'audiobook-player',
|
||||
'Audiobook Player',
|
||||
'Manage and play audiobook collections.',
|
||||
['AUDIOBOOK_DIR'],
|
||||
['play', 'pause', 'bookmark', 'list', 'progress'],
|
||||
),
|
||||
makeSkill(
|
||||
'music-lib',
|
||||
'Music Library',
|
||||
'Organize and query local music libraries.',
|
||||
['MUSIC_DIR'],
|
||||
['scan', 'search', 'play', 'playlist', 'stats'],
|
||||
),
|
||||
|
||||
// Smart Home (8)
|
||||
makeSkill(
|
||||
'hass-control',
|
||||
'Home Assistant Control',
|
||||
'Control Home Assistant entities and automations.',
|
||||
['HASS_URL', 'HASS_TOKEN'],
|
||||
['entities', 'services', 'automations', 'scenes', 'history'],
|
||||
),
|
||||
makeSkill(
|
||||
'zigbee-mqtt',
|
||||
'Zigbee2MQTT',
|
||||
'Manage Zigbee devices via MQTT.',
|
||||
['MQTT_BROKER', 'ZIGBEE_TOPIC'],
|
||||
['devices', 'pair', 'remove', 'rename', 'groups'],
|
||||
),
|
||||
makeSkill(
|
||||
'tasmota-ctl',
|
||||
'Tasmota Control',
|
||||
'Control Tasmota-flashed devices.',
|
||||
['TASMOTA_HOSTS'],
|
||||
['status', 'power', 'config', 'update', 'backup'],
|
||||
),
|
||||
makeSkill(
|
||||
'esphome-mgr',
|
||||
'ESPHome Manager',
|
||||
'Manage ESPHome device configurations.',
|
||||
['ESPHOME_DIR'],
|
||||
['compile', 'upload', 'logs', 'dashboard', 'config'],
|
||||
),
|
||||
makeSkill(
|
||||
'mqtt-broker',
|
||||
'MQTT Broker',
|
||||
'Interact with MQTT brokers for IoT messaging.',
|
||||
['MQTT_BROKER', 'MQTT_USER'],
|
||||
['pub', 'sub', 'topics', 'clients', 'stats'],
|
||||
),
|
||||
makeSkill(
|
||||
'hue-lights',
|
||||
'Philips Hue',
|
||||
'Control Philips Hue lights and scenes.',
|
||||
['HUE_BRIDGE_IP', 'HUE_API_KEY'],
|
||||
['lights', 'scenes', 'groups', 'schedules', 'sensors'],
|
||||
),
|
||||
makeSkill(
|
||||
'smart-thermo',
|
||||
'Smart Thermostat',
|
||||
'Control smart thermostats and HVAC systems.',
|
||||
['THERMOSTAT_API_KEY'],
|
||||
['status', 'set', 'schedule', 'history', 'zones'],
|
||||
),
|
||||
makeSkill(
|
||||
'cam-viewer',
|
||||
'Camera Viewer',
|
||||
'View and manage security camera feeds.',
|
||||
['CAMERA_URLS'],
|
||||
['list', 'snapshot', 'stream', 'record', 'events'],
|
||||
),
|
||||
|
||||
// Finance (5)
|
||||
makeSkill(
|
||||
'budget-track',
|
||||
'Budget Tracker',
|
||||
'Track budgets and spending across categories.',
|
||||
['BUDGET_DB'],
|
||||
['summary', 'add', 'categories', 'report', 'goals'],
|
||||
),
|
||||
makeSkill(
|
||||
'crypto-watch',
|
||||
'Crypto Watcher',
|
||||
'Monitor cryptocurrency prices and portfolios.',
|
||||
['CRYPTO_API_KEY'],
|
||||
['prices', 'portfolio', 'alerts', 'history', 'convert'],
|
||||
),
|
||||
makeSkill(
|
||||
'stock-alerts',
|
||||
'Stock Alerts',
|
||||
'Set up stock price alerts and notifications.',
|
||||
['STOCK_API_KEY'],
|
||||
['quote', 'watch', 'alerts', 'portfolio', 'news'],
|
||||
),
|
||||
makeSkill(
|
||||
'expense-cat',
|
||||
'Expense Categorizer',
|
||||
'Automatically categorize expenses.',
|
||||
['EXPENSE_DB'],
|
||||
['import', 'categorize', 'report', 'rules', 'export'],
|
||||
),
|
||||
makeSkill(
|
||||
'invoice-gen',
|
||||
'Invoice Generator',
|
||||
'Generate and manage invoices.',
|
||||
['INVOICE_DIR', 'COMPANY_INFO'],
|
||||
['create', 'list', 'send', 'paid', 'overdue'],
|
||||
),
|
||||
|
||||
// Communication (5)
|
||||
makeSkill(
|
||||
'slack-bot',
|
||||
'Slack Bot',
|
||||
'Interact with Slack channels and messages.',
|
||||
['SLACK_TOKEN'],
|
||||
['send', 'channels', 'users', 'search', 'files'],
|
||||
),
|
||||
makeSkill(
|
||||
'discord-mgr',
|
||||
'Discord Manager',
|
||||
'Manage Discord servers and messages.',
|
||||
['DISCORD_TOKEN'],
|
||||
['send', 'servers', 'channels', 'members', 'roles'],
|
||||
),
|
||||
makeSkill(
|
||||
'telegram-bot',
|
||||
'Telegram Bot',
|
||||
'Send and receive Telegram messages.',
|
||||
['TELEGRAM_BOT_TOKEN'],
|
||||
['send', 'receive', 'chats', 'files', 'inline'],
|
||||
),
|
||||
makeSkill(
|
||||
'matrix-cli',
|
||||
'Matrix CLI',
|
||||
'Interact with Matrix chat rooms.',
|
||||
['MATRIX_HOMESERVER', 'MATRIX_TOKEN'],
|
||||
['send', 'rooms', 'join', 'leave', 'sync'],
|
||||
),
|
||||
makeSkill(
|
||||
'irc-bridge',
|
||||
'IRC Bridge',
|
||||
'Bridge IRC channels to other platforms.',
|
||||
['IRC_SERVER', 'IRC_NICK'],
|
||||
['connect', 'join', 'send', 'channels', 'users'],
|
||||
),
|
||||
|
||||
// Data & Analytics (5)
|
||||
makeSkill(
|
||||
'pg-queries',
|
||||
'PostgreSQL Queries',
|
||||
'Execute PostgreSQL queries and manage databases.',
|
||||
['DATABASE_URL'],
|
||||
['query', 'tables', 'schema', 'backup', 'restore'],
|
||||
),
|
||||
makeSkill(
|
||||
'clickhouse-ql',
|
||||
'ClickHouse Queries',
|
||||
'Run ClickHouse analytics queries.',
|
||||
['CLICKHOUSE_URL'],
|
||||
['query', 'tables', 'insert', 'system', 'optimize'],
|
||||
),
|
||||
makeSkill(
|
||||
'redis-cli',
|
||||
'Redis CLI',
|
||||
'Interact with Redis cache and data structures.',
|
||||
['REDIS_URL'],
|
||||
['get', 'set', 'keys', 'info', 'flush'],
|
||||
),
|
||||
makeSkill(
|
||||
'elastic-search',
|
||||
'Elasticsearch',
|
||||
'Search and manage Elasticsearch indices.',
|
||||
['ELASTICSEARCH_URL'],
|
||||
['search', 'index', 'mapping', 'cluster', 'aliases'],
|
||||
),
|
||||
makeSkill(
|
||||
'mongo-shell',
|
||||
'MongoDB Shell',
|
||||
'Query and manage MongoDB collections.',
|
||||
['MONGODB_URI'],
|
||||
['find', 'insert', 'update', 'delete', 'aggregate'],
|
||||
),
|
||||
|
||||
// Security (3)
|
||||
makeSkill(
|
||||
'vault-secrets',
|
||||
'Vault Secrets',
|
||||
'Manage secrets in HashiCorp Vault.',
|
||||
['VAULT_ADDR', 'VAULT_TOKEN'],
|
||||
['read', 'write', 'list', 'delete', 'seal'],
|
||||
),
|
||||
makeSkill(
|
||||
'gpg-keys',
|
||||
'GPG Keys',
|
||||
'Manage GPG keys and encryption.',
|
||||
['GNUPGHOME'],
|
||||
['list', 'generate', 'export', 'import', 'encrypt'],
|
||||
),
|
||||
makeSkill(
|
||||
'ssh-rotate',
|
||||
'SSH Key Rotator',
|
||||
'Rotate and manage SSH keys.',
|
||||
['SSH_KEY_DIR'],
|
||||
['generate', 'rotate', 'deploy', 'list', 'revoke'],
|
||||
),
|
||||
]
|
||||
|
||||
function injectMetadata(rawSkillMd: string, metadata: Record<string, unknown>) {
|
||||
const frontmatterEnd = rawSkillMd.indexOf('\n---', 3)
|
||||
if (frontmatterEnd === -1) return rawSkillMd
|
||||
return `${rawSkillMd.slice(0, frontmatterEnd)}\nmetadata: ${JSON.stringify(
|
||||
metadata,
|
||||
)}${rawSkillMd.slice(frontmatterEnd)}`
|
||||
}
|
||||
|
||||
function randomStats() {
|
||||
return {
|
||||
downloads: Math.floor(Math.random() * 5000),
|
||||
stars: Math.floor(Math.random() * 500),
|
||||
installsCurrent: Math.floor(Math.random() * 200),
|
||||
installsAllTime: Math.floor(Math.random() * 1000),
|
||||
}
|
||||
}
|
||||
|
||||
export const applyRandomStats = internalMutation({
|
||||
args: {
|
||||
skillId: v.id('skills'),
|
||||
stats: v.object({
|
||||
downloads: v.number(),
|
||||
stars: v.number(),
|
||||
installsCurrent: v.number(),
|
||||
installsAllTime: v.number(),
|
||||
}),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
await ctx.db.patch(args.skillId, {
|
||||
statsDownloads: args.stats.downloads,
|
||||
statsStars: args.stats.stars,
|
||||
statsInstallsCurrent: args.stats.installsCurrent,
|
||||
statsInstallsAllTime: args.stats.installsAllTime,
|
||||
stats: {
|
||||
downloads: args.stats.downloads,
|
||||
stars: args.stats.stars,
|
||||
installsCurrent: args.stats.installsCurrent,
|
||||
installsAllTime: args.stats.installsAllTime,
|
||||
versions: 1,
|
||||
comments: 0,
|
||||
},
|
||||
})
|
||||
},
|
||||
})
|
||||
|
||||
export const seedExtraSkillsInternal = internalAction({
|
||||
args: {
|
||||
reset: v.optional(v.boolean()),
|
||||
},
|
||||
handler: async (ctx: ActionCtx, args) => {
|
||||
const results: Array<{ slug: string; ok: boolean; skipped?: boolean }> = []
|
||||
|
||||
for (const spec of EXTRA_SEED_SKILLS) {
|
||||
const skillMd = injectMetadata(spec.rawSkillMd, spec.metadata)
|
||||
const frontmatter = parseFrontmatter(skillMd)
|
||||
const clawdis = parseClawdisMetadata(frontmatter)
|
||||
const storageId = await ctx.storage.store(new Blob([skillMd], { type: 'text/markdown' }))
|
||||
|
||||
const result = (await ctx.runMutation(internal.devSeed.seedSkillMutation, {
|
||||
reset: args.reset,
|
||||
storageId,
|
||||
metadata: spec.metadata,
|
||||
frontmatter,
|
||||
clawdis,
|
||||
skillMd,
|
||||
slug: spec.slug,
|
||||
displayName: spec.displayName,
|
||||
summary: spec.summary,
|
||||
version: spec.version,
|
||||
})) as { ok: boolean; skipped?: boolean; skillId?: string }
|
||||
|
||||
// Apply random stats after creation (only if not skipped)
|
||||
if (result.skillId && !result.skipped) {
|
||||
const stats = randomStats()
|
||||
await ctx.runMutation(internal.devSeedExtra.applyRandomStats, {
|
||||
skillId: result.skillId as Id<'skills'>,
|
||||
stats,
|
||||
})
|
||||
}
|
||||
|
||||
results.push({ slug: spec.slug, ok: result.ok, skipped: result.skipped })
|
||||
}
|
||||
|
||||
const created = results.filter((r) => !r.skipped).length
|
||||
const skipped = results.filter((r) => r.skipped).length
|
||||
|
||||
return { ok: true, total: results.length, created, skipped }
|
||||
},
|
||||
})
|
||||
+4
-3
@@ -2,6 +2,7 @@ import { v } from 'convex/values'
|
||||
import { zipSync } from 'fflate'
|
||||
import { api } from './_generated/api'
|
||||
import { httpAction, mutation } from './_generated/server'
|
||||
import { insertStatEvent } from './skillStatEvents'
|
||||
|
||||
export const downloadZip = httpAction(async (ctx, request) => {
|
||||
const url = new URL(request.url)
|
||||
@@ -69,9 +70,9 @@ export const increment = mutation({
|
||||
handler: async (ctx, args) => {
|
||||
const skill = await ctx.db.get(args.skillId)
|
||||
if (!skill) return
|
||||
await ctx.db.patch(skill._id, {
|
||||
stats: { ...skill.stats, downloads: skill.stats.downloads + 1 },
|
||||
updatedAt: Date.now(),
|
||||
await insertStatEvent(ctx, {
|
||||
skillId: skill._id,
|
||||
kind: 'download',
|
||||
})
|
||||
},
|
||||
})
|
||||
|
||||
@@ -0,0 +1,317 @@
|
||||
import { ConvexError, v } from 'convex/values'
|
||||
import { unzipSync } from 'fflate'
|
||||
import semver from 'semver'
|
||||
import { api, internal } from './_generated/api'
|
||||
import type { Id } from './_generated/dataModel'
|
||||
import type { ActionCtx } from './_generated/server'
|
||||
import { action } from './_generated/server'
|
||||
import { requireUserFromAction } from './lib/access'
|
||||
import {
|
||||
buildGitHubImportFileList,
|
||||
computeDefaultSelectedPaths,
|
||||
detectGitHubImportCandidates,
|
||||
fetchGitHubZipBytes,
|
||||
listTextFilesUnderCandidate,
|
||||
normalizeRepoPath,
|
||||
parseGitHubImportUrl,
|
||||
resolveGitHubCommit,
|
||||
stripGitHubZipRoot,
|
||||
suggestDisplayName,
|
||||
suggestVersion,
|
||||
} from './lib/githubImport'
|
||||
import { publishVersionForUser } from './lib/skillPublish'
|
||||
import { sanitizePath } from './lib/skills'
|
||||
|
||||
const MAX_SELECTED_BYTES = 50 * 1024 * 1024
|
||||
const MAX_UNZIPPED_BYTES = 80 * 1024 * 1024
|
||||
const MAX_FILE_COUNT = 7_500
|
||||
const MAX_SINGLE_FILE_BYTES = 10 * 1024 * 1024
|
||||
|
||||
export const previewGitHubImport = action({
|
||||
args: { url: v.string() },
|
||||
handler: async (ctx, args) => {
|
||||
await requireUserFromAction(ctx)
|
||||
|
||||
const parsed = parseGitHubImportUrl(args.url)
|
||||
const resolved = await resolveGitHubCommit(parsed, fetch)
|
||||
const zipBytes = await fetchGitHubZipBytes(resolved, fetch)
|
||||
const entries = unzipToEntries(zipBytes)
|
||||
const stripped = stripGitHubZipRoot(entries)
|
||||
const candidates = detectGitHubImportCandidates(stripped).filter((candidate) =>
|
||||
isCandidateUnderResolvedPath(candidate.path, resolved.path),
|
||||
)
|
||||
if (candidates.length === 0) throw new ConvexError('No SKILL.md found in this repo')
|
||||
|
||||
return {
|
||||
resolved,
|
||||
candidates: candidates.map((candidate) => ({
|
||||
path: candidate.path,
|
||||
readmePath: candidate.readmePath,
|
||||
name: candidate.name ?? null,
|
||||
description: candidate.description ?? null,
|
||||
})),
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
export const previewGitHubImportCandidate = action({
|
||||
args: { url: v.string(), candidatePath: v.string() },
|
||||
handler: async (ctx, args) => {
|
||||
const { userId } = await requireUserFromAction(ctx)
|
||||
|
||||
const parsed = parseGitHubImportUrl(args.url)
|
||||
const resolved = await resolveGitHubCommit(parsed, fetch)
|
||||
const zipBytes = await fetchGitHubZipBytes(resolved, fetch)
|
||||
const entries = unzipToEntries(zipBytes)
|
||||
const stripped = stripGitHubZipRoot(entries)
|
||||
|
||||
const normalizedCandidatePath = normalizeRepoPath(args.candidatePath)
|
||||
if (!isCandidateUnderResolvedPath(normalizedCandidatePath, resolved.path)) {
|
||||
throw new ConvexError('Candidate path is outside the requested import scope')
|
||||
}
|
||||
|
||||
const candidates = detectGitHubImportCandidates(stripped).filter((candidate) =>
|
||||
isCandidateUnderResolvedPath(candidate.path, resolved.path),
|
||||
)
|
||||
|
||||
const candidate = candidates.find((item) => item.path === normalizedCandidatePath)
|
||||
if (!candidate) throw new ConvexError('Candidate not found')
|
||||
|
||||
const files = listTextFilesUnderCandidate(stripped, candidate.path)
|
||||
const defaultSelectedPaths = computeDefaultSelectedPaths({ candidate, files })
|
||||
const fileList = buildGitHubImportFileList({
|
||||
candidate,
|
||||
files,
|
||||
defaultSelectedPaths,
|
||||
})
|
||||
|
||||
const baseForNaming = candidate.path ? (candidate.path.split('/').at(-1) ?? '') : resolved.repo
|
||||
const suggestedDisplayName = suggestDisplayName(candidate, baseForNaming)
|
||||
|
||||
const rawSlugBase = sanitizeSlug(candidate.path ? baseForNaming : resolved.repo)
|
||||
const suggestedSlug = await suggestAvailableSlug(ctx, userId, rawSlugBase)
|
||||
|
||||
const existing = await ctx.runQuery(api.skills.getBySlug, { slug: suggestedSlug })
|
||||
const existingLatest =
|
||||
existing?.skill && existing.skill.ownerUserId === userId
|
||||
? (existing.latestVersion?.version ?? null)
|
||||
: null
|
||||
const suggestedVersion = suggestVersion(existingLatest)
|
||||
|
||||
return {
|
||||
resolved,
|
||||
candidate: {
|
||||
path: candidate.path,
|
||||
readmePath: candidate.readmePath,
|
||||
name: candidate.name ?? null,
|
||||
description: candidate.description ?? null,
|
||||
},
|
||||
defaults: {
|
||||
selectedPaths: defaultSelectedPaths,
|
||||
slug: suggestedSlug,
|
||||
displayName: suggestedDisplayName,
|
||||
version: suggestedVersion,
|
||||
tags: ['latest'],
|
||||
},
|
||||
files: fileList,
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
export const importGitHubSkill = action({
|
||||
args: {
|
||||
url: v.string(),
|
||||
commit: v.string(),
|
||||
candidatePath: v.string(),
|
||||
selectedPaths: v.array(v.string()),
|
||||
slug: v.optional(v.string()),
|
||||
displayName: v.optional(v.string()),
|
||||
version: v.optional(v.string()),
|
||||
tags: v.optional(v.array(v.string())),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const { userId } = await requireUserFromAction(ctx)
|
||||
|
||||
const parsed = parseGitHubImportUrl(args.url)
|
||||
const resolved = await resolveGitHubCommit(parsed, fetch)
|
||||
if (!/^[a-f0-9]{40}$/i.test(args.commit)) throw new ConvexError('Invalid commit')
|
||||
if (args.commit.toLowerCase() !== resolved.commit.toLowerCase()) {
|
||||
throw new ConvexError('Import is out of date. Re-run preview.')
|
||||
}
|
||||
|
||||
const normalizedCandidatePath = normalizeRepoPath(args.candidatePath)
|
||||
if (!isCandidateUnderResolvedPath(normalizedCandidatePath, resolved.path)) {
|
||||
throw new ConvexError('Candidate path is outside the requested import scope')
|
||||
}
|
||||
|
||||
const zipBytes = await fetchGitHubZipBytes(resolved, fetch)
|
||||
const entries = stripGitHubZipRoot(unzipToEntries(zipBytes))
|
||||
|
||||
const candidates = detectGitHubImportCandidates(entries).filter((candidate) =>
|
||||
isCandidateUnderResolvedPath(candidate.path, resolved.path),
|
||||
)
|
||||
const candidate = candidates.find((item) => item.path === normalizedCandidatePath)
|
||||
if (!candidate) throw new ConvexError('Candidate not found')
|
||||
|
||||
const filesUnderCandidate = listTextFilesUnderCandidate(entries, candidate.path)
|
||||
const byPath = new Map(filesUnderCandidate.map((file) => [file.path, file.bytes]))
|
||||
|
||||
const selected = Array.from(
|
||||
new Set(args.selectedPaths.map((path) => normalizeRepoPath(path)).filter(Boolean)),
|
||||
)
|
||||
if (selected.length === 0) throw new ConvexError('No files selected')
|
||||
|
||||
const candidateRoot = candidate.path ? `${candidate.path}/` : ''
|
||||
const normalizedReadmePath = normalizeRepoPath(candidate.readmePath)
|
||||
if (!selected.includes(normalizedReadmePath)) {
|
||||
throw new ConvexError('SKILL.md must be selected')
|
||||
}
|
||||
|
||||
let totalBytes = 0
|
||||
const storedFiles: Array<{
|
||||
path: string
|
||||
size: number
|
||||
storageId: Id<'_storage'>
|
||||
sha256: string
|
||||
contentType?: string
|
||||
}> = []
|
||||
|
||||
for (const path of selected.sort()) {
|
||||
if (candidateRoot && !path.startsWith(candidateRoot)) {
|
||||
throw new ConvexError('Selected file is outside the chosen skill folder')
|
||||
}
|
||||
|
||||
const bytes = byPath.get(path)
|
||||
if (!bytes) continue
|
||||
totalBytes += bytes.byteLength
|
||||
if (totalBytes > MAX_SELECTED_BYTES) throw new ConvexError('Selected files exceed 50MB limit')
|
||||
|
||||
const relPath = candidateRoot ? path.slice(candidateRoot.length) : path
|
||||
const sanitized = sanitizePath(relPath)
|
||||
if (!sanitized) throw new ConvexError('Invalid file paths')
|
||||
|
||||
const sha256 = await sha256Hex(bytes)
|
||||
const safeBytes = new Uint8Array(bytes)
|
||||
const storageId = await ctx.storage.store(new Blob([safeBytes], { type: 'text/plain' }))
|
||||
storedFiles.push({
|
||||
path: sanitized,
|
||||
size: bytes.byteLength,
|
||||
storageId,
|
||||
sha256,
|
||||
contentType: 'text/plain',
|
||||
})
|
||||
}
|
||||
|
||||
if (storedFiles.length === 0) throw new ConvexError('No files selected')
|
||||
|
||||
const slugBase = (args.slug ?? '').trim().toLowerCase()
|
||||
const displayName = (args.displayName ?? '').trim()
|
||||
const tags = (args.tags ?? ['latest']).map((tag) => tag.trim()).filter(Boolean)
|
||||
const version = (args.version ?? '').trim()
|
||||
|
||||
if (!slugBase) throw new ConvexError('Slug required')
|
||||
if (!displayName) throw new ConvexError('Display name required')
|
||||
if (!version || !semver.valid(version)) throw new ConvexError('Version must be valid semver')
|
||||
|
||||
const result = await publishVersionForUser(ctx, userId, {
|
||||
slug: slugBase,
|
||||
displayName,
|
||||
version,
|
||||
changelog: '',
|
||||
tags,
|
||||
files: storedFiles,
|
||||
source: {
|
||||
kind: 'github',
|
||||
url: resolved.originalUrl,
|
||||
repo: `${resolved.owner}/${resolved.repo}`,
|
||||
ref: resolved.ref,
|
||||
commit: resolved.commit,
|
||||
path: candidate.path,
|
||||
importedAt: Date.now(),
|
||||
},
|
||||
})
|
||||
|
||||
return { ok: true, slug: slugBase, version, ...result }
|
||||
},
|
||||
})
|
||||
|
||||
function unzipToEntries(zipBytes: Uint8Array) {
|
||||
const entries = unzipSync(zipBytes)
|
||||
const out: Record<string, Uint8Array> = {}
|
||||
const rawPaths = Object.keys(entries)
|
||||
if (rawPaths.length > MAX_FILE_COUNT) throw new ConvexError('Repo archive has too many files')
|
||||
let totalBytes = 0
|
||||
for (const [rawPath, bytes] of Object.entries(entries)) {
|
||||
const normalizedPath = normalizeZipPath(rawPath)
|
||||
if (!normalizedPath) continue
|
||||
if (isJunkPath(normalizedPath)) continue
|
||||
if (!bytes) continue
|
||||
if (bytes.byteLength > MAX_SINGLE_FILE_BYTES) continue
|
||||
totalBytes += bytes.byteLength
|
||||
if (totalBytes > MAX_UNZIPPED_BYTES) throw new ConvexError('Repo archive is too large')
|
||||
out[normalizedPath] = bytes
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
function isCandidateUnderResolvedPath(candidatePath: string, resolvedPath: string) {
|
||||
const root = normalizeRepoPath(resolvedPath)
|
||||
if (!root) return true
|
||||
if (!candidatePath) return false
|
||||
if (candidatePath === root) return true
|
||||
return candidatePath.startsWith(`${root}/`)
|
||||
}
|
||||
|
||||
function sanitizeSlug(value: string) {
|
||||
return value
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9-]+/g, '-')
|
||||
.replace(/^-+/, '')
|
||||
.replace(/-+$/, '')
|
||||
.replace(/--+/g, '-')
|
||||
}
|
||||
|
||||
async function suggestAvailableSlug(ctx: ActionCtx, userId: Id<'users'>, base: string) {
|
||||
const cleaned = sanitizeSlug(base)
|
||||
if (!cleaned) throw new ConvexError('Could not derive slug')
|
||||
for (let i = 0; i < 50; i += 1) {
|
||||
const candidate = i === 0 ? cleaned : `${cleaned}-${i + 1}`
|
||||
const existing = await ctx.runQuery(internal.skills.getSkillBySlugInternal, { slug: candidate })
|
||||
if (!existing) return candidate
|
||||
if (existing.ownerUserId === userId) return candidate
|
||||
}
|
||||
throw new ConvexError('Could not find an available slug')
|
||||
}
|
||||
|
||||
async function sha256Hex(bytes: Uint8Array) {
|
||||
const normalized = new Uint8Array(bytes)
|
||||
const digest = await crypto.subtle.digest('SHA-256', normalized.buffer)
|
||||
return toHex(new Uint8Array(digest))
|
||||
}
|
||||
|
||||
function toHex(bytes: Uint8Array) {
|
||||
let out = ''
|
||||
for (const byte of bytes) out += byte.toString(16).padStart(2, '0')
|
||||
return out
|
||||
}
|
||||
|
||||
function normalizeZipPath(path: string) {
|
||||
const normalized = path
|
||||
.replaceAll('\u0000', '')
|
||||
.replaceAll('\\', '/')
|
||||
.trim()
|
||||
.replace(/^\.\/+/, '')
|
||||
.replace(/^\/+/, '')
|
||||
if (!normalized) return ''
|
||||
if (normalized.includes('..')) return ''
|
||||
return normalized
|
||||
}
|
||||
|
||||
function isJunkPath(path: string) {
|
||||
const normalized = path.toLowerCase()
|
||||
if (normalized.startsWith('__macosx/')) return true
|
||||
if (normalized.endsWith('/.ds_store')) return true
|
||||
if (normalized === '.ds_store') return true
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
import { v } from 'convex/values'
|
||||
import { internal } from './_generated/api'
|
||||
import type { Doc, Id } from './_generated/dataModel'
|
||||
import { action, internalMutation, internalQuery } from './_generated/server'
|
||||
import { assertRole, requireUserFromAction } from './lib/access'
|
||||
|
||||
const DEFAULT_BATCH_SIZE = 50
|
||||
const MAX_BATCH_SIZE = 200
|
||||
const SYNC_STATE_KEY = 'souls'
|
||||
|
||||
type BackupPageItem =
|
||||
| {
|
||||
kind: 'ok'
|
||||
soulId: Id<'souls'>
|
||||
versionId: Id<'soulVersions'>
|
||||
slug: string
|
||||
displayName: string
|
||||
version: string
|
||||
ownerHandle: string
|
||||
files: Doc<'soulVersions'>['files']
|
||||
publishedAt: number
|
||||
}
|
||||
| { kind: 'missingLatestVersion'; soulId: Id<'souls'> }
|
||||
| { kind: 'missingVersionDoc'; soulId: Id<'souls'>; versionId: Id<'soulVersions'> }
|
||||
| { kind: 'missingOwner'; soulId: Id<'souls'>; ownerUserId: Id<'users'> }
|
||||
|
||||
type BackupPageResult = {
|
||||
items: BackupPageItem[]
|
||||
cursor: string | null
|
||||
isDone: boolean
|
||||
}
|
||||
|
||||
type BackupSyncState = {
|
||||
cursor: string | null
|
||||
}
|
||||
|
||||
export type SyncGitHubSoulBackupsResult = {
|
||||
stats: {
|
||||
soulsScanned: number
|
||||
soulsSkipped: number
|
||||
soulsBackedUp: number
|
||||
soulsMissingVersion: number
|
||||
soulsMissingOwner: number
|
||||
errors: number
|
||||
}
|
||||
cursor: string | null
|
||||
isDone: boolean
|
||||
}
|
||||
|
||||
export const getGitHubSoulBackupPageInternal = internalQuery({
|
||||
args: {
|
||||
cursor: v.optional(v.string()),
|
||||
batchSize: v.optional(v.number()),
|
||||
},
|
||||
handler: async (ctx, args): Promise<BackupPageResult> => {
|
||||
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
|
||||
const { page, isDone, continueCursor } = await ctx.db
|
||||
.query('souls')
|
||||
.order('asc')
|
||||
.paginate({ cursor: args.cursor ?? null, numItems: batchSize })
|
||||
|
||||
const items: BackupPageItem[] = []
|
||||
for (const soul of page) {
|
||||
if (soul.softDeletedAt) continue
|
||||
if (!soul.latestVersionId) {
|
||||
items.push({ kind: 'missingLatestVersion', soulId: soul._id })
|
||||
continue
|
||||
}
|
||||
|
||||
const version = await ctx.db.get(soul.latestVersionId)
|
||||
if (!version) {
|
||||
items.push({
|
||||
kind: 'missingVersionDoc',
|
||||
soulId: soul._id,
|
||||
versionId: soul.latestVersionId,
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
const owner = await ctx.db.get(soul.ownerUserId)
|
||||
if (!owner || owner.deletedAt) {
|
||||
items.push({ kind: 'missingOwner', soulId: soul._id, ownerUserId: soul.ownerUserId })
|
||||
continue
|
||||
}
|
||||
|
||||
items.push({
|
||||
kind: 'ok',
|
||||
soulId: soul._id,
|
||||
versionId: version._id,
|
||||
slug: soul.slug,
|
||||
displayName: soul.displayName,
|
||||
version: version.version,
|
||||
ownerHandle: owner.handle ?? owner._id,
|
||||
files: version.files,
|
||||
publishedAt: version.createdAt,
|
||||
})
|
||||
}
|
||||
|
||||
return { items, cursor: continueCursor, isDone }
|
||||
},
|
||||
})
|
||||
|
||||
export const getGitHubSoulBackupSyncStateInternal = internalQuery({
|
||||
args: {},
|
||||
handler: async (ctx): Promise<BackupSyncState> => {
|
||||
const state = await ctx.db
|
||||
.query('githubBackupSyncState')
|
||||
.withIndex('by_key', (q) => q.eq('key', SYNC_STATE_KEY))
|
||||
.unique()
|
||||
return { cursor: state?.cursor ?? null }
|
||||
},
|
||||
})
|
||||
|
||||
export const setGitHubSoulBackupSyncStateInternal = internalMutation({
|
||||
args: {
|
||||
cursor: v.optional(v.string()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const now = Date.now()
|
||||
const state = await ctx.db
|
||||
.query('githubBackupSyncState')
|
||||
.withIndex('by_key', (q) => q.eq('key', SYNC_STATE_KEY))
|
||||
.unique()
|
||||
|
||||
if (!state) {
|
||||
await ctx.db.insert('githubBackupSyncState', {
|
||||
key: SYNC_STATE_KEY,
|
||||
cursor: args.cursor,
|
||||
updatedAt: now,
|
||||
})
|
||||
return { ok: true as const }
|
||||
}
|
||||
|
||||
await ctx.db.patch(state._id, {
|
||||
cursor: args.cursor,
|
||||
updatedAt: now,
|
||||
})
|
||||
|
||||
return { ok: true as const }
|
||||
},
|
||||
})
|
||||
|
||||
export const syncGitHubSoulBackups: ReturnType<typeof action> = action({
|
||||
args: {
|
||||
dryRun: v.optional(v.boolean()),
|
||||
batchSize: v.optional(v.number()),
|
||||
maxBatches: v.optional(v.number()),
|
||||
resetCursor: v.optional(v.boolean()),
|
||||
},
|
||||
handler: async (ctx, args): Promise<SyncGitHubSoulBackupsResult> => {
|
||||
const { user } = await requireUserFromAction(ctx)
|
||||
assertRole(user, ['admin'])
|
||||
|
||||
if (args.resetCursor && !args.dryRun) {
|
||||
await ctx.runMutation(internal.githubSoulBackups.setGitHubSoulBackupSyncStateInternal, {
|
||||
cursor: undefined,
|
||||
})
|
||||
}
|
||||
|
||||
return ctx.runAction(internal.githubSoulBackupsNode.syncGitHubSoulBackupsInternal, {
|
||||
dryRun: args.dryRun,
|
||||
batchSize: args.batchSize,
|
||||
maxBatches: args.maxBatches,
|
||||
}) as Promise<SyncGitHubSoulBackupsResult>
|
||||
},
|
||||
})
|
||||
|
||||
function clampInt(value: number, min: number, max: number) {
|
||||
return Math.max(min, Math.min(max, Math.floor(value)))
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
'use node'
|
||||
|
||||
import { v } from 'convex/values'
|
||||
import { internal } from './_generated/api'
|
||||
import type { Doc } from './_generated/dataModel'
|
||||
import type { ActionCtx } from './_generated/server'
|
||||
import { internalAction } from './_generated/server'
|
||||
import {
|
||||
backupSoulToGitHub,
|
||||
fetchGitHubSoulMeta,
|
||||
getGitHubSoulBackupContext,
|
||||
isGitHubSoulBackupConfigured,
|
||||
} from './lib/githubSoulBackup'
|
||||
|
||||
const DEFAULT_BATCH_SIZE = 50
|
||||
const MAX_BATCH_SIZE = 200
|
||||
const DEFAULT_MAX_BATCHES = 5
|
||||
const MAX_MAX_BATCHES = 200
|
||||
|
||||
type BackupPageItem =
|
||||
| {
|
||||
kind: 'ok'
|
||||
slug: string
|
||||
version: string
|
||||
displayName: string
|
||||
ownerHandle: string
|
||||
files: Doc<'soulVersions'>['files']
|
||||
publishedAt: number
|
||||
}
|
||||
| { kind: 'missingLatestVersion' }
|
||||
| { kind: 'missingVersionDoc' }
|
||||
| { kind: 'missingOwner' }
|
||||
|
||||
export type GitHubSoulBackupSyncStats = {
|
||||
soulsScanned: number
|
||||
soulsSkipped: number
|
||||
soulsBackedUp: number
|
||||
soulsMissingVersion: number
|
||||
soulsMissingOwner: number
|
||||
errors: number
|
||||
}
|
||||
|
||||
export type SyncGitHubSoulBackupsInternalArgs = {
|
||||
dryRun?: boolean
|
||||
batchSize?: number
|
||||
maxBatches?: number
|
||||
}
|
||||
|
||||
export type SyncGitHubSoulBackupsInternalResult = {
|
||||
stats: GitHubSoulBackupSyncStats
|
||||
cursor: string | null
|
||||
isDone: boolean
|
||||
}
|
||||
|
||||
export const backupSoulForPublishInternal = internalAction({
|
||||
args: {
|
||||
slug: v.string(),
|
||||
version: v.string(),
|
||||
displayName: v.string(),
|
||||
ownerHandle: v.string(),
|
||||
files: v.array(
|
||||
v.object({
|
||||
path: v.string(),
|
||||
size: v.number(),
|
||||
storageId: v.id('_storage'),
|
||||
sha256: v.string(),
|
||||
contentType: v.optional(v.string()),
|
||||
}),
|
||||
),
|
||||
publishedAt: v.number(),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
if (!isGitHubSoulBackupConfigured()) {
|
||||
return { skipped: true as const }
|
||||
}
|
||||
await backupSoulToGitHub(ctx, args)
|
||||
return { skipped: false as const }
|
||||
},
|
||||
})
|
||||
|
||||
export async function syncGitHubSoulBackupsInternalHandler(
|
||||
ctx: ActionCtx,
|
||||
args: SyncGitHubSoulBackupsInternalArgs,
|
||||
): Promise<SyncGitHubSoulBackupsInternalResult> {
|
||||
const dryRun = Boolean(args.dryRun)
|
||||
const stats: GitHubSoulBackupSyncStats = {
|
||||
soulsScanned: 0,
|
||||
soulsSkipped: 0,
|
||||
soulsBackedUp: 0,
|
||||
soulsMissingVersion: 0,
|
||||
soulsMissingOwner: 0,
|
||||
errors: 0,
|
||||
}
|
||||
|
||||
if (!isGitHubSoulBackupConfigured()) {
|
||||
return { stats, cursor: null, isDone: true }
|
||||
}
|
||||
|
||||
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
|
||||
const maxBatches = clampInt(args.maxBatches ?? DEFAULT_MAX_BATCHES, 1, MAX_MAX_BATCHES)
|
||||
const context = await getGitHubSoulBackupContext()
|
||||
|
||||
const state = dryRun
|
||||
? { cursor: null as string | null }
|
||||
: ((await ctx.runQuery(
|
||||
internal.githubSoulBackups.getGitHubSoulBackupSyncStateInternal,
|
||||
{},
|
||||
)) as {
|
||||
cursor: string | null
|
||||
})
|
||||
|
||||
let cursor: string | null = state.cursor
|
||||
let isDone = false
|
||||
|
||||
for (let batch = 0; batch < maxBatches; batch++) {
|
||||
const page = (await ctx.runQuery(internal.githubSoulBackups.getGitHubSoulBackupPageInternal, {
|
||||
cursor: cursor ?? undefined,
|
||||
batchSize,
|
||||
})) as { items: BackupPageItem[]; cursor: string | null; isDone: boolean }
|
||||
|
||||
cursor = page.cursor
|
||||
isDone = page.isDone
|
||||
|
||||
for (const item of page.items) {
|
||||
if (item.kind !== 'ok') {
|
||||
if (item.kind === 'missingLatestVersion' || item.kind === 'missingVersionDoc') {
|
||||
stats.soulsMissingVersion += 1
|
||||
} else if (item.kind === 'missingOwner') {
|
||||
stats.soulsMissingOwner += 1
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
stats.soulsScanned += 1
|
||||
try {
|
||||
const meta = await fetchGitHubSoulMeta(context, item.ownerHandle, item.slug)
|
||||
if (meta?.latest?.version === item.version) {
|
||||
stats.soulsSkipped += 1
|
||||
continue
|
||||
}
|
||||
|
||||
if (!dryRun) {
|
||||
await backupSoulToGitHub(
|
||||
ctx,
|
||||
{
|
||||
slug: item.slug,
|
||||
version: item.version,
|
||||
displayName: item.displayName,
|
||||
ownerHandle: item.ownerHandle,
|
||||
files: item.files,
|
||||
publishedAt: item.publishedAt,
|
||||
},
|
||||
context,
|
||||
)
|
||||
stats.soulsBackedUp += 1
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('GitHub soul backup sync failed', error)
|
||||
stats.errors += 1
|
||||
}
|
||||
}
|
||||
|
||||
if (!dryRun) {
|
||||
await ctx.runMutation(internal.githubSoulBackups.setGitHubSoulBackupSyncStateInternal, {
|
||||
cursor: isDone ? undefined : (cursor ?? undefined),
|
||||
})
|
||||
}
|
||||
|
||||
if (isDone) break
|
||||
}
|
||||
|
||||
return { stats, cursor, isDone }
|
||||
}
|
||||
|
||||
export const syncGitHubSoulBackupsInternal = internalAction({
|
||||
args: {
|
||||
dryRun: v.optional(v.boolean()),
|
||||
batchSize: v.optional(v.number()),
|
||||
maxBatches: v.optional(v.number()),
|
||||
},
|
||||
handler: syncGitHubSoulBackupsInternalHandler,
|
||||
})
|
||||
|
||||
function clampInt(value: number, min: number, max: number) {
|
||||
return Math.max(min, Math.min(max, Math.floor(value)))
|
||||
}
|
||||
+50
-1
@@ -1,4 +1,4 @@
|
||||
import { ApiRoutes, LegacyApiRoutes } from 'clawdhub-schema'
|
||||
import { ApiRoutes, LegacyApiRoutes } from 'clawhub-schema'
|
||||
import { httpRouter } from 'convex/server'
|
||||
import { auth } from './auth'
|
||||
import { downloadZip } from './downloads'
|
||||
@@ -15,12 +15,19 @@ import {
|
||||
} from './httpApi'
|
||||
import {
|
||||
listSkillsV1Http,
|
||||
listSoulsV1Http,
|
||||
publishSkillV1Http,
|
||||
publishSoulV1Http,
|
||||
resolveSkillVersionV1Http,
|
||||
searchSkillsV1Http,
|
||||
skillsDeleteRouterV1Http,
|
||||
skillsGetRouterV1Http,
|
||||
skillsPostRouterV1Http,
|
||||
soulsDeleteRouterV1Http,
|
||||
soulsGetRouterV1Http,
|
||||
soulsPostRouterV1Http,
|
||||
starsDeleteRouterV1Http,
|
||||
starsPostRouterV1Http,
|
||||
whoamiV1Http,
|
||||
} from './httpApiV1'
|
||||
|
||||
@@ -76,12 +83,54 @@ http.route({
|
||||
handler: skillsDeleteRouterV1Http,
|
||||
})
|
||||
|
||||
http.route({
|
||||
pathPrefix: `${ApiRoutes.stars}/`,
|
||||
method: 'POST',
|
||||
handler: starsPostRouterV1Http,
|
||||
})
|
||||
|
||||
http.route({
|
||||
pathPrefix: `${ApiRoutes.stars}/`,
|
||||
method: 'DELETE',
|
||||
handler: starsDeleteRouterV1Http,
|
||||
})
|
||||
|
||||
http.route({
|
||||
path: ApiRoutes.whoami,
|
||||
method: 'GET',
|
||||
handler: whoamiV1Http,
|
||||
})
|
||||
|
||||
http.route({
|
||||
path: ApiRoutes.souls,
|
||||
method: 'GET',
|
||||
handler: listSoulsV1Http,
|
||||
})
|
||||
|
||||
http.route({
|
||||
pathPrefix: `${ApiRoutes.souls}/`,
|
||||
method: 'GET',
|
||||
handler: soulsGetRouterV1Http,
|
||||
})
|
||||
|
||||
http.route({
|
||||
path: ApiRoutes.souls,
|
||||
method: 'POST',
|
||||
handler: publishSoulV1Http,
|
||||
})
|
||||
|
||||
http.route({
|
||||
pathPrefix: `${ApiRoutes.souls}/`,
|
||||
method: 'POST',
|
||||
handler: soulsPostRouterV1Http,
|
||||
})
|
||||
|
||||
http.route({
|
||||
pathPrefix: `${ApiRoutes.souls}/`,
|
||||
method: 'DELETE',
|
||||
handler: soulsDeleteRouterV1Http,
|
||||
})
|
||||
|
||||
// TODO: remove legacy /api routes after deprecation window.
|
||||
http.route({
|
||||
path: LegacyApiRoutes.download,
|
||||
|
||||
@@ -33,7 +33,7 @@ describe('httpApi handlers', () => {
|
||||
expect(await response.json()).toEqual({ results: [] })
|
||||
})
|
||||
|
||||
it('searchSkillsHttp forwards args', async () => {
|
||||
it('searchSkillsHttp forwards args (approvedOnly alias)', async () => {
|
||||
const runAction = vi.fn().mockResolvedValue([
|
||||
{
|
||||
score: 1,
|
||||
@@ -48,13 +48,39 @@ describe('httpApi handlers', () => {
|
||||
expect(runAction).toHaveBeenCalledWith(expect.anything(), {
|
||||
query: 'test',
|
||||
limit: 5,
|
||||
approvedOnly: true,
|
||||
highlightedOnly: true,
|
||||
})
|
||||
expect(response.status).toBe(200)
|
||||
const json = await response.json()
|
||||
expect(json.results[0].slug).toBe('a')
|
||||
})
|
||||
|
||||
it('searchSkillsHttp forwards highlightedOnly', async () => {
|
||||
const runAction = vi.fn().mockResolvedValue([])
|
||||
await __handlers.searchSkillsHandler(
|
||||
makeCtx({ runAction }),
|
||||
new Request('https://example.com/api/search?q=test&highlightedOnly=true'),
|
||||
)
|
||||
expect(runAction).toHaveBeenCalledWith(expect.anything(), {
|
||||
query: 'test',
|
||||
limit: undefined,
|
||||
highlightedOnly: true,
|
||||
})
|
||||
})
|
||||
|
||||
it('searchSkillsHttp omits highlightedOnly when approvedOnly is false', async () => {
|
||||
const runAction = vi.fn().mockResolvedValue([])
|
||||
await __handlers.searchSkillsHandler(
|
||||
makeCtx({ runAction }),
|
||||
new Request('https://example.com/api/search?q=test&approvedOnly=false'),
|
||||
)
|
||||
expect(runAction).toHaveBeenCalledWith(expect.anything(), {
|
||||
query: 'test',
|
||||
limit: undefined,
|
||||
highlightedOnly: undefined,
|
||||
})
|
||||
})
|
||||
|
||||
it('getSkillHttp validates slug', async () => {
|
||||
const response = await __handlers.getSkillHandler(
|
||||
makeCtx({ runQuery: vi.fn() }),
|
||||
@@ -104,6 +130,30 @@ describe('httpApi handlers', () => {
|
||||
expect(json.owner.handle).toBe('p')
|
||||
})
|
||||
|
||||
it('getSkillHttp returns payload with null owner/latestVersion', async () => {
|
||||
const runQuery = vi.fn().mockResolvedValue({
|
||||
skill: {
|
||||
slug: 'demo',
|
||||
displayName: 'Demo',
|
||||
summary: null,
|
||||
tags: {},
|
||||
stats: {},
|
||||
createdAt: 1,
|
||||
updatedAt: 2,
|
||||
},
|
||||
latestVersion: null,
|
||||
owner: null,
|
||||
})
|
||||
const response = await __handlers.getSkillHandler(
|
||||
makeCtx({ runQuery }),
|
||||
new Request('https://example.com/api/skill?slug=demo'),
|
||||
)
|
||||
expect(response.status).toBe(200)
|
||||
const json = await response.json()
|
||||
expect(json.latestVersion).toBeNull()
|
||||
expect(json.owner).toBeNull()
|
||||
})
|
||||
|
||||
it('resolveSkillVersionHttp validates hash', async () => {
|
||||
const response = await __handlers.resolveSkillVersionHandler(
|
||||
makeCtx({ runQuery: vi.fn() }),
|
||||
@@ -186,6 +236,65 @@ describe('httpApi handlers', () => {
|
||||
expect(runMutation).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('cliTelemetrySyncHttp returns 400 on invalid payload', async () => {
|
||||
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: 'users:1' } as never)
|
||||
const response = await __handlers.cliTelemetrySyncHandler(
|
||||
makeCtx({ runMutation: vi.fn() }),
|
||||
new Request('https://x/api/cli/telemetry/sync', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ roots: 'nope' }),
|
||||
}),
|
||||
)
|
||||
expect(response.status).toBe(400)
|
||||
})
|
||||
|
||||
it('cliTelemetrySyncHttp forwards skill versions when provided', async () => {
|
||||
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: 'users:1' } as never)
|
||||
const runMutation = vi.fn().mockResolvedValue(null)
|
||||
await __handlers.cliTelemetrySyncHandler(
|
||||
makeCtx({ runMutation }),
|
||||
new Request('https://x/api/cli/telemetry/sync', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
roots: [
|
||||
{
|
||||
rootId: 'abc',
|
||||
label: '~/skills',
|
||||
skills: [{ slug: 'weather', version: '1.0.0' }],
|
||||
},
|
||||
],
|
||||
}),
|
||||
}),
|
||||
)
|
||||
expect(runMutation).toHaveBeenCalledWith(expect.anything(), {
|
||||
userId: 'users:1',
|
||||
roots: [
|
||||
{ rootId: 'abc', label: '~/skills', skills: [{ slug: 'weather', version: '1.0.0' }] },
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
it('cliTelemetrySyncHttp returns 400 on invalid json', async () => {
|
||||
const request = new Request('https://x/api/cli/telemetry/sync', { method: 'POST', body: '{' })
|
||||
const response = await __handlers.cliTelemetrySyncHandler(makeCtx({}), request)
|
||||
expect(response.status).toBe(400)
|
||||
})
|
||||
|
||||
it('cliTelemetrySyncHttp returns 401 when unauthorized', async () => {
|
||||
vi.mocked(requireApiTokenUser).mockRejectedValueOnce(new Error('Unauthorized'))
|
||||
const response = await __handlers.cliTelemetrySyncHandler(
|
||||
makeCtx({}),
|
||||
new Request('https://x/api/cli/telemetry/sync', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ roots: [] }),
|
||||
}),
|
||||
)
|
||||
expect(response.status).toBe(401)
|
||||
})
|
||||
|
||||
it('cliUploadUrlHttp returns uploadUrl', async () => {
|
||||
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: 'user1' } as never)
|
||||
const runMutation = vi.fn().mockResolvedValue('https://upload.local')
|
||||
@@ -316,6 +425,50 @@ describe('httpApi handlers', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('cliSkillUndeleteHttp calls delete handler with deleted=false', async () => {
|
||||
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: 'user1' } as never)
|
||||
const runMutation = vi.fn().mockResolvedValue({ ok: true })
|
||||
const response = await __handlers.cliSkillDeleteHandler(
|
||||
makeCtx({ runMutation }),
|
||||
new Request('https://x/api/cli/skill/undelete', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ slug: 'demo' }),
|
||||
}),
|
||||
false,
|
||||
)
|
||||
expect(response.status).toBe(200)
|
||||
expect(runMutation).toHaveBeenCalledWith(expect.anything(), {
|
||||
userId: 'user1',
|
||||
slug: 'demo',
|
||||
deleted: false,
|
||||
})
|
||||
warnSpy.mockRestore()
|
||||
})
|
||||
|
||||
it('cliSkillDeleteHttp calls delete handler with deleted=true', async () => {
|
||||
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: 'user1' } as never)
|
||||
const runMutation = vi.fn().mockResolvedValue({ ok: true })
|
||||
const response = await __handlers.cliSkillDeleteHandler(
|
||||
makeCtx({ runMutation }),
|
||||
new Request('https://x/api/cli/skill/delete', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ slug: 'demo' }),
|
||||
}),
|
||||
true,
|
||||
)
|
||||
expect(response.status).toBe(200)
|
||||
expect(runMutation).toHaveBeenCalledWith(expect.anything(), {
|
||||
userId: 'user1',
|
||||
slug: 'demo',
|
||||
deleted: true,
|
||||
})
|
||||
warnSpy.mockRestore()
|
||||
})
|
||||
|
||||
it('cliSkillDeleteHandler returns 400 on invalid json', async () => {
|
||||
const request = new Request('https://x/api/cli/skill/delete', { method: 'POST', body: '{' })
|
||||
const response = await __handlers.cliSkillDeleteHandler(makeCtx({}), request, true)
|
||||
|
||||
@@ -25,6 +25,29 @@ describe('httpApi', () => {
|
||||
expect(parsed.files[0]?.path).toBe('SKILL.md')
|
||||
})
|
||||
|
||||
it('normalizes optional fields in publish payload', () => {
|
||||
const parsed = __test.parsePublishBody({
|
||||
slug: 'cool-skill',
|
||||
displayName: 'Cool Skill',
|
||||
version: '1.2.3',
|
||||
changelog: '',
|
||||
tags: [],
|
||||
forkOf: { slug: 'base-skill' },
|
||||
files: [
|
||||
{
|
||||
path: 'SKILL.md',
|
||||
size: 5,
|
||||
storageId: 'fakeStorageId',
|
||||
sha256: 'abcd',
|
||||
contentType: 'text/markdown',
|
||||
},
|
||||
],
|
||||
})
|
||||
expect(parsed.tags).toBeUndefined()
|
||||
expect(parsed.source).toBeUndefined()
|
||||
expect(parsed.forkOf).toEqual({ slug: 'base-skill', version: undefined })
|
||||
})
|
||||
|
||||
it('rejects invalid publish payloads', () => {
|
||||
expect(() => __test.parsePublishBody(null)).toThrow(/Publish payload/i)
|
||||
expect(() =>
|
||||
|
||||
+4
-2
@@ -5,7 +5,7 @@ import {
|
||||
CliSkillDeleteRequestSchema,
|
||||
CliTelemetrySyncRequestSchema,
|
||||
parseArk,
|
||||
} from 'clawdhub-schema'
|
||||
} from 'clawhub-schema'
|
||||
import { api, internal } from './_generated/api'
|
||||
import type { Id } from './_generated/dataModel'
|
||||
import type { ActionCtx } from './_generated/server'
|
||||
@@ -44,13 +44,14 @@ async function searchSkillsHandler(ctx: ActionCtx, request: Request) {
|
||||
const query = url.searchParams.get('q')?.trim() ?? ''
|
||||
const limit = toOptionalNumber(url.searchParams.get('limit'))
|
||||
const approvedOnly = url.searchParams.get('approvedOnly') === 'true'
|
||||
const highlightedOnly = url.searchParams.get('highlightedOnly') === 'true' || approvedOnly
|
||||
|
||||
if (!query) return json({ results: [] })
|
||||
|
||||
const results = (await ctx.runAction(api.search.searchSkills, {
|
||||
query,
|
||||
limit,
|
||||
approvedOnly: approvedOnly || undefined,
|
||||
highlightedOnly: highlightedOnly || undefined,
|
||||
})) as SearchSkillEntry[]
|
||||
|
||||
return json({
|
||||
@@ -273,6 +274,7 @@ function parsePublishBody(body: unknown) {
|
||||
version: parsed.version,
|
||||
changelog: parsed.changelog,
|
||||
tags,
|
||||
source: parsed.source ?? undefined,
|
||||
forkOf: parsed.forkOf
|
||||
? {
|
||||
slug: parsed.forkOf.slug,
|
||||
|
||||
@@ -158,6 +158,31 @@ describe('httpApiV1 handlers', () => {
|
||||
expect(json.items[0].tags.latest).toBe('1.0.0')
|
||||
})
|
||||
|
||||
it('lists skills supports sort aliases', async () => {
|
||||
const checks: Array<[string, string]> = [
|
||||
['rating', 'stars'],
|
||||
['installs', 'installsCurrent'],
|
||||
['installs-all-time', 'installsAllTime'],
|
||||
['trending', 'trending'],
|
||||
]
|
||||
|
||||
for (const [input, expected] of checks) {
|
||||
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
|
||||
if ('sort' in args || 'cursor' in args || 'limit' in args) {
|
||||
expect(args.sort).toBe(expected)
|
||||
return { items: [], nextCursor: null }
|
||||
}
|
||||
return null
|
||||
})
|
||||
const runMutation = vi.fn().mockResolvedValue(okRate())
|
||||
const response = await __handlers.listSkillsV1Handler(
|
||||
makeCtx({ runQuery, runMutation }),
|
||||
new Request(`https://example.com/api/v1/skills?sort=${input}`),
|
||||
)
|
||||
expect(response.status).toBe(200)
|
||||
}
|
||||
})
|
||||
|
||||
it('get skill returns 404 when missing', async () => {
|
||||
const runQuery = vi.fn().mockResolvedValue(null)
|
||||
const runMutation = vi.fn().mockResolvedValue(okRate())
|
||||
@@ -498,4 +523,62 @@ describe('httpApiV1 handlers', () => {
|
||||
)
|
||||
expect(response2.status).toBe(200)
|
||||
})
|
||||
|
||||
it('stars require auth', async () => {
|
||||
vi.mocked(requireApiTokenUser).mockRejectedValueOnce(new Error('Unauthorized'))
|
||||
const runMutation = vi.fn().mockResolvedValue(okRate())
|
||||
const response = await __handlers.starsPostRouterV1Handler(
|
||||
makeCtx({ runMutation }),
|
||||
new Request('https://example.com/api/v1/stars/demo', { method: 'POST' }),
|
||||
)
|
||||
expect(response.status).toBe(401)
|
||||
})
|
||||
|
||||
it('stars add succeeds', async () => {
|
||||
vi.mocked(requireApiTokenUser).mockResolvedValue({
|
||||
userId: 'users:1',
|
||||
user: { handle: 'p' },
|
||||
} as never)
|
||||
const runQuery = vi.fn().mockResolvedValue({ _id: 'skills:1' })
|
||||
const runMutation = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(okRate())
|
||||
.mockResolvedValueOnce(okRate())
|
||||
.mockResolvedValueOnce({ ok: true, starred: true, alreadyStarred: false })
|
||||
const response = await __handlers.starsPostRouterV1Handler(
|
||||
makeCtx({ runQuery, runMutation }),
|
||||
new Request('https://example.com/api/v1/stars/demo', {
|
||||
method: 'POST',
|
||||
headers: { Authorization: 'Bearer clh_test' },
|
||||
}),
|
||||
)
|
||||
expect(response.status).toBe(200)
|
||||
const json = await response.json()
|
||||
expect(json.ok).toBe(true)
|
||||
expect(json.starred).toBe(true)
|
||||
})
|
||||
|
||||
it('stars delete succeeds', async () => {
|
||||
vi.mocked(requireApiTokenUser).mockResolvedValue({
|
||||
userId: 'users:1',
|
||||
user: { handle: 'p' },
|
||||
} as never)
|
||||
const runQuery = vi.fn().mockResolvedValue({ _id: 'skills:1' })
|
||||
const runMutation = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(okRate())
|
||||
.mockResolvedValueOnce(okRate())
|
||||
.mockResolvedValueOnce({ ok: true, unstarred: true, alreadyUnstarred: false })
|
||||
const response = await __handlers.starsDeleteRouterV1Handler(
|
||||
makeCtx({ runQuery, runMutation }),
|
||||
new Request('https://example.com/api/v1/stars/demo', {
|
||||
method: 'DELETE',
|
||||
headers: { Authorization: 'Bearer clh_test' },
|
||||
}),
|
||||
)
|
||||
expect(response.status).toBe(200)
|
||||
const json = await response.json()
|
||||
expect(json.ok).toBe(true)
|
||||
expect(json.unstarred).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
+471
-7
@@ -1,4 +1,4 @@
|
||||
import { CliPublishRequestSchema, parseArk } from 'clawdhub-schema'
|
||||
import { CliPublishRequestSchema, parseArk } from 'clawhub-schema'
|
||||
import { api, internal } from './_generated/api'
|
||||
import type { Doc, Id } from './_generated/dataModel'
|
||||
import type { ActionCtx } from './_generated/server'
|
||||
@@ -6,6 +6,7 @@ import { httpAction } from './_generated/server'
|
||||
import { requireApiTokenUser } from './lib/apiTokenAuth'
|
||||
import { hashToken } from './lib/tokens'
|
||||
import { publishVersionForUser } from './skills'
|
||||
import { publishSoulVersionForUser } from './souls'
|
||||
|
||||
const RATE_LIMIT_WINDOW_MS = 60_000
|
||||
const RATE_LIMITS = {
|
||||
@@ -43,6 +44,9 @@ type ListSkillsResult = {
|
||||
nextCursor: string | null
|
||||
}
|
||||
|
||||
type SkillFile = Doc<'skillVersions'>['files'][number]
|
||||
type SoulFile = Doc<'soulVersions'>['files'][number]
|
||||
|
||||
type GetBySlugResult = {
|
||||
skill: {
|
||||
_id: Id<'skills'>
|
||||
@@ -55,7 +59,7 @@ type GetBySlugResult = {
|
||||
updatedAt: number
|
||||
} | null
|
||||
latestVersion: Doc<'skillVersions'> | null
|
||||
owner: { handle?: string; displayName?: string; image?: string } | null
|
||||
owner: { _id: Id<'users'>; handle?: string; displayName?: string; image?: string } | null
|
||||
} | null
|
||||
|
||||
type ListVersionsResult = {
|
||||
@@ -76,6 +80,57 @@ type ListVersionsResult = {
|
||||
nextCursor: string | null
|
||||
}
|
||||
|
||||
type ListSoulsResult = {
|
||||
items: Array<{
|
||||
soul: {
|
||||
_id: Id<'souls'>
|
||||
slug: string
|
||||
displayName: string
|
||||
summary?: string
|
||||
tags: Record<string, Id<'soulVersions'>>
|
||||
stats: unknown
|
||||
createdAt: number
|
||||
updatedAt: number
|
||||
latestVersionId?: Id<'soulVersions'>
|
||||
}
|
||||
latestVersion: { version: string; createdAt: number; changelog: string } | null
|
||||
}>
|
||||
nextCursor: string | null
|
||||
}
|
||||
|
||||
type GetSoulBySlugResult = {
|
||||
soul: {
|
||||
_id: Id<'souls'>
|
||||
slug: string
|
||||
displayName: string
|
||||
summary?: string
|
||||
tags: Record<string, Id<'soulVersions'>>
|
||||
stats: unknown
|
||||
createdAt: number
|
||||
updatedAt: number
|
||||
} | null
|
||||
latestVersion: Doc<'soulVersions'> | null
|
||||
owner: { handle?: string; displayName?: string; image?: string } | null
|
||||
} | null
|
||||
|
||||
type ListSoulVersionsResult = {
|
||||
items: Array<{
|
||||
version: string
|
||||
createdAt: number
|
||||
changelog: string
|
||||
changelogSource?: 'auto' | 'user'
|
||||
files: Array<{
|
||||
path: string
|
||||
size: number
|
||||
storageId: Id<'_storage'>
|
||||
sha256: string
|
||||
contentType?: string
|
||||
}>
|
||||
softDeletedAt?: number
|
||||
}>
|
||||
nextCursor: string | null
|
||||
}
|
||||
|
||||
async function searchSkillsV1Handler(ctx: ActionCtx, request: Request) {
|
||||
const rate = await applyRateLimit(ctx, request, 'read')
|
||||
if (!rate.ok) return rate.response
|
||||
@@ -139,11 +194,14 @@ async function listSkillsV1Handler(ctx: ActionCtx, request: Request) {
|
||||
|
||||
const url = new URL(request.url)
|
||||
const limit = toOptionalNumber(url.searchParams.get('limit'))
|
||||
const cursor = url.searchParams.get('cursor')?.trim() || undefined
|
||||
const rawCursor = url.searchParams.get('cursor')?.trim() || undefined
|
||||
const sort = parseListSort(url.searchParams.get('sort'))
|
||||
const cursor = sort === 'updated' ? rawCursor : undefined
|
||||
|
||||
const result = (await ctx.runQuery(api.skills.listPublicPage, {
|
||||
limit,
|
||||
cursor,
|
||||
sort,
|
||||
})) as ListSkillsResult
|
||||
|
||||
const items = await Promise.all(
|
||||
@@ -209,6 +267,7 @@ async function skillsGetRouterV1Handler(ctx: ActionCtx, request: Request) {
|
||||
owner: result.owner
|
||||
? {
|
||||
handle: result.owner.handle ?? null,
|
||||
userId: result.owner._id,
|
||||
displayName: result.owner.displayName ?? null,
|
||||
image: result.owner.image ?? null,
|
||||
}
|
||||
@@ -263,7 +322,7 @@ async function skillsGetRouterV1Handler(ctx: ActionCtx, request: Request) {
|
||||
createdAt: version.createdAt,
|
||||
changelog: version.changelog,
|
||||
changelogSource: version.changelogSource ?? null,
|
||||
files: version.files.map((file) => ({
|
||||
files: version.files.map((file: SkillFile) => ({
|
||||
path: file.path,
|
||||
size: file.size,
|
||||
sha256: file.sha256,
|
||||
@@ -316,6 +375,9 @@ async function skillsGetRouterV1Handler(ctx: ActionCtx, request: Request) {
|
||||
if (!blob) return text('File missing in storage', 410, rate.headers)
|
||||
const textContent = await blob.text()
|
||||
|
||||
const isSvg =
|
||||
file.contentType?.toLowerCase().includes('svg') || file.path.toLowerCase().endsWith('.svg')
|
||||
|
||||
const headers = mergeHeaders(rate.headers, {
|
||||
'Content-Type': file.contentType
|
||||
? `${file.contentType}; charset=utf-8`
|
||||
@@ -324,6 +386,14 @@ async function skillsGetRouterV1Handler(ctx: ActionCtx, request: Request) {
|
||||
ETag: file.sha256,
|
||||
'X-Content-SHA256': file.sha256,
|
||||
'X-Content-Size': String(file.size),
|
||||
'X-Content-Type-Options': 'nosniff',
|
||||
'X-Frame-Options': 'DENY',
|
||||
// For any text response that a browser might try to render, lock it down.
|
||||
// In particular, this prevents SVG <foreignObject> script execution from
|
||||
// reading localStorage tokens on this origin.
|
||||
'Content-Security-Policy':
|
||||
"default-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'",
|
||||
...(isSvg ? { 'Content-Disposition': 'attachment' } : {}),
|
||||
})
|
||||
return new Response(textContent, { status: 200, headers })
|
||||
}
|
||||
@@ -506,14 +576,16 @@ async function parseMultipartPublish(
|
||||
files.push({ path, size, storageId, sha256, contentType })
|
||||
}
|
||||
|
||||
const forkOf = payload.forkOf && typeof payload.forkOf === 'object' ? payload.forkOf : undefined
|
||||
const body = {
|
||||
slug: payload.slug,
|
||||
displayName: payload.displayName,
|
||||
version: payload.version,
|
||||
changelog: typeof payload.changelog === 'string' ? payload.changelog : '',
|
||||
tags: Array.isArray(payload.tags) ? payload.tags : undefined,
|
||||
...(payload.source ? { source: payload.source } : {}),
|
||||
files,
|
||||
...(payload.forkOf === undefined ? {} : { forkOf: payload.forkOf }),
|
||||
...(forkOf ? { forkOf } : {}),
|
||||
}
|
||||
|
||||
return parsePublishBody(body)
|
||||
@@ -529,6 +601,7 @@ function parsePublishBody(body: unknown) {
|
||||
version: parsed.version,
|
||||
changelog: parsed.changelog,
|
||||
tags,
|
||||
source: parsed.source ?? undefined,
|
||||
forkOf: parsed.forkOf
|
||||
? {
|
||||
slug: parsed.forkOf.slug,
|
||||
@@ -542,6 +615,20 @@ function parsePublishBody(body: unknown) {
|
||||
}
|
||||
}
|
||||
|
||||
async function resolveSoulTags(
|
||||
ctx: ActionCtx,
|
||||
tags: Record<string, Id<'soulVersions'>>,
|
||||
): Promise<Record<string, string>> {
|
||||
const resolved: Record<string, string> = {}
|
||||
for (const [tag, versionId] of Object.entries(tags)) {
|
||||
const version = await ctx.runQuery(api.souls.getVersionById, { versionId })
|
||||
if (version && !version.softDeletedAt) {
|
||||
resolved[tag] = version.version
|
||||
}
|
||||
}
|
||||
return resolved
|
||||
}
|
||||
|
||||
async function resolveTags(
|
||||
ctx: ActionCtx,
|
||||
tags: Record<string, Id<'skillVersions'>>,
|
||||
@@ -684,9 +771,36 @@ function toOptionalNumber(value: string | null) {
|
||||
return Number.isFinite(parsed) ? parsed : undefined
|
||||
}
|
||||
|
||||
type SkillListSort =
|
||||
| 'updated'
|
||||
| 'downloads'
|
||||
| 'stars'
|
||||
| 'installsCurrent'
|
||||
| 'installsAllTime'
|
||||
| 'trending'
|
||||
|
||||
function parseListSort(value: string | null): SkillListSort {
|
||||
const normalized = value?.trim().toLowerCase()
|
||||
if (normalized === 'downloads') return 'downloads'
|
||||
if (normalized === 'stars' || normalized === 'rating') return 'stars'
|
||||
if (
|
||||
normalized === 'installs' ||
|
||||
normalized === 'install' ||
|
||||
normalized === 'installscurrent' ||
|
||||
normalized === 'installs-current'
|
||||
) {
|
||||
return 'installsCurrent'
|
||||
}
|
||||
if (normalized === 'installsalltime' || normalized === 'installs-all-time') {
|
||||
return 'installsAllTime'
|
||||
}
|
||||
if (normalized === 'trending') return 'trending'
|
||||
return 'updated'
|
||||
}
|
||||
|
||||
async function sha256Hex(bytes: Uint8Array) {
|
||||
const normalized = new Uint8Array(bytes)
|
||||
const digest = await crypto.subtle.digest('SHA-256', normalized.buffer)
|
||||
const data = new Uint8Array(bytes)
|
||||
const digest = await crypto.subtle.digest('SHA-256', data)
|
||||
return toHex(new Uint8Array(digest))
|
||||
}
|
||||
|
||||
@@ -696,6 +810,349 @@ function toHex(bytes: Uint8Array) {
|
||||
return out
|
||||
}
|
||||
|
||||
async function listSoulsV1Handler(ctx: ActionCtx, request: Request) {
|
||||
const rate = await applyRateLimit(ctx, request, 'read')
|
||||
if (!rate.ok) return rate.response
|
||||
|
||||
const url = new URL(request.url)
|
||||
const limit = toOptionalNumber(url.searchParams.get('limit'))
|
||||
const cursor = url.searchParams.get('cursor')?.trim() || undefined
|
||||
|
||||
const result = (await ctx.runQuery(api.souls.listPublicPage, {
|
||||
limit,
|
||||
cursor,
|
||||
})) as ListSoulsResult
|
||||
|
||||
const items = await Promise.all(
|
||||
result.items.map(async (item) => {
|
||||
const tags = await resolveSoulTags(ctx, item.soul.tags)
|
||||
return {
|
||||
slug: item.soul.slug,
|
||||
displayName: item.soul.displayName,
|
||||
summary: item.soul.summary ?? null,
|
||||
tags,
|
||||
stats: item.soul.stats,
|
||||
createdAt: item.soul.createdAt,
|
||||
updatedAt: item.soul.updatedAt,
|
||||
latestVersion: item.latestVersion
|
||||
? {
|
||||
version: item.latestVersion.version,
|
||||
createdAt: item.latestVersion.createdAt,
|
||||
changelog: item.latestVersion.changelog,
|
||||
}
|
||||
: null,
|
||||
}
|
||||
}),
|
||||
)
|
||||
|
||||
return json({ items, nextCursor: result.nextCursor ?? null }, 200, rate.headers)
|
||||
}
|
||||
|
||||
export const listSoulsV1Http = httpAction(listSoulsV1Handler)
|
||||
|
||||
async function soulsGetRouterV1Handler(ctx: ActionCtx, request: Request) {
|
||||
const rate = await applyRateLimit(ctx, request, 'read')
|
||||
if (!rate.ok) return rate.response
|
||||
|
||||
const segments = getPathSegments(request, '/api/v1/souls/')
|
||||
if (segments.length === 0) return text('Missing slug', 400, rate.headers)
|
||||
const slug = segments[0]?.trim().toLowerCase() ?? ''
|
||||
const second = segments[1]
|
||||
const third = segments[2]
|
||||
|
||||
if (segments.length === 1) {
|
||||
const result = (await ctx.runQuery(api.souls.getBySlug, { slug })) as GetSoulBySlugResult
|
||||
if (!result?.soul) return text('Soul not found', 404, rate.headers)
|
||||
|
||||
const tags = await resolveSoulTags(ctx, result.soul.tags)
|
||||
return json(
|
||||
{
|
||||
soul: {
|
||||
slug: result.soul.slug,
|
||||
displayName: result.soul.displayName,
|
||||
summary: result.soul.summary ?? null,
|
||||
tags,
|
||||
stats: result.soul.stats,
|
||||
createdAt: result.soul.createdAt,
|
||||
updatedAt: result.soul.updatedAt,
|
||||
},
|
||||
latestVersion: result.latestVersion
|
||||
? {
|
||||
version: result.latestVersion.version,
|
||||
createdAt: result.latestVersion.createdAt,
|
||||
changelog: result.latestVersion.changelog,
|
||||
}
|
||||
: null,
|
||||
owner: result.owner
|
||||
? {
|
||||
handle: result.owner.handle ?? null,
|
||||
displayName: result.owner.displayName ?? null,
|
||||
image: result.owner.image ?? null,
|
||||
}
|
||||
: null,
|
||||
},
|
||||
200,
|
||||
rate.headers,
|
||||
)
|
||||
}
|
||||
|
||||
if (second === 'versions' && segments.length === 2) {
|
||||
const soul = await ctx.runQuery(internal.souls.getSoulBySlugInternal, { slug })
|
||||
if (!soul || soul.softDeletedAt) return text('Soul not found', 404, rate.headers)
|
||||
|
||||
const url = new URL(request.url)
|
||||
const limit = toOptionalNumber(url.searchParams.get('limit'))
|
||||
const cursor = url.searchParams.get('cursor')?.trim() || undefined
|
||||
const result = (await ctx.runQuery(api.souls.listVersionsPage, {
|
||||
soulId: soul._id,
|
||||
limit,
|
||||
cursor,
|
||||
})) as ListSoulVersionsResult
|
||||
|
||||
const items = result.items
|
||||
.filter((version) => !version.softDeletedAt)
|
||||
.map((version) => ({
|
||||
version: version.version,
|
||||
createdAt: version.createdAt,
|
||||
changelog: version.changelog,
|
||||
changelogSource: version.changelogSource ?? null,
|
||||
}))
|
||||
|
||||
return json({ items, nextCursor: result.nextCursor ?? null }, 200, rate.headers)
|
||||
}
|
||||
|
||||
if (second === 'versions' && third && segments.length === 3) {
|
||||
const soul = await ctx.runQuery(internal.souls.getSoulBySlugInternal, { slug })
|
||||
if (!soul || soul.softDeletedAt) return text('Soul not found', 404, rate.headers)
|
||||
|
||||
const version = await ctx.runQuery(api.souls.getVersionBySoulAndVersion, {
|
||||
soulId: soul._id,
|
||||
version: third,
|
||||
})
|
||||
if (!version) return text('Version not found', 404, rate.headers)
|
||||
if (version.softDeletedAt) return text('Version not available', 410, rate.headers)
|
||||
|
||||
return json(
|
||||
{
|
||||
soul: { slug: soul.slug, displayName: soul.displayName },
|
||||
version: {
|
||||
version: version.version,
|
||||
createdAt: version.createdAt,
|
||||
changelog: version.changelog,
|
||||
changelogSource: version.changelogSource ?? null,
|
||||
files: version.files.map((file: SoulFile) => ({
|
||||
path: file.path,
|
||||
size: file.size,
|
||||
sha256: file.sha256,
|
||||
contentType: file.contentType ?? null,
|
||||
})),
|
||||
},
|
||||
},
|
||||
200,
|
||||
rate.headers,
|
||||
)
|
||||
}
|
||||
|
||||
if (second === 'file' && segments.length === 2) {
|
||||
const url = new URL(request.url)
|
||||
const path = url.searchParams.get('path')?.trim()
|
||||
if (!path) return text('Missing path', 400, rate.headers)
|
||||
const versionParam = url.searchParams.get('version')?.trim()
|
||||
const tagParam = url.searchParams.get('tag')?.trim()
|
||||
|
||||
const soulResult = (await ctx.runQuery(api.souls.getBySlug, {
|
||||
slug,
|
||||
})) as GetSoulBySlugResult
|
||||
if (!soulResult?.soul) return text('Soul not found', 404, rate.headers)
|
||||
|
||||
let version = soulResult.latestVersion
|
||||
if (versionParam) {
|
||||
version = await ctx.runQuery(api.souls.getVersionBySoulAndVersion, {
|
||||
soulId: soulResult.soul._id,
|
||||
version: versionParam,
|
||||
})
|
||||
} else if (tagParam) {
|
||||
const versionId = soulResult.soul.tags[tagParam]
|
||||
if (versionId) {
|
||||
version = await ctx.runQuery(api.souls.getVersionById, { versionId })
|
||||
}
|
||||
}
|
||||
|
||||
if (!version) return text('Version not found', 404, rate.headers)
|
||||
if (version.softDeletedAt) return text('Version not available', 410, rate.headers)
|
||||
|
||||
const normalized = path.trim()
|
||||
const normalizedLower = normalized.toLowerCase()
|
||||
const file =
|
||||
version.files.find((entry) => entry.path === normalized) ??
|
||||
version.files.find((entry) => entry.path.toLowerCase() === normalizedLower)
|
||||
if (!file) return text('File not found', 404, rate.headers)
|
||||
if (file.size > MAX_RAW_FILE_BYTES) return text('File exceeds 200KB limit', 413, rate.headers)
|
||||
|
||||
const blob = await ctx.storage.get(file.storageId)
|
||||
if (!blob) return text('File missing in storage', 410, rate.headers)
|
||||
const textContent = await blob.text()
|
||||
|
||||
void ctx.runMutation(api.soulDownloads.increment, { soulId: soulResult.soul._id })
|
||||
|
||||
const isSvg =
|
||||
file.contentType?.toLowerCase().includes('svg') || file.path.toLowerCase().endsWith('.svg')
|
||||
|
||||
const headers = mergeHeaders(rate.headers, {
|
||||
'Content-Type': file.contentType
|
||||
? `${file.contentType}; charset=utf-8`
|
||||
: 'text/plain; charset=utf-8',
|
||||
'Cache-Control': 'private, max-age=60',
|
||||
ETag: file.sha256,
|
||||
'X-Content-SHA256': file.sha256,
|
||||
'X-Content-Size': String(file.size),
|
||||
'X-Content-Type-Options': 'nosniff',
|
||||
'X-Frame-Options': 'DENY',
|
||||
// For any text response that a browser might try to render, lock it down.
|
||||
// In particular, this prevents SVG <foreignObject> script execution from
|
||||
// reading localStorage tokens on this origin.
|
||||
'Content-Security-Policy':
|
||||
"default-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'",
|
||||
...(isSvg ? { 'Content-Disposition': 'attachment' } : {}),
|
||||
})
|
||||
return new Response(textContent, { status: 200, headers })
|
||||
}
|
||||
|
||||
return text('Not found', 404, rate.headers)
|
||||
}
|
||||
|
||||
export const soulsGetRouterV1Http = httpAction(soulsGetRouterV1Handler)
|
||||
|
||||
async function publishSoulV1Handler(ctx: ActionCtx, request: Request) {
|
||||
const rate = await applyRateLimit(ctx, request, 'write')
|
||||
if (!rate.ok) return rate.response
|
||||
|
||||
try {
|
||||
if (!parseBearerToken(request)) return text('Unauthorized', 401, rate.headers)
|
||||
} catch {
|
||||
return text('Unauthorized', 401, rate.headers)
|
||||
}
|
||||
const { userId } = await requireApiTokenUser(ctx, request)
|
||||
|
||||
const contentType = request.headers.get('content-type') ?? ''
|
||||
try {
|
||||
if (contentType.includes('application/json')) {
|
||||
const body = await request.json()
|
||||
const payload = parsePublishBody(body)
|
||||
const result = await publishSoulVersionForUser(ctx, userId, payload)
|
||||
return json({ ok: true, ...result }, 200, rate.headers)
|
||||
}
|
||||
|
||||
if (contentType.includes('multipart/form-data')) {
|
||||
const payload = await parseMultipartPublish(ctx, request)
|
||||
const result = await publishSoulVersionForUser(ctx, userId, payload)
|
||||
return json({ ok: true, ...result }, 200, rate.headers)
|
||||
}
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : 'Publish failed'
|
||||
return text(message, 400, rate.headers)
|
||||
}
|
||||
|
||||
return text('Unsupported content type', 415, rate.headers)
|
||||
}
|
||||
|
||||
export const publishSoulV1Http = httpAction(publishSoulV1Handler)
|
||||
|
||||
async function soulsPostRouterV1Handler(ctx: ActionCtx, request: Request) {
|
||||
const rate = await applyRateLimit(ctx, request, 'write')
|
||||
if (!rate.ok) return rate.response
|
||||
|
||||
const segments = getPathSegments(request, '/api/v1/souls/')
|
||||
if (segments.length !== 2 || segments[1] !== 'undelete') {
|
||||
return text('Not found', 404, rate.headers)
|
||||
}
|
||||
const slug = segments[0]?.trim().toLowerCase() ?? ''
|
||||
try {
|
||||
const { userId } = await requireApiTokenUser(ctx, request)
|
||||
await ctx.runMutation(internal.souls.setSoulSoftDeletedInternal, {
|
||||
userId,
|
||||
slug,
|
||||
deleted: false,
|
||||
})
|
||||
return json({ ok: true }, 200, rate.headers)
|
||||
} catch {
|
||||
return text('Unauthorized', 401, rate.headers)
|
||||
}
|
||||
}
|
||||
|
||||
export const soulsPostRouterV1Http = httpAction(soulsPostRouterV1Handler)
|
||||
|
||||
async function soulsDeleteRouterV1Handler(ctx: ActionCtx, request: Request) {
|
||||
const rate = await applyRateLimit(ctx, request, 'write')
|
||||
if (!rate.ok) return rate.response
|
||||
|
||||
const segments = getPathSegments(request, '/api/v1/souls/')
|
||||
if (segments.length !== 1) return text('Not found', 404, rate.headers)
|
||||
const slug = segments[0]?.trim().toLowerCase() ?? ''
|
||||
try {
|
||||
const { userId } = await requireApiTokenUser(ctx, request)
|
||||
await ctx.runMutation(internal.souls.setSoulSoftDeletedInternal, {
|
||||
userId,
|
||||
slug,
|
||||
deleted: true,
|
||||
})
|
||||
return json({ ok: true }, 200, rate.headers)
|
||||
} catch {
|
||||
return text('Unauthorized', 401, rate.headers)
|
||||
}
|
||||
}
|
||||
|
||||
export const soulsDeleteRouterV1Http = httpAction(soulsDeleteRouterV1Handler)
|
||||
|
||||
async function starsPostRouterV1Handler(ctx: ActionCtx, request: Request) {
|
||||
const rate = await applyRateLimit(ctx, request, 'write')
|
||||
if (!rate.ok) return rate.response
|
||||
|
||||
const segments = getPathSegments(request, '/api/v1/stars/')
|
||||
if (segments.length !== 1) return text('Not found', 404, rate.headers)
|
||||
const slug = segments[0]?.trim().toLowerCase() ?? ''
|
||||
|
||||
try {
|
||||
const { userId } = await requireApiTokenUser(ctx, request)
|
||||
const skill = await ctx.runQuery(internal.skills.getSkillBySlugInternal, { slug })
|
||||
if (!skill) return text('Skill not found', 404, rate.headers)
|
||||
|
||||
const result = await ctx.runMutation(internal.stars.addStarInternal, {
|
||||
userId,
|
||||
skillId: skill._id,
|
||||
})
|
||||
return json(result, 200, rate.headers)
|
||||
} catch {
|
||||
return text('Unauthorized', 401, rate.headers)
|
||||
}
|
||||
}
|
||||
|
||||
export const starsPostRouterV1Http = httpAction(starsPostRouterV1Handler)
|
||||
|
||||
async function starsDeleteRouterV1Handler(ctx: ActionCtx, request: Request) {
|
||||
const rate = await applyRateLimit(ctx, request, 'write')
|
||||
if (!rate.ok) return rate.response
|
||||
|
||||
const segments = getPathSegments(request, '/api/v1/stars/')
|
||||
if (segments.length !== 1) return text('Not found', 404, rate.headers)
|
||||
const slug = segments[0]?.trim().toLowerCase() ?? ''
|
||||
|
||||
try {
|
||||
const { userId } = await requireApiTokenUser(ctx, request)
|
||||
const skill = await ctx.runQuery(internal.skills.getSkillBySlugInternal, { slug })
|
||||
if (!skill) return text('Skill not found', 404, rate.headers)
|
||||
|
||||
const result = await ctx.runMutation(internal.stars.removeStarInternal, {
|
||||
userId,
|
||||
skillId: skill._id,
|
||||
})
|
||||
return json(result, 200, rate.headers)
|
||||
} catch {
|
||||
return text('Unauthorized', 401, rate.headers)
|
||||
}
|
||||
}
|
||||
|
||||
export const starsDeleteRouterV1Http = httpAction(starsDeleteRouterV1Handler)
|
||||
export const __handlers = {
|
||||
searchSkillsV1Handler,
|
||||
resolveSkillVersionV1Handler,
|
||||
@@ -704,5 +1161,12 @@ export const __handlers = {
|
||||
publishSkillV1Handler,
|
||||
skillsPostRouterV1Handler,
|
||||
skillsDeleteRouterV1Handler,
|
||||
listSoulsV1Handler,
|
||||
soulsGetRouterV1Handler,
|
||||
publishSoulV1Handler,
|
||||
soulsPostRouterV1Handler,
|
||||
soulsDeleteRouterV1Handler,
|
||||
starsPostRouterV1Handler,
|
||||
starsDeleteRouterV1Handler,
|
||||
whoamiV1Handler,
|
||||
}
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
import { v } from 'convex/values'
|
||||
import { internalMutation } from './_generated/server'
|
||||
import { buildTrendingLeaderboard } from './lib/leaderboards'
|
||||
|
||||
const MAX_TRENDING_LIMIT = 200
|
||||
const KEEP_LEADERBOARD_ENTRIES = 3
|
||||
|
||||
export const rebuildTrendingLeaderboardInternal = internalMutation({
|
||||
args: { limit: v.optional(v.number()) },
|
||||
handler: async (ctx, args) => {
|
||||
const limit = clampInt(args.limit ?? MAX_TRENDING_LIMIT, 1, MAX_TRENDING_LIMIT)
|
||||
const now = Date.now()
|
||||
const { startDay, endDay, items } = await buildTrendingLeaderboard(ctx, { limit, now })
|
||||
|
||||
await ctx.db.insert('skillLeaderboards', {
|
||||
kind: 'trending',
|
||||
generatedAt: now,
|
||||
rangeStartDay: startDay,
|
||||
rangeEndDay: endDay,
|
||||
items,
|
||||
})
|
||||
|
||||
const recent = await ctx.db
|
||||
.query('skillLeaderboards')
|
||||
.withIndex('by_kind', (q) => q.eq('kind', 'trending'))
|
||||
.order('desc')
|
||||
.take(KEEP_LEADERBOARD_ENTRIES + 5)
|
||||
|
||||
for (const entry of recent.slice(KEEP_LEADERBOARD_ENTRIES)) {
|
||||
await ctx.db.delete(entry._id)
|
||||
}
|
||||
|
||||
return { ok: true as const, count: items.length }
|
||||
},
|
||||
})
|
||||
|
||||
function clampInt(value: number, min: number, max: number) {
|
||||
return Math.min(Math.max(value, min), max)
|
||||
}
|
||||
+10
-2
@@ -1,5 +1,5 @@
|
||||
import { getAuthUserId } from '@convex-dev/auth/server'
|
||||
import { api } from '../_generated/api'
|
||||
import { internal } from '../_generated/api'
|
||||
import type { Doc } from '../_generated/dataModel'
|
||||
import type { ActionCtx, MutationCtx, QueryCtx } from '../_generated/server'
|
||||
|
||||
@@ -16,7 +16,7 @@ export async function requireUser(ctx: MutationCtx | QueryCtx) {
|
||||
export async function requireUserFromAction(ctx: ActionCtx) {
|
||||
const userId = await getAuthUserId(ctx)
|
||||
if (!userId) throw new Error('Unauthorized')
|
||||
const user = await ctx.runQuery(api.users.getById, { userId })
|
||||
const user = await ctx.runQuery(internal.users.getByIdInternal, { userId })
|
||||
if (!user || user.deletedAt) throw new Error('User not found')
|
||||
return { userId, user: user as Doc<'users'> }
|
||||
}
|
||||
@@ -26,3 +26,11 @@ export function assertRole(user: Doc<'users'>, allowed: Role[]) {
|
||||
throw new Error('Forbidden')
|
||||
}
|
||||
}
|
||||
|
||||
export function assertAdmin(user: Doc<'users'>) {
|
||||
assertRole(user, ['admin'])
|
||||
}
|
||||
|
||||
export function assertModerator(user: Doc<'users'>) {
|
||||
assertRole(user, ['admin', 'moderator'])
|
||||
}
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
import type { Doc, Id } from '../_generated/dataModel'
|
||||
import type { QueryCtx } from '../_generated/server'
|
||||
|
||||
type BadgeKind = Doc<'skillBadges'>['kind']
|
||||
|
||||
export type SkillBadgeMap = Partial<Record<BadgeKind, { byUserId: Id<'users'>; at: number }>>
|
||||
|
||||
export type SkillBadgeSource = { badges?: SkillBadgeMap | null }
|
||||
|
||||
type BadgeCtx = Pick<QueryCtx, 'db'>
|
||||
|
||||
export function isSkillHighlighted(skill: SkillBadgeSource) {
|
||||
return Boolean(skill.badges?.highlighted)
|
||||
}
|
||||
|
||||
export function isSkillOfficial(skill: SkillBadgeSource) {
|
||||
return Boolean(skill.badges?.official)
|
||||
}
|
||||
|
||||
export function isSkillDeprecated(skill: SkillBadgeSource) {
|
||||
return Boolean(skill.badges?.deprecated)
|
||||
}
|
||||
|
||||
export function buildBadgeMap(records: Doc<'skillBadges'>[]): SkillBadgeMap {
|
||||
return records.reduce<SkillBadgeMap>((acc, record) => {
|
||||
acc[record.kind] = { byUserId: record.byUserId, at: record.at }
|
||||
return acc
|
||||
}, {})
|
||||
}
|
||||
|
||||
export async function getSkillBadgeMap(
|
||||
ctx: BadgeCtx,
|
||||
skillId: Id<'skills'>,
|
||||
): Promise<SkillBadgeMap> {
|
||||
const records = await ctx.db
|
||||
.query('skillBadges')
|
||||
.withIndex('by_skill', (q) => q.eq('skillId', skillId))
|
||||
.collect()
|
||||
return buildBadgeMap(records)
|
||||
}
|
||||
|
||||
export async function getSkillBadgeMaps(
|
||||
ctx: BadgeCtx,
|
||||
skillIds: Array<Id<'skills'>>,
|
||||
): Promise<Map<Id<'skills'>, SkillBadgeMap>> {
|
||||
const entries = await Promise.all(
|
||||
skillIds.map(async (skillId) => [skillId, await getSkillBadgeMap(ctx, skillId)] as const),
|
||||
)
|
||||
return new Map(entries)
|
||||
}
|
||||
@@ -1,9 +1,16 @@
|
||||
export const EMBEDDING_MODEL = 'text-embedding-3-small'
|
||||
export const EMBEDDING_DIMENSIONS = 1536
|
||||
|
||||
function emptyEmbedding() {
|
||||
return Array.from({ length: EMBEDDING_DIMENSIONS }, () => 0)
|
||||
}
|
||||
|
||||
export async function generateEmbedding(text: string) {
|
||||
const apiKey = process.env.OPENAI_API_KEY
|
||||
if (!apiKey) throw new Error('OPENAI_API_KEY is not configured')
|
||||
if (!apiKey) {
|
||||
console.warn('OPENAI_API_KEY is not configured; using zero embeddings')
|
||||
return emptyEmbedding()
|
||||
}
|
||||
|
||||
const response = await fetch('https://api.openai.com/v1/embeddings', {
|
||||
method: 'POST',
|
||||
|
||||
@@ -8,7 +8,7 @@ const GITHUB_API = 'https://api.github.com'
|
||||
const DEFAULT_REPO = 'clawdbot/skills'
|
||||
const DEFAULT_ROOT = 'skills'
|
||||
const META_FILENAME = '_meta.json'
|
||||
const USER_AGENT = 'clawdhub/skills-backup'
|
||||
const USER_AGENT = 'clawhub/skills-backup'
|
||||
|
||||
type BackupFile = {
|
||||
path: string
|
||||
|
||||
@@ -0,0 +1,247 @@
|
||||
/* @vitest-environment node */
|
||||
|
||||
import { unzipSync } from 'fflate'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
buildGitHubZipForTests,
|
||||
computeDefaultSelectedPaths,
|
||||
detectGitHubImportCandidates,
|
||||
extractMarkdownRelativeTargets,
|
||||
fetchGitHubZipBytes,
|
||||
parseGitHubImportUrl,
|
||||
resolveGitHubCommit,
|
||||
resolveMarkdownTarget,
|
||||
stripGitHubZipRoot,
|
||||
} from './githubImport'
|
||||
|
||||
function requestInfoToUrlString(input: RequestInfo | URL): string {
|
||||
if (typeof input === 'string') return input
|
||||
if (input instanceof URL) return input.toString()
|
||||
if (input instanceof Request) return input.url
|
||||
|
||||
throw new Error('Unexpected fetch input type')
|
||||
}
|
||||
|
||||
describe('github import', () => {
|
||||
it('parses repo root urls', () => {
|
||||
expect(parseGitHubImportUrl('https://github.com/visionik/ouracli')).toEqual({
|
||||
owner: 'visionik',
|
||||
repo: 'ouracli',
|
||||
originalUrl: 'https://github.com/visionik/ouracli',
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects non-https and non-github urls', () => {
|
||||
expect(() => parseGitHubImportUrl('http://github.com/a/b')).toThrow(/https/i)
|
||||
expect(() => parseGitHubImportUrl('https://example.com/a/b')).toThrow(/github\.com/i)
|
||||
expect(() => parseGitHubImportUrl('not-a-url')).toThrow(/Invalid URL/i)
|
||||
})
|
||||
|
||||
it('rejects malformed tree/blob urls', () => {
|
||||
expect(() => parseGitHubImportUrl('https://github.com/a/b/tree/')).toThrow(/Missing ref/i)
|
||||
expect(() => parseGitHubImportUrl('https://github.com/a/b/blob/main')).toThrow(/Missing path/i)
|
||||
expect(() => parseGitHubImportUrl('https://github.com/a/b/tree/main/bad%5cpath')).toThrow()
|
||||
})
|
||||
|
||||
it('parses tree urls with ref and path', () => {
|
||||
expect(parseGitHubImportUrl('https://github.com/a/b/tree/main/skills/foo')).toEqual({
|
||||
owner: 'a',
|
||||
repo: 'b',
|
||||
ref: 'main',
|
||||
path: 'skills/foo',
|
||||
originalUrl: 'https://github.com/a/b/tree/main/skills/foo',
|
||||
})
|
||||
})
|
||||
|
||||
it('parses blob urls and derives folder path', () => {
|
||||
expect(parseGitHubImportUrl('https://github.com/a/b/blob/main/skills/foo/SKILL.md')).toEqual({
|
||||
owner: 'a',
|
||||
repo: 'b',
|
||||
ref: 'main',
|
||||
path: 'skills/foo',
|
||||
originalUrl: 'https://github.com/a/b/blob/main/skills/foo/SKILL.md',
|
||||
})
|
||||
})
|
||||
|
||||
it('strips single top-level folder from GitHub zip entries', () => {
|
||||
const zip = buildGitHubZipForTests({
|
||||
'repo-1/skill/SKILL.md': 'Body',
|
||||
'repo-1/skill/a.txt': 'a',
|
||||
})
|
||||
const stripped = stripGitHubZipRoot(unzipSync(zip))
|
||||
expect(Object.keys(stripped).sort()).toEqual(['skill/SKILL.md', 'skill/a.txt'])
|
||||
})
|
||||
|
||||
it('keeps paths when zip has multiple top-level roots', () => {
|
||||
const zip = buildGitHubZipForTests({
|
||||
'a/SKILL.md': 'Body',
|
||||
'b/SKILL.md': 'Body',
|
||||
})
|
||||
const stripped = stripGitHubZipRoot(unzipSync(zip))
|
||||
expect(Object.keys(stripped).sort()).toEqual(['a/SKILL.md', 'b/SKILL.md'])
|
||||
})
|
||||
|
||||
it('detects candidates in a GitHub zip and strips the root folder', () => {
|
||||
const zip = buildGitHubZipForTests({
|
||||
'ouracli-123/SKILL.md': `---\nname: demo\ndescription: Hello\n---\nBody`,
|
||||
'ouracli-123/src/index.ts': 'export {}',
|
||||
})
|
||||
const stripped = stripGitHubZipRoot(unzipSync(zip))
|
||||
const candidates = detectGitHubImportCandidates(stripped)
|
||||
expect(candidates.map((c) => c.path)).toEqual([''])
|
||||
expect(candidates[0]?.name).toBe('demo')
|
||||
})
|
||||
|
||||
it('detects multiple candidates and supports skills.md', () => {
|
||||
const zip = buildGitHubZipForTests({
|
||||
'repo-1/alpha/SKILL.md': `---\nname: Alpha\n---\nBody`,
|
||||
'repo-1/beta/skills.md': `---\nname: Beta\n---\nBody`,
|
||||
'repo-1/readme.md': 'x',
|
||||
})
|
||||
const stripped = stripGitHubZipRoot(unzipSync(zip))
|
||||
const candidates = detectGitHubImportCandidates(stripped)
|
||||
expect(candidates.map((c) => c.path)).toEqual(['alpha', 'beta'])
|
||||
expect(candidates.map((c) => c.name)).toEqual(['Alpha', 'Beta'])
|
||||
})
|
||||
|
||||
it('computes default selection via markdown references', () => {
|
||||
const entries = {
|
||||
'skill/SKILL.md': `---\nname: demo\n---\nSee [usage](docs/usage.md) and .\nIgnore [web](https://example.com).`,
|
||||
'skill/docs/usage.md': `See [more](more.md)`,
|
||||
'skill/docs/more.md': `Ok`,
|
||||
'skill/img/logo.svg': `<svg/>`,
|
||||
'skill/extra.txt': 'not referenced',
|
||||
}
|
||||
const zip = buildGitHubZipForTests(
|
||||
Object.fromEntries(Object.entries(entries).map(([k, v]) => [`repo-1/${k}`, v])),
|
||||
)
|
||||
const raw = unzipSync(zip)
|
||||
const stripped = stripGitHubZipRoot(raw)
|
||||
const candidates = detectGitHubImportCandidates(stripped)
|
||||
const candidate = candidates.find((c) => c.path === 'skill')
|
||||
expect(candidate).toBeTruthy()
|
||||
if (!candidate) throw new Error('candidate not found')
|
||||
|
||||
const files = Object.entries(stripped)
|
||||
.filter(([path]) => path.startsWith('skill/'))
|
||||
.map(([path, bytes]) => ({ path, bytes }))
|
||||
const selected = computeDefaultSelectedPaths({ candidate, files })
|
||||
expect(selected).toContain('skill/SKILL.md')
|
||||
expect(selected).toContain('skill/docs/usage.md')
|
||||
expect(selected).toContain('skill/docs/more.md')
|
||||
expect(selected).toContain('skill/img/logo.svg')
|
||||
expect(selected).not.toContain('skill/extra.txt')
|
||||
})
|
||||
|
||||
it('does not select files outside skill folder (even when referenced)', () => {
|
||||
const entries = {
|
||||
'skill/SKILL.md': `See [outside](../outside.md) and [abs](/abs.md) and [mail](mailto:test@example.com).`,
|
||||
'outside.md': `secret`,
|
||||
'skill/docs/usage.md': `Ok`,
|
||||
}
|
||||
const zip = buildGitHubZipForTests(
|
||||
Object.fromEntries(Object.entries(entries).map(([k, v]) => [`repo-1/${k}`, v])),
|
||||
)
|
||||
const stripped = stripGitHubZipRoot(unzipSync(zip))
|
||||
const candidate = detectGitHubImportCandidates(stripped).find((c) => c.path === 'skill')
|
||||
expect(candidate).toBeTruthy()
|
||||
if (!candidate) throw new Error('candidate not found')
|
||||
const files = Object.entries(stripped).map(([path, bytes]) => ({ path, bytes }))
|
||||
const selected = computeDefaultSelectedPaths({ candidate, files })
|
||||
expect(selected).toContain('skill/SKILL.md')
|
||||
expect(selected).not.toContain('outside.md')
|
||||
})
|
||||
|
||||
it('extracts markdown targets with titles and angle brackets', () => {
|
||||
const targets = extractMarkdownRelativeTargets(
|
||||
`See [a](docs/usage.md "Title") and [b](<docs/my file.md>) and `,
|
||||
)
|
||||
expect(targets).toEqual(['docs/usage.md', 'docs/my file.md', 'img/logo.svg'])
|
||||
})
|
||||
|
||||
it('resolves markdown targets safely', () => {
|
||||
expect(resolveMarkdownTarget('a/SKILL.md', 'docs/usage.md')).toBe('a/docs/usage.md')
|
||||
expect(resolveMarkdownTarget('a/SKILL.md', '../oops.md')).toBeNull()
|
||||
expect(resolveMarkdownTarget('a/SKILL.md', '/abs.md')).toBeNull()
|
||||
expect(resolveMarkdownTarget('a/SKILL.md', 'docs/usage.md#section')).toBe('a/docs/usage.md')
|
||||
expect(resolveMarkdownTarget('a/SKILL.md', 'docs/usage.md?x=1')).toBe('a/docs/usage.md')
|
||||
})
|
||||
|
||||
it('resolves HEAD commit via redirect chain and refuses unexpected redirect hosts', async () => {
|
||||
const fetcher: typeof fetch = async (input) => {
|
||||
const url = requestInfoToUrlString(input)
|
||||
if (url.includes('/archive/HEAD.zip')) {
|
||||
return new Response(null, {
|
||||
status: 302,
|
||||
headers: {
|
||||
location:
|
||||
'https://codeload.github.com/a/b/zip/0123456789012345678901234567890123456789',
|
||||
},
|
||||
})
|
||||
}
|
||||
if (url.startsWith('https://codeload.github.com/a/b/zip/')) {
|
||||
return new Response(null, { status: 200 })
|
||||
}
|
||||
throw new Error(`Unexpected fetch: ${url}`)
|
||||
}
|
||||
const resolved = await resolveGitHubCommit(
|
||||
{ owner: 'a', repo: 'b', originalUrl: 'https://github.com/a/b' },
|
||||
fetcher,
|
||||
)
|
||||
expect(resolved.commit).toBe('0123456789012345678901234567890123456789')
|
||||
|
||||
const badFetcher: typeof fetch = async (input) => {
|
||||
const url = requestInfoToUrlString(input)
|
||||
if (url.includes('/archive/HEAD.zip')) {
|
||||
return new Response(null, {
|
||||
status: 302,
|
||||
headers: { location: 'https://evil.example/zip/abc' },
|
||||
})
|
||||
}
|
||||
throw new Error(`Unexpected fetch: ${url}`)
|
||||
}
|
||||
await expect(
|
||||
resolveGitHubCommit(
|
||||
{ owner: 'a', repo: 'b', originalUrl: 'https://github.com/a/b' },
|
||||
badFetcher,
|
||||
),
|
||||
).rejects.toThrow(/redirect/i)
|
||||
})
|
||||
|
||||
it('resolves explicit ref commit via GitHub API', async () => {
|
||||
const fetcher: typeof fetch = async (input) => {
|
||||
const url = requestInfoToUrlString(input)
|
||||
if (url.startsWith('https://api.github.com/repos/a/b/commits/')) {
|
||||
return new Response(JSON.stringify({ sha: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' }), {
|
||||
status: 200,
|
||||
})
|
||||
}
|
||||
throw new Error(`Unexpected fetch: ${url}`)
|
||||
}
|
||||
const resolved = await resolveGitHubCommit(
|
||||
{ owner: 'a', repo: 'b', ref: 'main', originalUrl: 'https://github.com/a/b' },
|
||||
fetcher,
|
||||
)
|
||||
expect(resolved.commit).toBe('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')
|
||||
})
|
||||
|
||||
it('enforces zip byte cap when content-length is too large', async () => {
|
||||
const resolved = {
|
||||
owner: 'a',
|
||||
repo: 'b',
|
||||
ref: 'main',
|
||||
commit: '0123456789012345678901234567890123456789',
|
||||
path: '',
|
||||
repoUrl: 'https://github.com/a/b',
|
||||
originalUrl: 'https://github.com/a/b',
|
||||
} as const
|
||||
const fetcher: typeof fetch = async () =>
|
||||
new Response(new Blob([new Uint8Array([1, 2, 3])]), {
|
||||
status: 200,
|
||||
headers: { 'content-length': String(999_999_999) },
|
||||
})
|
||||
await expect(fetchGitHubZipBytes(resolved, fetcher, { maxZipBytes: 10 })).rejects.toThrow(
|
||||
/too large/i,
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,425 @@
|
||||
import { TEXT_FILE_EXTENSION_SET } from 'clawhub-schema'
|
||||
import { zipSync } from 'fflate'
|
||||
import semver from 'semver'
|
||||
import { parseFrontmatter } from './skills'
|
||||
|
||||
export type GitHubImportUrl = {
|
||||
owner: string
|
||||
repo: string
|
||||
ref?: string
|
||||
path?: string
|
||||
originalUrl: string
|
||||
}
|
||||
|
||||
export type GitHubImportResolved = {
|
||||
owner: string
|
||||
repo: string
|
||||
ref: string
|
||||
commit: string
|
||||
path: string
|
||||
repoUrl: string
|
||||
originalUrl: string
|
||||
}
|
||||
|
||||
export type GitHubImportCandidate = {
|
||||
path: string
|
||||
readmePath: string
|
||||
name?: string
|
||||
description?: string
|
||||
}
|
||||
|
||||
export type GitHubImportFileEntry = {
|
||||
path: string
|
||||
size: number
|
||||
defaultSelected: boolean
|
||||
}
|
||||
|
||||
const MAX_REDIRECTS = 6
|
||||
const GITHUB_HOST = 'github.com'
|
||||
const CODELOAD_HOST = 'codeload.github.com'
|
||||
const SKILL_FILENAMES = ['skill.md', 'skills.md']
|
||||
|
||||
export function parseGitHubImportUrl(input: string): GitHubImportUrl {
|
||||
const originalUrl = input.trim()
|
||||
let url: URL
|
||||
try {
|
||||
url = new URL(originalUrl)
|
||||
} catch {
|
||||
throw new Error('Invalid URL')
|
||||
}
|
||||
if (url.protocol !== 'https:') throw new Error('Only https:// URLs are supported')
|
||||
if (url.hostname !== GITHUB_HOST) throw new Error('Only github.com URLs are supported')
|
||||
|
||||
const segments = url.pathname
|
||||
.split('/')
|
||||
.map((segment) => segment.trim())
|
||||
.filter(Boolean)
|
||||
.map((segment) => {
|
||||
try {
|
||||
return decodeURIComponent(segment)
|
||||
} catch {
|
||||
throw new Error('Invalid URL')
|
||||
}
|
||||
})
|
||||
|
||||
const owner = segments[0] ?? ''
|
||||
const repo = (segments[1] ?? '').replace(/\.git$/, '')
|
||||
if (!owner || !repo) throw new Error('GitHub URL must be /<owner>/<repo>')
|
||||
|
||||
const kind = segments[2] ?? ''
|
||||
if (!kind) return { owner, repo, originalUrl }
|
||||
if (kind !== 'tree' && kind !== 'blob') {
|
||||
return { owner, repo, originalUrl }
|
||||
}
|
||||
|
||||
const ref = segments[3] ?? ''
|
||||
if (!ref) throw new Error('Missing ref in GitHub URL')
|
||||
|
||||
const rest = segments.slice(4).join('/')
|
||||
const normalizedRest = normalizeRepoPath(rest)
|
||||
|
||||
if (kind === 'blob') {
|
||||
if (!rest) throw new Error('Missing path in GitHub URL')
|
||||
if (!normalizedRest) throw new Error('Invalid path in GitHub URL')
|
||||
const dir = normalizedRest.split('/').slice(0, -1).join('/')
|
||||
return { owner, repo, ref, path: dir || undefined, originalUrl }
|
||||
}
|
||||
|
||||
if (rest && !normalizedRest) throw new Error('Invalid path in GitHub URL')
|
||||
return { owner, repo, ref, path: normalizedRest || undefined, originalUrl }
|
||||
}
|
||||
|
||||
export async function resolveGitHubCommit(
|
||||
parsed: GitHubImportUrl,
|
||||
fetcher: typeof fetch,
|
||||
): Promise<GitHubImportResolved> {
|
||||
const repoUrl = `https://${GITHUB_HOST}/${parsed.owner}/${parsed.repo}`
|
||||
const ref = parsed.ref?.trim() || 'HEAD'
|
||||
const path = normalizeRepoPath(parsed.path ?? '')
|
||||
|
||||
const commit =
|
||||
ref === 'HEAD'
|
||||
? await resolveHeadCommit(parsed, fetcher)
|
||||
: await resolveRefCommit(parsed, ref, fetcher)
|
||||
|
||||
return {
|
||||
owner: parsed.owner,
|
||||
repo: parsed.repo,
|
||||
ref,
|
||||
commit,
|
||||
path,
|
||||
repoUrl,
|
||||
originalUrl: parsed.originalUrl,
|
||||
}
|
||||
}
|
||||
|
||||
async function resolveRefCommit(parsed: GitHubImportUrl, ref: string, fetcher: typeof fetch) {
|
||||
const apiUrl = `https://api.github.com/repos/${parsed.owner}/${parsed.repo}/commits/${encodeURIComponent(ref)}`
|
||||
const response = await fetcher(apiUrl, {
|
||||
headers: {
|
||||
Accept: 'application/vnd.github+json',
|
||||
'User-Agent': 'clawhub/github-import',
|
||||
},
|
||||
})
|
||||
if (!response.ok) throw new Error('GitHub ref not found')
|
||||
const body = (await response.json()) as { sha?: unknown }
|
||||
const sha = typeof body.sha === 'string' ? body.sha : ''
|
||||
if (!/^[a-f0-9]{40}$/i.test(sha)) throw new Error('GitHub commit sha missing')
|
||||
return sha.toLowerCase()
|
||||
}
|
||||
|
||||
async function resolveHeadCommit(parsed: GitHubImportUrl, fetcher: typeof fetch) {
|
||||
let url = `https://${GITHUB_HOST}/${parsed.owner}/${parsed.repo}/archive/HEAD.zip`
|
||||
for (let i = 0; i < MAX_REDIRECTS; i += 1) {
|
||||
const response = await fetcher(url, { redirect: 'manual' })
|
||||
const location = response.headers.get('location')
|
||||
if (!location) break
|
||||
const next = new URL(location, url)
|
||||
if (next.hostname !== GITHUB_HOST && next.hostname !== CODELOAD_HOST) {
|
||||
throw new Error('Unexpected redirect host')
|
||||
}
|
||||
url = next.toString()
|
||||
}
|
||||
|
||||
const maybe = url.split('/').at(-1) ?? ''
|
||||
if (!/^[a-f0-9]{40}$/i.test(maybe)) {
|
||||
throw new Error('Could not resolve commit for HEAD')
|
||||
}
|
||||
return maybe.toLowerCase()
|
||||
}
|
||||
|
||||
export async function fetchGitHubZipBytes(
|
||||
resolved: GitHubImportResolved,
|
||||
fetcher: typeof fetch,
|
||||
limits?: { maxZipBytes?: number },
|
||||
): Promise<Uint8Array> {
|
||||
const maxZipBytes = limits?.maxZipBytes ?? 25 * 1024 * 1024
|
||||
const url = `https://${CODELOAD_HOST}/${resolved.owner}/${resolved.repo}/zip/${resolved.commit}`
|
||||
const response = await fetcher(url, {
|
||||
headers: { 'User-Agent': 'clawhub/github-import' },
|
||||
})
|
||||
if (!response.ok) throw new Error('GitHub archive download failed')
|
||||
|
||||
const lengthHeader = response.headers.get('content-length')
|
||||
if (lengthHeader) {
|
||||
const contentLength = Number.parseInt(lengthHeader, 10)
|
||||
if (Number.isFinite(contentLength) && contentLength > maxZipBytes) {
|
||||
throw new Error('GitHub archive too large')
|
||||
}
|
||||
}
|
||||
|
||||
const reader = response.body?.getReader()
|
||||
if (!reader) {
|
||||
const buffer = new Uint8Array(await response.arrayBuffer())
|
||||
if (buffer.byteLength > maxZipBytes) throw new Error('GitHub archive too large')
|
||||
return buffer
|
||||
}
|
||||
|
||||
const chunks: Uint8Array[] = []
|
||||
let total = 0
|
||||
while (true) {
|
||||
const { done, value } = await reader.read()
|
||||
if (done) break
|
||||
if (!value) continue
|
||||
total += value.byteLength
|
||||
if (total > maxZipBytes) throw new Error('GitHub archive too large')
|
||||
chunks.push(value)
|
||||
}
|
||||
|
||||
const out = new Uint8Array(total)
|
||||
let offset = 0
|
||||
for (const chunk of chunks) {
|
||||
out.set(chunk, offset)
|
||||
offset += chunk.byteLength
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
export type ZipEntryMap = Record<string, Uint8Array>
|
||||
|
||||
export function buildGitHubZipForTests(entries: Record<string, string>) {
|
||||
const asBytes = Object.fromEntries(
|
||||
Object.entries(entries).map(([path, text]) => [path, new TextEncoder().encode(text)]),
|
||||
)
|
||||
return Uint8Array.from(zipSync(asBytes, { level: 1 }))
|
||||
}
|
||||
|
||||
export function stripGitHubZipRoot(entries: ZipEntryMap): ZipEntryMap {
|
||||
const paths = Object.keys(entries)
|
||||
if (paths.length === 0) return {}
|
||||
const first = paths[0] ?? ''
|
||||
const firstRoot = first.split('/')[0] ?? ''
|
||||
if (!firstRoot) return entries
|
||||
const prefix = `${firstRoot}/`
|
||||
if (!paths.every((path) => path.startsWith(prefix))) return entries
|
||||
const out: ZipEntryMap = {}
|
||||
for (const [path, data] of Object.entries(entries)) {
|
||||
const stripped = path.slice(prefix.length)
|
||||
if (!stripped) continue
|
||||
out[stripped] = data
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
export function detectGitHubImportCandidates(entries: ZipEntryMap): GitHubImportCandidate[] {
|
||||
const candidates: GitHubImportCandidate[] = []
|
||||
for (const path of Object.keys(entries)) {
|
||||
const normalized = normalizeRepoPath(path)
|
||||
const lower = normalized.toLowerCase()
|
||||
const isSkill = SKILL_FILENAMES.some((name) => lower === name || lower.endsWith(`/${name}`))
|
||||
if (!isSkill) continue
|
||||
const dir = normalized.split('/').slice(0, -1).join('/')
|
||||
const readmePath = normalized
|
||||
const raw = new TextDecoder().decode(entries[path] ?? new Uint8Array())
|
||||
const frontmatter = parseFrontmatter(raw)
|
||||
const name = typeof frontmatter.name === 'string' ? frontmatter.name : undefined
|
||||
const description =
|
||||
typeof frontmatter.description === 'string' ? frontmatter.description : undefined
|
||||
candidates.push({
|
||||
path: normalizeRepoPath(dir),
|
||||
readmePath,
|
||||
name: name?.trim() || undefined,
|
||||
description: description?.trim() || undefined,
|
||||
})
|
||||
}
|
||||
return uniqCandidates(candidates)
|
||||
}
|
||||
|
||||
function uniqCandidates(candidates: GitHubImportCandidate[]) {
|
||||
const seen = new Set<string>()
|
||||
const out: GitHubImportCandidate[] = []
|
||||
for (const candidate of candidates) {
|
||||
const key = `${candidate.path}::${candidate.readmePath}`
|
||||
if (seen.has(key)) continue
|
||||
seen.add(key)
|
||||
out.push(candidate)
|
||||
}
|
||||
return out.sort((a, b) => a.path.localeCompare(b.path))
|
||||
}
|
||||
|
||||
export function listTextFilesUnderCandidate(
|
||||
entries: ZipEntryMap,
|
||||
candidatePath: string,
|
||||
): Array<{ path: string; bytes: Uint8Array }> {
|
||||
const root = normalizeCandidateRoot(candidatePath)
|
||||
const out: Array<{ path: string; bytes: Uint8Array }> = []
|
||||
for (const [path, bytes] of Object.entries(entries)) {
|
||||
const normalized = normalizeRepoPath(path)
|
||||
if (!isUnderRoot(normalized, root)) continue
|
||||
if (!isTextPath(normalized)) continue
|
||||
out.push({ path: normalized, bytes })
|
||||
}
|
||||
return out.sort((a, b) => a.path.localeCompare(b.path))
|
||||
}
|
||||
|
||||
export function computeDefaultSelectedPaths(params: {
|
||||
candidate: GitHubImportCandidate
|
||||
files: Array<{ path: string; bytes: Uint8Array }>
|
||||
maxDepth?: number
|
||||
maxAdds?: number
|
||||
}) {
|
||||
const maxDepth = params.maxDepth ?? 4
|
||||
const maxAdds = params.maxAdds ?? 200
|
||||
const byPath = new Map(params.files.map((file) => [file.path, file.bytes]))
|
||||
const candidateRoot = normalizeCandidateRoot(params.candidate.path)
|
||||
const selected = new Set<string>()
|
||||
let added = 0
|
||||
|
||||
const add = (path: string) => {
|
||||
const normalized = normalizeRepoPath(path)
|
||||
if (!isUnderRoot(normalized, candidateRoot)) return
|
||||
if (!byPath.has(normalized)) return
|
||||
if (!selected.has(normalized)) {
|
||||
selected.add(normalized)
|
||||
added += 1
|
||||
}
|
||||
}
|
||||
|
||||
add(params.candidate.readmePath)
|
||||
|
||||
const visited = new Set<string>()
|
||||
const queue: Array<{ path: string; depth: number }> = [
|
||||
{ path: params.candidate.readmePath, depth: 0 },
|
||||
]
|
||||
|
||||
while (queue.length > 0) {
|
||||
const item = queue.shift()
|
||||
if (!item) break
|
||||
if (item.depth >= maxDepth) continue
|
||||
if (visited.has(item.path)) continue
|
||||
visited.add(item.path)
|
||||
|
||||
const bytes = byPath.get(item.path)
|
||||
if (!bytes) continue
|
||||
if (!item.path.toLowerCase().endsWith('.md')) continue
|
||||
|
||||
const text = new TextDecoder().decode(bytes)
|
||||
const refs = extractMarkdownRelativeTargets(text)
|
||||
for (const ref of refs) {
|
||||
if (added >= maxAdds) break
|
||||
const resolved = resolveMarkdownTarget(item.path, ref)
|
||||
if (!resolved) continue
|
||||
add(resolved)
|
||||
if (resolved.toLowerCase().endsWith('.md') && byPath.has(resolved)) {
|
||||
queue.push({ path: resolved, depth: item.depth + 1 })
|
||||
}
|
||||
}
|
||||
if (added >= maxAdds) break
|
||||
}
|
||||
|
||||
return Array.from(selected).sort()
|
||||
}
|
||||
|
||||
export function buildGitHubImportFileList(params: {
|
||||
candidate: GitHubImportCandidate
|
||||
files: Array<{ path: string; bytes: Uint8Array }>
|
||||
defaultSelectedPaths: string[]
|
||||
}): GitHubImportFileEntry[] {
|
||||
const selected = new Set(params.defaultSelectedPaths)
|
||||
return params.files.map((file) => ({
|
||||
path: file.path,
|
||||
size: file.bytes.byteLength,
|
||||
defaultSelected: selected.has(file.path),
|
||||
}))
|
||||
}
|
||||
|
||||
export function normalizeRepoPath(path: string) {
|
||||
const stripped = path.replace(/^\/+/, '').trim()
|
||||
if (!stripped) return ''
|
||||
const cleaned = stripped.split('/').filter(Boolean).join('/')
|
||||
if (!cleaned || cleaned.includes('\\') || cleaned.includes('..')) return ''
|
||||
return cleaned
|
||||
}
|
||||
|
||||
export function normalizeCandidateRoot(candidatePath: string) {
|
||||
const normalized = normalizeRepoPath(candidatePath)
|
||||
return normalized ? `${normalized}/` : ''
|
||||
}
|
||||
|
||||
function isUnderRoot(path: string, rootWithSlash: string) {
|
||||
if (!rootWithSlash) return true
|
||||
return path === rootWithSlash.slice(0, -1) || path.startsWith(rootWithSlash)
|
||||
}
|
||||
|
||||
function isTextPath(path: string) {
|
||||
const lower = path.toLowerCase()
|
||||
const ext = lower.split('.').at(-1) ?? ''
|
||||
if (!ext) return false
|
||||
return TEXT_FILE_EXTENSION_SET.has(ext)
|
||||
}
|
||||
|
||||
export function suggestDisplayName(candidate: GitHubImportCandidate, fallbackBase: string) {
|
||||
const base = candidate.name?.trim() || fallbackBase.trim()
|
||||
if (!base) return ''
|
||||
return base
|
||||
.replace(/[-_]+/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.replace(/\b\w/g, (char) => char.toUpperCase())
|
||||
}
|
||||
|
||||
export function suggestVersion(latestVersion?: string | null) {
|
||||
const latest = latestVersion?.trim() || ''
|
||||
if (latest && semver.valid(latest)) {
|
||||
return semver.inc(latest, 'patch') ?? '0.1.0'
|
||||
}
|
||||
return '0.1.0'
|
||||
}
|
||||
|
||||
export function extractMarkdownRelativeTargets(markdown: string): string[] {
|
||||
const out: string[] = []
|
||||
const pattern = /!?\[[^\]]*]\(([^)]+)\)/g
|
||||
for (const match of markdown.matchAll(pattern)) {
|
||||
const raw = (match[1] ?? '').trim()
|
||||
if (!raw) continue
|
||||
const isAngleWrapped = raw.startsWith('<') && raw.endsWith('>')
|
||||
const cleaned = raw.replace(/^<|>$/g, '').trim()
|
||||
if (!cleaned) continue
|
||||
const target = isAngleWrapped ? cleaned : (cleaned.split(/\s+/)[0] ?? '')
|
||||
if (!target) continue
|
||||
if (target.startsWith('#')) continue
|
||||
const lower = target.toLowerCase()
|
||||
if (lower.startsWith('http:') || lower.startsWith('https:')) continue
|
||||
if (lower.startsWith('mailto:')) continue
|
||||
out.push(target)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
export function resolveMarkdownTarget(fromPath: string, target: string) {
|
||||
const withoutHash = target.split('#')[0] ?? ''
|
||||
const withoutQuery = (withoutHash.split('?')[0] ?? '').trim()
|
||||
if (!withoutQuery) return null
|
||||
if (withoutQuery.startsWith('/')) return null
|
||||
if (withoutQuery.includes('\\') || withoutQuery.includes('..')) return null
|
||||
|
||||
const fromDirParts = normalizeRepoPath(fromPath).split('/').slice(0, -1)
|
||||
const targetParts = withoutQuery.split('/').filter(Boolean)
|
||||
const combined = [...fromDirParts, ...targetParts]
|
||||
const normalized: string[] = []
|
||||
for (const part of combined) {
|
||||
if (part === '.') continue
|
||||
if (part === '..') return null
|
||||
normalized.push(part)
|
||||
}
|
||||
return normalizeRepoPath(normalized.join('/')) || null
|
||||
}
|
||||
@@ -0,0 +1,443 @@
|
||||
'use node'
|
||||
|
||||
import { createPrivateKey, createSign } from 'node:crypto'
|
||||
import type { Id } from '../_generated/dataModel'
|
||||
import type { ActionCtx } from '../_generated/server'
|
||||
|
||||
const GITHUB_API = 'https://api.github.com'
|
||||
const DEFAULT_REPO = 'clawdbot/souls'
|
||||
const DEFAULT_ROOT = 'souls'
|
||||
const META_FILENAME = '_meta.json'
|
||||
const USER_AGENT = 'clawhub/souls-backup'
|
||||
|
||||
type BackupFile = {
|
||||
path: string
|
||||
size: number
|
||||
storageId: Id<'_storage'>
|
||||
sha256: string
|
||||
contentType?: string
|
||||
}
|
||||
|
||||
type BackupParams = {
|
||||
slug: string
|
||||
version: string
|
||||
displayName: string
|
||||
ownerHandle: string
|
||||
files: BackupFile[]
|
||||
publishedAt: number
|
||||
}
|
||||
|
||||
type RepoInfo = {
|
||||
default_branch?: string
|
||||
}
|
||||
|
||||
type GitRef = {
|
||||
object: { sha: string }
|
||||
}
|
||||
|
||||
type GitCommit = {
|
||||
sha: string
|
||||
tree: { sha: string }
|
||||
}
|
||||
|
||||
type GitTreeEntry = {
|
||||
path?: string
|
||||
type?: string
|
||||
}
|
||||
|
||||
type GitTree = {
|
||||
tree?: GitTreeEntry[]
|
||||
}
|
||||
|
||||
type MetaFile = {
|
||||
owner: string
|
||||
slug: string
|
||||
displayName: string
|
||||
latest: {
|
||||
version: string
|
||||
publishedAt: number
|
||||
commit: string | null
|
||||
}
|
||||
history: Array<{
|
||||
version: string
|
||||
publishedAt: number
|
||||
commit: string
|
||||
}>
|
||||
}
|
||||
|
||||
export type GitHubBackupContext = {
|
||||
token: string
|
||||
repo: string
|
||||
repoOwner: string
|
||||
repoName: string
|
||||
branch: string
|
||||
root: string
|
||||
}
|
||||
|
||||
export function isGitHubSoulBackupConfigured() {
|
||||
return Boolean(
|
||||
process.env.GITHUB_APP_ID &&
|
||||
process.env.GITHUB_APP_PRIVATE_KEY &&
|
||||
process.env.GITHUB_APP_INSTALLATION_ID,
|
||||
)
|
||||
}
|
||||
|
||||
export async function getGitHubSoulBackupContext(): Promise<GitHubBackupContext> {
|
||||
const repo = process.env.GITHUB_SOULS_REPO ?? DEFAULT_REPO
|
||||
const root = process.env.GITHUB_SOULS_ROOT ?? DEFAULT_ROOT
|
||||
const [repoOwner, repoName] = parseRepo(repo)
|
||||
const token = await createInstallationToken()
|
||||
const repoInfo = await githubGet<RepoInfo>(token, `/repos/${repoOwner}/${repoName}`)
|
||||
const branch = repoInfo.default_branch ?? 'main'
|
||||
|
||||
return { token, repo, repoOwner, repoName, branch, root }
|
||||
}
|
||||
|
||||
export async function fetchGitHubSoulMeta(
|
||||
context: GitHubBackupContext,
|
||||
ownerHandle: string,
|
||||
slug: string,
|
||||
): Promise<MetaFile | null> {
|
||||
const soulRoot = buildSoulRoot(context.root, ownerHandle, slug)
|
||||
return fetchMetaFile(
|
||||
context.token,
|
||||
context.repoOwner,
|
||||
context.repoName,
|
||||
`${soulRoot}/${META_FILENAME}`,
|
||||
context.branch,
|
||||
)
|
||||
}
|
||||
|
||||
export async function backupSoulToGitHub(
|
||||
ctx: ActionCtx,
|
||||
params: BackupParams,
|
||||
context?: GitHubBackupContext,
|
||||
) {
|
||||
if (!isGitHubSoulBackupConfigured()) return
|
||||
|
||||
const resolved = context ?? (await getGitHubSoulBackupContext())
|
||||
const soulRoot = buildSoulRoot(resolved.root, params.ownerHandle, params.slug)
|
||||
const ref = await githubGet<GitRef>(
|
||||
resolved.token,
|
||||
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/ref/heads/${resolved.branch}`,
|
||||
)
|
||||
const baseCommitSha = ref.object.sha
|
||||
const baseCommit = await githubGet<GitCommit>(
|
||||
resolved.token,
|
||||
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/commits/${baseCommitSha}`,
|
||||
)
|
||||
const baseTreeSha = baseCommit.tree.sha
|
||||
const existingTree = await githubGet<GitTree>(
|
||||
resolved.token,
|
||||
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/trees/${baseTreeSha}?recursive=1`,
|
||||
)
|
||||
|
||||
const prefix = `${soulRoot}/`
|
||||
const existingPaths = new Set(
|
||||
(existingTree.tree ?? [])
|
||||
.filter((entry) => entry.type === 'blob' && entry.path?.startsWith(prefix))
|
||||
.map((entry) => entry.path ?? ''),
|
||||
)
|
||||
|
||||
const newPaths = new Set<string>()
|
||||
const treeEntries: Array<{
|
||||
path: string
|
||||
mode: '100644'
|
||||
type: 'blob'
|
||||
sha: string | null
|
||||
}> = []
|
||||
|
||||
for (const file of params.files) {
|
||||
const content = await fetchStorageBase64(ctx, file.storageId)
|
||||
const blobSha = await createBlob(resolved.token, resolved.repoOwner, resolved.repoName, content)
|
||||
const path = `${soulRoot}/${file.path}`
|
||||
newPaths.add(path)
|
||||
treeEntries.push({ path, mode: '100644', type: 'blob', sha: blobSha })
|
||||
}
|
||||
|
||||
const existingMeta = await fetchMetaFile(
|
||||
resolved.token,
|
||||
resolved.repoOwner,
|
||||
resolved.repoName,
|
||||
`${soulRoot}/${META_FILENAME}`,
|
||||
resolved.branch,
|
||||
)
|
||||
const metaPath = `${soulRoot}/${META_FILENAME}`
|
||||
const metaDraft = buildMetaFile(params, existingMeta, resolved.repo, baseCommitSha, null)
|
||||
const metaDraftContent = `${JSON.stringify(metaDraft, null, 2)}\n`
|
||||
const metaDraftSha = await createBlob(
|
||||
resolved.token,
|
||||
resolved.repoOwner,
|
||||
resolved.repoName,
|
||||
toBase64(metaDraftContent),
|
||||
)
|
||||
newPaths.add(metaPath)
|
||||
treeEntries.push({ path: metaPath, mode: '100644', type: 'blob', sha: metaDraftSha })
|
||||
|
||||
for (const path of existingPaths) {
|
||||
if (newPaths.has(path)) continue
|
||||
treeEntries.push({ path, mode: '100644', type: 'blob', sha: null })
|
||||
}
|
||||
|
||||
const newTree = await githubPost<{ sha: string }>(
|
||||
resolved.token,
|
||||
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/trees`,
|
||||
{
|
||||
base_tree: baseTreeSha,
|
||||
tree: treeEntries,
|
||||
},
|
||||
)
|
||||
|
||||
const commit = await githubPost<GitCommit>(
|
||||
resolved.token,
|
||||
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/commits`,
|
||||
{
|
||||
message: `soul: ${params.slug} v${params.version}`,
|
||||
tree: newTree.sha,
|
||||
parents: [baseCommitSha],
|
||||
},
|
||||
)
|
||||
|
||||
const metaFinal = buildMetaFile(params, existingMeta, resolved.repo, baseCommitSha, commit.sha)
|
||||
const metaFinalContent = `${JSON.stringify(metaFinal, null, 2)}\n`
|
||||
const metaFinalSha = await createBlob(
|
||||
resolved.token,
|
||||
resolved.repoOwner,
|
||||
resolved.repoName,
|
||||
toBase64(metaFinalContent),
|
||||
)
|
||||
const metaTree = await githubPost<{ sha: string }>(
|
||||
resolved.token,
|
||||
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/trees`,
|
||||
{
|
||||
base_tree: commit.tree.sha,
|
||||
tree: [{ path: metaPath, mode: '100644', type: 'blob', sha: metaFinalSha }],
|
||||
},
|
||||
)
|
||||
const metaCommit = await githubPost<GitCommit>(
|
||||
resolved.token,
|
||||
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/commits`,
|
||||
{
|
||||
message: `meta: ${params.slug} v${params.version}`,
|
||||
tree: metaTree.sha,
|
||||
parents: [commit.sha],
|
||||
},
|
||||
)
|
||||
|
||||
await githubPatch(
|
||||
resolved.token,
|
||||
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/refs/heads/${resolved.branch}`,
|
||||
{
|
||||
sha: metaCommit.sha,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
function buildMetaFile(
|
||||
params: BackupParams,
|
||||
existing: MetaFile | null,
|
||||
repo: string,
|
||||
baseCommitSha: string,
|
||||
latestCommitSha: string | null,
|
||||
): MetaFile {
|
||||
let history = [...(existing?.history ?? [])]
|
||||
if (existing?.latest?.version) {
|
||||
const previousCommit = existing.latest.commit ?? commitUrl(repo, baseCommitSha)
|
||||
const previous = {
|
||||
version: existing.latest.version,
|
||||
publishedAt: existing.latest.publishedAt,
|
||||
commit: previousCommit,
|
||||
}
|
||||
history = [previous, ...history.filter((entry) => entry.version !== previous.version)]
|
||||
}
|
||||
|
||||
return {
|
||||
owner: normalizeOwner(params.ownerHandle),
|
||||
slug: params.slug,
|
||||
displayName: params.displayName,
|
||||
latest: {
|
||||
version: params.version,
|
||||
publishedAt: params.publishedAt,
|
||||
commit: latestCommitSha ? commitUrl(repo, latestCommitSha) : null,
|
||||
},
|
||||
history: history.slice(0, 200),
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchMetaFile(
|
||||
token: string,
|
||||
repoOwner: string,
|
||||
repoName: string,
|
||||
path: string,
|
||||
branch: string,
|
||||
): Promise<MetaFile | null> {
|
||||
try {
|
||||
const response = await githubGet<{ content?: string }>(
|
||||
token,
|
||||
`/repos/${repoOwner}/${repoName}/contents/${encodePath(path)}?ref=${branch}`,
|
||||
)
|
||||
if (!response.content) return null
|
||||
const raw = fromBase64(response.content)
|
||||
return JSON.parse(raw) as MetaFile
|
||||
} catch (error) {
|
||||
if (isNotFoundError(error)) return null
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchStorageBase64(ctx: ActionCtx, storageId: Id<'_storage'>) {
|
||||
const blob = await ctx.storage.get(storageId)
|
||||
if (!blob) throw new Error('File missing in storage')
|
||||
const buffer = Buffer.from(await blob.arrayBuffer())
|
||||
return buffer.toString('base64')
|
||||
}
|
||||
|
||||
async function createInstallationToken() {
|
||||
const appId = process.env.GITHUB_APP_ID
|
||||
const installationId = process.env.GITHUB_APP_INSTALLATION_ID
|
||||
if (!appId || !installationId) {
|
||||
throw new Error('GitHub App credentials missing')
|
||||
}
|
||||
const jwt = createAppJwt(appId)
|
||||
const response = await fetch(`${GITHUB_API}/app/installations/${installationId}/access_tokens`, {
|
||||
method: 'POST',
|
||||
headers: buildHeaders(jwt, true),
|
||||
})
|
||||
if (!response.ok) {
|
||||
const message = await response.text()
|
||||
throw new Error(`GitHub App token failed: ${message}`)
|
||||
}
|
||||
const payload = (await response.json()) as { token?: string }
|
||||
if (!payload.token) throw new Error('GitHub App token missing')
|
||||
return payload.token
|
||||
}
|
||||
|
||||
function createAppJwt(appId: string) {
|
||||
const privateKey = loadPrivateKey()
|
||||
const now = Math.floor(Date.now() / 1000)
|
||||
const header = { alg: 'RS256', typ: 'JWT' }
|
||||
const payload = { iat: now - 60, exp: now + 9 * 60, iss: appId }
|
||||
const encodedHeader = base64Url(JSON.stringify(header))
|
||||
const encodedPayload = base64Url(JSON.stringify(payload))
|
||||
const signingInput = `${encodedHeader}.${encodedPayload}`
|
||||
const sign = createSign('RSA-SHA256')
|
||||
sign.update(signingInput)
|
||||
sign.end()
|
||||
const signature = sign.sign(privateKey)
|
||||
return `${signingInput}.${base64Url(signature)}`
|
||||
}
|
||||
|
||||
function loadPrivateKey() {
|
||||
const raw = process.env.GITHUB_APP_PRIVATE_KEY
|
||||
if (!raw) throw new Error('GITHUB_APP_PRIVATE_KEY is not configured')
|
||||
const normalized = raw.replace(/\\n/g, '\n')
|
||||
return createPrivateKey(normalized)
|
||||
}
|
||||
|
||||
async function createBlob(token: string, repoOwner: string, repoName: string, content: string) {
|
||||
const result = await githubPost<{ sha: string }>(
|
||||
token,
|
||||
`/repos/${repoOwner}/${repoName}/git/blobs`,
|
||||
{
|
||||
content,
|
||||
encoding: 'base64',
|
||||
},
|
||||
)
|
||||
if (!result.sha) throw new Error('GitHub blob missing sha')
|
||||
return result.sha
|
||||
}
|
||||
|
||||
async function githubGet<T>(token: string, path: string): Promise<T> {
|
||||
const response = await fetch(`${GITHUB_API}${path}`, {
|
||||
headers: buildHeaders(token),
|
||||
})
|
||||
if (!response.ok) {
|
||||
const message = await response.text()
|
||||
throw new Error(`GitHub GET ${path} failed: ${message}`)
|
||||
}
|
||||
return (await response.json()) as T
|
||||
}
|
||||
|
||||
async function githubPost<T>(token: string, path: string, body: unknown): Promise<T> {
|
||||
const response = await fetch(`${GITHUB_API}${path}`, {
|
||||
method: 'POST',
|
||||
headers: buildHeaders(token),
|
||||
body: JSON.stringify(body),
|
||||
})
|
||||
if (!response.ok) {
|
||||
const message = await response.text()
|
||||
throw new Error(`GitHub POST ${path} failed: ${message}`)
|
||||
}
|
||||
return (await response.json()) as T
|
||||
}
|
||||
|
||||
async function githubPatch(token: string, path: string, body: unknown) {
|
||||
const response = await fetch(`${GITHUB_API}${path}`, {
|
||||
method: 'PATCH',
|
||||
headers: buildHeaders(token),
|
||||
body: JSON.stringify(body),
|
||||
})
|
||||
if (!response.ok) {
|
||||
const message = await response.text()
|
||||
throw new Error(`GitHub PATCH ${path} failed: ${message}`)
|
||||
}
|
||||
}
|
||||
|
||||
function buildHeaders(token: string, isAppJwt = false) {
|
||||
return {
|
||||
Authorization: `${isAppJwt ? 'Bearer' : 'token'} ${token}`,
|
||||
Accept: 'application/vnd.github+json',
|
||||
'User-Agent': USER_AGENT,
|
||||
}
|
||||
}
|
||||
|
||||
function parseRepo(repo: string) {
|
||||
const [owner, name] = repo.split('/')
|
||||
if (!owner || !name) throw new Error('GITHUB_SOULS_REPO must be owner/repo')
|
||||
return [owner, name] as const
|
||||
}
|
||||
|
||||
function normalizeOwner(value: string) {
|
||||
const normalized = value
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9-]/g, '-')
|
||||
.replace(/-+/g, '-')
|
||||
.replace(/^-+|-+$/g, '')
|
||||
return normalized || 'unknown'
|
||||
}
|
||||
|
||||
function commitUrl(repo: string, sha: string) {
|
||||
return `https://github.com/${repo}/commit/${sha}`
|
||||
}
|
||||
|
||||
function buildSoulRoot(root: string, ownerHandle: string, slug: string) {
|
||||
const ownerSegment = normalizeOwner(ownerHandle)
|
||||
return `${root}/${ownerSegment}/${slug}`
|
||||
}
|
||||
|
||||
function encodePath(path: string) {
|
||||
return path
|
||||
.split('/')
|
||||
.map((segment) => encodeURIComponent(segment))
|
||||
.join('/')
|
||||
}
|
||||
|
||||
function base64Url(value: string | Buffer) {
|
||||
const buffer = typeof value === 'string' ? Buffer.from(value) : value
|
||||
return buffer.toString('base64').replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/g, '')
|
||||
}
|
||||
|
||||
function toBase64(value: string) {
|
||||
return Buffer.from(value).toString('base64')
|
||||
}
|
||||
|
||||
function fromBase64(value: string) {
|
||||
return Buffer.from(value, 'base64').toString('utf8')
|
||||
}
|
||||
|
||||
function isNotFoundError(error: unknown) {
|
||||
return (
|
||||
error instanceof Error && (error.message.includes('404') || error.message.includes('Not Found'))
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
import type { Id } from '../_generated/dataModel'
|
||||
import type { MutationCtx, QueryCtx } from '../_generated/server'
|
||||
|
||||
const DAY_MS = 24 * 60 * 60 * 1000
|
||||
export const TRENDING_DAYS = 7
|
||||
|
||||
type LeaderboardEntry = {
|
||||
skillId: Id<'skills'>
|
||||
score: number
|
||||
installs: number
|
||||
downloads: number
|
||||
}
|
||||
|
||||
export function toDayKey(timestamp: number) {
|
||||
return Math.floor(timestamp / DAY_MS)
|
||||
}
|
||||
|
||||
export function getTrendingRange(now: number) {
|
||||
const endDay = toDayKey(now)
|
||||
const startDay = endDay - (TRENDING_DAYS - 1)
|
||||
return { startDay, endDay }
|
||||
}
|
||||
|
||||
export async function buildTrendingLeaderboard(
|
||||
ctx: QueryCtx | MutationCtx,
|
||||
params: { limit: number; now?: number },
|
||||
) {
|
||||
const now = params.now ?? Date.now()
|
||||
const { startDay, endDay } = getTrendingRange(now)
|
||||
const rows = await ctx.db
|
||||
.query('skillDailyStats')
|
||||
.withIndex('by_day', (q) => q.gte('day', startDay).lte('day', endDay))
|
||||
.collect()
|
||||
|
||||
const totals = new Map<Id<'skills'>, { installs: number; downloads: number }>()
|
||||
for (const row of rows) {
|
||||
const current = totals.get(row.skillId) ?? { installs: 0, downloads: 0 }
|
||||
current.installs += row.installs
|
||||
current.downloads += row.downloads
|
||||
totals.set(row.skillId, current)
|
||||
}
|
||||
|
||||
const entries = Array.from(totals, ([skillId, totalsEntry]) => ({
|
||||
skillId,
|
||||
installs: totalsEntry.installs,
|
||||
downloads: totalsEntry.downloads,
|
||||
score: totalsEntry.installs,
|
||||
}))
|
||||
|
||||
const items = topN(entries, params.limit, compareTrendingEntries).sort((a, b) =>
|
||||
compareTrendingEntries(b, a),
|
||||
)
|
||||
|
||||
return { startDay, endDay, items }
|
||||
}
|
||||
|
||||
function compareTrendingEntries(a: LeaderboardEntry, b: LeaderboardEntry) {
|
||||
if (a.score !== b.score) return a.score - b.score
|
||||
if (a.downloads !== b.downloads) return a.downloads - b.downloads
|
||||
return 0
|
||||
}
|
||||
|
||||
function topN<T>(entries: T[], limit: number, compare: (a: T, b: T) => number) {
|
||||
if (entries.length <= limit) return entries.slice()
|
||||
|
||||
const heap: T[] = []
|
||||
for (const entry of entries) {
|
||||
if (heap.length < limit) {
|
||||
heap.push(entry)
|
||||
siftUp(heap, heap.length - 1, compare)
|
||||
continue
|
||||
}
|
||||
if (compare(entry, heap[0]) <= 0) continue
|
||||
heap[0] = entry
|
||||
siftDown(heap, 0, compare)
|
||||
}
|
||||
return heap
|
||||
}
|
||||
|
||||
function siftUp<T>(heap: T[], index: number, compare: (a: T, b: T) => number) {
|
||||
let current = index
|
||||
while (current > 0) {
|
||||
const parent = Math.floor((current - 1) / 2)
|
||||
if (compare(heap[current], heap[parent]) >= 0) break
|
||||
;[heap[current], heap[parent]] = [heap[parent], heap[current]]
|
||||
current = parent
|
||||
}
|
||||
}
|
||||
|
||||
function siftDown<T>(heap: T[], index: number, compare: (a: T, b: T) => number) {
|
||||
let current = index
|
||||
const length = heap.length
|
||||
while (true) {
|
||||
const left = current * 2 + 1
|
||||
const right = current * 2 + 2
|
||||
let smallest = current
|
||||
if (left < length && compare(heap[left], heap[smallest]) < 0) smallest = left
|
||||
if (right < length && compare(heap[right], heap[smallest]) < 0) smallest = right
|
||||
if (smallest === current) break
|
||||
;[heap[current], heap[smallest]] = [heap[smallest], heap[current]]
|
||||
current = smallest
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import type { Doc } from '../_generated/dataModel'
|
||||
|
||||
const FLAG_RULES: Array<{ flag: string; pattern: RegExp }> = [
|
||||
// Known-bad / known-suspicious identifiers.
|
||||
// NOTE: keep these narrowly scoped; use staff review to confirm removals.
|
||||
{
|
||||
flag: 'blocked.malware',
|
||||
pattern: /(keepcold131\/ClawdAuthenticatorTool|ClawdAuthenticatorTool)/i,
|
||||
},
|
||||
|
||||
{ flag: 'suspicious.keyword', pattern: /(malware|stealer|phish|phishing|keylogger)/i },
|
||||
{ flag: 'suspicious.secrets', pattern: /(api[-_ ]?key|token|password|private key|secret)/i },
|
||||
{ flag: 'suspicious.crypto', pattern: /(wallet|seed phrase|mnemonic|crypto)/i },
|
||||
{ flag: 'suspicious.webhook', pattern: /(discord\.gg|webhook|hooks\.slack)/i },
|
||||
{ flag: 'suspicious.script', pattern: /(curl[^\n]+\|\s*(sh|bash))/i },
|
||||
{ flag: 'suspicious.url_shortener', pattern: /(bit\.ly|tinyurl\.com|t\.co|goo\.gl|is\.gd)/i },
|
||||
]
|
||||
|
||||
export function deriveModerationFlags({
|
||||
skill,
|
||||
parsed,
|
||||
files,
|
||||
}: {
|
||||
skill: Pick<Doc<'skills'>, 'slug' | 'displayName' | 'summary'>
|
||||
parsed: Doc<'skillVersions'>['parsed']
|
||||
files: Doc<'skillVersions'>['files']
|
||||
}) {
|
||||
const text = [
|
||||
skill.slug,
|
||||
skill.displayName,
|
||||
skill.summary ?? '',
|
||||
JSON.stringify(parsed?.frontmatter ?? {}),
|
||||
JSON.stringify(parsed?.metadata ?? {}),
|
||||
JSON.stringify((parsed as { moltbot?: unknown } | undefined)?.moltbot ?? {}),
|
||||
...files.map((file) => file.path),
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join('\n')
|
||||
|
||||
const flags = new Set<string>()
|
||||
|
||||
for (const rule of FLAG_RULES) {
|
||||
if (rule.pattern.test(text)) {
|
||||
flags.add(rule.flag)
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(flags)
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
import type { Doc } from '../_generated/dataModel'
|
||||
|
||||
export type PublicUser = Pick<
|
||||
Doc<'users'>,
|
||||
'_id' | '_creationTime' | 'handle' | 'name' | 'displayName' | 'image' | 'bio'
|
||||
>
|
||||
|
||||
export type PublicSkill = Pick<
|
||||
Doc<'skills'>,
|
||||
| '_id'
|
||||
| '_creationTime'
|
||||
| 'slug'
|
||||
| 'displayName'
|
||||
| 'summary'
|
||||
| 'ownerUserId'
|
||||
| 'canonicalSkillId'
|
||||
| 'forkOf'
|
||||
| 'latestVersionId'
|
||||
| 'tags'
|
||||
| 'badges'
|
||||
| 'stats'
|
||||
| 'createdAt'
|
||||
| 'updatedAt'
|
||||
>
|
||||
|
||||
export type PublicSoul = Pick<
|
||||
Doc<'souls'>,
|
||||
| '_id'
|
||||
| '_creationTime'
|
||||
| 'slug'
|
||||
| 'displayName'
|
||||
| 'summary'
|
||||
| 'ownerUserId'
|
||||
| 'latestVersionId'
|
||||
| 'tags'
|
||||
| 'stats'
|
||||
| 'createdAt'
|
||||
| 'updatedAt'
|
||||
>
|
||||
|
||||
export function toPublicUser(user: Doc<'users'> | null | undefined): PublicUser | null {
|
||||
if (!user || user.deletedAt) return null
|
||||
return {
|
||||
_id: user._id,
|
||||
_creationTime: user._creationTime,
|
||||
handle: user.handle,
|
||||
name: user.name,
|
||||
displayName: user.displayName,
|
||||
image: user.image,
|
||||
bio: user.bio,
|
||||
}
|
||||
}
|
||||
|
||||
export function toPublicSkill(skill: Doc<'skills'> | null | undefined): PublicSkill | null {
|
||||
if (!skill || skill.softDeletedAt) return null
|
||||
if (skill.moderationStatus && skill.moderationStatus !== 'active') return null
|
||||
if (skill.moderationFlags?.includes('blocked.malware')) return null
|
||||
return {
|
||||
_id: skill._id,
|
||||
_creationTime: skill._creationTime,
|
||||
slug: skill.slug,
|
||||
displayName: skill.displayName,
|
||||
summary: skill.summary,
|
||||
ownerUserId: skill.ownerUserId,
|
||||
canonicalSkillId: skill.canonicalSkillId,
|
||||
forkOf: skill.forkOf,
|
||||
latestVersionId: skill.latestVersionId,
|
||||
tags: skill.tags,
|
||||
badges: skill.badges,
|
||||
stats: skill.stats,
|
||||
createdAt: skill.createdAt,
|
||||
updatedAt: skill.updatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
export function toPublicSoul(soul: Doc<'souls'> | null | undefined): PublicSoul | null {
|
||||
if (!soul || soul.softDeletedAt) return null
|
||||
return {
|
||||
_id: soul._id,
|
||||
_creationTime: soul._creationTime,
|
||||
slug: soul.slug,
|
||||
displayName: soul.displayName,
|
||||
summary: soul.summary,
|
||||
ownerUserId: soul.ownerUserId,
|
||||
latestVersionId: soul.latestVersionId,
|
||||
tags: soul.tags,
|
||||
stats: soul.stats,
|
||||
createdAt: soul.createdAt,
|
||||
updatedAt: soul.updatedAt,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
/* @vitest-environment node */
|
||||
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { __test, matchesExactTokens, tokenize } from './searchText'
|
||||
|
||||
describe('searchText', () => {
|
||||
it('tokenize lowercases and splits on punctuation', () => {
|
||||
expect(tokenize('Minimax Usage /minimax-usage')).toEqual([
|
||||
'minimax',
|
||||
'usage',
|
||||
'minimax',
|
||||
'usage',
|
||||
])
|
||||
})
|
||||
|
||||
it('matchesExactTokens requires at least one query token to prefix-match', () => {
|
||||
const queryTokens = tokenize('Remind Me')
|
||||
expect(matchesExactTokens(queryTokens, ['Remind Me', '/remind-me', 'Short summary'])).toBe(true)
|
||||
// "Reminder" starts with "remind", so it matches with prefix matching
|
||||
expect(matchesExactTokens(queryTokens, ['Reminder tool', '/reminder', 'Short summary'])).toBe(
|
||||
true,
|
||||
)
|
||||
// Matches because "remind" token is present
|
||||
expect(matchesExactTokens(queryTokens, ['Remind tool', '/remind', 'Short summary'])).toBe(true)
|
||||
// No matching tokens at all
|
||||
expect(matchesExactTokens(queryTokens, ['Other tool', '/other', 'Short summary'])).toBe(false)
|
||||
})
|
||||
|
||||
it('matchesExactTokens supports prefix matching for partial queries', () => {
|
||||
// "go" should match "gohome" because "gohome" starts with "go"
|
||||
expect(matchesExactTokens(['go'], ['GoHome', '/gohome', 'Navigate home'])).toBe(true)
|
||||
// "pad" should match "padel"
|
||||
expect(matchesExactTokens(['pad'], ['Padel', '/padel', 'Tennis-like sport'])).toBe(true)
|
||||
// "xyz" should not match anything
|
||||
expect(matchesExactTokens(['xyz'], ['GoHome', '/gohome', 'Navigate home'])).toBe(false)
|
||||
})
|
||||
|
||||
it('matchesExactTokens ignores empty inputs', () => {
|
||||
expect(matchesExactTokens([], ['text'])).toBe(false)
|
||||
expect(matchesExactTokens(['token'], [' ', null, undefined])).toBe(false)
|
||||
})
|
||||
|
||||
it('normalize uses lowercase', () => {
|
||||
expect(__test.normalize('AbC')).toBe('abc')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,27 @@
|
||||
const WORD_RE = /[a-z0-9]+/g
|
||||
|
||||
function normalize(value: string) {
|
||||
return value.toLowerCase()
|
||||
}
|
||||
|
||||
export function tokenize(value: string): string[] {
|
||||
if (!value) return []
|
||||
return normalize(value).match(WORD_RE) ?? []
|
||||
}
|
||||
|
||||
export function matchesExactTokens(
|
||||
queryTokens: string[],
|
||||
parts: Array<string | null | undefined>,
|
||||
): boolean {
|
||||
if (queryTokens.length === 0) return false
|
||||
const text = parts.filter((part) => Boolean(part?.trim())).join(' ')
|
||||
if (!text) return false
|
||||
const textTokens = tokenize(text)
|
||||
if (textTokens.length === 0) return false
|
||||
// Require at least one token to prefix-match, allowing vector similarity to determine relevance
|
||||
return queryTokens.some((queryToken) =>
|
||||
textTokens.some((textToken) => textToken.includes(queryToken)),
|
||||
)
|
||||
}
|
||||
|
||||
export const __test = { normalize, tokenize, matchesExactTokens }
|
||||
@@ -0,0 +1,28 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { __test } from './skillPublish'
|
||||
|
||||
describe('skillPublish', () => {
|
||||
it('merges github source into metadata', () => {
|
||||
const merged = __test.mergeSourceIntoMetadata(
|
||||
{ clawdis: { emoji: 'x' } },
|
||||
{
|
||||
kind: 'github',
|
||||
url: 'https://github.com/a/b',
|
||||
repo: 'a/b',
|
||||
ref: 'main',
|
||||
commit: '0123456789012345678901234567890123456789',
|
||||
path: 'skills/demo',
|
||||
importedAt: 123,
|
||||
},
|
||||
)
|
||||
expect((merged as Record<string, unknown>).clawdis).toEqual({ emoji: 'x' })
|
||||
const source = (merged as Record<string, unknown>).source
|
||||
expect(source).toEqual(
|
||||
expect.objectContaining({
|
||||
kind: 'github',
|
||||
repo: 'a/b',
|
||||
path: 'skills/demo',
|
||||
}),
|
||||
)
|
||||
})
|
||||
})
|
||||
+70
-25
@@ -3,8 +3,10 @@ import semver from 'semver'
|
||||
import { api, internal } from '../_generated/api'
|
||||
import type { Doc, Id } from '../_generated/dataModel'
|
||||
import type { ActionCtx, MutationCtx } from '../_generated/server'
|
||||
import { getSkillBadgeMap, isSkillHighlighted } from './badges'
|
||||
import { generateChangelogForPublish } from './changelog'
|
||||
import { generateEmbedding } from './embeddings'
|
||||
import type { PublicUser } from './public'
|
||||
import {
|
||||
buildEmbeddingText,
|
||||
getFrontmatterMetadata,
|
||||
@@ -32,6 +34,15 @@ export type PublishVersionArgs = {
|
||||
changelog: string
|
||||
tags?: string[]
|
||||
forkOf?: { slug: string; version?: string }
|
||||
source?: {
|
||||
kind: 'github'
|
||||
url: string
|
||||
repo: string
|
||||
ref: string
|
||||
commit: string
|
||||
path: string
|
||||
importedAt: number
|
||||
}
|
||||
files: Array<{
|
||||
path: string
|
||||
size: number
|
||||
@@ -59,21 +70,27 @@ export async function publishVersionForUser(
|
||||
const suppliedChangelog = args.changelog.trim()
|
||||
const changelogSource = suppliedChangelog ? ('user' as const) : ('auto' as const)
|
||||
|
||||
const sanitizedFiles = args.files.map((file) => {
|
||||
const path = sanitizePath(file.path)
|
||||
if (!path) throw new ConvexError('Invalid file paths')
|
||||
if (!isTextFile(path, file.contentType ?? undefined)) {
|
||||
throw new ConvexError('Only text-based files are allowed')
|
||||
}
|
||||
return { ...file, path }
|
||||
})
|
||||
const sanitizedFiles = args.files.map((file) => ({
|
||||
...file,
|
||||
path: sanitizePath(file.path),
|
||||
}))
|
||||
if (sanitizedFiles.some((file) => !file.path)) {
|
||||
throw new ConvexError('Invalid file paths')
|
||||
}
|
||||
const safeFiles = sanitizedFiles.map((file) => ({
|
||||
...file,
|
||||
path: file.path as string,
|
||||
}))
|
||||
if (safeFiles.some((file) => !isTextFile(file.path, file.contentType ?? undefined))) {
|
||||
throw new ConvexError('Only text-based files are allowed')
|
||||
}
|
||||
|
||||
const totalBytes = sanitizedFiles.reduce((sum, file) => sum + file.size, 0)
|
||||
const totalBytes = safeFiles.reduce((sum, file) => sum + file.size, 0)
|
||||
if (totalBytes > MAX_TOTAL_BYTES) {
|
||||
throw new ConvexError('Skill bundle exceeds 50MB limit')
|
||||
}
|
||||
|
||||
const readmeFile = sanitizedFiles.find(
|
||||
const readmeFile = safeFiles.find(
|
||||
(file) => file.path?.toLowerCase() === 'skill.md' || file.path?.toLowerCase() === 'skills.md',
|
||||
)
|
||||
if (!readmeFile) throw new ConvexError('SKILL.md is required')
|
||||
@@ -81,10 +98,10 @@ export async function publishVersionForUser(
|
||||
const readmeText = await fetchText(ctx, readmeFile.storageId)
|
||||
const frontmatter = parseFrontmatter(readmeText)
|
||||
const clawdis = parseClawdisMetadata(frontmatter)
|
||||
const metadata = getFrontmatterMetadata(frontmatter)
|
||||
const metadata = mergeSourceIntoMetadata(getFrontmatterMetadata(frontmatter), args.source)
|
||||
|
||||
const otherFiles = [] as Array<{ path: string; content: string }>
|
||||
for (const file of sanitizedFiles) {
|
||||
for (const file of safeFiles) {
|
||||
if (!file.path || file.path.toLowerCase().endsWith('.md')) continue
|
||||
if (!isTextFile(file.path, file.contentType ?? undefined)) continue
|
||||
const content = await fetchText(ctx, file.storageId)
|
||||
@@ -98,11 +115,8 @@ export async function publishVersionForUser(
|
||||
otherFiles,
|
||||
})
|
||||
|
||||
const fingerprint = await hashSkillFiles(
|
||||
sanitizedFiles.map((file) => ({
|
||||
path: file.path ?? '',
|
||||
sha256: file.sha256,
|
||||
})),
|
||||
const fingerprintPromise = hashSkillFiles(
|
||||
safeFiles.map((file) => ({ path: file.path, sha256: file.sha256 })),
|
||||
)
|
||||
|
||||
const changelogPromise =
|
||||
@@ -112,12 +126,13 @@ export async function publishVersionForUser(
|
||||
slug,
|
||||
version,
|
||||
readmeText,
|
||||
files: sanitizedFiles.map((file) => ({ path: file.path ?? '', sha256: file.sha256 })),
|
||||
files: safeFiles.map((file) => ({ path: file.path, sha256: file.sha256 })),
|
||||
})
|
||||
|
||||
const embeddingPromise = generateEmbedding(embeddingText)
|
||||
|
||||
const [changelogText, embedding] = await Promise.all([
|
||||
const [fingerprint, changelogText, embedding] = await Promise.all([
|
||||
fingerprintPromise,
|
||||
changelogPromise,
|
||||
embeddingPromise.catch((error) => {
|
||||
throw new ConvexError(formatEmbeddingError(error))
|
||||
@@ -139,7 +154,10 @@ export async function publishVersionForUser(
|
||||
version: args.forkOf.version?.trim() || undefined,
|
||||
}
|
||||
: undefined,
|
||||
files: sanitizedFiles,
|
||||
files: safeFiles.map((file) => ({
|
||||
...file,
|
||||
path: file.path,
|
||||
})),
|
||||
parsed: {
|
||||
frontmatter,
|
||||
metadata,
|
||||
@@ -148,13 +166,18 @@ export async function publishVersionForUser(
|
||||
embedding,
|
||||
})) as PublishResult
|
||||
|
||||
const owner = (await ctx.runQuery(internal.users.getByIdInternal, {
|
||||
userId,
|
||||
})) as Doc<'users'> | null
|
||||
const ownerHandle = owner?.handle ?? owner?.displayName ?? owner?.name ?? 'unknown'
|
||||
|
||||
void ctx.scheduler
|
||||
.runAfter(0, internal.githubBackupsNode.backupSkillForPublishInternal, {
|
||||
slug,
|
||||
version,
|
||||
displayName,
|
||||
ownerHandle: userId,
|
||||
files: sanitizedFiles,
|
||||
ownerHandle,
|
||||
files: safeFiles,
|
||||
publishedAt: Date.now(),
|
||||
})
|
||||
.catch((error) => {
|
||||
@@ -170,19 +193,41 @@ export async function publishVersionForUser(
|
||||
return publishResult
|
||||
}
|
||||
|
||||
function mergeSourceIntoMetadata(metadata: unknown, source: PublishVersionArgs['source']) {
|
||||
if (!source) return metadata === undefined ? undefined : metadata
|
||||
const sourceValue = {
|
||||
kind: source.kind,
|
||||
url: source.url,
|
||||
repo: source.repo,
|
||||
ref: source.ref,
|
||||
commit: source.commit,
|
||||
path: source.path,
|
||||
importedAt: source.importedAt,
|
||||
}
|
||||
|
||||
if (!metadata) return { source: sourceValue }
|
||||
if (typeof metadata !== 'object' || Array.isArray(metadata)) return { source: sourceValue }
|
||||
return { ...(metadata as Record<string, unknown>), source: sourceValue }
|
||||
}
|
||||
|
||||
export const __test = {
|
||||
mergeSourceIntoMetadata,
|
||||
}
|
||||
|
||||
export async function queueHighlightedWebhook(ctx: MutationCtx, skillId: Id<'skills'>) {
|
||||
const skill = await ctx.db.get(skillId)
|
||||
if (!skill) return
|
||||
const owner = await ctx.db.get(skill.ownerUserId)
|
||||
const latestVersion = skill.latestVersionId ? await ctx.db.get(skill.latestVersionId) : null
|
||||
|
||||
const badges = await getSkillBadgeMap(ctx, skillId)
|
||||
const payload: WebhookSkillPayload = {
|
||||
slug: skill.slug,
|
||||
displayName: skill.displayName,
|
||||
summary: skill.summary ?? undefined,
|
||||
version: latestVersion?.version ?? undefined,
|
||||
ownerHandle: owner?.handle ?? owner?.name ?? undefined,
|
||||
batch: skill.batch ?? undefined,
|
||||
highlighted: isSkillHighlighted({ badges }),
|
||||
tags: Object.keys(skill.tags ?? {}),
|
||||
}
|
||||
|
||||
@@ -219,7 +264,7 @@ async function schedulePublishWebhook(
|
||||
) {
|
||||
const result = (await ctx.runQuery(api.skills.getBySlug, {
|
||||
slug: params.slug,
|
||||
})) as { skill: Doc<'skills'>; owner: Doc<'users'> | null } | null
|
||||
})) as { skill: Doc<'skills'>; owner: PublicUser | null } | null
|
||||
if (!result?.skill) return
|
||||
|
||||
const payload: WebhookSkillPayload = {
|
||||
@@ -228,7 +273,7 @@ async function schedulePublishWebhook(
|
||||
summary: result.skill.summary ?? undefined,
|
||||
version: params.version,
|
||||
ownerHandle: result.owner?.handle ?? result.owner?.name ?? undefined,
|
||||
batch: result.skill.batch ?? undefined,
|
||||
highlighted: isSkillHighlighted(result.skill),
|
||||
tags: Object.keys(result.skill.tags ?? {}),
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
import type { Doc, Id } from '../_generated/dataModel'
|
||||
import type { MutationCtx } from '../_generated/server'
|
||||
import { toDayKey } from './leaderboards'
|
||||
|
||||
type SkillStatDeltas = {
|
||||
downloads?: number
|
||||
stars?: number
|
||||
installsCurrent?: number
|
||||
installsAllTime?: number
|
||||
}
|
||||
|
||||
export function applySkillStatDeltas(skill: Doc<'skills'>, deltas: SkillStatDeltas) {
|
||||
const currentDownloads =
|
||||
typeof skill.statsDownloads === 'number' ? skill.statsDownloads : skill.stats.downloads
|
||||
const currentStars = typeof skill.statsStars === 'number' ? skill.statsStars : skill.stats.stars
|
||||
const currentInstallsCurrent =
|
||||
typeof skill.statsInstallsCurrent === 'number'
|
||||
? skill.statsInstallsCurrent
|
||||
: (skill.stats.installsCurrent ?? 0)
|
||||
const currentInstallsAllTime =
|
||||
typeof skill.statsInstallsAllTime === 'number'
|
||||
? skill.statsInstallsAllTime
|
||||
: (skill.stats.installsAllTime ?? 0)
|
||||
|
||||
const nextDownloads = Math.max(0, currentDownloads + (deltas.downloads ?? 0))
|
||||
const nextStars = Math.max(0, currentStars + (deltas.stars ?? 0))
|
||||
const nextInstallsCurrent = Math.max(0, currentInstallsCurrent + (deltas.installsCurrent ?? 0))
|
||||
const nextInstallsAllTime = Math.max(0, currentInstallsAllTime + (deltas.installsAllTime ?? 0))
|
||||
|
||||
return {
|
||||
statsDownloads: nextDownloads,
|
||||
statsStars: nextStars,
|
||||
statsInstallsCurrent: nextInstallsCurrent,
|
||||
statsInstallsAllTime: nextInstallsAllTime,
|
||||
stats: {
|
||||
...skill.stats,
|
||||
downloads: nextDownloads,
|
||||
stars: nextStars,
|
||||
installsCurrent: nextInstallsCurrent,
|
||||
installsAllTime: nextInstallsAllTime,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export async function bumpDailySkillStats(
|
||||
ctx: MutationCtx,
|
||||
params: {
|
||||
skillId: Id<'skills'>
|
||||
now: number
|
||||
downloads?: number
|
||||
installs?: number
|
||||
},
|
||||
) {
|
||||
const downloads = params.downloads ?? 0
|
||||
const installs = params.installs ?? 0
|
||||
if (downloads === 0 && installs === 0) return
|
||||
|
||||
const day = toDayKey(params.now)
|
||||
const existing = await ctx.db
|
||||
.query('skillDailyStats')
|
||||
.withIndex('by_skill_day', (q) => q.eq('skillId', params.skillId).eq('day', day))
|
||||
.unique()
|
||||
|
||||
if (existing) {
|
||||
await ctx.db.patch(existing._id, {
|
||||
downloads: Math.max(0, existing.downloads + downloads),
|
||||
installs: Math.max(0, existing.installs + installs),
|
||||
updatedAt: params.now,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
await ctx.db.insert('skillDailyStats', {
|
||||
skillId: params.skillId,
|
||||
day,
|
||||
downloads: Math.max(0, downloads),
|
||||
installs: Math.max(0, installs),
|
||||
updatedAt: params.now,
|
||||
})
|
||||
}
|
||||
@@ -107,6 +107,35 @@ describe('skills utils', () => {
|
||||
expect(clawdis?.requires?.anyBins).toEqual(['rg', 'fd'])
|
||||
})
|
||||
|
||||
it('parses clawdbot metadata with nix plugin pointer', () => {
|
||||
const frontmatter = parseFrontmatter(
|
||||
`---\nmetadata: {"clawdbot":{"nix":{"plugin":"github:clawdbot/nix-steipete-tools?dir=tools/peekaboo","systems":["aarch64-darwin"]}}}\n---\nBody`,
|
||||
)
|
||||
const clawdis = parseClawdisMetadata(frontmatter)
|
||||
expect(clawdis?.nix?.plugin).toBe('github:clawdbot/nix-steipete-tools?dir=tools/peekaboo')
|
||||
expect(clawdis?.nix?.systems).toEqual(['aarch64-darwin'])
|
||||
})
|
||||
|
||||
it('parses clawdbot config requirements with example', () => {
|
||||
const frontmatter = parseFrontmatter(
|
||||
`---\nmetadata: {"clawdbot":{"config":{"requiredEnv":["PADEL_AUTH_FILE"],"stateDirs":[".config/padel"],"example":"config = { env = { PADEL_AUTH_FILE = \\"/run/agenix/padel-auth\\"; }; };"}}}\n---\nBody`,
|
||||
)
|
||||
const clawdis = parseClawdisMetadata(frontmatter)
|
||||
expect(clawdis?.config?.requiredEnv).toEqual(['PADEL_AUTH_FILE'])
|
||||
expect(clawdis?.config?.stateDirs).toEqual(['.config/padel'])
|
||||
expect(clawdis?.config?.example).toBe(
|
||||
'config = { env = { PADEL_AUTH_FILE = "/run/agenix/padel-auth"; }; };',
|
||||
)
|
||||
})
|
||||
|
||||
it('parses cli help output', () => {
|
||||
const frontmatter = parseFrontmatter(
|
||||
`---\nmetadata: {"clawdbot":{"cliHelp":"padel --help\\nUsage: padel [command]\\n"}}\n---\nBody`,
|
||||
)
|
||||
const clawdis = parseClawdisMetadata(frontmatter)
|
||||
expect(clawdis?.cliHelp).toBe('padel --help\nUsage: padel [command]')
|
||||
})
|
||||
|
||||
it('sanitizes file paths', () => {
|
||||
expect(sanitizePath('good/file.md')).toBe('good/file.md')
|
||||
expect(sanitizePath('../bad/file.md')).toBeNull()
|
||||
|
||||
+44
-4
@@ -1,11 +1,13 @@
|
||||
import {
|
||||
type ClawdbotConfigSpec,
|
||||
type ClawdisSkillMetadata,
|
||||
ClawdisSkillMetadataSchema,
|
||||
isTextContentType,
|
||||
type NixPluginSpec,
|
||||
parseArk,
|
||||
type SkillInstallSpec,
|
||||
TEXT_FILE_EXTENSION_SET,
|
||||
} from 'clawdhub-schema'
|
||||
} from 'clawhub-schema'
|
||||
import { parse as parseYaml } from 'yaml'
|
||||
|
||||
export type ParsedSkillFrontmatter = Record<string, unknown>
|
||||
@@ -59,11 +61,19 @@ export function getFrontmatterMetadata(frontmatter: ParsedSkillFrontmatter) {
|
||||
|
||||
export function parseClawdisMetadata(frontmatter: ParsedSkillFrontmatter) {
|
||||
const metadata = getFrontmatterMetadata(frontmatter)
|
||||
const clawdisFromMetadata =
|
||||
const metadataRecord =
|
||||
metadata && typeof metadata === 'object' && !Array.isArray(metadata)
|
||||
? (metadata as Record<string, unknown>).clawdis
|
||||
? (metadata as Record<string, unknown>)
|
||||
: undefined
|
||||
const clawdisRaw = clawdisFromMetadata ?? frontmatter.clawdis
|
||||
const clawdbotMeta = metadataRecord?.clawdbot
|
||||
const clawdisMeta = metadataRecord?.clawdis
|
||||
const metadataSource =
|
||||
clawdbotMeta && typeof clawdbotMeta === 'object' && !Array.isArray(clawdbotMeta)
|
||||
? (clawdbotMeta as Record<string, unknown>)
|
||||
: clawdisMeta && typeof clawdisMeta === 'object' && !Array.isArray(clawdisMeta)
|
||||
? (clawdisMeta as Record<string, unknown>)
|
||||
: undefined
|
||||
const clawdisRaw = metadataSource ?? frontmatter.clawdis
|
||||
if (!clawdisRaw || typeof clawdisRaw !== 'object' || Array.isArray(clawdisRaw)) return undefined
|
||||
|
||||
try {
|
||||
@@ -84,6 +94,7 @@ export function parseClawdisMetadata(frontmatter: ParsedSkillFrontmatter) {
|
||||
if (typeof clawdisObj.homepage === 'string') metadata.homepage = clawdisObj.homepage
|
||||
if (typeof clawdisObj.skillKey === 'string') metadata.skillKey = clawdisObj.skillKey
|
||||
if (typeof clawdisObj.primaryEnv === 'string') metadata.primaryEnv = clawdisObj.primaryEnv
|
||||
if (typeof clawdisObj.cliHelp === 'string') metadata.cliHelp = clawdisObj.cliHelp
|
||||
if (osRaw.length > 0) metadata.os = osRaw
|
||||
|
||||
if (requiresRaw) {
|
||||
@@ -101,6 +112,10 @@ export function parseClawdisMetadata(frontmatter: ParsedSkillFrontmatter) {
|
||||
}
|
||||
|
||||
if (install.length > 0) metadata.install = install
|
||||
const nix = parseNixPluginSpec(clawdisObj.nix)
|
||||
if (nix) metadata.nix = nix
|
||||
const config = parseClawdbotConfigSpec(clawdisObj.config)
|
||||
if (config) metadata.config = config
|
||||
|
||||
return parseArk(ClawdisSkillMetadataSchema, metadata, 'Clawdis metadata')
|
||||
} catch {
|
||||
@@ -223,6 +238,31 @@ function parseInstallSpec(input: unknown): SkillInstallSpec | undefined {
|
||||
return spec
|
||||
}
|
||||
|
||||
function parseNixPluginSpec(input: unknown): NixPluginSpec | undefined {
|
||||
if (!input || typeof input !== 'object') return undefined
|
||||
const raw = input as Record<string, unknown>
|
||||
if (typeof raw.plugin !== 'string') return undefined
|
||||
const plugin = raw.plugin.trim()
|
||||
if (!plugin) return undefined
|
||||
const systems = normalizeStringList(raw.systems)
|
||||
const spec: NixPluginSpec = { plugin }
|
||||
if (systems.length > 0) spec.systems = systems
|
||||
return spec
|
||||
}
|
||||
|
||||
function parseClawdbotConfigSpec(input: unknown): ClawdbotConfigSpec | undefined {
|
||||
if (!input || typeof input !== 'object') return undefined
|
||||
const raw = input as Record<string, unknown>
|
||||
const requiredEnv = normalizeStringList(raw.requiredEnv)
|
||||
const stateDirs = normalizeStringList(raw.stateDirs)
|
||||
const example = typeof raw.example === 'string' ? raw.example.trim() : ''
|
||||
const spec: ClawdbotConfigSpec = {}
|
||||
if (requiredEnv.length > 0) spec.requiredEnv = requiredEnv
|
||||
if (stateDirs.length > 0) spec.stateDirs = stateDirs
|
||||
if (example) spec.example = example
|
||||
return Object.keys(spec).length > 0 ? spec : undefined
|
||||
}
|
||||
|
||||
function toHex(bytes: Uint8Array) {
|
||||
let out = ''
|
||||
for (const byte of bytes) out += byte.toString(16).padStart(2, '0')
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
import { internal } from '../_generated/api'
|
||||
import type { Doc } from '../_generated/dataModel'
|
||||
import type { ActionCtx } from '../_generated/server'
|
||||
|
||||
const CHANGELOG_MODEL = process.env.OPENAI_CHANGELOG_MODEL ?? 'gpt-4.1'
|
||||
const MAX_README_CHARS = 8_000
|
||||
const MAX_PATHS_IN_PROMPT = 30
|
||||
|
||||
type FileMeta = { path: string; sha256?: string }
|
||||
|
||||
type FileDiffSummary = {
|
||||
added: string[]
|
||||
removed: string[]
|
||||
changed: string[]
|
||||
}
|
||||
|
||||
function clampText(value: string, maxChars: number) {
|
||||
const trimmed = value.trim()
|
||||
if (trimmed.length <= maxChars) return trimmed
|
||||
return `${trimmed.slice(0, maxChars).trimEnd()}\n…`
|
||||
}
|
||||
|
||||
function summarizeFileDiff(oldFiles: FileMeta[], nextFiles: FileMeta[]): FileDiffSummary {
|
||||
const oldByPath = new Map(oldFiles.map((f) => [f.path, f] as const))
|
||||
const nextByPath = new Map(nextFiles.map((f) => [f.path, f] as const))
|
||||
|
||||
const added: string[] = []
|
||||
const removed: string[] = []
|
||||
const changed: string[] = []
|
||||
|
||||
for (const [path, file] of nextByPath.entries()) {
|
||||
const prev = oldByPath.get(path)
|
||||
if (!prev) {
|
||||
added.push(path)
|
||||
continue
|
||||
}
|
||||
if (file.sha256 && prev.sha256 && file.sha256 !== prev.sha256) changed.push(path)
|
||||
}
|
||||
for (const path of oldByPath.keys()) {
|
||||
if (!nextByPath.has(path)) removed.push(path)
|
||||
}
|
||||
|
||||
added.sort()
|
||||
removed.sort()
|
||||
changed.sort()
|
||||
return { added, removed, changed }
|
||||
}
|
||||
|
||||
function formatDiffSummary(diff: FileDiffSummary) {
|
||||
const parts: string[] = []
|
||||
if (diff.added.length) parts.push(`${diff.added.length} added`)
|
||||
if (diff.changed.length) parts.push(`${diff.changed.length} changed`)
|
||||
if (diff.removed.length) parts.push(`${diff.removed.length} removed`)
|
||||
return parts.join(', ') || 'no file changes detected'
|
||||
}
|
||||
|
||||
function pickPaths(values: string[]) {
|
||||
if (values.length <= MAX_PATHS_IN_PROMPT) return values
|
||||
return values.slice(0, MAX_PATHS_IN_PROMPT)
|
||||
}
|
||||
|
||||
function extractResponseText(payload: unknown) {
|
||||
if (!payload || typeof payload !== 'object') return null
|
||||
const output = (payload as { output?: unknown }).output
|
||||
if (!Array.isArray(output)) return null
|
||||
const chunks: string[] = []
|
||||
for (const item of output) {
|
||||
if (!item || typeof item !== 'object') continue
|
||||
if ((item as { type?: unknown }).type !== 'message') continue
|
||||
const content = (item as { content?: unknown }).content
|
||||
if (!Array.isArray(content)) continue
|
||||
for (const part of content) {
|
||||
if (!part || typeof part !== 'object') continue
|
||||
if ((part as { type?: unknown }).type !== 'output_text') continue
|
||||
const text = (part as { text?: unknown }).text
|
||||
if (typeof text === 'string' && text.trim()) chunks.push(text)
|
||||
}
|
||||
}
|
||||
const joined = chunks.join('\n').trim()
|
||||
return joined || null
|
||||
}
|
||||
|
||||
async function generateWithOpenAI(args: {
|
||||
slug: string
|
||||
version: string
|
||||
oldReadme: string | null
|
||||
nextReadme: string
|
||||
fileDiff: FileDiffSummary | null
|
||||
}) {
|
||||
const apiKey = process.env.OPENAI_API_KEY
|
||||
if (!apiKey) return null
|
||||
|
||||
const oldReadme = args.oldReadme ? clampText(args.oldReadme, MAX_README_CHARS) : ''
|
||||
const nextReadme = clampText(args.nextReadme, MAX_README_CHARS)
|
||||
|
||||
const fileDiff = args.fileDiff
|
||||
const diffSummary = fileDiff ? formatDiffSummary(fileDiff) : 'unknown'
|
||||
const changedPaths = fileDiff ? pickPaths(fileDiff.changed) : []
|
||||
const addedPaths = fileDiff ? pickPaths(fileDiff.added) : []
|
||||
const removedPaths = fileDiff ? pickPaths(fileDiff.removed) : []
|
||||
|
||||
const input = [
|
||||
`Soul: ${args.slug}`,
|
||||
`Version: ${args.version}`,
|
||||
`File changes: ${diffSummary}`,
|
||||
changedPaths.length ? `Changed files (sample): ${changedPaths.join(', ')}` : null,
|
||||
addedPaths.length ? `Added files (sample): ${addedPaths.join(', ')}` : null,
|
||||
removedPaths.length ? `Removed files (sample): ${removedPaths.join(', ')}` : null,
|
||||
oldReadme ? `Previous SOUL.md:\n${oldReadme}` : null,
|
||||
`New SOUL.md:\n${nextReadme}`,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join('\n\n')
|
||||
|
||||
const response = await fetch('https://api.openai.com/v1/responses', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: CHANGELOG_MODEL,
|
||||
instructions:
|
||||
'Write a concise changelog for this soul version. Audience: everyone. Output plain text. Prefer 2–6 bullet points. If it is a big change, include a short 1-line summary first, then bullets. Don’t mention that you are AI. Don’t invent details; only use the inputs.',
|
||||
input,
|
||||
max_output_tokens: 220,
|
||||
}),
|
||||
})
|
||||
|
||||
if (!response.ok) return null
|
||||
const payload = (await response.json()) as unknown
|
||||
return extractResponseText(payload)
|
||||
}
|
||||
|
||||
function generateFallback(args: {
|
||||
slug: string
|
||||
version: string
|
||||
oldReadme: string | null
|
||||
nextReadme: string
|
||||
fileDiff: FileDiffSummary | null
|
||||
}) {
|
||||
const lines: string[] = []
|
||||
if (!args.oldReadme) {
|
||||
lines.push(`- Initial release.`)
|
||||
return lines.join('\n')
|
||||
}
|
||||
|
||||
const diff = args.fileDiff
|
||||
if (diff) {
|
||||
const parts: string[] = []
|
||||
if (diff.added.length) parts.push(`added ${diff.added.length}`)
|
||||
if (diff.changed.length) parts.push(`updated ${diff.changed.length}`)
|
||||
if (diff.removed.length) parts.push(`removed ${diff.removed.length}`)
|
||||
if (parts.length) lines.push(`- ${parts.join(', ')} file(s).`)
|
||||
}
|
||||
|
||||
lines.push(`- Updated SOUL.md.`)
|
||||
return lines.join('\n')
|
||||
}
|
||||
|
||||
export async function generateSoulChangelogForPublish(
|
||||
ctx: ActionCtx,
|
||||
args: { slug: string; version: string; readmeText: string; files: FileMeta[] },
|
||||
): Promise<string> {
|
||||
try {
|
||||
const soul = (await ctx.runQuery(internal.souls.getSoulBySlugInternal, {
|
||||
slug: args.slug,
|
||||
})) as Doc<'souls'> | null
|
||||
const previous: Doc<'soulVersions'> | null =
|
||||
soul?.latestVersionId && !soul.softDeletedAt
|
||||
? ((await ctx.runQuery(internal.souls.getVersionByIdInternal, {
|
||||
versionId: soul.latestVersionId,
|
||||
})) as Doc<'soulVersions'> | null)
|
||||
: null
|
||||
|
||||
const oldReadmeText: string | null = previous
|
||||
? await readReadmeFromVersion(ctx, previous)
|
||||
: null
|
||||
const oldFiles = previous
|
||||
? previous.files.map((file) => ({ path: file.path, sha256: file.sha256 }))
|
||||
: []
|
||||
const fileDiff = previous ? summarizeFileDiff(oldFiles, args.files) : null
|
||||
|
||||
const ai = await generateWithOpenAI({
|
||||
slug: args.slug,
|
||||
version: args.version,
|
||||
oldReadme: oldReadmeText,
|
||||
nextReadme: args.readmeText,
|
||||
fileDiff,
|
||||
}).catch(() => null)
|
||||
|
||||
return (
|
||||
ai ??
|
||||
generateFallback({
|
||||
slug: args.slug,
|
||||
version: args.version,
|
||||
oldReadme: oldReadmeText,
|
||||
nextReadme: args.readmeText,
|
||||
fileDiff,
|
||||
})
|
||||
)
|
||||
} catch {
|
||||
return '- Updated soul.'
|
||||
}
|
||||
}
|
||||
|
||||
export async function generateSoulChangelogPreview(
|
||||
ctx: ActionCtx,
|
||||
args: {
|
||||
slug: string
|
||||
version: string
|
||||
readmeText: string
|
||||
filePaths?: string[]
|
||||
},
|
||||
): Promise<string> {
|
||||
try {
|
||||
const soul = (await ctx.runQuery(internal.souls.getSoulBySlugInternal, {
|
||||
slug: args.slug,
|
||||
})) as Doc<'souls'> | null
|
||||
const previous: Doc<'soulVersions'> | null =
|
||||
soul?.latestVersionId && !soul.softDeletedAt
|
||||
? ((await ctx.runQuery(internal.souls.getVersionByIdInternal, {
|
||||
versionId: soul.latestVersionId,
|
||||
})) as Doc<'soulVersions'> | null)
|
||||
: null
|
||||
|
||||
const oldReadmeText: string | null = previous
|
||||
? await readReadmeFromVersion(ctx, previous)
|
||||
: null
|
||||
const oldPaths = previous ? previous.files.map((file) => file.path) : []
|
||||
const nextPaths = args.filePaths ?? []
|
||||
const diff = previous ? summarizeFileDiffFromPaths(oldPaths, nextPaths) : null
|
||||
|
||||
const ai = await generateWithOpenAI({
|
||||
slug: args.slug,
|
||||
version: args.version,
|
||||
oldReadme: oldReadmeText,
|
||||
nextReadme: args.readmeText,
|
||||
fileDiff: diff,
|
||||
}).catch(() => null)
|
||||
|
||||
return (
|
||||
ai ??
|
||||
generateFallback({
|
||||
slug: args.slug,
|
||||
version: args.version,
|
||||
oldReadme: oldReadmeText,
|
||||
nextReadme: args.readmeText,
|
||||
fileDiff: diff,
|
||||
})
|
||||
)
|
||||
} catch {
|
||||
return '- Updated soul.'
|
||||
}
|
||||
}
|
||||
|
||||
async function readReadmeFromVersion(ctx: ActionCtx, version: Doc<'soulVersions'>) {
|
||||
const file = version.files.find((entry) => entry.path.toLowerCase() === 'soul.md')
|
||||
if (!file) return null
|
||||
const blob = await ctx.storage.get(file.storageId)
|
||||
if (!blob) return null
|
||||
return blob.text()
|
||||
}
|
||||
|
||||
function summarizeFileDiffFromPaths(oldPaths: string[], nextPaths: string[]) {
|
||||
const oldFiles = oldPaths.map((path) => ({ path }))
|
||||
const nextFiles = nextPaths.map((path) => ({ path }))
|
||||
return summarizeFileDiff(oldFiles, nextFiles)
|
||||
}
|
||||
|
||||
export const __test = {
|
||||
summarizeFileDiff,
|
||||
}
|
||||
@@ -0,0 +1,236 @@
|
||||
import { ConvexError } from 'convex/values'
|
||||
import semver from 'semver'
|
||||
import { internal } from '../_generated/api'
|
||||
import type { Doc, Id } from '../_generated/dataModel'
|
||||
import type { ActionCtx } from '../_generated/server'
|
||||
import { generateEmbedding } from './embeddings'
|
||||
import {
|
||||
buildEmbeddingText,
|
||||
getFrontmatterMetadata,
|
||||
getFrontmatterValue,
|
||||
hashSkillFiles,
|
||||
isTextFile,
|
||||
parseFrontmatter,
|
||||
sanitizePath,
|
||||
} from './skills'
|
||||
import { generateSoulChangelogForPublish } from './soulChangelog'
|
||||
|
||||
const MAX_TOTAL_BYTES = 50 * 1024 * 1024
|
||||
|
||||
const MAX_SUMMARY_LENGTH = 160
|
||||
|
||||
function deriveSoulSummary(readmeText: string) {
|
||||
const lines = readmeText.split(/\r?\n/)
|
||||
let inFrontmatter = false
|
||||
for (const raw of lines) {
|
||||
const trimmed = raw.trim()
|
||||
if (!trimmed) continue
|
||||
if (!inFrontmatter && trimmed === '---') {
|
||||
inFrontmatter = true
|
||||
continue
|
||||
}
|
||||
if (inFrontmatter) {
|
||||
if (trimmed === '---') {
|
||||
inFrontmatter = false
|
||||
}
|
||||
continue
|
||||
}
|
||||
const cleaned = trimmed.replace(/^#+\s*/, '')
|
||||
if (!cleaned) continue
|
||||
if (cleaned.length > MAX_SUMMARY_LENGTH) {
|
||||
return `${cleaned.slice(0, MAX_SUMMARY_LENGTH - 3).trimEnd()}...`
|
||||
}
|
||||
return cleaned
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
export type PublishResult = {
|
||||
soulId: Id<'souls'>
|
||||
versionId: Id<'soulVersions'>
|
||||
embeddingId: Id<'soulEmbeddings'>
|
||||
}
|
||||
|
||||
export type PublishVersionArgs = {
|
||||
slug: string
|
||||
displayName: string
|
||||
version: string
|
||||
changelog: string
|
||||
tags?: string[]
|
||||
source?: {
|
||||
kind: 'github'
|
||||
url: string
|
||||
repo: string
|
||||
ref: string
|
||||
commit: string
|
||||
path: string
|
||||
importedAt: number
|
||||
}
|
||||
files: Array<{
|
||||
path: string
|
||||
size: number
|
||||
storageId: Id<'_storage'>
|
||||
sha256: string
|
||||
contentType?: string
|
||||
}>
|
||||
}
|
||||
|
||||
export async function publishSoulVersionForUser(
|
||||
ctx: ActionCtx,
|
||||
userId: Id<'users'>,
|
||||
args: PublishVersionArgs,
|
||||
): Promise<PublishResult> {
|
||||
const version = args.version.trim()
|
||||
const slug = args.slug.trim().toLowerCase()
|
||||
const displayName = args.displayName.trim()
|
||||
if (!slug || !displayName) throw new ConvexError('Slug and display name required')
|
||||
if (!/^[a-z0-9][a-z0-9-]*$/.test(slug)) {
|
||||
throw new ConvexError('Slug must be lowercase and url-safe')
|
||||
}
|
||||
if (!semver.valid(version)) {
|
||||
throw new ConvexError('Version must be valid semver')
|
||||
}
|
||||
const suppliedChangelog = args.changelog.trim()
|
||||
const changelogSource = suppliedChangelog ? ('user' as const) : ('auto' as const)
|
||||
|
||||
const sanitizedFiles = args.files.map((file) => {
|
||||
const path = sanitizePath(file.path)
|
||||
if (!path) throw new ConvexError('Invalid file paths')
|
||||
if (!isTextFile(path, file.contentType ?? undefined)) {
|
||||
throw new ConvexError('Only text-based files are allowed')
|
||||
}
|
||||
return { ...file, path }
|
||||
})
|
||||
|
||||
const totalBytes = sanitizedFiles.reduce((sum, file) => sum + file.size, 0)
|
||||
if (totalBytes > MAX_TOTAL_BYTES) {
|
||||
throw new ConvexError('Soul bundle exceeds 50MB limit')
|
||||
}
|
||||
|
||||
const isSoulFile = (path: string) => path.toLowerCase() === 'soul.md'
|
||||
const readmeFile = sanitizedFiles.find((file) => isSoulFile(file.path))
|
||||
if (!readmeFile) throw new ConvexError('SOUL.md is required')
|
||||
|
||||
const nonSoulFiles = sanitizedFiles.filter((file) => !isSoulFile(file.path))
|
||||
if (nonSoulFiles.length > 0) {
|
||||
throw new ConvexError('Only SOUL.md is allowed for soul bundles')
|
||||
}
|
||||
|
||||
const readmeText = await fetchText(ctx, readmeFile.storageId)
|
||||
const frontmatter = parseFrontmatter(readmeText)
|
||||
const summary = getFrontmatterValue(frontmatter, 'description') ?? deriveSoulSummary(readmeText)
|
||||
const metadata = mergeSourceIntoMetadata(getFrontmatterMetadata(frontmatter), args.source)
|
||||
|
||||
const embeddingText = buildEmbeddingText({
|
||||
frontmatter,
|
||||
readme: readmeText,
|
||||
otherFiles: [],
|
||||
})
|
||||
|
||||
const fingerprint = await hashSkillFiles(
|
||||
sanitizedFiles.map((file) => ({
|
||||
path: file.path ?? '',
|
||||
sha256: file.sha256,
|
||||
})),
|
||||
)
|
||||
|
||||
const changelogPromise =
|
||||
changelogSource === 'user'
|
||||
? Promise.resolve(suppliedChangelog)
|
||||
: generateSoulChangelogForPublish(ctx, {
|
||||
slug,
|
||||
version,
|
||||
readmeText,
|
||||
files: sanitizedFiles.map((file) => ({ path: file.path ?? '', sha256: file.sha256 })),
|
||||
})
|
||||
|
||||
const embeddingPromise = generateEmbedding(embeddingText)
|
||||
|
||||
const [changelogText, embedding] = await Promise.all([
|
||||
changelogPromise,
|
||||
embeddingPromise.catch((error) => {
|
||||
throw new ConvexError(formatEmbeddingError(error))
|
||||
}),
|
||||
])
|
||||
|
||||
const publishResult = (await ctx.runMutation(internal.souls.insertVersion, {
|
||||
userId,
|
||||
slug,
|
||||
displayName,
|
||||
version,
|
||||
changelog: changelogText,
|
||||
changelogSource,
|
||||
tags: args.tags?.map((tag) => tag.trim()).filter(Boolean),
|
||||
fingerprint,
|
||||
files: sanitizedFiles,
|
||||
parsed: {
|
||||
frontmatter,
|
||||
metadata,
|
||||
},
|
||||
summary,
|
||||
embedding,
|
||||
})) as PublishResult
|
||||
|
||||
const owner = (await ctx.runQuery(internal.users.getByIdInternal, {
|
||||
userId,
|
||||
})) as Doc<'users'> | null
|
||||
const ownerHandle = owner?.handle ?? owner?.name ?? userId
|
||||
|
||||
void ctx.scheduler
|
||||
.runAfter(0, internal.githubSoulBackupsNode.backupSoulForPublishInternal, {
|
||||
slug,
|
||||
version,
|
||||
displayName,
|
||||
ownerHandle,
|
||||
files: sanitizedFiles,
|
||||
publishedAt: Date.now(),
|
||||
})
|
||||
.catch((error) => {
|
||||
console.error('GitHub soul backup scheduling failed', error)
|
||||
})
|
||||
|
||||
return publishResult
|
||||
}
|
||||
|
||||
function mergeSourceIntoMetadata(metadata: unknown, source: PublishVersionArgs['source']) {
|
||||
if (!source) return metadata === undefined ? undefined : metadata
|
||||
const sourceValue = {
|
||||
kind: source.kind,
|
||||
url: source.url,
|
||||
repo: source.repo,
|
||||
ref: source.ref,
|
||||
commit: source.commit,
|
||||
path: source.path,
|
||||
importedAt: source.importedAt,
|
||||
}
|
||||
|
||||
if (!metadata) return { source: sourceValue }
|
||||
if (typeof metadata !== 'object' || Array.isArray(metadata)) return { source: sourceValue }
|
||||
return { ...(metadata as Record<string, unknown>), source: sourceValue }
|
||||
}
|
||||
|
||||
export async function fetchText(
|
||||
ctx: { storage: { get: (id: Id<'_storage'>) => Promise<Blob | null> } },
|
||||
storageId: Id<'_storage'>,
|
||||
) {
|
||||
const blob = await ctx.storage.get(storageId)
|
||||
if (!blob) throw new Error('File missing in storage')
|
||||
return blob.text()
|
||||
}
|
||||
|
||||
function formatEmbeddingError(error: unknown) {
|
||||
if (error instanceof Error) {
|
||||
if (error.message.includes('OPENAI_API_KEY')) {
|
||||
return 'OPENAI_API_KEY is not configured.'
|
||||
}
|
||||
if (error.message.startsWith('Embedding failed')) {
|
||||
return error.message
|
||||
}
|
||||
}
|
||||
return 'Embedding failed. Please try again.'
|
||||
}
|
||||
|
||||
export const __test = {
|
||||
getSummary: (frontmatter: Record<string, unknown>) =>
|
||||
getFrontmatterValue(frontmatter, 'description'),
|
||||
}
|
||||
+26
-13
@@ -1,20 +1,33 @@
|
||||
/* @vitest-environment node */
|
||||
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { API_TOKEN_PREFIX, generateToken, hashToken } from './tokens'
|
||||
import { __test, generateToken, hashToken } from './tokens'
|
||||
|
||||
describe('tokens', () => {
|
||||
it('generates token with prefix and url-safe chars', () => {
|
||||
const { token, prefix } = generateToken()
|
||||
expect(token.startsWith(API_TOKEN_PREFIX)).toBe(true)
|
||||
expect(prefix).toBe(token.slice(0, 12))
|
||||
expect(token).toMatch(/^[a-z0-9_-]+$/i)
|
||||
it('hashToken returns sha256 hex', async () => {
|
||||
await expect(hashToken('test')).resolves.toBe(
|
||||
'9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08',
|
||||
)
|
||||
})
|
||||
|
||||
it('hashes tokens deterministically', async () => {
|
||||
const a = await hashToken('clh_test')
|
||||
const b = await hashToken('clh_test')
|
||||
const c = await hashToken('clh_other')
|
||||
expect(a).toBe(b)
|
||||
expect(a).not.toBe(c)
|
||||
expect(a).toMatch(/^[a-f0-9]{64}$/)
|
||||
it('generateToken returns token + prefix', () => {
|
||||
const { token, prefix } = generateToken()
|
||||
expect(token).toMatch(/^clh_[A-Za-z0-9_-]+$/)
|
||||
expect(prefix).toBe(token.slice(0, 12))
|
||||
})
|
||||
|
||||
it('toHex encodes bytes', () => {
|
||||
expect(__test.toHex(new Uint8Array([0, 15, 255]))).toBe('000fff')
|
||||
})
|
||||
|
||||
it('toBase64 encodes 1/2/3-byte tails', () => {
|
||||
expect(__test.toBase64(new Uint8Array([0xff]))).toBe('/w==')
|
||||
expect(__test.toBase64(new Uint8Array([0xff, 0xee]))).toBe('/+4=')
|
||||
expect(__test.toBase64(new Uint8Array([0xff, 0xee, 0xdd]))).toBe('/+7d')
|
||||
})
|
||||
|
||||
it('toBase64Url replaces alphabet and strips padding', () => {
|
||||
expect(__test.toBase64Url(new Uint8Array([0xff]))).toBe('_w')
|
||||
expect(__test.toBase64Url(new Uint8Array([0xfa, 0x00, 0x00]))).toBe('-gAA')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -43,3 +43,9 @@ function toBase64(bytes: Uint8Array) {
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
export const __test = {
|
||||
toHex,
|
||||
toBase64,
|
||||
toBase64Url,
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@ describe('webhook config', () => {
|
||||
delete process.env.SITE_URL
|
||||
process.env.DISCORD_WEBHOOK_URL = 'https://example.com'
|
||||
const config = getWebhookConfig()
|
||||
expect(config.siteUrl).toBe('https://clawdhub.com')
|
||||
expect(config.siteUrl).toBe('https://clawhub.ai')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -36,11 +36,11 @@ describe('webhook filtering', () => {
|
||||
const config = {
|
||||
url: 'https://example.com',
|
||||
highlightedOnly: true,
|
||||
siteUrl: 'https://clawdhub.com',
|
||||
siteUrl: 'https://clawhub.ai',
|
||||
}
|
||||
const allowed = shouldSendWebhook(
|
||||
'skill.publish',
|
||||
{ slug: 'demo', displayName: 'Demo', batch: 'latest' },
|
||||
{ slug: 'demo', displayName: 'Demo', highlighted: false },
|
||||
config,
|
||||
)
|
||||
expect(allowed).toBe(false)
|
||||
@@ -50,11 +50,11 @@ describe('webhook filtering', () => {
|
||||
const config = {
|
||||
url: 'https://example.com',
|
||||
highlightedOnly: true,
|
||||
siteUrl: 'https://clawdhub.com',
|
||||
siteUrl: 'https://clawhub.ai',
|
||||
}
|
||||
const allowed = shouldSendWebhook(
|
||||
'skill.highlighted',
|
||||
{ slug: 'demo', displayName: 'Demo', batch: 'latest' },
|
||||
{ slug: 'demo', displayName: 'Demo', highlighted: true },
|
||||
config,
|
||||
)
|
||||
expect(allowed).toBe(true)
|
||||
@@ -65,9 +65,9 @@ describe('payload building', () => {
|
||||
it('builds canonical url with owner', () => {
|
||||
const url = buildSkillUrl(
|
||||
{ slug: 'beeper', displayName: 'Beeper', ownerHandle: 'KrauseFx' },
|
||||
'https://clawdhub.com',
|
||||
'https://clawhub.ai',
|
||||
)
|
||||
expect(url).toBe('https://clawdhub.com/KrauseFx/beeper')
|
||||
expect(url).toBe('https://clawhub.ai/KrauseFx/beeper')
|
||||
})
|
||||
|
||||
it('builds a publish embed', () => {
|
||||
@@ -81,7 +81,7 @@ describe('payload building', () => {
|
||||
ownerHandle: 'steipete',
|
||||
tags: ['latest', 'discord'],
|
||||
},
|
||||
{ url: 'https://example.com', highlightedOnly: false, siteUrl: 'https://clawdhub.com' },
|
||||
{ url: 'https://example.com', highlightedOnly: false, siteUrl: 'https://clawhub.ai' },
|
||||
)
|
||||
const embed = payload.embeds[0]
|
||||
expect(embed.title).toBe('Demo Skill')
|
||||
|
||||
@@ -6,7 +6,7 @@ export type WebhookSkillPayload = {
|
||||
summary?: string
|
||||
version?: string
|
||||
ownerHandle?: string
|
||||
batch?: string
|
||||
highlighted?: boolean
|
||||
tags?: string[]
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ export type WebhookConfig = {
|
||||
siteUrl: string
|
||||
}
|
||||
|
||||
const DEFAULT_SITE_URL = 'https://clawdhub.com'
|
||||
const DEFAULT_SITE_URL = 'https://clawhub.ai'
|
||||
|
||||
export function getWebhookConfig(env: NodeJS.ProcessEnv = process.env): WebhookConfig {
|
||||
const url = env.DISCORD_WEBHOOK_URL?.trim() || null
|
||||
@@ -33,7 +33,7 @@ export function shouldSendWebhook(
|
||||
if (!config.url) return false
|
||||
if (!config.highlightedOnly) return true
|
||||
if (event === 'skill.highlighted') return true
|
||||
return skill.batch === 'highlighted'
|
||||
return Boolean(skill.highlighted)
|
||||
}
|
||||
|
||||
export function buildDiscordPayload(
|
||||
@@ -72,7 +72,7 @@ export function buildDiscordPayload(
|
||||
},
|
||||
],
|
||||
footer: {
|
||||
text: 'ClawdHub',
|
||||
text: 'OpenClaw',
|
||||
},
|
||||
timestamp: new Date().toISOString(),
|
||||
},
|
||||
@@ -89,9 +89,9 @@ export function buildSkillUrl(skill: WebhookSkillPayload, siteUrl: string) {
|
||||
function buildDescription(event: WebhookEvent, skill: WebhookSkillPayload) {
|
||||
const summary = (skill.summary ?? '').trim()
|
||||
if (summary) return truncate(summary, 200)
|
||||
if (event === 'skill.highlighted') return 'Newly highlighted skill on ClawdHub.'
|
||||
if (skill.version) return `New version v${skill.version} published on ClawdHub.`
|
||||
return 'New skill published on ClawdHub.'
|
||||
if (event === 'skill.highlighted') return 'Newly highlighted skill on OpenClaw.'
|
||||
if (skill.version) return `New version v${skill.version} published on OpenClaw.`
|
||||
return 'New skill published on OpenClaw.'
|
||||
}
|
||||
|
||||
function parseBoolean(value?: string) {
|
||||
|
||||
@@ -265,6 +265,34 @@ type FingerprintBackfillPageResult = {
|
||||
isDone: boolean
|
||||
}
|
||||
|
||||
type BadgeBackfillStats = {
|
||||
skillsScanned: number
|
||||
skillsPatched: number
|
||||
highlightsPatched: number
|
||||
}
|
||||
|
||||
type SkillBadgeTableBackfillStats = {
|
||||
skillsScanned: number
|
||||
recordsInserted: number
|
||||
}
|
||||
|
||||
type BadgeBackfillPageItem = {
|
||||
skillId: Id<'skills'>
|
||||
ownerUserId: Id<'users'>
|
||||
createdAt?: number
|
||||
updatedAt?: number
|
||||
batch?: string
|
||||
badges?: Doc<'skills'>['badges']
|
||||
}
|
||||
|
||||
type BadgeBackfillPageResult = {
|
||||
items: BadgeBackfillPageItem[]
|
||||
cursor: string | null
|
||||
isDone: boolean
|
||||
}
|
||||
|
||||
type BadgeKind = Doc<'skillBadges'>['kind']
|
||||
|
||||
export const getSkillFingerprintBackfillPageInternal = internalQuery({
|
||||
args: {
|
||||
cursor: v.optional(v.string()),
|
||||
@@ -467,6 +495,344 @@ export const scheduleBackfillSkillFingerprints: ReturnType<typeof action> = acti
|
||||
},
|
||||
})
|
||||
|
||||
export const getSkillBadgeBackfillPageInternal = internalQuery({
|
||||
args: {
|
||||
cursor: v.optional(v.string()),
|
||||
batchSize: v.optional(v.number()),
|
||||
},
|
||||
handler: async (ctx, args): Promise<BadgeBackfillPageResult> => {
|
||||
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
|
||||
const { page, isDone, continueCursor } = await ctx.db
|
||||
.query('skills')
|
||||
.order('asc')
|
||||
.paginate({ cursor: args.cursor ?? null, numItems: batchSize })
|
||||
|
||||
const items: BadgeBackfillPageItem[] = page.map((skill) => ({
|
||||
skillId: skill._id,
|
||||
ownerUserId: skill.ownerUserId,
|
||||
createdAt: skill.createdAt ?? undefined,
|
||||
updatedAt: skill.updatedAt ?? undefined,
|
||||
batch: skill.batch ?? undefined,
|
||||
badges: skill.badges ?? undefined,
|
||||
}))
|
||||
|
||||
return { items, cursor: continueCursor, isDone }
|
||||
},
|
||||
})
|
||||
|
||||
export const applySkillBadgeBackfillPatchInternal = internalMutation({
|
||||
args: {
|
||||
skillId: v.id('skills'),
|
||||
badges: v.optional(
|
||||
v.object({
|
||||
redactionApproved: v.optional(
|
||||
v.object({
|
||||
byUserId: v.id('users'),
|
||||
at: v.number(),
|
||||
}),
|
||||
),
|
||||
highlighted: v.optional(
|
||||
v.object({
|
||||
byUserId: v.id('users'),
|
||||
at: v.number(),
|
||||
}),
|
||||
),
|
||||
official: v.optional(
|
||||
v.object({
|
||||
byUserId: v.id('users'),
|
||||
at: v.number(),
|
||||
}),
|
||||
),
|
||||
deprecated: v.optional(
|
||||
v.object({
|
||||
byUserId: v.id('users'),
|
||||
at: v.number(),
|
||||
}),
|
||||
),
|
||||
}),
|
||||
),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
await ctx.db.patch(args.skillId, { badges: args.badges ?? undefined, updatedAt: Date.now() })
|
||||
return { ok: true as const }
|
||||
},
|
||||
})
|
||||
|
||||
export const upsertSkillBadgeRecordInternal = internalMutation({
|
||||
args: {
|
||||
skillId: v.id('skills'),
|
||||
kind: v.union(
|
||||
v.literal('highlighted'),
|
||||
v.literal('official'),
|
||||
v.literal('deprecated'),
|
||||
v.literal('redactionApproved'),
|
||||
),
|
||||
byUserId: v.id('users'),
|
||||
at: v.number(),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const existing = await ctx.db
|
||||
.query('skillBadges')
|
||||
.withIndex('by_skill_kind', (q) => q.eq('skillId', args.skillId).eq('kind', args.kind))
|
||||
.unique()
|
||||
if (existing) return { inserted: false as const }
|
||||
await ctx.db.insert('skillBadges', {
|
||||
skillId: args.skillId,
|
||||
kind: args.kind,
|
||||
byUserId: args.byUserId,
|
||||
at: args.at,
|
||||
})
|
||||
return { inserted: true as const }
|
||||
},
|
||||
})
|
||||
|
||||
export type BadgeBackfillActionArgs = {
|
||||
dryRun?: boolean
|
||||
batchSize?: number
|
||||
maxBatches?: number
|
||||
}
|
||||
|
||||
export type BadgeBackfillActionResult = { ok: true; stats: BadgeBackfillStats }
|
||||
|
||||
export async function backfillSkillBadgesInternalHandler(
|
||||
ctx: ActionCtx,
|
||||
args: BadgeBackfillActionArgs,
|
||||
): Promise<BadgeBackfillActionResult> {
|
||||
const dryRun = Boolean(args.dryRun)
|
||||
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
|
||||
const maxBatches = clampInt(args.maxBatches ?? DEFAULT_MAX_BATCHES, 1, MAX_MAX_BATCHES)
|
||||
|
||||
const totals: BadgeBackfillStats = {
|
||||
skillsScanned: 0,
|
||||
skillsPatched: 0,
|
||||
highlightsPatched: 0,
|
||||
}
|
||||
|
||||
let cursor: string | null = null
|
||||
let isDone = false
|
||||
|
||||
for (let i = 0; i < maxBatches; i++) {
|
||||
const page = (await ctx.runQuery(internal.maintenance.getSkillBadgeBackfillPageInternal, {
|
||||
cursor: cursor ?? undefined,
|
||||
batchSize,
|
||||
})) as BadgeBackfillPageResult
|
||||
|
||||
cursor = page.cursor
|
||||
isDone = page.isDone
|
||||
|
||||
for (const item of page.items) {
|
||||
totals.skillsScanned++
|
||||
|
||||
const shouldHighlight = item.batch === 'highlighted' && !item.badges?.highlighted
|
||||
if (!shouldHighlight) continue
|
||||
|
||||
totals.skillsPatched++
|
||||
totals.highlightsPatched++
|
||||
|
||||
if (dryRun) continue
|
||||
|
||||
const at = item.updatedAt ?? item.createdAt ?? Date.now()
|
||||
await ctx.runMutation(internal.maintenance.applySkillBadgeBackfillPatchInternal, {
|
||||
skillId: item.skillId,
|
||||
badges: {
|
||||
...item.badges,
|
||||
highlighted: {
|
||||
byUserId: item.ownerUserId,
|
||||
at,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
if (isDone) break
|
||||
}
|
||||
|
||||
if (!isDone) {
|
||||
throw new ConvexError('Backfill incomplete (maxBatches reached)')
|
||||
}
|
||||
|
||||
return { ok: true as const, stats: totals }
|
||||
}
|
||||
|
||||
export const backfillSkillBadgesInternal = internalAction({
|
||||
args: {
|
||||
dryRun: v.optional(v.boolean()),
|
||||
batchSize: v.optional(v.number()),
|
||||
maxBatches: v.optional(v.number()),
|
||||
},
|
||||
handler: backfillSkillBadgesInternalHandler,
|
||||
})
|
||||
|
||||
export const backfillSkillBadges: ReturnType<typeof action> = action({
|
||||
args: {
|
||||
dryRun: v.optional(v.boolean()),
|
||||
batchSize: v.optional(v.number()),
|
||||
maxBatches: v.optional(v.number()),
|
||||
},
|
||||
handler: async (ctx, args): Promise<BadgeBackfillActionResult> => {
|
||||
const { user } = await requireUserFromAction(ctx)
|
||||
assertRole(user, ['admin'])
|
||||
return ctx.runAction(
|
||||
internal.maintenance.backfillSkillBadgesInternal,
|
||||
args,
|
||||
) as Promise<BadgeBackfillActionResult>
|
||||
},
|
||||
})
|
||||
|
||||
export const scheduleBackfillSkillBadges: ReturnType<typeof action> = action({
|
||||
args: { dryRun: v.optional(v.boolean()) },
|
||||
handler: async (ctx, args) => {
|
||||
const { user } = await requireUserFromAction(ctx)
|
||||
assertRole(user, ['admin'])
|
||||
await ctx.scheduler.runAfter(0, internal.maintenance.backfillSkillBadgesInternal, {
|
||||
dryRun: Boolean(args.dryRun),
|
||||
batchSize: DEFAULT_BATCH_SIZE,
|
||||
maxBatches: DEFAULT_MAX_BATCHES,
|
||||
})
|
||||
return { ok: true as const }
|
||||
},
|
||||
})
|
||||
|
||||
export type SkillBadgeTableBackfillActionResult = {
|
||||
ok: true
|
||||
stats: SkillBadgeTableBackfillStats
|
||||
}
|
||||
|
||||
export async function backfillSkillBadgeTableInternalHandler(
|
||||
ctx: ActionCtx,
|
||||
args: BadgeBackfillActionArgs,
|
||||
): Promise<SkillBadgeTableBackfillActionResult> {
|
||||
const dryRun = Boolean(args.dryRun)
|
||||
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
|
||||
const maxBatches = clampInt(args.maxBatches ?? DEFAULT_MAX_BATCHES, 1, MAX_MAX_BATCHES)
|
||||
|
||||
const totals: SkillBadgeTableBackfillStats = {
|
||||
skillsScanned: 0,
|
||||
recordsInserted: 0,
|
||||
}
|
||||
|
||||
let cursor: string | null = null
|
||||
let isDone = false
|
||||
|
||||
for (let i = 0; i < maxBatches; i++) {
|
||||
const page = (await ctx.runQuery(internal.maintenance.getSkillBadgeBackfillPageInternal, {
|
||||
cursor: cursor ?? undefined,
|
||||
batchSize,
|
||||
})) as BadgeBackfillPageResult
|
||||
|
||||
cursor = page.cursor
|
||||
isDone = page.isDone
|
||||
|
||||
for (const item of page.items) {
|
||||
totals.skillsScanned++
|
||||
const badges = item.badges ?? {}
|
||||
const entries: Array<{ kind: BadgeKind; byUserId: Id<'users'>; at: number }> = []
|
||||
|
||||
if (badges.redactionApproved) {
|
||||
entries.push({
|
||||
kind: 'redactionApproved',
|
||||
byUserId: badges.redactionApproved.byUserId,
|
||||
at: badges.redactionApproved.at,
|
||||
})
|
||||
}
|
||||
|
||||
if (badges.official) {
|
||||
entries.push({
|
||||
kind: 'official',
|
||||
byUserId: badges.official.byUserId,
|
||||
at: badges.official.at,
|
||||
})
|
||||
}
|
||||
|
||||
if (badges.deprecated) {
|
||||
entries.push({
|
||||
kind: 'deprecated',
|
||||
byUserId: badges.deprecated.byUserId,
|
||||
at: badges.deprecated.at,
|
||||
})
|
||||
}
|
||||
|
||||
const highlighted =
|
||||
badges.highlighted ??
|
||||
(item.batch === 'highlighted'
|
||||
? {
|
||||
byUserId: item.ownerUserId,
|
||||
at: item.updatedAt ?? item.createdAt ?? Date.now(),
|
||||
}
|
||||
: undefined)
|
||||
|
||||
if (highlighted) {
|
||||
entries.push({
|
||||
kind: 'highlighted',
|
||||
byUserId: highlighted.byUserId,
|
||||
at: highlighted.at,
|
||||
})
|
||||
}
|
||||
|
||||
if (dryRun) continue
|
||||
|
||||
for (const entry of entries) {
|
||||
const result = await ctx.runMutation(internal.maintenance.upsertSkillBadgeRecordInternal, {
|
||||
skillId: item.skillId,
|
||||
kind: entry.kind,
|
||||
byUserId: entry.byUserId,
|
||||
at: entry.at,
|
||||
})
|
||||
if (result.inserted) {
|
||||
totals.recordsInserted++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (isDone) break
|
||||
}
|
||||
|
||||
if (!isDone) {
|
||||
throw new ConvexError('Backfill incomplete (maxBatches reached)')
|
||||
}
|
||||
|
||||
return { ok: true as const, stats: totals }
|
||||
}
|
||||
|
||||
export const backfillSkillBadgeTableInternal = internalAction({
|
||||
args: {
|
||||
dryRun: v.optional(v.boolean()),
|
||||
batchSize: v.optional(v.number()),
|
||||
maxBatches: v.optional(v.number()),
|
||||
},
|
||||
handler: backfillSkillBadgeTableInternalHandler,
|
||||
})
|
||||
|
||||
export const backfillSkillBadgeTable: ReturnType<typeof action> = action({
|
||||
args: {
|
||||
dryRun: v.optional(v.boolean()),
|
||||
batchSize: v.optional(v.number()),
|
||||
maxBatches: v.optional(v.number()),
|
||||
},
|
||||
handler: async (ctx, args): Promise<SkillBadgeTableBackfillActionResult> => {
|
||||
const { user } = await requireUserFromAction(ctx)
|
||||
assertRole(user, ['admin'])
|
||||
return ctx.runAction(
|
||||
internal.maintenance.backfillSkillBadgeTableInternal,
|
||||
args,
|
||||
) as Promise<SkillBadgeTableBackfillActionResult>
|
||||
},
|
||||
})
|
||||
|
||||
export const scheduleBackfillSkillBadgeTable: ReturnType<typeof action> = action({
|
||||
args: { dryRun: v.optional(v.boolean()) },
|
||||
handler: async (ctx, args) => {
|
||||
const { user } = await requireUserFromAction(ctx)
|
||||
assertRole(user, ['admin'])
|
||||
await ctx.scheduler.runAfter(0, internal.maintenance.backfillSkillBadgeTableInternal, {
|
||||
dryRun: Boolean(args.dryRun),
|
||||
batchSize: DEFAULT_BATCH_SIZE,
|
||||
maxBatches: DEFAULT_MAX_BATCHES,
|
||||
})
|
||||
return { ok: true as const }
|
||||
},
|
||||
})
|
||||
|
||||
function clampInt(value: number, min: number, max: number) {
|
||||
const rounded = Math.trunc(value)
|
||||
if (!Number.isFinite(rounded)) return min
|
||||
|
||||
+244
-7
@@ -3,6 +3,8 @@ import { defineSchema, defineTable } from 'convex/server'
|
||||
import { v } from 'convex/values'
|
||||
import { EMBEDDING_DIMENSIONS } from './lib/embeddings'
|
||||
|
||||
const authSchema = authTables as unknown as Record<string, ReturnType<typeof defineTable>>
|
||||
|
||||
const users = defineTable({
|
||||
name: v.optional(v.string()),
|
||||
image: v.optional(v.string()),
|
||||
@@ -47,8 +49,41 @@ const skills = defineTable({
|
||||
at: v.number(),
|
||||
}),
|
||||
),
|
||||
highlighted: v.optional(
|
||||
v.object({
|
||||
byUserId: v.id('users'),
|
||||
at: v.number(),
|
||||
}),
|
||||
),
|
||||
official: v.optional(
|
||||
v.object({
|
||||
byUserId: v.id('users'),
|
||||
at: v.number(),
|
||||
}),
|
||||
),
|
||||
deprecated: v.optional(
|
||||
v.object({
|
||||
byUserId: v.id('users'),
|
||||
at: v.number(),
|
||||
}),
|
||||
),
|
||||
}),
|
||||
moderationStatus: v.optional(
|
||||
v.union(v.literal('active'), v.literal('hidden'), v.literal('removed')),
|
||||
),
|
||||
moderationNotes: v.optional(v.string()),
|
||||
moderationReason: v.optional(v.string()),
|
||||
moderationFlags: v.optional(v.array(v.string())),
|
||||
lastReviewedAt: v.optional(v.number()),
|
||||
hiddenAt: v.optional(v.number()),
|
||||
hiddenBy: v.optional(v.id('users')),
|
||||
reportCount: v.optional(v.number()),
|
||||
lastReportedAt: v.optional(v.number()),
|
||||
batch: v.optional(v.string()),
|
||||
statsDownloads: v.optional(v.number()),
|
||||
statsStars: v.optional(v.number()),
|
||||
statsInstallsCurrent: v.optional(v.number()),
|
||||
statsInstallsAllTime: v.optional(v.number()),
|
||||
stats: v.object({
|
||||
downloads: v.number(),
|
||||
installsCurrent: v.optional(v.number()),
|
||||
@@ -63,7 +98,33 @@ const skills = defineTable({
|
||||
.index('by_slug', ['slug'])
|
||||
.index('by_owner', ['ownerUserId'])
|
||||
.index('by_updated', ['updatedAt'])
|
||||
.index('by_stats_downloads', ['statsDownloads', 'updatedAt'])
|
||||
.index('by_stats_stars', ['statsStars', 'updatedAt'])
|
||||
.index('by_stats_installs_current', ['statsInstallsCurrent', 'updatedAt'])
|
||||
.index('by_stats_installs_all_time', ['statsInstallsAllTime', 'updatedAt'])
|
||||
.index('by_batch', ['batch'])
|
||||
.index('by_active_updated', ['softDeletedAt', 'updatedAt'])
|
||||
|
||||
const souls = defineTable({
|
||||
slug: v.string(),
|
||||
displayName: v.string(),
|
||||
summary: v.optional(v.string()),
|
||||
ownerUserId: v.id('users'),
|
||||
latestVersionId: v.optional(v.id('soulVersions')),
|
||||
tags: v.record(v.string(), v.id('soulVersions')),
|
||||
softDeletedAt: v.optional(v.number()),
|
||||
stats: v.object({
|
||||
downloads: v.number(),
|
||||
stars: v.number(),
|
||||
versions: v.number(),
|
||||
comments: v.number(),
|
||||
}),
|
||||
createdAt: v.number(),
|
||||
updatedAt: v.number(),
|
||||
})
|
||||
.index('by_slug', ['slug'])
|
||||
.index('by_owner', ['ownerUserId'])
|
||||
.index('by_updated', ['updatedAt'])
|
||||
|
||||
const skillVersions = defineTable({
|
||||
skillId: v.id('skills'),
|
||||
@@ -84,6 +145,7 @@ const skillVersions = defineTable({
|
||||
frontmatter: v.record(v.string(), v.any()),
|
||||
metadata: v.optional(v.any()),
|
||||
clawdis: v.optional(v.any()),
|
||||
moltbot: v.optional(v.any()),
|
||||
}),
|
||||
createdBy: v.id('users'),
|
||||
createdAt: v.number(),
|
||||
@@ -92,6 +154,34 @@ const skillVersions = defineTable({
|
||||
.index('by_skill', ['skillId'])
|
||||
.index('by_skill_version', ['skillId', 'version'])
|
||||
|
||||
const soulVersions = defineTable({
|
||||
soulId: v.id('souls'),
|
||||
version: v.string(),
|
||||
fingerprint: v.optional(v.string()),
|
||||
changelog: v.string(),
|
||||
changelogSource: v.optional(v.union(v.literal('auto'), v.literal('user'))),
|
||||
files: v.array(
|
||||
v.object({
|
||||
path: v.string(),
|
||||
size: v.number(),
|
||||
storageId: v.id('_storage'),
|
||||
sha256: v.string(),
|
||||
contentType: v.optional(v.string()),
|
||||
}),
|
||||
),
|
||||
parsed: v.object({
|
||||
frontmatter: v.record(v.string(), v.any()),
|
||||
metadata: v.optional(v.any()),
|
||||
clawdis: v.optional(v.any()),
|
||||
moltbot: v.optional(v.any()),
|
||||
}),
|
||||
createdBy: v.id('users'),
|
||||
createdAt: v.number(),
|
||||
softDeletedAt: v.optional(v.number()),
|
||||
})
|
||||
.index('by_soul', ['soulId'])
|
||||
.index('by_soul_version', ['soulId', 'version'])
|
||||
|
||||
const skillVersionFingerprints = defineTable({
|
||||
skillId: v.id('skills'),
|
||||
versionId: v.id('skillVersions'),
|
||||
@@ -102,6 +192,31 @@ const skillVersionFingerprints = defineTable({
|
||||
.index('by_fingerprint', ['fingerprint'])
|
||||
.index('by_skill_fingerprint', ['skillId', 'fingerprint'])
|
||||
|
||||
const skillBadges = defineTable({
|
||||
skillId: v.id('skills'),
|
||||
kind: v.union(
|
||||
v.literal('highlighted'),
|
||||
v.literal('official'),
|
||||
v.literal('deprecated'),
|
||||
v.literal('redactionApproved'),
|
||||
),
|
||||
byUserId: v.id('users'),
|
||||
at: v.number(),
|
||||
})
|
||||
.index('by_skill', ['skillId'])
|
||||
.index('by_skill_kind', ['skillId', 'kind'])
|
||||
.index('by_kind_at', ['kind', 'at'])
|
||||
|
||||
const soulVersionFingerprints = defineTable({
|
||||
soulId: v.id('souls'),
|
||||
versionId: v.id('soulVersions'),
|
||||
fingerprint: v.string(),
|
||||
createdAt: v.number(),
|
||||
})
|
||||
.index('by_version', ['versionId'])
|
||||
.index('by_fingerprint', ['fingerprint'])
|
||||
.index('by_soul_fingerprint', ['soulId', 'fingerprint'])
|
||||
|
||||
const skillEmbeddings = defineTable({
|
||||
skillId: v.id('skills'),
|
||||
versionId: v.id('skillVersions'),
|
||||
@@ -120,6 +235,85 @@ const skillEmbeddings = defineTable({
|
||||
filterFields: ['visibility'],
|
||||
})
|
||||
|
||||
const skillDailyStats = defineTable({
|
||||
skillId: v.id('skills'),
|
||||
day: v.number(),
|
||||
downloads: v.number(),
|
||||
installs: v.number(),
|
||||
updatedAt: v.number(),
|
||||
})
|
||||
.index('by_skill_day', ['skillId', 'day'])
|
||||
.index('by_day', ['day'])
|
||||
|
||||
const skillLeaderboards = defineTable({
|
||||
kind: v.string(),
|
||||
generatedAt: v.number(),
|
||||
rangeStartDay: v.number(),
|
||||
rangeEndDay: v.number(),
|
||||
items: v.array(
|
||||
v.object({
|
||||
skillId: v.id('skills'),
|
||||
score: v.number(),
|
||||
installs: v.number(),
|
||||
downloads: v.number(),
|
||||
}),
|
||||
),
|
||||
}).index('by_kind', ['kind', 'generatedAt'])
|
||||
|
||||
const skillStatBackfillState = defineTable({
|
||||
key: v.string(),
|
||||
cursor: v.optional(v.string()),
|
||||
doneAt: v.optional(v.number()),
|
||||
updatedAt: v.number(),
|
||||
}).index('by_key', ['key'])
|
||||
|
||||
const skillStatEvents = defineTable({
|
||||
skillId: v.id('skills'),
|
||||
kind: v.union(
|
||||
v.literal('download'),
|
||||
v.literal('star'),
|
||||
v.literal('unstar'),
|
||||
v.literal('install_new'),
|
||||
v.literal('install_reactivate'),
|
||||
v.literal('install_deactivate'),
|
||||
v.literal('install_clear'),
|
||||
),
|
||||
delta: v.optional(
|
||||
v.object({
|
||||
allTime: v.number(),
|
||||
current: v.number(),
|
||||
}),
|
||||
),
|
||||
occurredAt: v.number(),
|
||||
processedAt: v.optional(v.number()),
|
||||
})
|
||||
.index('by_unprocessed', ['processedAt'])
|
||||
.index('by_skill', ['skillId'])
|
||||
|
||||
const skillStatUpdateCursors = defineTable({
|
||||
key: v.string(),
|
||||
cursorCreationTime: v.optional(v.number()),
|
||||
updatedAt: v.number(),
|
||||
}).index('by_key', ['key'])
|
||||
|
||||
const soulEmbeddings = defineTable({
|
||||
soulId: v.id('souls'),
|
||||
versionId: v.id('soulVersions'),
|
||||
ownerId: v.id('users'),
|
||||
embedding: v.array(v.number()),
|
||||
isLatest: v.boolean(),
|
||||
isApproved: v.boolean(),
|
||||
visibility: v.string(),
|
||||
updatedAt: v.number(),
|
||||
})
|
||||
.index('by_soul', ['soulId'])
|
||||
.index('by_version', ['versionId'])
|
||||
.vectorIndex('by_embedding', {
|
||||
vectorField: 'embedding',
|
||||
dimensions: EMBEDDING_DIMENSIONS,
|
||||
filterFields: ['visibility'],
|
||||
})
|
||||
|
||||
const comments = defineTable({
|
||||
skillId: v.id('skills'),
|
||||
userId: v.id('users'),
|
||||
@@ -131,6 +325,27 @@ const comments = defineTable({
|
||||
.index('by_skill', ['skillId'])
|
||||
.index('by_user', ['userId'])
|
||||
|
||||
const skillReports = defineTable({
|
||||
skillId: v.id('skills'),
|
||||
userId: v.id('users'),
|
||||
reason: v.optional(v.string()),
|
||||
createdAt: v.number(),
|
||||
})
|
||||
.index('by_skill', ['skillId'])
|
||||
.index('by_user', ['userId'])
|
||||
.index('by_skill_user', ['skillId', 'userId'])
|
||||
|
||||
const soulComments = defineTable({
|
||||
soulId: v.id('souls'),
|
||||
userId: v.id('users'),
|
||||
body: v.string(),
|
||||
createdAt: v.number(),
|
||||
softDeletedAt: v.optional(v.number()),
|
||||
deletedBy: v.optional(v.id('users')),
|
||||
})
|
||||
.index('by_soul', ['soulId'])
|
||||
.index('by_user', ['userId'])
|
||||
|
||||
const stars = defineTable({
|
||||
skillId: v.id('skills'),
|
||||
userId: v.id('users'),
|
||||
@@ -140,6 +355,15 @@ const stars = defineTable({
|
||||
.index('by_user', ['userId'])
|
||||
.index('by_skill_user', ['skillId', 'userId'])
|
||||
|
||||
const soulStars = defineTable({
|
||||
soulId: v.id('souls'),
|
||||
userId: v.id('users'),
|
||||
createdAt: v.number(),
|
||||
})
|
||||
.index('by_soul', ['soulId'])
|
||||
.index('by_user', ['userId'])
|
||||
.index('by_soul_user', ['soulId', 'userId'])
|
||||
|
||||
const auditLogs = defineTable({
|
||||
actorUserId: v.id('users'),
|
||||
action: v.string(),
|
||||
@@ -173,6 +397,12 @@ const rateLimits = defineTable({
|
||||
.index('by_key_window', ['key', 'windowStart'])
|
||||
.index('by_key', ['key'])
|
||||
|
||||
const githubBackupSyncState = defineTable({
|
||||
key: v.string(),
|
||||
cursor: v.optional(v.string()),
|
||||
updatedAt: v.number(),
|
||||
}).index('by_key', ['key'])
|
||||
|
||||
const userSyncRoots = defineTable({
|
||||
userId: v.id('users'),
|
||||
rootId: v.string(),
|
||||
@@ -211,21 +441,28 @@ const userSkillRootInstalls = defineTable({
|
||||
.index('by_user_skill', ['userId', 'skillId'])
|
||||
.index('by_skill', ['skillId'])
|
||||
|
||||
const githubBackupSyncState = defineTable({
|
||||
key: v.string(),
|
||||
cursor: v.optional(v.string()),
|
||||
updatedAt: v.number(),
|
||||
}).index('by_key', ['key'])
|
||||
|
||||
export default defineSchema({
|
||||
...authTables,
|
||||
...authSchema,
|
||||
users,
|
||||
skills,
|
||||
souls,
|
||||
skillVersions,
|
||||
soulVersions,
|
||||
skillVersionFingerprints,
|
||||
skillBadges,
|
||||
soulVersionFingerprints,
|
||||
skillEmbeddings,
|
||||
soulEmbeddings,
|
||||
skillDailyStats,
|
||||
skillLeaderboards,
|
||||
skillStatBackfillState,
|
||||
skillStatEvents,
|
||||
skillStatUpdateCursors,
|
||||
comments,
|
||||
skillReports,
|
||||
soulComments,
|
||||
stars,
|
||||
soulStars,
|
||||
auditLogs,
|
||||
apiTokens,
|
||||
rateLimits,
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
/* @vitest-environment node */
|
||||
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { __test } from './search'
|
||||
|
||||
describe('search helpers', () => {
|
||||
it('advances candidate limit until max', () => {
|
||||
expect(__test.getNextCandidateLimit(50, 1000)).toBe(100)
|
||||
expect(__test.getNextCandidateLimit(800, 1000)).toBe(1000)
|
||||
expect(__test.getNextCandidateLimit(1000, 1000)).toBeNull()
|
||||
})
|
||||
})
|
||||
+207
-21
@@ -2,16 +2,25 @@ import { v } from 'convex/values'
|
||||
import { internal } from './_generated/api'
|
||||
import type { Doc, Id } from './_generated/dataModel'
|
||||
import { action, internalQuery } from './_generated/server'
|
||||
import { getSkillBadgeMaps, isSkillHighlighted, type SkillBadgeMap } from './lib/badges'
|
||||
import { generateEmbedding } from './lib/embeddings'
|
||||
import { toPublicSkill, toPublicSoul } from './lib/public'
|
||||
import { matchesExactTokens, tokenize } from './lib/searchText'
|
||||
|
||||
type HydratedEntry = {
|
||||
embeddingId: Id<'skillEmbeddings'>
|
||||
skill: Doc<'skills'> | null
|
||||
skill: NonNullable<ReturnType<typeof toPublicSkill>>
|
||||
version: Doc<'skillVersions'> | null
|
||||
ownerHandle: string | null
|
||||
}
|
||||
|
||||
type SearchResult = HydratedEntry & { score: number }
|
||||
|
||||
function getNextCandidateLimit(current: number, max: number) {
|
||||
const next = Math.min(current * 2, max)
|
||||
return next > current ? next : null
|
||||
}
|
||||
|
||||
export const searchSkills: ReturnType<typeof action> = action({
|
||||
args: {
|
||||
query: v.string(),
|
||||
@@ -21,48 +30,225 @@ export const searchSkills: ReturnType<typeof action> = action({
|
||||
handler: async (ctx, args): Promise<SearchResult[]> => {
|
||||
const query = args.query.trim()
|
||||
if (!query) return []
|
||||
const vector = await generateEmbedding(query)
|
||||
const results = await ctx.vectorSearch('skillEmbeddings', 'by_embedding', {
|
||||
vector,
|
||||
limit: args.limit ?? 10,
|
||||
filter: (q) => q.or(q.eq('visibility', 'latest'), q.eq('visibility', 'latest-approved')),
|
||||
})
|
||||
const queryTokens = tokenize(query)
|
||||
if (queryTokens.length === 0) return []
|
||||
let vector: number[]
|
||||
try {
|
||||
vector = await generateEmbedding(query)
|
||||
} catch (error) {
|
||||
console.warn('Search embedding generation failed', error)
|
||||
return []
|
||||
}
|
||||
const limit = args.limit ?? 10
|
||||
// Convex vectorSearch max limit is 256; clamp candidate sizes accordingly.
|
||||
const maxCandidate = Math.min(Math.max(limit * 10, 200), 256)
|
||||
let candidateLimit = Math.min(Math.max(limit * 3, 50), 256)
|
||||
let hydrated: HydratedEntry[] = []
|
||||
let scoreById = new Map<Id<'skillEmbeddings'>, number>()
|
||||
let exactMatches: HydratedEntry[] = []
|
||||
|
||||
const hydrated = (await ctx.runQuery(internal.search.hydrateResults, {
|
||||
embeddingIds: results.map((result) => result._id),
|
||||
})) as HydratedEntry[]
|
||||
while (candidateLimit <= maxCandidate) {
|
||||
const results = await ctx.vectorSearch('skillEmbeddings', 'by_embedding', {
|
||||
vector,
|
||||
limit: candidateLimit,
|
||||
filter: (q) => q.or(q.eq('visibility', 'latest'), q.eq('visibility', 'latest-approved')),
|
||||
})
|
||||
|
||||
const scoreById = new Map<Id<'skillEmbeddings'>, number>(
|
||||
results.map((result) => [result._id, result._score]),
|
||||
)
|
||||
hydrated = (await ctx.runQuery(internal.search.hydrateResults, {
|
||||
embeddingIds: results.map((result) => result._id),
|
||||
})) as HydratedEntry[]
|
||||
|
||||
const filtered = args.highlightedOnly
|
||||
? hydrated.filter((entry) => entry.skill?.batch === 'highlighted')
|
||||
: hydrated
|
||||
scoreById = new Map<Id<'skillEmbeddings'>, number>(
|
||||
results.map((result) => [result._id, result._score]),
|
||||
)
|
||||
|
||||
return filtered
|
||||
const badgeMapEntries = (await ctx.runQuery(internal.search.getSkillBadgeMapsInternal, {
|
||||
skillIds: hydrated.map((entry) => entry.skill._id),
|
||||
})) as Array<[Id<'skills'>, SkillBadgeMap]>
|
||||
const badgeMapBySkillId = new Map(badgeMapEntries)
|
||||
const hydratedWithBadges = hydrated.map((entry) => ({
|
||||
...entry,
|
||||
skill: {
|
||||
...entry.skill,
|
||||
badges: badgeMapBySkillId.get(entry.skill._id) ?? {},
|
||||
},
|
||||
}))
|
||||
|
||||
const filtered = args.highlightedOnly
|
||||
? hydratedWithBadges.filter((entry) => isSkillHighlighted(entry.skill))
|
||||
: hydratedWithBadges
|
||||
|
||||
exactMatches = filtered.filter((entry) =>
|
||||
matchesExactTokens(queryTokens, [
|
||||
entry.skill.displayName,
|
||||
entry.skill.slug,
|
||||
entry.skill.summary,
|
||||
]),
|
||||
)
|
||||
|
||||
if (exactMatches.length >= limit || results.length < candidateLimit) {
|
||||
break
|
||||
}
|
||||
|
||||
const nextLimit = getNextCandidateLimit(candidateLimit, maxCandidate)
|
||||
if (!nextLimit) break
|
||||
candidateLimit = nextLimit
|
||||
}
|
||||
|
||||
return exactMatches
|
||||
.map((entry) => ({
|
||||
...entry,
|
||||
score: scoreById.get(entry.embeddingId) ?? 0,
|
||||
}))
|
||||
.filter((entry) => entry.skill)
|
||||
.slice(0, limit)
|
||||
},
|
||||
})
|
||||
|
||||
export const getBadgeMapsForSkills = internalQuery({
|
||||
args: { skillIds: v.array(v.id('skills')) },
|
||||
handler: async (ctx, args): Promise<Array<[Id<'skills'>, SkillBadgeMap]>> => {
|
||||
const badgeMap = await getSkillBadgeMaps(ctx, args.skillIds)
|
||||
return Array.from(badgeMap.entries())
|
||||
},
|
||||
})
|
||||
|
||||
export const hydrateResults = internalQuery({
|
||||
args: { embeddingIds: v.array(v.id('skillEmbeddings')) },
|
||||
handler: async (ctx, args): Promise<HydratedEntry[]> => {
|
||||
const entries: HydratedEntry[] = []
|
||||
const ownerHandleCache = new Map<Id<'users'>, Promise<string | null>>()
|
||||
|
||||
const getOwnerHandle = (ownerUserId: Id<'users'>) => {
|
||||
const cached = ownerHandleCache.get(ownerUserId)
|
||||
if (cached) return cached
|
||||
const handlePromise = ctx.db
|
||||
.get(ownerUserId)
|
||||
.then((owner) => owner?.handle ?? owner?._id ?? null)
|
||||
ownerHandleCache.set(ownerUserId, handlePromise)
|
||||
return handlePromise
|
||||
}
|
||||
|
||||
const entries = await Promise.all(
|
||||
args.embeddingIds.map(async (embeddingId) => {
|
||||
const embedding = await ctx.db.get(embeddingId)
|
||||
if (!embedding) return null
|
||||
const skill = await ctx.db.get(embedding.skillId)
|
||||
if (!skill || skill.softDeletedAt) return null
|
||||
const [version, ownerHandle] = await Promise.all([
|
||||
ctx.db.get(embedding.versionId),
|
||||
getOwnerHandle(skill.ownerUserId),
|
||||
])
|
||||
const publicSkill = toPublicSkill(skill)
|
||||
if (!publicSkill) return null
|
||||
return { embeddingId, skill: publicSkill, version, ownerHandle }
|
||||
}),
|
||||
)
|
||||
|
||||
return entries.filter((entry): entry is HydratedEntry => entry !== null)
|
||||
},
|
||||
})
|
||||
|
||||
type HydratedSoulEntry = {
|
||||
embeddingId: Id<'soulEmbeddings'>
|
||||
soul: NonNullable<ReturnType<typeof toPublicSoul>>
|
||||
version: Doc<'soulVersions'> | null
|
||||
}
|
||||
|
||||
type SoulSearchResult = HydratedSoulEntry & { score: number }
|
||||
|
||||
export const searchSouls: ReturnType<typeof action> = action({
|
||||
args: {
|
||||
query: v.string(),
|
||||
limit: v.optional(v.number()),
|
||||
},
|
||||
handler: async (ctx, args): Promise<SoulSearchResult[]> => {
|
||||
const query = args.query.trim()
|
||||
if (!query) return []
|
||||
const queryTokens = tokenize(query)
|
||||
if (queryTokens.length === 0) return []
|
||||
let vector: number[]
|
||||
try {
|
||||
vector = await generateEmbedding(query)
|
||||
} catch (error) {
|
||||
console.warn('Search embedding generation failed', error)
|
||||
return []
|
||||
}
|
||||
const limit = args.limit ?? 10
|
||||
// Convex vectorSearch max limit is 256; clamp candidate sizes accordingly.
|
||||
const maxCandidate = Math.min(Math.max(limit * 10, 200), 256)
|
||||
let candidateLimit = Math.min(Math.max(limit * 3, 50), 256)
|
||||
let hydrated: HydratedSoulEntry[] = []
|
||||
let scoreById = new Map<Id<'soulEmbeddings'>, number>()
|
||||
let exactMatches: HydratedSoulEntry[] = []
|
||||
|
||||
while (candidateLimit <= maxCandidate) {
|
||||
const results = await ctx.vectorSearch('soulEmbeddings', 'by_embedding', {
|
||||
vector,
|
||||
limit: candidateLimit,
|
||||
filter: (q) => q.or(q.eq('visibility', 'latest'), q.eq('visibility', 'latest-approved')),
|
||||
})
|
||||
|
||||
hydrated = (await ctx.runQuery(internal.search.hydrateSoulResults, {
|
||||
embeddingIds: results.map((result) => result._id),
|
||||
})) as HydratedSoulEntry[]
|
||||
|
||||
scoreById = new Map<Id<'soulEmbeddings'>, number>(
|
||||
results.map((result) => [result._id, result._score]),
|
||||
)
|
||||
|
||||
exactMatches = hydrated.filter((entry) =>
|
||||
matchesExactTokens(queryTokens, [
|
||||
entry.soul.displayName,
|
||||
entry.soul.slug,
|
||||
entry.soul.summary,
|
||||
]),
|
||||
)
|
||||
|
||||
if (exactMatches.length >= limit || results.length < candidateLimit) {
|
||||
break
|
||||
}
|
||||
|
||||
const nextLimit = getNextCandidateLimit(candidateLimit, maxCandidate)
|
||||
if (!nextLimit) break
|
||||
candidateLimit = nextLimit
|
||||
}
|
||||
|
||||
return exactMatches
|
||||
.map((entry) => ({
|
||||
...entry,
|
||||
score: scoreById.get(entry.embeddingId) ?? 0,
|
||||
}))
|
||||
.filter((entry) => entry.soul)
|
||||
.slice(0, limit)
|
||||
},
|
||||
})
|
||||
|
||||
export const hydrateSoulResults = internalQuery({
|
||||
args: { embeddingIds: v.array(v.id('soulEmbeddings')) },
|
||||
handler: async (ctx, args): Promise<HydratedSoulEntry[]> => {
|
||||
const entries: HydratedSoulEntry[] = []
|
||||
|
||||
for (const embeddingId of args.embeddingIds) {
|
||||
const embedding = await ctx.db.get(embeddingId)
|
||||
if (!embedding) continue
|
||||
const skill = await ctx.db.get(embedding.skillId)
|
||||
if (skill?.softDeletedAt) continue
|
||||
const soul = await ctx.db.get(embedding.soulId)
|
||||
if (soul?.softDeletedAt) continue
|
||||
const version = await ctx.db.get(embedding.versionId)
|
||||
entries.push({ embeddingId, skill, version })
|
||||
const publicSoul = toPublicSoul(soul)
|
||||
if (!publicSoul) continue
|
||||
entries.push({ embeddingId, soul: publicSoul, version })
|
||||
}
|
||||
|
||||
return entries
|
||||
},
|
||||
})
|
||||
|
||||
export const getSkillBadgeMapsInternal = internalQuery({
|
||||
args: { skillIds: v.array(v.id('skills')) },
|
||||
handler: async (ctx, args) => {
|
||||
const badgeMap = await getSkillBadgeMaps(ctx, args.skillIds)
|
||||
return Array.from(badgeMap.entries())
|
||||
},
|
||||
})
|
||||
|
||||
export const __test = { getNextCandidateLimit }
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import type { Doc } from './_generated/dataModel'
|
||||
import { decideSeedStart } from './seed'
|
||||
|
||||
function seedState(cursor: string, updatedAt: number) {
|
||||
return { cursor, updatedAt } as unknown as Doc<'githubBackupSyncState'>
|
||||
}
|
||||
|
||||
describe('decideSeedStart', () => {
|
||||
it('returns done when done', () => {
|
||||
expect(decideSeedStart(seedState('done', Date.now()), Date.now())).toEqual({
|
||||
started: false,
|
||||
reason: 'done',
|
||||
})
|
||||
})
|
||||
|
||||
it('returns running when lock fresh', () => {
|
||||
const now = Date.now()
|
||||
expect(decideSeedStart(seedState('running', now), now + 1000)).toEqual({
|
||||
started: false,
|
||||
reason: 'running',
|
||||
})
|
||||
})
|
||||
|
||||
it('starts when lock stale', () => {
|
||||
const now = Date.now()
|
||||
const stale = now - 10 * 60 * 1000 - 1
|
||||
expect(decideSeedStart(seedState('running', stale), now)).toEqual({
|
||||
started: true,
|
||||
reason: 'patched',
|
||||
})
|
||||
})
|
||||
|
||||
it('starts when missing', () => {
|
||||
expect(decideSeedStart(null, Date.now())).toEqual({ started: true, reason: 'inserted' })
|
||||
})
|
||||
})
|
||||
+254
@@ -0,0 +1,254 @@
|
||||
import { v } from 'convex/values'
|
||||
import { internal } from './_generated/api'
|
||||
import type { Doc, Id } from './_generated/dataModel'
|
||||
import type { ActionCtx, DatabaseReader, DatabaseWriter } from './_generated/server'
|
||||
import { action, internalMutation, internalQuery } from './_generated/server'
|
||||
import { publishSoulVersionForUser } from './lib/soulPublish'
|
||||
import { SOUL_SEED_DISPLAY_NAME, SOUL_SEED_HANDLE, SOUL_SEED_KEY, SOUL_SEEDS } from './seedSouls'
|
||||
|
||||
const SEED_LOCK_STALE_MS = 10 * 60 * 1000
|
||||
|
||||
type SeedStateDoc = Doc<'githubBackupSyncState'>
|
||||
|
||||
type SeedStartDecision = {
|
||||
started: boolean
|
||||
reason: 'done' | 'running' | 'patched' | 'inserted'
|
||||
}
|
||||
|
||||
async function getSeedState(ctx: { db: DatabaseReader }): Promise<SeedStateDoc | null> {
|
||||
const entries = (await ctx.db
|
||||
.query('githubBackupSyncState')
|
||||
.withIndex('by_key', (q) => q.eq('key', SOUL_SEED_KEY))
|
||||
.order('desc')
|
||||
.take(2)) as SeedStateDoc[]
|
||||
return entries[0] ?? null
|
||||
}
|
||||
|
||||
async function cleanupSeedState(ctx: { db: DatabaseWriter }, keepId: Id<'githubBackupSyncState'>) {
|
||||
const entries = (await ctx.db
|
||||
.query('githubBackupSyncState')
|
||||
.withIndex('by_key', (q) => q.eq('key', SOUL_SEED_KEY))
|
||||
.order('desc')
|
||||
.take(50)) as SeedStateDoc[]
|
||||
|
||||
for (const entry of entries) {
|
||||
if (entry._id === keepId) continue
|
||||
await ctx.db.delete(entry._id)
|
||||
}
|
||||
}
|
||||
|
||||
export function decideSeedStart(existing: SeedStateDoc | null, now: number): SeedStartDecision {
|
||||
const cursor = existing?.cursor ?? null
|
||||
if (cursor === 'done') return { started: false, reason: 'done' }
|
||||
if (cursor === 'running' && existing && now - existing.updatedAt < SEED_LOCK_STALE_MS) {
|
||||
return { started: false, reason: 'running' }
|
||||
}
|
||||
return existing ? { started: true, reason: 'patched' } : { started: true, reason: 'inserted' }
|
||||
}
|
||||
|
||||
export const getSoulSeedStateInternal = internalQuery({
|
||||
args: {},
|
||||
handler: async (ctx) => getSeedState(ctx),
|
||||
})
|
||||
|
||||
export const setSoulSeedStateInternal = internalMutation({
|
||||
args: { status: v.string() },
|
||||
handler: async (ctx, args) => {
|
||||
const existing = await getSeedState(ctx)
|
||||
const now = Date.now()
|
||||
if (existing) {
|
||||
await ctx.db.patch(existing._id, { cursor: args.status, updatedAt: now })
|
||||
await cleanupSeedState(ctx, existing._id)
|
||||
return existing._id
|
||||
}
|
||||
const id = await ctx.db.insert('githubBackupSyncState', {
|
||||
key: SOUL_SEED_KEY,
|
||||
cursor: args.status,
|
||||
updatedAt: now,
|
||||
})
|
||||
await cleanupSeedState(ctx, id)
|
||||
return id
|
||||
},
|
||||
})
|
||||
|
||||
export const tryStartSoulSeedInternal = internalMutation({
|
||||
args: {},
|
||||
handler: async (ctx) => {
|
||||
const now = Date.now()
|
||||
const existing = await getSeedState(ctx)
|
||||
const decision = decideSeedStart(existing, now)
|
||||
|
||||
if (!decision.started) return decision
|
||||
|
||||
if (existing) {
|
||||
await ctx.db.patch(existing._id, { cursor: 'running', updatedAt: now })
|
||||
await cleanupSeedState(ctx, existing._id)
|
||||
return { started: true, reason: 'patched' as const }
|
||||
}
|
||||
|
||||
const id = await ctx.db.insert('githubBackupSyncState', {
|
||||
key: SOUL_SEED_KEY,
|
||||
cursor: 'running',
|
||||
updatedAt: now,
|
||||
})
|
||||
await cleanupSeedState(ctx, id)
|
||||
return { started: true, reason: 'inserted' as const }
|
||||
},
|
||||
})
|
||||
|
||||
export const hasAnySoulsInternal = internalQuery({
|
||||
args: {},
|
||||
handler: async (ctx) => {
|
||||
const entry = await ctx.db.query('souls').take(1)
|
||||
return entry.length > 0
|
||||
},
|
||||
})
|
||||
|
||||
export const ensureSoulSeeds = action({
|
||||
args: {},
|
||||
handler: async (ctx) => {
|
||||
const started = (await ctx.runMutation(internal.seed.tryStartSoulSeedInternal, {})) as {
|
||||
started: boolean
|
||||
reason: 'done' | 'running' | 'patched' | 'inserted'
|
||||
}
|
||||
if (!started.started) {
|
||||
if (started.reason === 'done') return { seeded: false, reason: 'already-seeded' as const }
|
||||
return { seeded: false, reason: 'in-progress' as const }
|
||||
}
|
||||
|
||||
const hasSouls = (await ctx.runQuery(internal.seed.hasAnySoulsInternal, {})) as boolean
|
||||
if (hasSouls) {
|
||||
await ctx.runMutation(internal.seed.setSoulSeedStateInternal, { status: 'done' })
|
||||
return { seeded: false, reason: 'souls-exist' as const }
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await runSeed(ctx)
|
||||
await ctx.runMutation(internal.seed.setSoulSeedStateInternal, { status: 'done' })
|
||||
return { seeded: true, reason: 'seeded' as const, ...result }
|
||||
} catch (error) {
|
||||
await ctx.runMutation(internal.seed.setSoulSeedStateInternal, { status: 'error' })
|
||||
throw error
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
export const seed = action({
|
||||
args: {},
|
||||
handler: async (ctx) => runSeed(ctx),
|
||||
})
|
||||
|
||||
async function runSeed(ctx: ActionCtx) {
|
||||
const userId = (await ctx.runMutation(internal.seed.ensureSeedUserInternal, {
|
||||
handle: SOUL_SEED_HANDLE,
|
||||
displayName: SOUL_SEED_DISPLAY_NAME,
|
||||
})) as Id<'users'>
|
||||
|
||||
const created: string[] = []
|
||||
const skipped: string[] = []
|
||||
|
||||
for (const seedEntry of SOUL_SEEDS) {
|
||||
const existing = (await ctx.runQuery(internal.souls.getSoulBySlugInternal, {
|
||||
slug: seedEntry.slug,
|
||||
})) as Doc<'souls'> | null
|
||||
if (existing) {
|
||||
if (existing.softDeletedAt && existing.ownerUserId === userId) {
|
||||
await ctx.runMutation(internal.souls.setSoulSoftDeletedInternal, {
|
||||
userId,
|
||||
slug: seedEntry.slug,
|
||||
deleted: false,
|
||||
})
|
||||
}
|
||||
skipped.push(seedEntry.slug)
|
||||
continue
|
||||
}
|
||||
|
||||
const body = seedEntry.readme
|
||||
if (!body) {
|
||||
skipped.push(seedEntry.slug)
|
||||
continue
|
||||
}
|
||||
|
||||
const bytes = new TextEncoder().encode(body)
|
||||
const sha256 = await sha256Hex(bytes)
|
||||
const storageId = await ctx.storage.store(new Blob([bytes], { type: 'text/markdown' }))
|
||||
|
||||
try {
|
||||
await publishSoulVersionForUser(ctx, userId, {
|
||||
slug: seedEntry.slug,
|
||||
displayName: seedEntry.displayName,
|
||||
version: seedEntry.version,
|
||||
changelog: '',
|
||||
tags: seedEntry.tags,
|
||||
files: [
|
||||
{
|
||||
path: 'SOUL.md',
|
||||
size: bytes.byteLength,
|
||||
storageId,
|
||||
sha256,
|
||||
contentType: 'text/markdown',
|
||||
},
|
||||
],
|
||||
})
|
||||
created.push(seedEntry.slug)
|
||||
} catch (error) {
|
||||
if (!isExpectedSeedSkipError(error)) throw error
|
||||
skipped.push(seedEntry.slug)
|
||||
}
|
||||
}
|
||||
|
||||
return { created, skipped }
|
||||
}
|
||||
|
||||
function isExpectedSeedSkipError(error: unknown) {
|
||||
const message = error instanceof Error ? error.message : String(error)
|
||||
return (
|
||||
message.includes('Version already exists') || message.includes('Only the owner can publish')
|
||||
)
|
||||
}
|
||||
|
||||
export const ensureSeedUserInternal = internalMutation({
|
||||
args: {
|
||||
handle: v.string(),
|
||||
displayName: v.string(),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const baseHandle = args.handle.trim()
|
||||
const displayName = args.displayName.trim()
|
||||
const candidates = [baseHandle, `${baseHandle}-bot`]
|
||||
for (let i = 2; i <= 6; i += 1) candidates.push(`${baseHandle}-bot-${i}`)
|
||||
|
||||
for (const candidate of candidates) {
|
||||
const existing = await ctx.db
|
||||
.query('users')
|
||||
.withIndex('handle', (q) => q.eq('handle', candidate))
|
||||
.take(2)
|
||||
const user = (existing[0] ?? null) as Doc<'users'> | null
|
||||
if (user) {
|
||||
if ((user.displayName ?? user.name) === displayName) return user._id
|
||||
continue
|
||||
}
|
||||
|
||||
return ctx.db.insert('users', {
|
||||
handle: candidate,
|
||||
displayName,
|
||||
createdAt: Date.now(),
|
||||
updatedAt: Date.now(),
|
||||
})
|
||||
}
|
||||
|
||||
throw new Error('Unable to allocate seed user handle')
|
||||
},
|
||||
})
|
||||
|
||||
async function sha256Hex(bytes: Uint8Array) {
|
||||
const data = new Uint8Array(bytes)
|
||||
const digest = await crypto.subtle.digest('SHA-256', data)
|
||||
return toHex(new Uint8Array(digest))
|
||||
}
|
||||
|
||||
function toHex(bytes: Uint8Array) {
|
||||
let out = ''
|
||||
for (const byte of bytes) out += byte.toString(16).padStart(2, '0')
|
||||
return out
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,568 @@
|
||||
/**
|
||||
* Skill Stat Events - Event-sourced stats processing for skills
|
||||
*
|
||||
* Instead of updating skill stats synchronously in the hot path (which can cause
|
||||
* contention when multiple users download/star/install the same skill), we insert
|
||||
* lightweight event records and process them in batches via a cron job.
|
||||
*
|
||||
* Flow:
|
||||
* 1. User action (download, star, install) → insertStatEvent() writes to skillStatEvents table
|
||||
* 2. Cron job runs every 5 minutes → processSkillStatEventsInternal() processes batches
|
||||
* 3. Events are aggregated per-skill to minimize database operations
|
||||
* 4. Stats are applied to skill documents and daily stats tables
|
||||
* 5. Events are marked as processed (kept forever for auditing)
|
||||
*/
|
||||
|
||||
import { v } from 'convex/values'
|
||||
import { internal } from './_generated/api'
|
||||
import type { Doc, Id } from './_generated/dataModel'
|
||||
import type { MutationCtx } from './_generated/server'
|
||||
import { internalAction, internalMutation, internalQuery } from './_generated/server'
|
||||
import { applySkillStatDeltas, bumpDailySkillStats } from './lib/skillStats'
|
||||
|
||||
/**
|
||||
* Event types that affect skill stats:
|
||||
*
|
||||
* - download: User downloaded skill as zip (+1 downloads)
|
||||
* - star: User starred the skill (+1 stars)
|
||||
* - unstar: User removed their star (-1 stars)
|
||||
* - install_new: First time this user installed this skill (+1 installsAllTime, +1 installsCurrent)
|
||||
* - install_reactivate: User re-added skill after removing it (+1 installsCurrent only)
|
||||
* - install_deactivate: User removed skill from all projects (-1 installsCurrent)
|
||||
* - install_clear: User cleared all telemetry data (custom delta for both allTime and current)
|
||||
*/
|
||||
export type StatEventKind =
|
||||
| 'download'
|
||||
| 'star'
|
||||
| 'unstar'
|
||||
| 'install_new'
|
||||
| 'install_reactivate'
|
||||
| 'install_deactivate'
|
||||
| 'install_clear'
|
||||
|
||||
/**
|
||||
* Insert a stat event to be processed later by the cron job.
|
||||
*
|
||||
* This is called from the hot path (downloads, stars, telemetry) instead of
|
||||
* directly updating skill stats. It's a single insert with no read-modify-write
|
||||
* cycle, so it's fast and doesn't contend with other operations on the same skill.
|
||||
*
|
||||
* @param ctx - Mutation context
|
||||
* @param params.skillId - The skill being affected
|
||||
* @param params.kind - Type of event (download, star, install_new, etc.)
|
||||
* @param params.occurredAt - When the event happened (defaults to now). Important for
|
||||
* daily stats bucketing - we want downloads at 11:55 PM Monday
|
||||
* to count toward Monday's stats even if processed on Tuesday.
|
||||
* @param params.delta - Only used for install_clear events, specifies exact delta amounts
|
||||
*/
|
||||
export async function insertStatEvent(
|
||||
ctx: MutationCtx,
|
||||
params: {
|
||||
skillId: Id<'skills'>
|
||||
kind: StatEventKind
|
||||
occurredAt?: number
|
||||
delta?: { allTime: number; current: number }
|
||||
},
|
||||
) {
|
||||
await ctx.db.insert('skillStatEvents', {
|
||||
skillId: params.skillId,
|
||||
kind: params.kind,
|
||||
delta: params.delta,
|
||||
occurredAt: params.occurredAt ?? Date.now(),
|
||||
processedAt: undefined,
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Aggregated deltas for a single skill after processing multiple events.
|
||||
*
|
||||
* When we process a batch of 100 events, many might be for the same skill.
|
||||
* Instead of updating the skill document once per event, we aggregate all
|
||||
* events for each skill and apply a single update.
|
||||
*
|
||||
* The downloadEvents and installNewEvents arrays store the original timestamps
|
||||
* so we can update daily stats with the correct day bucket for each event.
|
||||
*/
|
||||
type AggregatedDeltas = {
|
||||
downloads: number
|
||||
stars: number
|
||||
installsAllTime: number
|
||||
installsCurrent: number
|
||||
/** Original timestamps for each download event (for daily stats bucketing) */
|
||||
downloadEvents: number[]
|
||||
/** Original timestamps for each new install event (for daily stats bucketing) */
|
||||
installNewEvents: number[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Aggregate multiple events for a single skill into net deltas.
|
||||
*
|
||||
* Example: If a skill has these events in the batch:
|
||||
* - download (Mon 11pm)
|
||||
* - download (Tue 1am)
|
||||
* - star
|
||||
* - unstar
|
||||
* - star
|
||||
*
|
||||
* The result would be:
|
||||
* - downloads: 2
|
||||
* - stars: 1 (net: +1 -1 +1 = +1)
|
||||
* - downloadEvents: [<Mon 11pm timestamp>, <Tue 1am timestamp>]
|
||||
*
|
||||
* This aggregation reduces the number of database operations from N events
|
||||
* to 1 skill update + N daily stat updates (which themselves may coalesce
|
||||
* if multiple events fall on the same day).
|
||||
*/
|
||||
function aggregateEvents(events: Doc<'skillStatEvents'>[]): AggregatedDeltas {
|
||||
const result: AggregatedDeltas = {
|
||||
downloads: 0,
|
||||
stars: 0,
|
||||
installsAllTime: 0,
|
||||
installsCurrent: 0,
|
||||
downloadEvents: [],
|
||||
installNewEvents: [],
|
||||
}
|
||||
|
||||
for (const event of events) {
|
||||
switch (event.kind) {
|
||||
case 'download':
|
||||
result.downloads += 1
|
||||
result.downloadEvents.push(event.occurredAt)
|
||||
break
|
||||
case 'star':
|
||||
result.stars += 1
|
||||
break
|
||||
case 'unstar':
|
||||
result.stars -= 1
|
||||
break
|
||||
case 'install_new':
|
||||
// New user installing for the first time: count toward both lifetime and current
|
||||
result.installsAllTime += 1
|
||||
result.installsCurrent += 1
|
||||
result.installNewEvents.push(event.occurredAt)
|
||||
break
|
||||
case 'install_reactivate':
|
||||
// User re-added skill after removing: only affects current count
|
||||
result.installsCurrent += 1
|
||||
break
|
||||
case 'install_deactivate':
|
||||
// User removed skill from all projects: only affects current count
|
||||
result.installsCurrent -= 1
|
||||
break
|
||||
case 'install_clear':
|
||||
// User cleared telemetry: uses custom delta values (typically negative)
|
||||
if (event.delta) {
|
||||
result.installsAllTime += event.delta.allTime
|
||||
result.installsCurrent += event.delta.current
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a batch of unprocessed stat events.
|
||||
*
|
||||
* Called by cron every 5 minutes. Processes up to batchSize events (default 100).
|
||||
* If the batch is full, schedules an immediate follow-up run to drain the queue.
|
||||
*
|
||||
* Processing steps:
|
||||
* 1. Query unprocessed events (processedAt is undefined)
|
||||
* 2. Group events by skillId to minimize skill document fetches
|
||||
* 3. For each skill:
|
||||
* a. Fetch the skill document once
|
||||
* b. Aggregate all events for this skill into net deltas
|
||||
* c. Apply deltas to skill stats (downloads, stars, installs)
|
||||
* d. Update daily stats for trending (using original event timestamps)
|
||||
* e. Mark all events as processed
|
||||
* 4. If batch was full, schedule another run immediately
|
||||
*
|
||||
* Aggregation levels:
|
||||
* - Level 1: Batch of 100 events from the queue
|
||||
* - Level 2: Group by skillId (e.g., 100 events → 30 unique skills)
|
||||
* - Level 3: Aggregate events per skill (e.g., 5 events → 1 skill update)
|
||||
* - Level 4: Daily stats may coalesce (e.g., 3 downloads same day → 1 upsert)
|
||||
*/
|
||||
export const processSkillStatEventsInternal = internalMutation({
|
||||
args: { batchSize: v.optional(v.number()) },
|
||||
handler: async (ctx, args) => {
|
||||
const batchSize = args.batchSize ?? 100
|
||||
const now = Date.now()
|
||||
|
||||
// Level 1: Fetch a batch of unprocessed events
|
||||
const events = await ctx.db
|
||||
.query('skillStatEvents')
|
||||
.withIndex('by_unprocessed', (q) => q.eq('processedAt', undefined))
|
||||
.take(batchSize)
|
||||
|
||||
if (events.length === 0) {
|
||||
return { processed: 0 }
|
||||
}
|
||||
|
||||
// Level 2: Group events by skillId to minimize database reads
|
||||
// Instead of fetching the same skill document multiple times,
|
||||
// we fetch it once and process all its events together
|
||||
const eventsBySkill = new Map<Id<'skills'>, Doc<'skillStatEvents'>[]>()
|
||||
for (const event of events) {
|
||||
const existing = eventsBySkill.get(event.skillId) ?? []
|
||||
existing.push(event)
|
||||
eventsBySkill.set(event.skillId, existing)
|
||||
}
|
||||
|
||||
// Process each skill's events
|
||||
for (const [skillId, skillEvents] of eventsBySkill) {
|
||||
const skill = await ctx.db.get(skillId)
|
||||
|
||||
// Skill was deleted - just mark events as processed
|
||||
if (!skill) {
|
||||
for (const event of skillEvents) {
|
||||
await ctx.db.patch(event._id, { processedAt: now })
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// Level 3: Aggregate all events for this skill into net deltas
|
||||
// e.g., 3 downloads + 2 stars - 1 unstar → { downloads: 3, stars: 1 }
|
||||
const deltas = aggregateEvents(skillEvents)
|
||||
|
||||
// Apply aggregated deltas to skill stats (single update per skill)
|
||||
if (
|
||||
deltas.downloads !== 0 ||
|
||||
deltas.stars !== 0 ||
|
||||
deltas.installsAllTime !== 0 ||
|
||||
deltas.installsCurrent !== 0
|
||||
) {
|
||||
const patch = applySkillStatDeltas(skill, {
|
||||
downloads: deltas.downloads,
|
||||
stars: deltas.stars,
|
||||
installsAllTime: deltas.installsAllTime,
|
||||
installsCurrent: deltas.installsCurrent,
|
||||
})
|
||||
await ctx.db.patch(skill._id, {
|
||||
...patch,
|
||||
updatedAt: now,
|
||||
})
|
||||
}
|
||||
|
||||
// Update daily stats for trending/leaderboards
|
||||
// We use the ORIGINAL event timestamp (occurredAt) so that:
|
||||
// - A download at Mon 11:55 PM counts toward Monday's stats
|
||||
// - Even if the cron processes it on Tuesday
|
||||
//
|
||||
// Level 4: bumpDailySkillStats does its own coalescing - multiple
|
||||
// events on the same day will update the same daily record
|
||||
for (const occurredAt of deltas.downloadEvents) {
|
||||
await bumpDailySkillStats(ctx, { skillId, now: occurredAt, downloads: 1 })
|
||||
}
|
||||
for (const occurredAt of deltas.installNewEvents) {
|
||||
await bumpDailySkillStats(ctx, { skillId, now: occurredAt, installs: 1 })
|
||||
}
|
||||
|
||||
// Mark all events for this skill as processed
|
||||
for (const event of skillEvents) {
|
||||
await ctx.db.patch(event._id, { processedAt: now })
|
||||
}
|
||||
}
|
||||
|
||||
// If we hit the batch limit, there may be more events waiting.
|
||||
// Schedule an immediate follow-up run to drain the queue.
|
||||
// This ensures high-volume periods don't create a backlog.
|
||||
if (events.length === batchSize) {
|
||||
await ctx.scheduler.runAfter(0, internal.skillStatEvents.processSkillStatEventsInternal, {
|
||||
batchSize,
|
||||
})
|
||||
}
|
||||
|
||||
return { processed: events.length }
|
||||
},
|
||||
})
|
||||
|
||||
// ============================================================================
|
||||
// Action-based processing (cursor-based, runs outside transaction window)
|
||||
// ============================================================================
|
||||
|
||||
const CURSOR_KEY = 'skill_stat_events'
|
||||
const EVENT_BATCH_SIZE = 500
|
||||
const MAX_SKILLS_PER_RUN = 500
|
||||
|
||||
/**
|
||||
* Fetch a batch of events after the given cursor (by _creationTime).
|
||||
* Returns events sorted by _creationTime ascending.
|
||||
*/
|
||||
export const getUnprocessedEventBatch = internalQuery({
|
||||
args: {
|
||||
cursorCreationTime: v.optional(v.number()),
|
||||
limit: v.optional(v.number()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const limit = args.limit ?? EVENT_BATCH_SIZE
|
||||
const cursor = args.cursorCreationTime
|
||||
|
||||
// Query events after the cursor using the built-in creation time index
|
||||
const events = await ctx.db
|
||||
.query('skillStatEvents')
|
||||
.withIndex('by_creation_time', (q) =>
|
||||
cursor !== undefined ? q.gt('_creationTime', cursor) : q,
|
||||
)
|
||||
.take(limit)
|
||||
return events
|
||||
},
|
||||
})
|
||||
|
||||
/**
|
||||
* Get the current cursor position from the cursors table.
|
||||
*/
|
||||
export const getStatEventCursor = internalQuery({
|
||||
args: {},
|
||||
handler: async (ctx) => {
|
||||
const cursor = await ctx.db
|
||||
.query('skillStatUpdateCursors')
|
||||
.withIndex('by_key', (q) => q.eq('key', CURSOR_KEY))
|
||||
.unique()
|
||||
return cursor?.cursorCreationTime
|
||||
},
|
||||
})
|
||||
|
||||
/**
|
||||
* Validator for skill deltas passed to the mutation.
|
||||
*/
|
||||
const skillDeltaValidator = v.object({
|
||||
skillId: v.id('skills'),
|
||||
downloads: v.number(),
|
||||
stars: v.number(),
|
||||
installsAllTime: v.number(),
|
||||
installsCurrent: v.number(),
|
||||
downloadEvents: v.array(v.number()),
|
||||
installNewEvents: v.array(v.number()),
|
||||
})
|
||||
|
||||
/**
|
||||
* Apply aggregated stats to skills and update the cursor.
|
||||
* This is a single atomic mutation that:
|
||||
* 1. Updates all affected skills with their aggregated deltas
|
||||
* 2. Updates daily stats for trending
|
||||
* 3. Advances the cursor to the new position
|
||||
*/
|
||||
export const applyAggregatedStatsAndUpdateCursor = internalMutation({
|
||||
args: {
|
||||
skillDeltas: v.array(skillDeltaValidator),
|
||||
newCursor: v.number(),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const now = Date.now()
|
||||
|
||||
// Process each skill's aggregated deltas
|
||||
for (const delta of args.skillDeltas) {
|
||||
const skill = await ctx.db.get(delta.skillId)
|
||||
|
||||
// Skill was deleted - skip
|
||||
if (!skill) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Apply aggregated deltas to skill stats
|
||||
if (
|
||||
delta.downloads !== 0 ||
|
||||
delta.stars !== 0 ||
|
||||
delta.installsAllTime !== 0 ||
|
||||
delta.installsCurrent !== 0
|
||||
) {
|
||||
const patch = applySkillStatDeltas(skill, {
|
||||
downloads: delta.downloads,
|
||||
stars: delta.stars,
|
||||
installsAllTime: delta.installsAllTime,
|
||||
installsCurrent: delta.installsCurrent,
|
||||
})
|
||||
await ctx.db.patch(skill._id, {
|
||||
...patch,
|
||||
updatedAt: now,
|
||||
})
|
||||
}
|
||||
|
||||
// Update daily stats for trending/leaderboards
|
||||
for (const occurredAt of delta.downloadEvents) {
|
||||
await bumpDailySkillStats(ctx, { skillId: delta.skillId, now: occurredAt, downloads: 1 })
|
||||
}
|
||||
for (const occurredAt of delta.installNewEvents) {
|
||||
await bumpDailySkillStats(ctx, { skillId: delta.skillId, now: occurredAt, installs: 1 })
|
||||
}
|
||||
}
|
||||
|
||||
// Update cursor position (upsert)
|
||||
const existingCursor = await ctx.db
|
||||
.query('skillStatUpdateCursors')
|
||||
.withIndex('by_key', (q) => q.eq('key', CURSOR_KEY))
|
||||
.unique()
|
||||
|
||||
if (existingCursor) {
|
||||
await ctx.db.patch(existingCursor._id, {
|
||||
cursorCreationTime: args.newCursor,
|
||||
updatedAt: now,
|
||||
})
|
||||
} else {
|
||||
await ctx.db.insert('skillStatUpdateCursors', {
|
||||
key: CURSOR_KEY,
|
||||
cursorCreationTime: args.newCursor,
|
||||
updatedAt: now,
|
||||
})
|
||||
}
|
||||
|
||||
return { skillsUpdated: args.skillDeltas.length }
|
||||
},
|
||||
})
|
||||
|
||||
/**
|
||||
* Action that processes skill stat events in batches outside the transaction window.
|
||||
*
|
||||
* Algorithm:
|
||||
* 1. Get current cursor position
|
||||
* 2. Fetch events in batches of 500, aggregating as we go
|
||||
* 3. Stop when we have >= 500 unique skills OR run out of events
|
||||
* 4. Call mutation to apply all deltas and update cursor atomically
|
||||
* 5. Self-schedule if we stopped due to skill limit (not exhaustion)
|
||||
*/
|
||||
export const processSkillStatEventsAction = internalAction({
|
||||
args: {},
|
||||
handler: async (ctx) => {
|
||||
// Get current cursor position (convert null to undefined for consistency)
|
||||
const cursorResult = await ctx.runQuery(internal.skillStatEvents.getStatEventCursor)
|
||||
let cursor: number | undefined = cursorResult ?? undefined
|
||||
|
||||
console.log(`[STAT-AGG] Starting aggregation, cursor=${cursor ?? 'none'}`)
|
||||
|
||||
// Aggregated deltas per skill
|
||||
const aggregatedBySkill = new Map<
|
||||
Id<'skills'>,
|
||||
{
|
||||
downloads: number
|
||||
stars: number
|
||||
installsAllTime: number
|
||||
installsCurrent: number
|
||||
downloadEvents: number[]
|
||||
installNewEvents: number[]
|
||||
}
|
||||
>()
|
||||
|
||||
let maxCreationTime: number | undefined = cursor
|
||||
let exhausted = false
|
||||
let totalEventsFetched = 0
|
||||
|
||||
// Fetch and aggregate until we have enough skills or run out of events
|
||||
while (aggregatedBySkill.size < MAX_SKILLS_PER_RUN) {
|
||||
const events = await ctx.runQuery(internal.skillStatEvents.getUnprocessedEventBatch, {
|
||||
cursorCreationTime: cursor,
|
||||
limit: EVENT_BATCH_SIZE,
|
||||
})
|
||||
|
||||
if (events.length === 0) {
|
||||
exhausted = true
|
||||
break
|
||||
}
|
||||
|
||||
totalEventsFetched += events.length
|
||||
const skillsBefore = aggregatedBySkill.size
|
||||
|
||||
// Aggregate events into per-skill deltas
|
||||
for (const event of events) {
|
||||
let skillDelta = aggregatedBySkill.get(event.skillId)
|
||||
if (!skillDelta) {
|
||||
skillDelta = {
|
||||
downloads: 0,
|
||||
stars: 0,
|
||||
installsAllTime: 0,
|
||||
installsCurrent: 0,
|
||||
downloadEvents: [],
|
||||
installNewEvents: [],
|
||||
}
|
||||
aggregatedBySkill.set(event.skillId, skillDelta)
|
||||
}
|
||||
|
||||
// Apply event to aggregated deltas
|
||||
switch (event.kind) {
|
||||
case 'download':
|
||||
skillDelta.downloads += 1
|
||||
skillDelta.downloadEvents.push(event.occurredAt)
|
||||
break
|
||||
case 'star':
|
||||
skillDelta.stars += 1
|
||||
break
|
||||
case 'unstar':
|
||||
skillDelta.stars -= 1
|
||||
break
|
||||
case 'install_new':
|
||||
skillDelta.installsAllTime += 1
|
||||
skillDelta.installsCurrent += 1
|
||||
skillDelta.installNewEvents.push(event.occurredAt)
|
||||
break
|
||||
case 'install_reactivate':
|
||||
skillDelta.installsCurrent += 1
|
||||
break
|
||||
case 'install_deactivate':
|
||||
skillDelta.installsCurrent -= 1
|
||||
break
|
||||
case 'install_clear':
|
||||
if (event.delta) {
|
||||
skillDelta.installsAllTime += event.delta.allTime
|
||||
skillDelta.installsCurrent += event.delta.current
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
// Track highest _creationTime seen
|
||||
if (maxCreationTime === undefined || event._creationTime > maxCreationTime) {
|
||||
maxCreationTime = event._creationTime
|
||||
}
|
||||
}
|
||||
|
||||
// Update cursor for next batch fetch
|
||||
cursor = events[events.length - 1]._creationTime
|
||||
|
||||
console.log(
|
||||
`[STAT-AGG] Fetched ${events.length} events, ${aggregatedBySkill.size - skillsBefore} new skills (${aggregatedBySkill.size} total)`,
|
||||
)
|
||||
|
||||
// If we got fewer than requested, we've exhausted the events
|
||||
if (events.length < EVENT_BATCH_SIZE) {
|
||||
exhausted = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// If we have nothing to process, we're done
|
||||
if (aggregatedBySkill.size === 0 || maxCreationTime === undefined) {
|
||||
console.log('[STAT-AGG] No events to process, done')
|
||||
return { processed: 0, skillsUpdated: 0, exhausted: true }
|
||||
}
|
||||
|
||||
// Convert map to array for mutation
|
||||
const skillDeltas = Array.from(aggregatedBySkill.entries()).map(([skillId, delta]) => ({
|
||||
skillId,
|
||||
...delta,
|
||||
}))
|
||||
|
||||
console.log(
|
||||
`[STAT-AGG] Running mutation for ${skillDeltas.length} skills (${totalEventsFetched} total events)`,
|
||||
)
|
||||
|
||||
// Apply all deltas and update cursor atomically
|
||||
await ctx.runMutation(internal.skillStatEvents.applyAggregatedStatsAndUpdateCursor, {
|
||||
skillDeltas,
|
||||
newCursor: maxCreationTime,
|
||||
})
|
||||
|
||||
// Self-schedule if we stopped because of skill limit, not exhaustion
|
||||
if (!exhausted) {
|
||||
console.log('[STAT-AGG] More events remaining, self-scheduling')
|
||||
await ctx.scheduler.runAfter(0, internal.skillStatEvents.processSkillStatEventsAction, {})
|
||||
} else {
|
||||
console.log('[STAT-AGG] All events processed, done')
|
||||
}
|
||||
|
||||
return {
|
||||
skillsUpdated: skillDeltas.length,
|
||||
exhausted,
|
||||
}
|
||||
},
|
||||
})
|
||||
+930
-51
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,88 @@
|
||||
import { v } from 'convex/values'
|
||||
import type { Doc } from './_generated/dataModel'
|
||||
import { mutation, query } from './_generated/server'
|
||||
import { assertModerator, requireUser } from './lib/access'
|
||||
import { type PublicUser, toPublicUser } from './lib/public'
|
||||
|
||||
export const listBySoul = query({
|
||||
args: { soulId: v.id('souls'), limit: v.optional(v.number()) },
|
||||
handler: async (ctx, args) => {
|
||||
const limit = args.limit ?? 50
|
||||
const comments = await ctx.db
|
||||
.query('soulComments')
|
||||
.withIndex('by_soul', (q) => q.eq('soulId', args.soulId))
|
||||
.order('desc')
|
||||
.take(limit)
|
||||
|
||||
const results: Array<{ comment: Doc<'soulComments'>; user: PublicUser | null }> = []
|
||||
for (const comment of comments) {
|
||||
if (comment.softDeletedAt) continue
|
||||
const user = toPublicUser(await ctx.db.get(comment.userId))
|
||||
results.push({ comment, user })
|
||||
}
|
||||
return results
|
||||
},
|
||||
})
|
||||
|
||||
export const add = mutation({
|
||||
args: { soulId: v.id('souls'), body: v.string() },
|
||||
handler: async (ctx, args) => {
|
||||
const { userId } = await requireUser(ctx)
|
||||
const body = args.body.trim()
|
||||
if (!body) throw new Error('Comment body required')
|
||||
|
||||
const soul = await ctx.db.get(args.soulId)
|
||||
if (!soul) throw new Error('Soul not found')
|
||||
|
||||
await ctx.db.insert('soulComments', {
|
||||
soulId: args.soulId,
|
||||
userId,
|
||||
body,
|
||||
createdAt: Date.now(),
|
||||
softDeletedAt: undefined,
|
||||
deletedBy: undefined,
|
||||
})
|
||||
|
||||
await ctx.db.patch(soul._id, {
|
||||
stats: { ...soul.stats, comments: soul.stats.comments + 1 },
|
||||
updatedAt: Date.now(),
|
||||
})
|
||||
},
|
||||
})
|
||||
|
||||
export const remove = mutation({
|
||||
args: { commentId: v.id('soulComments') },
|
||||
handler: async (ctx, args) => {
|
||||
const { user } = await requireUser(ctx)
|
||||
const comment = await ctx.db.get(args.commentId)
|
||||
if (!comment) throw new Error('Comment not found')
|
||||
if (comment.softDeletedAt) return
|
||||
|
||||
const isOwner = comment.userId === user._id
|
||||
if (!isOwner) {
|
||||
assertModerator(user)
|
||||
}
|
||||
|
||||
await ctx.db.patch(comment._id, {
|
||||
softDeletedAt: Date.now(),
|
||||
deletedBy: user._id,
|
||||
})
|
||||
|
||||
const soul = await ctx.db.get(comment.soulId)
|
||||
if (soul) {
|
||||
await ctx.db.patch(soul._id, {
|
||||
stats: { ...soul.stats, comments: Math.max(0, soul.stats.comments - 1) },
|
||||
updatedAt: Date.now(),
|
||||
})
|
||||
}
|
||||
|
||||
await ctx.db.insert('auditLogs', {
|
||||
actorUserId: user._id,
|
||||
action: 'soul.comment.delete',
|
||||
targetType: 'soulComment',
|
||||
targetId: comment._id,
|
||||
metadata: { soulId: comment.soulId },
|
||||
createdAt: Date.now(),
|
||||
})
|
||||
},
|
||||
})
|
||||
@@ -0,0 +1,14 @@
|
||||
import { v } from 'convex/values'
|
||||
import { mutation } from './_generated/server'
|
||||
|
||||
export const increment = mutation({
|
||||
args: { soulId: v.id('souls') },
|
||||
handler: async (ctx, args) => {
|
||||
const soul = await ctx.db.get(args.soulId)
|
||||
if (!soul) return
|
||||
await ctx.db.patch(soul._id, {
|
||||
stats: { ...soul.stats, downloads: soul.stats.downloads + 1 },
|
||||
updatedAt: Date.now(),
|
||||
})
|
||||
},
|
||||
})
|
||||
@@ -0,0 +1,71 @@
|
||||
import { v } from 'convex/values'
|
||||
import { mutation, query } from './_generated/server'
|
||||
import { requireUser } from './lib/access'
|
||||
import { toPublicSoul } from './lib/public'
|
||||
|
||||
export const isStarred = query({
|
||||
args: { soulId: v.id('souls') },
|
||||
handler: async (ctx, args) => {
|
||||
const { userId } = await requireUser(ctx)
|
||||
const existing = await ctx.db
|
||||
.query('soulStars')
|
||||
.withIndex('by_soul_user', (q) => q.eq('soulId', args.soulId).eq('userId', userId))
|
||||
.unique()
|
||||
return Boolean(existing)
|
||||
},
|
||||
})
|
||||
|
||||
export const toggle = mutation({
|
||||
args: { soulId: v.id('souls') },
|
||||
handler: async (ctx, args) => {
|
||||
const { userId } = await requireUser(ctx)
|
||||
const soul = await ctx.db.get(args.soulId)
|
||||
if (!soul) throw new Error('Soul not found')
|
||||
|
||||
const existing = await ctx.db
|
||||
.query('soulStars')
|
||||
.withIndex('by_soul_user', (q) => q.eq('soulId', args.soulId).eq('userId', userId))
|
||||
.unique()
|
||||
|
||||
if (existing) {
|
||||
await ctx.db.delete(existing._id)
|
||||
await ctx.db.patch(soul._id, {
|
||||
stats: { ...soul.stats, stars: Math.max(0, soul.stats.stars - 1) },
|
||||
updatedAt: Date.now(),
|
||||
})
|
||||
return { starred: false }
|
||||
}
|
||||
|
||||
await ctx.db.insert('soulStars', {
|
||||
soulId: args.soulId,
|
||||
userId,
|
||||
createdAt: Date.now(),
|
||||
})
|
||||
await ctx.db.patch(soul._id, {
|
||||
stats: { ...soul.stats, stars: soul.stats.stars + 1 },
|
||||
updatedAt: Date.now(),
|
||||
})
|
||||
return { starred: true }
|
||||
},
|
||||
})
|
||||
|
||||
export const listByUser = query({
|
||||
args: { userId: v.id('users'), limit: v.optional(v.number()) },
|
||||
handler: async (ctx, args) => {
|
||||
const limit = args.limit ?? 50
|
||||
const stars = await ctx.db
|
||||
.query('soulStars')
|
||||
.withIndex('by_user', (q) => q.eq('userId', args.userId))
|
||||
.order('desc')
|
||||
.take(limit)
|
||||
|
||||
const souls: NonNullable<ReturnType<typeof toPublicSoul>>[] = []
|
||||
for (const star of stars) {
|
||||
const soul = await ctx.db.get(star.soulId)
|
||||
const publicSoul = toPublicSoul(soul)
|
||||
if (!publicSoul) continue
|
||||
souls.push(publicSoul)
|
||||
}
|
||||
return souls
|
||||
},
|
||||
})
|
||||
+570
@@ -0,0 +1,570 @@
|
||||
import { ConvexError, v } from 'convex/values'
|
||||
import { internal } from './_generated/api'
|
||||
import type { Doc, Id } from './_generated/dataModel'
|
||||
import { action, internalMutation, internalQuery, mutation, query } from './_generated/server'
|
||||
import { assertModerator, requireUser, requireUserFromAction } from './lib/access'
|
||||
import { toPublicSoul, toPublicUser } from './lib/public'
|
||||
import { getFrontmatterValue, hashSkillFiles } from './lib/skills'
|
||||
import { generateSoulChangelogPreview } from './lib/soulChangelog'
|
||||
import { fetchText, type PublishResult, publishSoulVersionForUser } from './lib/soulPublish'
|
||||
|
||||
export { publishSoulVersionForUser } from './lib/soulPublish'
|
||||
|
||||
type ReadmeResult = { path: string; text: string }
|
||||
|
||||
type FileTextResult = { path: string; text: string; size: number; sha256: string }
|
||||
|
||||
const MAX_DIFF_FILE_BYTES = 200 * 1024
|
||||
const MAX_LIST_LIMIT = 50
|
||||
|
||||
export const getBySlug = query({
|
||||
args: { slug: v.string() },
|
||||
handler: async (ctx, args) => {
|
||||
const matches = await ctx.db
|
||||
.query('souls')
|
||||
.withIndex('by_slug', (q) => q.eq('slug', args.slug))
|
||||
.order('desc')
|
||||
.take(2)
|
||||
const soul = matches[0] ?? null
|
||||
if (!soul || soul.softDeletedAt) return null
|
||||
const latestVersion = soul.latestVersionId ? await ctx.db.get(soul.latestVersionId) : null
|
||||
const owner = toPublicUser(await ctx.db.get(soul.ownerUserId))
|
||||
const publicSoul = toPublicSoul(soul)
|
||||
if (!publicSoul) return null
|
||||
|
||||
return { soul: publicSoul, latestVersion, owner }
|
||||
},
|
||||
})
|
||||
|
||||
export const getSoulBySlugInternal = internalQuery({
|
||||
args: { slug: v.string() },
|
||||
handler: async (ctx, args) => {
|
||||
const matches = await ctx.db
|
||||
.query('souls')
|
||||
.withIndex('by_slug', (q) => q.eq('slug', args.slug))
|
||||
.order('desc')
|
||||
.take(2)
|
||||
return matches[0] ?? null
|
||||
},
|
||||
})
|
||||
|
||||
export const list = query({
|
||||
args: {
|
||||
ownerUserId: v.optional(v.id('users')),
|
||||
limit: v.optional(v.number()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const limit = args.limit ?? 24
|
||||
const ownerUserId = args.ownerUserId
|
||||
if (ownerUserId) {
|
||||
const entries = await ctx.db
|
||||
.query('souls')
|
||||
.withIndex('by_owner', (q) => q.eq('ownerUserId', ownerUserId))
|
||||
.order('desc')
|
||||
.take(limit * 5)
|
||||
return entries
|
||||
.filter((soul) => !soul.softDeletedAt)
|
||||
.slice(0, limit)
|
||||
.map((soul) => toPublicSoul(soul))
|
||||
.filter((soul): soul is NonNullable<typeof soul> => Boolean(soul))
|
||||
}
|
||||
const entries = await ctx.db
|
||||
.query('souls')
|
||||
.order('desc')
|
||||
.take(limit * 5)
|
||||
return entries
|
||||
.filter((soul) => !soul.softDeletedAt)
|
||||
.slice(0, limit)
|
||||
.map((soul) => toPublicSoul(soul))
|
||||
.filter((soul): soul is NonNullable<typeof soul> => Boolean(soul))
|
||||
},
|
||||
})
|
||||
|
||||
export const listPublicPage = query({
|
||||
args: {
|
||||
cursor: v.optional(v.string()),
|
||||
limit: v.optional(v.number()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const limit = clampInt(args.limit ?? 24, 1, MAX_LIST_LIMIT)
|
||||
const { page, isDone, continueCursor } = await ctx.db
|
||||
.query('souls')
|
||||
.withIndex('by_updated', (q) => q)
|
||||
.order('desc')
|
||||
.paginate({ cursor: args.cursor ?? null, numItems: limit })
|
||||
|
||||
const items: Array<{
|
||||
soul: NonNullable<ReturnType<typeof toPublicSoul>>
|
||||
latestVersion: Doc<'soulVersions'> | null
|
||||
}> = []
|
||||
|
||||
for (const soul of page) {
|
||||
if (soul.softDeletedAt) continue
|
||||
const latestVersion = soul.latestVersionId ? await ctx.db.get(soul.latestVersionId) : null
|
||||
const publicSoul = toPublicSoul(soul)
|
||||
if (!publicSoul) continue
|
||||
items.push({ soul: publicSoul, latestVersion })
|
||||
}
|
||||
|
||||
return { items, nextCursor: isDone ? null : continueCursor }
|
||||
},
|
||||
})
|
||||
|
||||
export const listVersions = query({
|
||||
args: { soulId: v.id('souls'), limit: v.optional(v.number()) },
|
||||
handler: async (ctx, args) => {
|
||||
const limit = args.limit ?? 20
|
||||
return ctx.db
|
||||
.query('soulVersions')
|
||||
.withIndex('by_soul', (q) => q.eq('soulId', args.soulId))
|
||||
.order('desc')
|
||||
.take(limit)
|
||||
},
|
||||
})
|
||||
|
||||
export const listVersionsPage = query({
|
||||
args: {
|
||||
soulId: v.id('souls'),
|
||||
cursor: v.optional(v.string()),
|
||||
limit: v.optional(v.number()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const limit = clampInt(args.limit ?? 20, 1, MAX_LIST_LIMIT)
|
||||
const { page, isDone, continueCursor } = await ctx.db
|
||||
.query('soulVersions')
|
||||
.withIndex('by_soul', (q) => q.eq('soulId', args.soulId))
|
||||
.order('desc')
|
||||
.paginate({ cursor: args.cursor ?? null, numItems: limit })
|
||||
const items = page.filter((version) => !version.softDeletedAt)
|
||||
return { items, nextCursor: isDone ? null : continueCursor }
|
||||
},
|
||||
})
|
||||
|
||||
export const getVersionById = query({
|
||||
args: { versionId: v.id('soulVersions') },
|
||||
handler: async (ctx, args) => ctx.db.get(args.versionId),
|
||||
})
|
||||
|
||||
export const getVersionByIdInternal = internalQuery({
|
||||
args: { versionId: v.id('soulVersions') },
|
||||
handler: async (ctx, args) => ctx.db.get(args.versionId),
|
||||
})
|
||||
|
||||
export const getVersionBySoulAndVersion = query({
|
||||
args: { soulId: v.id('souls'), version: v.string() },
|
||||
handler: async (ctx, args) => {
|
||||
return ctx.db
|
||||
.query('soulVersions')
|
||||
.withIndex('by_soul_version', (q) => q.eq('soulId', args.soulId).eq('version', args.version))
|
||||
.unique()
|
||||
},
|
||||
})
|
||||
|
||||
export const publishVersion: ReturnType<typeof action> = action({
|
||||
args: {
|
||||
slug: v.string(),
|
||||
displayName: v.string(),
|
||||
version: v.string(),
|
||||
changelog: v.string(),
|
||||
tags: v.optional(v.array(v.string())),
|
||||
source: v.optional(
|
||||
v.object({
|
||||
kind: v.literal('github'),
|
||||
url: v.string(),
|
||||
repo: v.string(),
|
||||
ref: v.string(),
|
||||
commit: v.string(),
|
||||
path: v.string(),
|
||||
importedAt: v.number(),
|
||||
}),
|
||||
),
|
||||
files: v.array(
|
||||
v.object({
|
||||
path: v.string(),
|
||||
size: v.number(),
|
||||
storageId: v.id('_storage'),
|
||||
sha256: v.string(),
|
||||
contentType: v.optional(v.string()),
|
||||
}),
|
||||
),
|
||||
},
|
||||
handler: async (ctx, args): Promise<PublishResult> => {
|
||||
const { userId } = await requireUserFromAction(ctx)
|
||||
return publishSoulVersionForUser(ctx, userId, args)
|
||||
},
|
||||
})
|
||||
|
||||
export const generateChangelogPreview = action({
|
||||
args: {
|
||||
slug: v.string(),
|
||||
version: v.string(),
|
||||
readmeText: v.string(),
|
||||
filePaths: v.optional(v.array(v.string())),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
await requireUserFromAction(ctx)
|
||||
const changelog = await generateSoulChangelogPreview(ctx, {
|
||||
slug: args.slug.trim().toLowerCase(),
|
||||
version: args.version.trim(),
|
||||
readmeText: args.readmeText,
|
||||
filePaths: args.filePaths?.map((value) => value.trim()).filter(Boolean),
|
||||
})
|
||||
return { changelog, source: 'auto' as const }
|
||||
},
|
||||
})
|
||||
|
||||
export const getReadme: ReturnType<typeof action> = action({
|
||||
args: { versionId: v.id('soulVersions') },
|
||||
handler: async (ctx, args): Promise<ReadmeResult> => {
|
||||
const version = (await ctx.runQuery(internal.souls.getVersionByIdInternal, {
|
||||
versionId: args.versionId,
|
||||
})) as Doc<'soulVersions'> | null
|
||||
if (!version) throw new ConvexError('Version not found')
|
||||
const readmeFile = version.files.find((file) => file.path.toLowerCase() === 'soul.md')
|
||||
if (!readmeFile) throw new ConvexError('SOUL.md not found')
|
||||
const text = await fetchText(ctx, readmeFile.storageId)
|
||||
return { path: readmeFile.path, text }
|
||||
},
|
||||
})
|
||||
|
||||
export const getFileText: ReturnType<typeof action> = action({
|
||||
args: { versionId: v.id('soulVersions'), path: v.string() },
|
||||
handler: async (ctx, args): Promise<FileTextResult> => {
|
||||
const version = (await ctx.runQuery(internal.souls.getVersionByIdInternal, {
|
||||
versionId: args.versionId,
|
||||
})) as Doc<'soulVersions'> | null
|
||||
if (!version) throw new ConvexError('Version not found')
|
||||
|
||||
const normalizedPath = args.path.trim()
|
||||
const normalizedLower = normalizedPath.toLowerCase()
|
||||
const file =
|
||||
version.files.find((entry) => entry.path === normalizedPath) ??
|
||||
version.files.find((entry) => entry.path.toLowerCase() === normalizedLower)
|
||||
if (!file) throw new ConvexError('File not found')
|
||||
if (file.size > MAX_DIFF_FILE_BYTES) {
|
||||
throw new ConvexError('File exceeds 200KB limit')
|
||||
}
|
||||
|
||||
const text = await fetchText(ctx, file.storageId)
|
||||
return { path: file.path, text, size: file.size, sha256: file.sha256 }
|
||||
},
|
||||
})
|
||||
|
||||
export const resolveVersionByHash = query({
|
||||
args: { slug: v.string(), hash: v.string() },
|
||||
handler: async (ctx, args) => {
|
||||
const slug = args.slug.trim().toLowerCase()
|
||||
const hash = args.hash.trim().toLowerCase()
|
||||
if (!slug || !/^[a-f0-9]{64}$/.test(hash)) return null
|
||||
|
||||
const soulMatches = await ctx.db
|
||||
.query('souls')
|
||||
.withIndex('by_slug', (q) => q.eq('slug', slug))
|
||||
.order('desc')
|
||||
.take(2)
|
||||
const soul = soulMatches[0] ?? null
|
||||
if (!soul || soul.softDeletedAt) return null
|
||||
|
||||
const latestVersion = soul.latestVersionId ? await ctx.db.get(soul.latestVersionId) : null
|
||||
|
||||
const fingerprintMatches = await ctx.db
|
||||
.query('soulVersionFingerprints')
|
||||
.withIndex('by_soul_fingerprint', (q) => q.eq('soulId', soul._id).eq('fingerprint', hash))
|
||||
.take(25)
|
||||
|
||||
let match: { version: string } | null = null
|
||||
if (fingerprintMatches.length > 0) {
|
||||
const newest = fingerprintMatches.reduce(
|
||||
(best, entry) => (entry.createdAt > best.createdAt ? entry : best),
|
||||
fingerprintMatches[0] as (typeof fingerprintMatches)[number],
|
||||
)
|
||||
const version = await ctx.db.get(newest.versionId)
|
||||
if (version && !version.softDeletedAt) {
|
||||
match = { version: version.version }
|
||||
}
|
||||
}
|
||||
|
||||
if (!match) {
|
||||
const versions = await ctx.db
|
||||
.query('soulVersions')
|
||||
.withIndex('by_soul', (q) => q.eq('soulId', soul._id))
|
||||
.order('desc')
|
||||
.take(200)
|
||||
|
||||
for (const version of versions) {
|
||||
if (version.softDeletedAt) continue
|
||||
if (typeof version.fingerprint === 'string' && version.fingerprint === hash) {
|
||||
match = { version: version.version }
|
||||
break
|
||||
}
|
||||
|
||||
const fingerprint = await hashSkillFiles(
|
||||
version.files.map((file) => ({ path: file.path, sha256: file.sha256 })),
|
||||
)
|
||||
if (fingerprint === hash) {
|
||||
match = { version: version.version }
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
match,
|
||||
latestVersion: latestVersion ? { version: latestVersion.version } : null,
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
export const updateTags = mutation({
|
||||
args: {
|
||||
soulId: v.id('souls'),
|
||||
tags: v.array(v.object({ tag: v.string(), versionId: v.id('soulVersions') })),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const { user } = await requireUser(ctx)
|
||||
const soul = await ctx.db.get(args.soulId)
|
||||
if (!soul) throw new Error('Soul not found')
|
||||
if (soul.ownerUserId !== user._id) {
|
||||
assertModerator(user)
|
||||
}
|
||||
|
||||
const nextTags = { ...soul.tags }
|
||||
for (const entry of args.tags) {
|
||||
nextTags[entry.tag] = entry.versionId
|
||||
}
|
||||
|
||||
const latestEntry = args.tags.find((entry) => entry.tag === 'latest')
|
||||
await ctx.db.patch(soul._id, {
|
||||
tags: nextTags,
|
||||
latestVersionId: latestEntry ? latestEntry.versionId : soul.latestVersionId,
|
||||
updatedAt: Date.now(),
|
||||
})
|
||||
|
||||
if (latestEntry) {
|
||||
const embeddings = await ctx.db
|
||||
.query('soulEmbeddings')
|
||||
.withIndex('by_soul', (q) => q.eq('soulId', soul._id))
|
||||
.collect()
|
||||
for (const embedding of embeddings) {
|
||||
const isLatest = embedding.versionId === latestEntry.versionId
|
||||
await ctx.db.patch(embedding._id, {
|
||||
isLatest,
|
||||
visibility: visibilityFor(isLatest, embedding.isApproved),
|
||||
updatedAt: Date.now(),
|
||||
})
|
||||
}
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
export const insertVersion = internalMutation({
|
||||
args: {
|
||||
userId: v.id('users'),
|
||||
slug: v.string(),
|
||||
displayName: v.string(),
|
||||
version: v.string(),
|
||||
changelog: v.string(),
|
||||
changelogSource: v.optional(v.union(v.literal('auto'), v.literal('user'))),
|
||||
tags: v.optional(v.array(v.string())),
|
||||
fingerprint: v.string(),
|
||||
summary: v.optional(v.string()),
|
||||
files: v.array(
|
||||
v.object({
|
||||
path: v.string(),
|
||||
size: v.number(),
|
||||
storageId: v.id('_storage'),
|
||||
sha256: v.string(),
|
||||
contentType: v.optional(v.string()),
|
||||
}),
|
||||
),
|
||||
parsed: v.object({
|
||||
frontmatter: v.record(v.string(), v.any()),
|
||||
metadata: v.optional(v.any()),
|
||||
}),
|
||||
embedding: v.array(v.number()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const userId = args.userId
|
||||
const user = await ctx.db.get(userId)
|
||||
if (!user || user.deletedAt) throw new Error('User not found')
|
||||
|
||||
const soulMatches = await ctx.db
|
||||
.query('souls')
|
||||
.withIndex('by_slug', (q) => q.eq('slug', args.slug))
|
||||
.order('desc')
|
||||
.take(2)
|
||||
let soul: Doc<'souls'> | null = soulMatches[0] ?? null
|
||||
|
||||
if (soul && soul.ownerUserId !== userId) {
|
||||
throw new Error('Only the owner can publish updates')
|
||||
}
|
||||
|
||||
const now = Date.now()
|
||||
if (!soul) {
|
||||
const summary = args.summary ?? getFrontmatterValue(args.parsed.frontmatter, 'description')
|
||||
const soulId = await ctx.db.insert('souls', {
|
||||
slug: args.slug,
|
||||
displayName: args.displayName,
|
||||
summary: summary ?? undefined,
|
||||
ownerUserId: userId,
|
||||
latestVersionId: undefined,
|
||||
tags: {},
|
||||
softDeletedAt: undefined,
|
||||
stats: {
|
||||
downloads: 0,
|
||||
stars: 0,
|
||||
versions: 0,
|
||||
comments: 0,
|
||||
},
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
})
|
||||
soul = await ctx.db.get(soulId)
|
||||
}
|
||||
|
||||
if (!soul) throw new Error('Soul creation failed')
|
||||
|
||||
const existingVersion = await ctx.db
|
||||
.query('soulVersions')
|
||||
.withIndex('by_soul_version', (q) => q.eq('soulId', soul._id).eq('version', args.version))
|
||||
.unique()
|
||||
if (existingVersion) {
|
||||
throw new Error('Version already exists')
|
||||
}
|
||||
|
||||
const versionId = await ctx.db.insert('soulVersions', {
|
||||
soulId: soul._id,
|
||||
version: args.version,
|
||||
fingerprint: args.fingerprint,
|
||||
changelog: args.changelog,
|
||||
changelogSource: args.changelogSource,
|
||||
files: args.files,
|
||||
parsed: args.parsed,
|
||||
createdBy: userId,
|
||||
createdAt: now,
|
||||
softDeletedAt: undefined,
|
||||
})
|
||||
|
||||
const nextTags: Record<string, Id<'soulVersions'>> = { ...soul.tags }
|
||||
nextTags.latest = versionId
|
||||
for (const tag of args.tags ?? []) {
|
||||
nextTags[tag] = versionId
|
||||
}
|
||||
|
||||
const latestBefore = soul.latestVersionId
|
||||
|
||||
await ctx.db.patch(soul._id, {
|
||||
displayName: args.displayName,
|
||||
summary:
|
||||
args.summary ?? getFrontmatterValue(args.parsed.frontmatter, 'description') ?? soul.summary,
|
||||
latestVersionId: versionId,
|
||||
tags: nextTags,
|
||||
stats: { ...soul.stats, versions: soul.stats.versions + 1 },
|
||||
softDeletedAt: undefined,
|
||||
updatedAt: now,
|
||||
})
|
||||
|
||||
const embeddingId = await ctx.db.insert('soulEmbeddings', {
|
||||
soulId: soul._id,
|
||||
versionId,
|
||||
ownerId: userId,
|
||||
embedding: args.embedding,
|
||||
isLatest: true,
|
||||
isApproved: true,
|
||||
visibility: visibilityFor(true, true),
|
||||
updatedAt: now,
|
||||
})
|
||||
|
||||
if (latestBefore) {
|
||||
const previousEmbedding = await ctx.db
|
||||
.query('soulEmbeddings')
|
||||
.withIndex('by_version', (q) => q.eq('versionId', latestBefore))
|
||||
.unique()
|
||||
if (previousEmbedding) {
|
||||
await ctx.db.patch(previousEmbedding._id, {
|
||||
isLatest: false,
|
||||
visibility: visibilityFor(false, previousEmbedding.isApproved),
|
||||
updatedAt: now,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
await ctx.db.insert('soulVersionFingerprints', {
|
||||
soulId: soul._id,
|
||||
versionId,
|
||||
fingerprint: args.fingerprint,
|
||||
createdAt: now,
|
||||
})
|
||||
|
||||
return { soulId: soul._id, versionId, embeddingId }
|
||||
},
|
||||
})
|
||||
|
||||
export const setSoulSoftDeletedInternal = internalMutation({
|
||||
args: {
|
||||
userId: v.id('users'),
|
||||
slug: v.string(),
|
||||
deleted: v.boolean(),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const user = await ctx.db.get(args.userId)
|
||||
if (!user || user.deletedAt) throw new Error('User not found')
|
||||
|
||||
const slug = args.slug.trim().toLowerCase()
|
||||
if (!slug) throw new Error('Slug required')
|
||||
|
||||
const soulMatches = await ctx.db
|
||||
.query('souls')
|
||||
.withIndex('by_slug', (q) => q.eq('slug', slug))
|
||||
.order('desc')
|
||||
.take(2)
|
||||
const soul = soulMatches[0] ?? null
|
||||
if (!soul) throw new Error('Soul not found')
|
||||
|
||||
if (soul.ownerUserId !== args.userId) {
|
||||
assertModerator(user)
|
||||
}
|
||||
|
||||
const now = Date.now()
|
||||
await ctx.db.patch(soul._id, {
|
||||
softDeletedAt: args.deleted ? now : undefined,
|
||||
updatedAt: now,
|
||||
})
|
||||
|
||||
const embeddings = await ctx.db
|
||||
.query('soulEmbeddings')
|
||||
.withIndex('by_soul', (q) => q.eq('soulId', soul._id))
|
||||
.collect()
|
||||
for (const embedding of embeddings) {
|
||||
await ctx.db.patch(embedding._id, {
|
||||
visibility: args.deleted
|
||||
? 'deleted'
|
||||
: visibilityFor(embedding.isLatest, embedding.isApproved),
|
||||
updatedAt: now,
|
||||
})
|
||||
}
|
||||
|
||||
await ctx.db.insert('auditLogs', {
|
||||
actorUserId: args.userId,
|
||||
action: args.deleted ? 'soul.delete' : 'soul.undelete',
|
||||
targetType: 'soul',
|
||||
targetId: soul._id,
|
||||
metadata: { slug, softDeletedAt: args.deleted ? now : null },
|
||||
createdAt: now,
|
||||
})
|
||||
|
||||
return { ok: true as const }
|
||||
},
|
||||
})
|
||||
|
||||
function visibilityFor(isLatest: boolean, isApproved: boolean) {
|
||||
if (isLatest && isApproved) return 'latest-approved'
|
||||
if (isLatest) return 'latest'
|
||||
if (isApproved) return 'archived-approved'
|
||||
return 'archived'
|
||||
}
|
||||
|
||||
function clampInt(value: number, min: number, max: number) {
|
||||
const rounded = Number.isFinite(value) ? Math.round(value) : min
|
||||
return Math.min(max, Math.max(min, rounded))
|
||||
}
|
||||
+50
-12
@@ -1,7 +1,8 @@
|
||||
import { v } from 'convex/values'
|
||||
import type { Doc } from './_generated/dataModel'
|
||||
import { mutation, query } from './_generated/server'
|
||||
import { internalMutation, mutation, query } from './_generated/server'
|
||||
import { requireUser } from './lib/access'
|
||||
import { toPublicSkill } from './lib/public'
|
||||
import { insertStatEvent } from './skillStatEvents'
|
||||
|
||||
export const isStarred = query({
|
||||
args: { skillId: v.id('skills') },
|
||||
@@ -29,10 +30,7 @@ export const toggle = mutation({
|
||||
|
||||
if (existing) {
|
||||
await ctx.db.delete(existing._id)
|
||||
await ctx.db.patch(skill._id, {
|
||||
stats: { ...skill.stats, stars: Math.max(0, skill.stats.stars - 1) },
|
||||
updatedAt: Date.now(),
|
||||
})
|
||||
await insertStatEvent(ctx, { skillId: skill._id, kind: 'unstar' })
|
||||
return { starred: false }
|
||||
}
|
||||
|
||||
@@ -42,10 +40,7 @@ export const toggle = mutation({
|
||||
createdAt: Date.now(),
|
||||
})
|
||||
|
||||
await ctx.db.patch(skill._id, {
|
||||
stats: { ...skill.stats, stars: skill.stats.stars + 1 },
|
||||
updatedAt: Date.now(),
|
||||
})
|
||||
await insertStatEvent(ctx, { skillId: skill._id, kind: 'star' })
|
||||
|
||||
return { starred: true }
|
||||
},
|
||||
@@ -60,11 +55,54 @@ export const listByUser = query({
|
||||
.withIndex('by_user', (q) => q.eq('userId', args.userId))
|
||||
.order('desc')
|
||||
.take(limit)
|
||||
const skills: Doc<'skills'>[] = []
|
||||
const skills: NonNullable<ReturnType<typeof toPublicSkill>>[] = []
|
||||
for (const star of stars) {
|
||||
const skill = await ctx.db.get(star.skillId)
|
||||
if (skill) skills.push(skill)
|
||||
const publicSkill = toPublicSkill(skill)
|
||||
if (!publicSkill) continue
|
||||
skills.push(publicSkill)
|
||||
}
|
||||
return skills
|
||||
},
|
||||
})
|
||||
|
||||
export const addStarInternal = internalMutation({
|
||||
args: { userId: v.id('users'), skillId: v.id('skills') },
|
||||
handler: async (ctx, args) => {
|
||||
const skill = await ctx.db.get(args.skillId)
|
||||
if (!skill) throw new Error('Skill not found')
|
||||
const existing = await ctx.db
|
||||
.query('stars')
|
||||
.withIndex('by_skill_user', (q) => q.eq('skillId', args.skillId).eq('userId', args.userId))
|
||||
.unique()
|
||||
if (existing) return { ok: true as const, starred: true, alreadyStarred: true }
|
||||
|
||||
await ctx.db.insert('stars', {
|
||||
skillId: args.skillId,
|
||||
userId: args.userId,
|
||||
createdAt: Date.now(),
|
||||
})
|
||||
|
||||
await insertStatEvent(ctx, { skillId: skill._id, kind: 'star' })
|
||||
|
||||
return { ok: true as const, starred: true, alreadyStarred: false }
|
||||
},
|
||||
})
|
||||
|
||||
export const removeStarInternal = internalMutation({
|
||||
args: { userId: v.id('users'), skillId: v.id('skills') },
|
||||
handler: async (ctx, args) => {
|
||||
const skill = await ctx.db.get(args.skillId)
|
||||
if (!skill) throw new Error('Skill not found')
|
||||
const existing = await ctx.db
|
||||
.query('stars')
|
||||
.withIndex('by_skill_user', (q) => q.eq('skillId', args.skillId).eq('userId', args.userId))
|
||||
.unique()
|
||||
if (!existing) return { ok: true as const, unstarred: false, alreadyUnstarred: true }
|
||||
|
||||
await ctx.db.delete(existing._id)
|
||||
await insertStatEvent(ctx, { skillId: skill._id, kind: 'unstar' })
|
||||
|
||||
return { ok: true as const, unstarred: true, alreadyUnstarred: false }
|
||||
},
|
||||
})
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
import { v } from 'convex/values'
|
||||
import { internal } from './_generated/api'
|
||||
import type { Doc } from './_generated/dataModel'
|
||||
import type { ActionCtx } from './_generated/server'
|
||||
import { internalAction, internalMutation, internalQuery } from './_generated/server'
|
||||
|
||||
const DEFAULT_BATCH_SIZE = 200
|
||||
const MAX_BATCH_SIZE = 1000
|
||||
const DEFAULT_MAX_BATCHES = 5
|
||||
const MAX_MAX_BATCHES = 50
|
||||
const BACKFILL_STATE_KEY = 'default'
|
||||
|
||||
export const backfillSkillStatFieldsInternal = internalMutation({
|
||||
args: {
|
||||
cursor: v.optional(v.string()),
|
||||
batchSize: v.optional(v.number()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
|
||||
const { page, isDone, continueCursor } = await ctx.db
|
||||
.query('skills')
|
||||
.order('asc')
|
||||
.paginate({ cursor: args.cursor ?? null, numItems: batchSize })
|
||||
|
||||
let patched = 0
|
||||
for (const skill of page) {
|
||||
const next = buildSkillStatPatch(skill)
|
||||
if (!next) continue
|
||||
await ctx.db.patch(skill._id, next)
|
||||
patched += 1
|
||||
}
|
||||
|
||||
return {
|
||||
ok: true as const,
|
||||
scanned: page.length,
|
||||
patched,
|
||||
cursor: isDone ? null : continueCursor,
|
||||
isDone,
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
type BackfillState = {
|
||||
cursor: string | null
|
||||
doneAt?: number
|
||||
}
|
||||
|
||||
type BackfillActionArgs = {
|
||||
batchSize?: number
|
||||
maxBatches?: number
|
||||
resetCursor?: boolean
|
||||
}
|
||||
|
||||
type BackfillStats = {
|
||||
scanned: number
|
||||
patched: number
|
||||
batches: number
|
||||
}
|
||||
|
||||
type BackfillActionResult = {
|
||||
ok: true
|
||||
isDone: boolean
|
||||
cursor: string | null
|
||||
stats: BackfillStats
|
||||
}
|
||||
|
||||
export const getSkillStatBackfillStateInternal = internalQuery({
|
||||
args: {},
|
||||
handler: async (ctx): Promise<BackfillState> => {
|
||||
const state = await ctx.db
|
||||
.query('skillStatBackfillState')
|
||||
.withIndex('by_key', (q) => q.eq('key', BACKFILL_STATE_KEY))
|
||||
.unique()
|
||||
return { cursor: state?.cursor ?? null, doneAt: state?.doneAt }
|
||||
},
|
||||
})
|
||||
|
||||
export const setSkillStatBackfillStateInternal = internalMutation({
|
||||
args: {
|
||||
cursor: v.optional(v.string()),
|
||||
doneAt: v.optional(v.number()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const now = Date.now()
|
||||
const state = await ctx.db
|
||||
.query('skillStatBackfillState')
|
||||
.withIndex('by_key', (q) => q.eq('key', BACKFILL_STATE_KEY))
|
||||
.unique()
|
||||
|
||||
if (!state) {
|
||||
await ctx.db.insert('skillStatBackfillState', {
|
||||
key: BACKFILL_STATE_KEY,
|
||||
cursor: args.cursor,
|
||||
doneAt: args.doneAt,
|
||||
updatedAt: now,
|
||||
})
|
||||
return { ok: true as const }
|
||||
}
|
||||
|
||||
await ctx.db.patch(state._id, {
|
||||
cursor: args.cursor,
|
||||
doneAt: args.doneAt,
|
||||
updatedAt: now,
|
||||
})
|
||||
|
||||
return { ok: true as const }
|
||||
},
|
||||
})
|
||||
|
||||
async function runSkillStatBackfillInternalHandler(
|
||||
ctx: ActionCtx,
|
||||
args: BackfillActionArgs,
|
||||
): Promise<BackfillActionResult> {
|
||||
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
|
||||
const maxBatches = clampInt(args.maxBatches ?? DEFAULT_MAX_BATCHES, 1, MAX_MAX_BATCHES)
|
||||
|
||||
if (args.resetCursor) {
|
||||
await ctx.runMutation(internal.statsMaintenance.setSkillStatBackfillStateInternal, {
|
||||
cursor: undefined,
|
||||
doneAt: undefined,
|
||||
})
|
||||
}
|
||||
|
||||
const state = (await ctx.runQuery(
|
||||
internal.statsMaintenance.getSkillStatBackfillStateInternal,
|
||||
{},
|
||||
)) as BackfillState
|
||||
if (state.doneAt && !args.resetCursor) {
|
||||
return {
|
||||
ok: true,
|
||||
isDone: true,
|
||||
cursor: null,
|
||||
stats: { scanned: 0, patched: 0, batches: 0 },
|
||||
}
|
||||
}
|
||||
|
||||
let cursor: string | null = state.cursor ?? null
|
||||
const stats: BackfillStats = { scanned: 0, patched: 0, batches: 0 }
|
||||
|
||||
for (let i = 0; i < maxBatches; i += 1) {
|
||||
const result = (await ctx.runMutation(
|
||||
internal.statsMaintenance.backfillSkillStatFieldsInternal,
|
||||
{
|
||||
cursor: cursor ?? undefined,
|
||||
batchSize,
|
||||
},
|
||||
)) as { scanned: number; patched: number; cursor: string | null; isDone: boolean }
|
||||
stats.scanned += result.scanned
|
||||
stats.patched += result.patched
|
||||
stats.batches += 1
|
||||
cursor = result.cursor
|
||||
|
||||
if (result.isDone) {
|
||||
await ctx.runMutation(internal.statsMaintenance.setSkillStatBackfillStateInternal, {
|
||||
cursor: undefined,
|
||||
doneAt: Date.now(),
|
||||
})
|
||||
return { ok: true, isDone: true, cursor: null, stats }
|
||||
}
|
||||
|
||||
await ctx.runMutation(internal.statsMaintenance.setSkillStatBackfillStateInternal, {
|
||||
cursor: cursor ?? undefined,
|
||||
doneAt: undefined,
|
||||
})
|
||||
}
|
||||
|
||||
return { ok: true, isDone: false, cursor, stats }
|
||||
}
|
||||
|
||||
export const runSkillStatBackfillInternal: ReturnType<typeof internalAction> = internalAction({
|
||||
args: {
|
||||
batchSize: v.optional(v.number()),
|
||||
maxBatches: v.optional(v.number()),
|
||||
resetCursor: v.optional(v.boolean()),
|
||||
},
|
||||
handler: runSkillStatBackfillInternalHandler,
|
||||
})
|
||||
|
||||
function buildSkillStatPatch(skill: Doc<'skills'>) {
|
||||
const stats = skill.stats
|
||||
const nextDownloads = stats.downloads
|
||||
const nextStars = stats.stars
|
||||
const nextInstallsCurrent = stats.installsCurrent ?? 0
|
||||
const nextInstallsAllTime = stats.installsAllTime ?? 0
|
||||
|
||||
if (
|
||||
skill.statsDownloads === nextDownloads &&
|
||||
skill.statsStars === nextStars &&
|
||||
skill.statsInstallsCurrent === nextInstallsCurrent &&
|
||||
skill.statsInstallsAllTime === nextInstallsAllTime
|
||||
) {
|
||||
return null
|
||||
}
|
||||
|
||||
return {
|
||||
statsDownloads: nextDownloads,
|
||||
statsStars: nextStars,
|
||||
statsInstallsCurrent: nextInstallsCurrent,
|
||||
statsInstallsAllTime: nextInstallsAllTime,
|
||||
}
|
||||
}
|
||||
|
||||
function clampInt(value: number, min: number, max: number) {
|
||||
return Math.min(Math.max(value, min), max)
|
||||
}
|
||||
+28
-37
@@ -4,6 +4,7 @@ import type { Id } from './_generated/dataModel'
|
||||
import type { MutationCtx, QueryCtx } from './_generated/server'
|
||||
import { internalMutation, mutation, query } from './_generated/server'
|
||||
import { requireUser } from './lib/access'
|
||||
import { insertStatEvent } from './skillStatEvents'
|
||||
|
||||
const TELEMETRY_STALE_MS = 120 * 24 * 60 * 60 * 1000
|
||||
|
||||
@@ -87,7 +88,13 @@ export const getMyInstalled = query({
|
||||
lastSeenAt: number
|
||||
expiredAt?: number
|
||||
skills: Array<{
|
||||
skill: { slug: string; displayName: string; summary?: string; stats: unknown }
|
||||
skill: {
|
||||
slug: string
|
||||
displayName: string
|
||||
summary?: string
|
||||
stats: unknown
|
||||
ownerUserId: Id<'users'>
|
||||
}
|
||||
firstSeenAt: number
|
||||
lastSeenAt: number
|
||||
lastVersion?: string
|
||||
@@ -104,7 +111,13 @@ export const getMyInstalled = query({
|
||||
|
||||
const filtered = includeRemoved ? installs : installs.filter((entry) => !entry.removedAt)
|
||||
const skills: Array<{
|
||||
skill: { slug: string; displayName: string; summary?: string; stats: unknown }
|
||||
skill: {
|
||||
slug: string
|
||||
displayName: string
|
||||
summary?: string
|
||||
stats: unknown
|
||||
ownerUserId: Id<'users'>
|
||||
}
|
||||
firstSeenAt: number
|
||||
lastSeenAt: number
|
||||
lastVersion?: string
|
||||
@@ -120,6 +133,7 @@ export const getMyInstalled = query({
|
||||
displayName: skill.displayName,
|
||||
summary: skill.summary,
|
||||
stats: skill.stats,
|
||||
ownerUserId: skill.ownerUserId,
|
||||
},
|
||||
firstSeenAt: entry.firstSeenAt,
|
||||
lastSeenAt: entry.lastSeenAt,
|
||||
@@ -157,24 +171,13 @@ async function clearTelemetryForUser(ctx: MutationCtx, params: { userId: Id<'use
|
||||
await ctx.db.delete(entry._id)
|
||||
continue
|
||||
}
|
||||
const stats = skill.stats as {
|
||||
downloads: number
|
||||
installsCurrent?: number
|
||||
installsAllTime?: number
|
||||
stars: number
|
||||
versions: number
|
||||
comments: number
|
||||
}
|
||||
await ctx.db.patch(skill._id, {
|
||||
stats: {
|
||||
...stats,
|
||||
installsCurrent: Math.max(
|
||||
0,
|
||||
(stats.installsCurrent ?? 0) - (entry.activeRoots > 0 ? 1 : 0),
|
||||
),
|
||||
installsAllTime: Math.max(0, (stats.installsAllTime ?? 0) - 1),
|
||||
await insertStatEvent(ctx, {
|
||||
skillId: skill._id,
|
||||
kind: 'install_clear',
|
||||
delta: {
|
||||
allTime: -1,
|
||||
current: entry.activeRoots > 0 ? -1 : 0,
|
||||
},
|
||||
updatedAt: Date.now(),
|
||||
})
|
||||
await ctx.db.delete(entry._id)
|
||||
}
|
||||
@@ -381,25 +384,13 @@ async function bumpSkillInstallCounts(
|
||||
ctx: MutationCtx,
|
||||
params: { skillId: Id<'skills'>; deltaAllTime: number; deltaCurrent: number },
|
||||
) {
|
||||
const skill = await ctx.db.get(params.skillId)
|
||||
if (!skill) return
|
||||
const stats = skill.stats as {
|
||||
downloads: number
|
||||
installsCurrent?: number
|
||||
installsAllTime?: number
|
||||
stars: number
|
||||
versions: number
|
||||
comments: number
|
||||
if (params.deltaAllTime === 1 && params.deltaCurrent === 1) {
|
||||
await insertStatEvent(ctx, { skillId: params.skillId, kind: 'install_new' })
|
||||
} else if (params.deltaAllTime === 0 && params.deltaCurrent === 1) {
|
||||
await insertStatEvent(ctx, { skillId: params.skillId, kind: 'install_reactivate' })
|
||||
} else if (params.deltaAllTime === 0 && params.deltaCurrent === -1) {
|
||||
await insertStatEvent(ctx, { skillId: params.skillId, kind: 'install_deactivate' })
|
||||
}
|
||||
|
||||
await ctx.db.patch(skill._id, {
|
||||
stats: {
|
||||
...stats,
|
||||
installsAllTime: Math.max(0, (stats.installsAllTime ?? 0) + params.deltaAllTime),
|
||||
installsCurrent: Math.max(0, (stats.installsCurrent ?? 0) + params.deltaCurrent),
|
||||
},
|
||||
updatedAt: Date.now(),
|
||||
})
|
||||
}
|
||||
|
||||
async function expireStaleRoots(
|
||||
|
||||
+12
-5
@@ -1,13 +1,19 @@
|
||||
import { getAuthUserId } from '@convex-dev/auth/server'
|
||||
import { v } from 'convex/values'
|
||||
import { internal } from './_generated/api'
|
||||
import { mutation, query } from './_generated/server'
|
||||
import { assertRole, requireUser } from './lib/access'
|
||||
import { internalQuery, mutation, query } from './_generated/server'
|
||||
import { assertAdmin, requireUser } from './lib/access'
|
||||
import { toPublicUser } from './lib/public'
|
||||
|
||||
const DEFAULT_ROLE = 'user'
|
||||
const ADMIN_HANDLE = 'steipete'
|
||||
|
||||
export const getById = query({
|
||||
args: { userId: v.id('users') },
|
||||
handler: async (ctx, args) => toPublicUser(await ctx.db.get(args.userId)),
|
||||
})
|
||||
|
||||
export const getByIdInternal = internalQuery({
|
||||
args: { userId: v.id('users') },
|
||||
handler: async (ctx, args) => ctx.db.get(args.userId),
|
||||
})
|
||||
@@ -78,7 +84,7 @@ export const list = query({
|
||||
args: { limit: v.optional(v.number()) },
|
||||
handler: async (ctx, args) => {
|
||||
const { user } = await requireUser(ctx)
|
||||
assertRole(user, ['admin'])
|
||||
assertAdmin(user)
|
||||
const limit = args.limit ?? 50
|
||||
return ctx.db.query('users').order('desc').take(limit)
|
||||
},
|
||||
@@ -87,10 +93,11 @@ export const list = query({
|
||||
export const getByHandle = query({
|
||||
args: { handle: v.string() },
|
||||
handler: async (ctx, args) => {
|
||||
return ctx.db
|
||||
const user = await ctx.db
|
||||
.query('users')
|
||||
.withIndex('handle', (q) => q.eq('handle', args.handle))
|
||||
.unique()
|
||||
return toPublicUser(user)
|
||||
},
|
||||
})
|
||||
|
||||
@@ -101,7 +108,7 @@ export const setRole = mutation({
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const { user } = await requireUser(ctx)
|
||||
assertRole(user, ['admin'])
|
||||
assertAdmin(user)
|
||||
await ctx.db.patch(args.userId, { role: args.role, updatedAt: Date.now() })
|
||||
await ctx.db.insert('auditLogs', {
|
||||
actorUserId: user._id,
|
||||
|
||||
+2
-2
@@ -11,7 +11,7 @@ export const sendDiscordWebhook = internalAction({
|
||||
summary: v.optional(v.string()),
|
||||
version: v.optional(v.string()),
|
||||
ownerHandle: v.optional(v.string()),
|
||||
batch: v.optional(v.string()),
|
||||
highlighted: v.optional(v.boolean()),
|
||||
tags: v.optional(v.array(v.string())),
|
||||
}),
|
||||
},
|
||||
@@ -21,7 +21,7 @@ export const sendDiscordWebhook = internalAction({
|
||||
event: args.event,
|
||||
slug: args.skill.slug,
|
||||
version: args.skill.version ?? null,
|
||||
batch: args.skill.batch ?? null,
|
||||
highlighted: args.skill.highlighted ?? false,
|
||||
highlightedOnly: config.highlightedOnly,
|
||||
}
|
||||
if (!shouldSendWebhook(args.event, args.skill, config)) {
|
||||
|
||||
+1
-1
@@ -22,7 +22,7 @@ Reading order (new contributor):
|
||||
Feature/ops docs (already present):
|
||||
|
||||
- `docs/spec.md`: product + implementation spec (data model + flows).
|
||||
- `docs/telemetry.md`: what `clawdhub sync` reports; opt-out.
|
||||
- `docs/telemetry.md`: what `clawhub sync` reports; opt-out.
|
||||
- `docs/webhook.md`: Discord webhook events/payload.
|
||||
- `docs/diffing.md`: version-to-version diff UI spec.
|
||||
- `docs/manual-testing.md`: CLI smoke scripts.
|
||||
|
||||
+3
-2
@@ -7,7 +7,7 @@ read_when:
|
||||
|
||||
# API v1
|
||||
|
||||
Base: `https://clawdhub.com`
|
||||
Base: `https://clawhub.ai`
|
||||
|
||||
OpenAPI: `/api/v1/openapi.json`
|
||||
|
||||
@@ -30,7 +30,8 @@ Headers: `X-RateLimit-Limit`, `X-RateLimit-Remaining`, `X-RateLimit-Reset`, `Ret
|
||||
Public read:
|
||||
|
||||
- `GET /api/v1/search?q=...`
|
||||
- `GET /api/v1/skills?limit=&cursor=`
|
||||
- `GET /api/v1/skills?limit=&cursor=&sort=`
|
||||
- `sort`: `updated` (default), `downloads`, `stars` (`rating`), `installsCurrent` (`installs`), `installsAllTime`, `trending`
|
||||
- `GET /api/v1/skills/{slug}`
|
||||
- `GET /api/v1/skills/{slug}/versions?limit=&cursor=`
|
||||
- `GET /api/v1/skills/{slug}/versions/{version}`
|
||||
|
||||
@@ -11,8 +11,8 @@ read_when:
|
||||
|
||||
- Web app: TanStack Start (React) under `src/`.
|
||||
- Backend: Convex under `convex/` (DB, storage, actions, HTTP routes).
|
||||
- CLI: `packages/clawdhub/` (published as `clawdhub`).
|
||||
- Shared schemas/routes: `packages/schema/` (`clawdhub-schema`).
|
||||
- CLI: `packages/clawdhub/` (published as `clawhub`, legacy `clawdhub`).
|
||||
- Shared schemas/routes: `packages/schema/` (`clawhub-schema`).
|
||||
|
||||
## Data + storage
|
||||
|
||||
@@ -38,8 +38,8 @@ read_when:
|
||||
- Download zip via `/api/v1/download?slug=...&version=...`.
|
||||
- Extract into `./skills/<slug>` (default).
|
||||
- Persist install state:
|
||||
- `./.clawdhub/lock.json` (per workdir)
|
||||
- `./skills/<slug>/.clawdhub/origin.json` (per skill folder)
|
||||
- `./.clawhub/lock.json` (per workdir, legacy `.clawdhub`)
|
||||
- `./skills/<slug>/.clawhub/origin.json` (per skill folder, legacy `.clawdhub`)
|
||||
|
||||
### Update (CLI)
|
||||
|
||||
|
||||
+4
-4
@@ -23,7 +23,7 @@ The CLI uses a long-lived API token (Bearer token) for publish/sync/delete.
|
||||
|
||||
### Browser flow (default)
|
||||
|
||||
`clawdhub login` does:
|
||||
`clawhub login` does:
|
||||
|
||||
1. Starts a loopback HTTP server on `127.0.0.1` (random port).
|
||||
2. Opens `<site>/cli/auth?redirect_uri=http://127.0.0.1:<port>/callback&state=...`.
|
||||
@@ -35,18 +35,18 @@ The CLI uses a long-lived API token (Bearer token) for publish/sync/delete.
|
||||
Create a token in the web UI (Settings → API tokens) and paste it:
|
||||
|
||||
```bash
|
||||
clawdhub login --token clh_...
|
||||
clawhub login --token clh_...
|
||||
```
|
||||
|
||||
### Token storage
|
||||
|
||||
Default global config path:
|
||||
|
||||
- macOS: `~/Library/Application Support/clawdhub/config.json`
|
||||
- macOS: `~/Library/Application Support/clawhub/config.json`
|
||||
|
||||
Override:
|
||||
|
||||
- `CLAWDHUB_CONFIG_PATH=/path/to/config.json`
|
||||
- `CLAWHUB_CONFIG_PATH=/path/to/config.json` (legacy `CLAWDHUB_CONFIG_PATH`)
|
||||
|
||||
### Revocation
|
||||
|
||||
|
||||
+37
-14
@@ -7,61 +7,77 @@ read_when:
|
||||
|
||||
# CLI
|
||||
|
||||
CLI package: `packages/clawdhub/` (bin: `clawdhub`).
|
||||
CLI package: `packages/clawdhub/` (published as `clawhub`, bin: `clawhub`).
|
||||
|
||||
From this repo you can run it via the wrapper script:
|
||||
|
||||
```bash
|
||||
bun clawdhub --help
|
||||
bun clawhub --help
|
||||
```
|
||||
|
||||
## Global flags
|
||||
|
||||
- `--workdir <dir>`: working directory (default: cwd)
|
||||
- `--workdir <dir>`: working directory (default: cwd; falls back to Clawdbot workspace if configured)
|
||||
- `--dir <dir>`: install dir under workdir (default: `skills`)
|
||||
- `--site <url>`: base URL for browser login (default: `https://clawdhub.com`)
|
||||
- `--registry <url>`: API base URL (default: discovered, else `https://clawdhub.com`)
|
||||
- `--site <url>`: base URL for browser login (default: `https://clawhub.ai`)
|
||||
- `--registry <url>`: API base URL (default: discovered, else `https://clawhub.ai`)
|
||||
- `--no-input`: disable prompts
|
||||
|
||||
Env equivalents:
|
||||
|
||||
- `CLAWDHUB_SITE`
|
||||
- `CLAWDHUB_REGISTRY`
|
||||
- `CLAWHUB_SITE` (legacy `CLAWDHUB_SITE`)
|
||||
- `CLAWHUB_REGISTRY` (legacy `CLAWDHUB_REGISTRY`)
|
||||
- `CLAWHUB_WORKDIR` (legacy `CLAWDHUB_WORKDIR`)
|
||||
|
||||
## Config file
|
||||
|
||||
Stores your API token + cached registry URL.
|
||||
|
||||
- macOS: `~/Library/Application Support/clawdhub/config.json`
|
||||
- override: `CLAWDHUB_CONFIG_PATH`
|
||||
- macOS: `~/Library/Application Support/clawhub/config.json`
|
||||
- override: `CLAWHUB_CONFIG_PATH` (legacy `CLAWDHUB_CONFIG_PATH`)
|
||||
|
||||
## Commands
|
||||
|
||||
### `login` / `auth login`
|
||||
|
||||
- Default: opens browser to `<site>/cli/auth` and completes via loopback callback.
|
||||
- Headless: `clawdhub login --token clh_...`
|
||||
- Headless: `clawhub login --token clh_...`
|
||||
|
||||
### `whoami`
|
||||
|
||||
- Verifies the stored token via `/api/v1/whoami`.
|
||||
|
||||
### `star <slug>` / `unstar <slug>`
|
||||
|
||||
- Adds/removes a skill from your highlights.
|
||||
- Calls `POST /api/v1/stars/<slug>` and `DELETE /api/v1/stars/<slug>`.
|
||||
- `--yes` skips confirmation.
|
||||
|
||||
### `search <query...>`
|
||||
|
||||
- Calls `/api/v1/search?q=...`.
|
||||
|
||||
### `explore`
|
||||
|
||||
- Lists latest updated skills via `/api/v1/skills?limit=...` (sorted by `updatedAt` desc).
|
||||
- Flags:
|
||||
- `--limit <n>` (1–200, default: 25)
|
||||
- `--sort newest|downloads|rating|installs|installsAllTime|trending` (default: newest)
|
||||
- `--json` (machine-readable output)
|
||||
- Output: `<slug> v<version> <age> <summary>` (summary truncated to 50 chars).
|
||||
|
||||
### `install <slug>`
|
||||
|
||||
- Resolves latest version via `/api/v1/skills/<slug>`.
|
||||
- Downloads zip via `/api/v1/download`.
|
||||
- Extracts into `<workdir>/<dir>/<slug>`.
|
||||
- Writes:
|
||||
- `<workdir>/.clawdhub/lock.json`
|
||||
- `<skill>/.clawdhub/origin.json`
|
||||
- `<workdir>/.clawhub/lock.json` (legacy `.clawdhub`)
|
||||
- `<skill>/.clawhub/origin.json` (legacy `.clawdhub`)
|
||||
|
||||
### `list`
|
||||
|
||||
- Reads `<workdir>/.clawdhub/lock.json`.
|
||||
- Reads `<workdir>/.clawhub/lock.json` (legacy `.clawdhub`).
|
||||
|
||||
### `update [slug]` / `update --all`
|
||||
|
||||
@@ -79,6 +95,13 @@ Stores your API token + cached registry URL.
|
||||
### `sync`
|
||||
|
||||
- Scans for local skill folders and publishes new/changed ones.
|
||||
- Roots can be any folder: a skills directory or a single skill folder with `SKILL.md`.
|
||||
- Auto-adds Clawdbot skill roots when `~/.clawdbot/clawdbot.json` is present:
|
||||
- `agent.workspace/skills` (main agent)
|
||||
- `routing.agents.*.workspace/skills` (per-agent)
|
||||
- `~/.clawdbot/skills` (shared)
|
||||
- `skills.load.extraDirs` (shared packs)
|
||||
- Respects `CLAWDBOT_CONFIG_PATH` / `CLAWDBOT_STATE_DIR` and `OPENCLAW_CONFIG_PATH` / `OPENCLAW_STATE_DIR`.
|
||||
- Flags:
|
||||
- `--root <dir...>` extra scan roots
|
||||
- `--all` upload without prompting
|
||||
@@ -90,5 +113,5 @@ Stores your API token + cached registry URL.
|
||||
|
||||
Telemetry:
|
||||
|
||||
- Sent during `sync` when logged in, unless `CLAWDHUB_DISABLE_TELEMETRY=1`.
|
||||
- Sent during `sync` when logged in, unless `CLAWHUB_DISABLE_TELEMETRY=1` (legacy `CLAWDHUB_DISABLE_TELEMETRY=1`).
|
||||
- Details: `docs/telemetry.md`.
|
||||
|
||||
+6
-5
@@ -7,7 +7,7 @@ read_when:
|
||||
|
||||
# Deploy
|
||||
|
||||
ClawdHub is two deployables:
|
||||
OpenClaw is two deployables:
|
||||
|
||||
- Web app (TanStack Start) → typically Vercel.
|
||||
- Convex backend → Convex deployment (serves `/api/...` routes).
|
||||
@@ -55,12 +55,13 @@ For self-host:
|
||||
|
||||
The CLI can discover the API base from:
|
||||
|
||||
- `/.well-known/clawdhub.json`
|
||||
- `/.well-known/clawhub.json` (preferred)
|
||||
- `/.well-known/clawdhub.json` (legacy)
|
||||
|
||||
If you don’t serve that file, users must set:
|
||||
|
||||
```bash
|
||||
export CLAWDHUB_REGISTRY=https://your-site.example
|
||||
export CLAWHUB_REGISTRY=https://your-site.example
|
||||
```
|
||||
|
||||
## 5) Post-deploy checks
|
||||
@@ -73,6 +74,6 @@ curl -i "https://<site>/api/v1/skills/gifgrep"
|
||||
Then:
|
||||
|
||||
```bash
|
||||
clawdhub login --site https://<site>
|
||||
clawdhub whoami
|
||||
clawhub login --site https://<site>
|
||||
clawhub whoami
|
||||
```
|
||||
|
||||
+2
-2
@@ -10,7 +10,7 @@ read_when:
|
||||
## Goals
|
||||
- Compare any file between two versions.
|
||||
- Default compare: `latest` vs `previous` (SemVer precedence).
|
||||
- UX feels native to ClawdHub (theme + typography + motion).
|
||||
- UX feels native to OpenClaw (theme + typography + motion).
|
||||
- Inline or side-by-side toggle.
|
||||
- Public access.
|
||||
|
||||
@@ -57,7 +57,7 @@ Optional helper action:
|
||||
- Feed into Monaco diff editor.
|
||||
|
||||
## Monaco theming
|
||||
- Define `clawdhub-light` / `clawdhub-dark` via `monaco.editor.defineTheme`.
|
||||
- Define `clawhub-light` / `clawhub-dark` via `monaco.editor.defineTheme`.
|
||||
- Derive colors from CSS variables on `document.documentElement`:
|
||||
- `--surface`, `--surface-muted`, `--ink`, `--ink-soft`, `--line`, `--accent`.
|
||||
- Apply theme on load + when theme changes (`data-theme`).
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
---
|
||||
summary: 'Feature spec: import a skill from a public GitHub URL (auto-detect SKILL.md, selective file upload, provenance).'
|
||||
read_when:
|
||||
- Adding GitHub import (web + API)
|
||||
- Reviewing safety limits (SSRF/zip-bombs)
|
||||
- Implementing provenance + canonical-claim flows
|
||||
---
|
||||
|
||||
# GitHub import (public repos)
|
||||
|
||||
Goal: paste a GitHub URL → auto-detect skill → preview files → publish (selective) → persist provenance.
|
||||
|
||||
Non-goal (v1): private repos (no OAuth/PAT support).
|
||||
|
||||
Related:
|
||||
- `docs/skill-format.md` (what counts as a skill; text-only limits)
|
||||
- `docs/api.md` / `docs/http-api.md` (REST patterns + auth)
|
||||
|
||||
## UX
|
||||
|
||||
Upload page: “Import from GitHub” mode.
|
||||
|
||||
Flow:
|
||||
1) URL input
|
||||
2) Detect skill candidates (SKILL.md)
|
||||
3) If multiple candidates: choose one
|
||||
4) File picker: check/uncheck; smart-select referenced files
|
||||
5) Confirm slug/name/version/tags
|
||||
6) Import → publish
|
||||
|
||||
## Accepted URLs
|
||||
|
||||
Allowlist: `https://github.com/...` only.
|
||||
|
||||
Supported shapes:
|
||||
- Repo root: `https://github.com/<owner>/<repo>`
|
||||
- Tree path: `https://github.com/<owner>/<repo>/tree/<ref>/<path>`
|
||||
- Blob path (file): `https://github.com/<owner>/<repo>/blob/<ref>/<path>`
|
||||
|
||||
Normalization:
|
||||
- Strip query/hash for fetch.
|
||||
- From `blob/.../SKILL.md` derive `path` as parent folder.
|
||||
- If `ref` missing: use `HEAD`.
|
||||
|
||||
Reject:
|
||||
- Non-GitHub hosts.
|
||||
- Unknown URL patterns.
|
||||
- Paths containing `..` after normalization.
|
||||
|
||||
## Fetch strategy (public)
|
||||
|
||||
Download archive:
|
||||
- `https://github.com/<owner>/<repo>/archive/<ref>.zip`
|
||||
- Follow redirects. Final redirect usually pins a commit via `codeload.github.com/.../zip/<sha-or-branch>`.
|
||||
|
||||
Unzip server-side (Node or Convex node action). Scan for skill candidates.
|
||||
|
||||
Skill candidate definition:
|
||||
- Any folder containing `SKILL.md` or `skill.md` (also accept `skills.md` for compatibility).
|
||||
- Treat repo root as a folder too.
|
||||
|
||||
Multiple skills:
|
||||
- Return candidate list: `{ path, frontmatter.name, frontmatter.description }`.
|
||||
- User chooses one.
|
||||
|
||||
## Smart file selection
|
||||
|
||||
Defaults:
|
||||
- Always select `SKILL.md` (or chosen readme file).
|
||||
- Prefer selecting only within chosen skill folder; allow “include out-of-folder refs” if explicitly toggled.
|
||||
|
||||
Referenced file expansion:
|
||||
- Parse Markdown links/images from selected `.md` files:
|
||||
- `[](<rel>)`, ``, `<rel>` only when relative.
|
||||
- Ignore `http(s):`, `mailto:`, `#anchors`.
|
||||
- Strip query/hash from relative targets.
|
||||
- Resolve against the current file’s directory.
|
||||
- Normalize, reject escapes (`..`).
|
||||
- Add referenced file if present in archive and is text-allowed.
|
||||
- Recurse for newly added `.md` files.
|
||||
|
||||
Hard caps:
|
||||
- Max recursion depth (e.g. 4).
|
||||
- Max referenced additions (e.g. 200).
|
||||
|
||||
UI affordances:
|
||||
- “Select referenced”
|
||||
- “Select all text”
|
||||
- “Clear”
|
||||
- Search/filter by path
|
||||
|
||||
## Publish behavior
|
||||
|
||||
Server publishes using existing pipeline:
|
||||
- Text-only enforced (see `docs/skill-format.md`).
|
||||
- Total ≤ 50MB (selected set).
|
||||
- Must include `SKILL.md` (or accepted variant).
|
||||
|
||||
Suggested defaults (UI):
|
||||
- `displayName`: frontmatter `name` else folder basename → title case.
|
||||
- `slug`: sanitize folder basename; if collision, suffix (`-2`, `-3`, …).
|
||||
- `version`: if new skill → `0.1.0`; if updating own existing skill → bump patch.
|
||||
- `tags`: default `latest`.
|
||||
|
||||
## Provenance (persist source)
|
||||
|
||||
Persist on each published version (server-side injection; no mutation of imported files):
|
||||
- Store in `skillVersions.parsed.metadata.source`:
|
||||
|
||||
Example:
|
||||
```json
|
||||
{
|
||||
"kind": "github",
|
||||
"url": "https://github.com/visionik/ouracli",
|
||||
"repo": "visionik/ouracli",
|
||||
"ref": "HEAD",
|
||||
"commit": "66ac8fb266b7c5ff6519431862be6a375bbfb883",
|
||||
"path": "",
|
||||
"importedAt": 1767930000000
|
||||
}
|
||||
```
|
||||
|
||||
Why `parsed.metadata`:
|
||||
- Already optional and stored with each version.
|
||||
- No schema churn for v1.
|
||||
|
||||
Future: canonical-claim
|
||||
- “claim canonical” can key off `{ kind:'github', repo, path }`.
|
||||
- Prefer commit-pinned provenance for auditability; allow UI to show “Imported from …”.
|
||||
|
||||
## API sketch (internal actions)
|
||||
|
||||
Two-step (recommended):
|
||||
- `previewGitHubImport(url)` → `{ commit, candidates:[...], files:[...], defaults:{...} }`
|
||||
- `importGitHubSkill({ url, commit, candidatePath, selectedPaths, slug, displayName, version, tags })`
|
||||
|
||||
Notes:
|
||||
- `importGitHubSkill` should re-fetch by pinned `commit` (not floating branch), to avoid TOCTOU.
|
||||
- Validate `selectedPaths` subset of fetched archive manifest.
|
||||
|
||||
## Security / abuse controls
|
||||
|
||||
SSRF:
|
||||
- Only `github.com` (+ `codeload.github.com` during redirect follow).
|
||||
- No arbitrary redirects to other hosts.
|
||||
|
||||
Zip safety:
|
||||
- Max compressed bytes (from `Content-Length` if present; else streaming cap).
|
||||
- Max uncompressed total bytes.
|
||||
- Max file count.
|
||||
- Max single file size.
|
||||
- Reject symlinks; reject absolute paths; reject `..` segments.
|
||||
|
||||
Rate limits:
|
||||
- Tie to existing write limits (import == publish).
|
||||
- Cache preview results briefly (e.g. 60s) keyed by `{repo, commit}`.
|
||||
|
||||
Error UX:
|
||||
- “No SKILL.md found.”
|
||||
- “Multiple skills found; pick one.”
|
||||
- “Repo too large / too many files.”
|
||||
- “Selected files exceed 50MB.”
|
||||
|
||||
## Manual test checklist
|
||||
|
||||
- Repo root skill (`SKILL.md` at root).
|
||||
- Nested skill (`skills/foo/SKILL.md`).
|
||||
- Multi-skill repo (two SKILL.md).
|
||||
- SKILL.md references `docs/usage.md` + images; smart-select picks `.md` and referenced text files; ignores external links.
|
||||
- Huge repo → clean “too large” error.
|
||||
- Redirect pinning → import stores commit sha in provenance.
|
||||
+27
-7
@@ -7,7 +7,7 @@ read_when:
|
||||
|
||||
# HTTP API
|
||||
|
||||
Base URL: `https://clawdhub.com` (default).
|
||||
Base URL: `https://clawhub.ai` (default).
|
||||
|
||||
All v1 paths are under `/api/v1/...` and implemented by Convex HTTP routes (`convex/http.ts`).
|
||||
Legacy `/api/...` and `/api/cli/...` remain for compatibility (see `DEPRECATIONS.md`).
|
||||
@@ -44,8 +44,13 @@ Response:
|
||||
|
||||
Query params:
|
||||
|
||||
- `limit` (optional): integer
|
||||
- `cursor` (optional): pagination cursor
|
||||
- `limit` (optional): integer (1–200)
|
||||
- `cursor` (optional): pagination cursor (only for `sort=updated`)
|
||||
- `sort` (optional): `updated` (default), `downloads`, `stars` (alias: `rating`), `installsCurrent` (alias: `installs`), `installsAllTime`, `trending`
|
||||
|
||||
Notes:
|
||||
|
||||
- `trending` ranks by installs in the last 7 days (telemetry-based).
|
||||
|
||||
Response:
|
||||
|
||||
@@ -140,6 +145,20 @@ Publishes a new version.
|
||||
|
||||
Soft-delete / restore a skill (owner/admin only).
|
||||
|
||||
### `POST /api/v1/stars/{slug}` / `DELETE /api/v1/stars/{slug}`
|
||||
|
||||
Add/remove a star (highlights). Both endpoints are idempotent.
|
||||
|
||||
Responses:
|
||||
|
||||
```json
|
||||
{ "ok": true, "starred": true, "alreadyStarred": false }
|
||||
```
|
||||
|
||||
```json
|
||||
{ "ok": true, "unstarred": true, "alreadyUnstarred": false }
|
||||
```
|
||||
|
||||
## Legacy CLI endpoints (deprecated)
|
||||
|
||||
Still supported for older CLI versions:
|
||||
@@ -153,16 +172,17 @@ Still supported for older CLI versions:
|
||||
|
||||
See `DEPRECATIONS.md` for removal plan.
|
||||
|
||||
## Registry discovery (`/.well-known/clawdhub.json`)
|
||||
## Registry discovery (`/.well-known/clawhub.json`)
|
||||
|
||||
The CLI can discover registry/auth settings from the site:
|
||||
|
||||
- `/.well-known/clawdhub.json` (JSON)
|
||||
- `/.well-known/clawhub.json` (JSON, preferred)
|
||||
- `/.well-known/clawdhub.json` (legacy)
|
||||
|
||||
Schema:
|
||||
|
||||
```json
|
||||
{ "apiBase": "https://clawdhub.com", "authBase": "https://clawdhub.com", "minCliVersion": "0.0.5" }
|
||||
{ "apiBase": "https://clawhub.ai", "authBase": "https://clawhub.ai", "minCliVersion": "0.0.5" }
|
||||
```
|
||||
|
||||
If you self-host, serve this file (or set `CLAWDHUB_REGISTRY` explicitly).
|
||||
If you self-host, serve this file (or set `CLAWHUB_REGISTRY` explicitly; legacy `CLAWDHUB_REGISTRY`).
|
||||
|
||||
+33
-19
@@ -8,43 +8,57 @@ read_when:
|
||||
# Manual testing (CLI)
|
||||
|
||||
## Setup
|
||||
- Ensure logged in: `bun clawdhub whoami` (or `bun clawdhub login`).
|
||||
- Ensure logged in: `bun clawhub whoami` (or `bun clawhub login`).
|
||||
- Optional: set env
|
||||
- `CLAWDHUB_SITE=https://clawdhub.com`
|
||||
- `CLAWDHUB_REGISTRY=https://clawdhub.com`
|
||||
- `CLAWHUB_SITE=https://clawhub.ai`
|
||||
- `CLAWHUB_REGISTRY=https://clawhub.ai`
|
||||
|
||||
## Smoke
|
||||
- `bun clawdhub --help`
|
||||
- `bun clawdhub --cli-version`
|
||||
- `bun clawdhub whoami`
|
||||
- `bun clawhub --help`
|
||||
- `bun clawhub --cli-version`
|
||||
- `bun clawhub whoami`
|
||||
|
||||
## Search
|
||||
- `bun clawdhub search gif --limit 5`
|
||||
- `bun clawhub search gif --limit 5`
|
||||
|
||||
## Install / list / update
|
||||
- `mkdir -p /tmp/clawdhub-manual && cd /tmp/clawdhub-manual`
|
||||
- `bunx clawdhub@beta install gifgrep --force`
|
||||
- `bunx clawdhub@beta list`
|
||||
- `bunx clawdhub@beta update gifgrep --force`
|
||||
- `mkdir -p /tmp/clawhub-manual && cd /tmp/clawhub-manual`
|
||||
- `bunx clawhub@beta install gifgrep --force`
|
||||
- `bunx clawhub@beta list`
|
||||
- `bunx clawhub@beta update gifgrep --force`
|
||||
|
||||
## Publish (changelog optional)
|
||||
- `mkdir -p /tmp/clawdhub-skill-demo/SKILL && cd /tmp/clawdhub-skill-demo`
|
||||
- `mkdir -p /tmp/clawhub-skill-demo/SKILL && cd /tmp/clawhub-skill-demo`
|
||||
- Create files:
|
||||
- `SKILL.md`
|
||||
- `notes.md`
|
||||
- Publish:
|
||||
- `bun clawdhub publish . --slug clawdhub-manual-<ts> --name "Manual <ts>" --version 1.0.0 --tags latest`
|
||||
- `bun clawhub publish . --slug clawhub-manual-<ts> --name "Manual <ts>" --version 1.0.0 --tags latest`
|
||||
- Publish update with empty changelog:
|
||||
- `bun clawdhub publish . --slug clawdhub-manual-<ts> --name "Manual <ts>" --version 1.0.1 --tags latest`
|
||||
- `bun clawhub publish . --slug clawhub-manual-<ts> --name "Manual <ts>" --version 1.0.1 --tags latest`
|
||||
|
||||
## Delete / undelete (owner/admin)
|
||||
- `bun clawdhub delete clawdhub-manual-<ts> --yes`
|
||||
- `bun clawhub delete clawhub-manual-<ts> --yes`
|
||||
- Verify hidden:
|
||||
- `curl -i "https://clawdhub.com/api/v1/skills/clawdhub-manual-<ts>"`
|
||||
- `curl -i "https://clawhub.ai/api/v1/skills/clawhub-manual-<ts>"`
|
||||
- Restore:
|
||||
- `bun clawdhub undelete clawdhub-manual-<ts> --yes`
|
||||
- `bun clawhub undelete clawhub-manual-<ts> --yes`
|
||||
- Cleanup:
|
||||
- `bun clawdhub delete clawdhub-manual-<ts> --yes`
|
||||
- `bun clawhub delete clawhub-manual-<ts> --yes`
|
||||
|
||||
## Sync
|
||||
- `bun clawdhub sync --dry-run --all`
|
||||
- `bun clawhub sync --dry-run --all`
|
||||
|
||||
## Playwright (menu smoke)
|
||||
|
||||
Run against prod:
|
||||
|
||||
```
|
||||
PLAYWRIGHT_BASE_URL=https://clawhub.ai bun run test:pw
|
||||
```
|
||||
|
||||
Run against a local preview server:
|
||||
|
||||
```
|
||||
bun run test:e2e:local
|
||||
```
|
||||
|
||||
+1
-1
@@ -20,7 +20,7 @@ Example (starter):
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "ClawdHub",
|
||||
"name": "OpenClaw",
|
||||
"logo": "public/logo.svg",
|
||||
"navigation": [
|
||||
{ "group": "Start", "pages": ["docs/README", "docs/quickstart"] },
|
||||
|
||||
+19
-13
@@ -51,23 +51,29 @@ Then paste the printed `JWT_PRIVATE_KEY` + `JWKS` into `.env.local` (and ensure
|
||||
From this repo:
|
||||
|
||||
```bash
|
||||
bun clawdhub --help
|
||||
bun clawdhub login
|
||||
bun clawdhub whoami
|
||||
bun clawdhub search gif --limit 5
|
||||
bun clawhub --help
|
||||
bun clawhub login
|
||||
bun clawhub whoami
|
||||
bun clawhub search gif --limit 5
|
||||
```
|
||||
|
||||
Install a skill into `./skills/<slug>`:
|
||||
Install a skill into `./skills/<slug>` (if Clawdbot is configured, installs into that workspace instead):
|
||||
|
||||
```bash
|
||||
bun clawdhub install <slug>
|
||||
bun clawdhub list
|
||||
bun clawhub install <slug>
|
||||
bun clawhub list
|
||||
```
|
||||
|
||||
You can also install into any folder:
|
||||
|
||||
```bash
|
||||
bun clawhub install <slug> --workdir /tmp/clawhub-demo --dir skills
|
||||
```
|
||||
|
||||
Update:
|
||||
|
||||
```bash
|
||||
bun clawdhub update --all
|
||||
bun clawhub update --all
|
||||
```
|
||||
|
||||
## 4) Publish a skill
|
||||
@@ -75,7 +81,7 @@ bun clawdhub update --all
|
||||
Create a folder containing `SKILL.md` (required) plus any supporting text files:
|
||||
|
||||
```bash
|
||||
mkdir -p /tmp/clawdhub-skill-demo && cd /tmp/clawdhub-skill-demo
|
||||
mkdir -p /tmp/clawhub-skill-demo && cd /tmp/clawhub-skill-demo
|
||||
cat > SKILL.md <<'EOF'
|
||||
---
|
||||
name: Demo Skill
|
||||
@@ -91,8 +97,8 @@ EOF
|
||||
Publish:
|
||||
|
||||
```bash
|
||||
bun clawdhub publish . \
|
||||
--slug clawdhub-demo-$(date +%s) \
|
||||
bun clawhub publish . \
|
||||
--slug clawhub-demo-$(date +%s) \
|
||||
--name "Demo $(date +%s)" \
|
||||
--version 1.0.0 \
|
||||
--tags latest \
|
||||
@@ -104,11 +110,11 @@ bun clawdhub publish . \
|
||||
`sync` scans for local skill folders and publishes the ones that aren’t “synced” yet.
|
||||
|
||||
```bash
|
||||
bun clawdhub sync
|
||||
bun clawhub sync
|
||||
```
|
||||
|
||||
Dry run + non-interactive:
|
||||
|
||||
```bash
|
||||
bun clawdhub sync --all --dry-run --no-input
|
||||
bun clawhub sync --all --dry-run --no-input
|
||||
```
|
||||
|
||||
@@ -18,16 +18,16 @@ Required:
|
||||
Optional:
|
||||
|
||||
- any supporting *text-based* files (see “Allowed files”)
|
||||
- `.clawdhubignore` (ignore patterns for publish/sync)
|
||||
- `.clawhubignore` (ignore patterns for publish/sync, legacy `.clawdhubignore`)
|
||||
- `.gitignore` (also honored)
|
||||
|
||||
Local install metadata (written by the CLI):
|
||||
|
||||
- `<skill>/.clawdhub/origin.json`
|
||||
- `<skill>/.clawhub/origin.json` (legacy `.clawdhub`)
|
||||
|
||||
Workdir install state (written by the CLI):
|
||||
|
||||
- `<workdir>/.clawdhub/lock.json`
|
||||
- `<workdir>/.clawhub/lock.json` (legacy `.clawdhub`)
|
||||
|
||||
## `SKILL.md`
|
||||
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
---
|
||||
summary: 'Soul bundle format, required files, limits.'
|
||||
read_when:
|
||||
- Publishing souls
|
||||
- Debugging soul publish failures
|
||||
---
|
||||
|
||||
# Soul format
|
||||
|
||||
## On disk
|
||||
|
||||
A soul is a single file:
|
||||
|
||||
- `SOUL.md` (or `soul.md`)
|
||||
|
||||
For now, onlycrabs.ai rejects any extra files.
|
||||
|
||||
## `SOUL.md`
|
||||
|
||||
- Markdown with optional YAML frontmatter.
|
||||
- The server extracts metadata from frontmatter during publish.
|
||||
- `description` is used as the soul summary in the UI/search.
|
||||
|
||||
## Limits
|
||||
|
||||
- Total bundle size: 50MB.
|
||||
- Embedding text includes `SOUL.md` only.
|
||||
|
||||
## Slugs
|
||||
|
||||
- Derived from folder name by default.
|
||||
- Must be lowercase and URL-safe: `^[a-z0-9][a-z0-9-]*$`.
|
||||
|
||||
## Versioning + tags
|
||||
|
||||
- Each publish creates a new version (semver).
|
||||
- Tags are string pointers to a version; `latest` is commonly used.
|
||||
+56
-10
@@ -1,14 +1,15 @@
|
||||
---
|
||||
summary: "ClawdHub spec: skills registry, versioning, vector search, moderation"
|
||||
summary: "OpenClaw spec: skills registry, versioning, vector search, moderation"
|
||||
read_when:
|
||||
- Bootstrapping ClawdHub
|
||||
- Bootstrapping OpenClaw
|
||||
- Implementing schema/auth/search/versioning
|
||||
- Reviewing API and upload/download flows
|
||||
---
|
||||
|
||||
# ClawdHub — product + implementation spec (v1)
|
||||
# OpenClaw — product + implementation spec (v1)
|
||||
|
||||
## Goals
|
||||
- onlycrabs.ai mode for sharing `SOUL.md` bundles (host-based entry point).
|
||||
- Minimal, fast SPA for browsing and publishing agent skills.
|
||||
- Skills stored in Convex (files + metadata + versions + stats).
|
||||
- GitHub OAuth login; GitHub App backs up skills to `clawdbot/skills`.
|
||||
@@ -28,7 +29,7 @@ read_when:
|
||||
- `handle` (GitHub login)
|
||||
- `name`, `bio`
|
||||
- `avatarUrl` (GitHub, fallback gravatar)
|
||||
- `role`: `admin | moderator | user`
|
||||
- `role`: `admin | moderator | user` (moderators can soft-delete and flag; admins can hard-delete + change owners)
|
||||
- `createdAt`, `updatedAt`
|
||||
|
||||
### Skill
|
||||
@@ -39,9 +40,14 @@ read_when:
|
||||
- `latestVersionId`
|
||||
- `latestTagVersionId` (for `latest` tag)
|
||||
- `tags` map: `{ tag -> versionId }`
|
||||
- `badges`: `{ redactionApproved?: { byUserId, at } }`
|
||||
- `badges`: `{ redactionApproved?: { byUserId, at }, highlighted?: { byUserId, at }, official?: { byUserId, at }, deprecated?: { byUserId, at } }`
|
||||
- `official` marks admin-verified/official skills.
|
||||
- `deprecated` marks skills that should not be used for new integrations.
|
||||
- `moderationStatus`: `active | hidden | removed`
|
||||
- `moderationFlags`: `string[]` (automatic detection)
|
||||
- `moderationNotes`, `moderationReason`
|
||||
- `hiddenAt`, `hiddenBy`, `lastReviewedAt`, `reportCount`
|
||||
- `stats`: `{ downloads, stars, versions, comments }`
|
||||
- `status`: `active` only (soft-delete on version/comment only)
|
||||
- `createdAt`, `updatedAt`
|
||||
|
||||
### SkillVersion
|
||||
@@ -60,9 +66,45 @@ read_when:
|
||||
From SKILL.md frontmatter + AgentSkills + Clawdis extensions:
|
||||
- `name`, `description`, `homepage`, `website`, `url`, `emoji`
|
||||
- `metadata.clawdis`: `always`, `skillKey`, `primaryEnv`, `emoji`, `homepage`, `os`,
|
||||
`requires` (`bins`, `anyBins`, `env`, `config`), `install[]`
|
||||
`requires` (`bins`, `anyBins`, `env`, `config`), `install[]`, `nix` (`plugin`, `systems`),
|
||||
`config` (`requiredEnv`, `stateDirs`, `example`), `cliHelp` (string; `cli --help` output)
|
||||
- `metadata.clawdbot`: alias of `metadata.clawdis` (preferred for nix-clawdbot plugin pointers)
|
||||
- Nix plugins are different from regular skills; they bundle the skill pack, the CLI binary, and config flags/requirements together.
|
||||
- `metadata` in frontmatter is YAML (object) preferred; legacy JSON-string accepted.
|
||||
|
||||
|
||||
|
||||
### Soul
|
||||
- `slug` (unique)
|
||||
- `displayName`
|
||||
- `ownerUserId`
|
||||
- `summary` (from SOUL.md frontmatter `description`)
|
||||
- `latestVersionId`
|
||||
- `tags` map: `{ tag -> versionId }`
|
||||
- `stats`: `{ downloads, stars, versions, comments }`
|
||||
- `status`: `active` only (soft-delete on version/comment only)
|
||||
- `createdAt`, `updatedAt`
|
||||
|
||||
### SoulVersion
|
||||
- `soulId`
|
||||
- `version` (semver string)
|
||||
- `tag` (string, optional; `latest` always maintained separately)
|
||||
- `changelog` (required)
|
||||
- `files`: list of file metadata (SOUL.md only)
|
||||
- `path`, `size`, `storageId`, `sha256`
|
||||
- `parsed` (metadata extracted from SOUL.md)
|
||||
- `vectorDocId` (if using RAG component) OR `embeddingId`
|
||||
- `createdBy`, `createdAt`
|
||||
- `softDeletedAt` (nullable)
|
||||
|
||||
### SoulComment
|
||||
- `soulId`, `userId`, `body`
|
||||
- `softDeletedAt`, `deletedBy`
|
||||
- `createdAt`
|
||||
|
||||
### SoulStar
|
||||
- `soulId`, `userId`, `createdAt`
|
||||
|
||||
### Comment
|
||||
- `skillId`, `userId`, `body`
|
||||
- `softDeletedAt`, `deletedBy`
|
||||
@@ -81,7 +123,8 @@ From SKILL.md frontmatter + AgentSkills + Clawdis extensions:
|
||||
## Auth + roles
|
||||
- Convex Auth with GitHub OAuth App.
|
||||
- Default role `user`; bootstrap `steipete` to `admin` on first login.
|
||||
- Admin UI to promote/demote roles; all changes logged.
|
||||
- Management console: moderators can hide/restore skills + mark duplicates; admins can change owners, approve badges, and hard-delete.
|
||||
- Role changes are admin-only and audited.
|
||||
|
||||
## Upload flow (50MB per version)
|
||||
1) Client requests upload session.
|
||||
@@ -94,6 +137,9 @@ From SKILL.md frontmatter + AgentSkills + Clawdis extensions:
|
||||
- version uniqueness
|
||||
5) Server stores files + metadata, sets `latest` tag, updates stats.
|
||||
|
||||
Soul upload flow: same as skills, but only `SOUL.md` is allowed in the bundle.
|
||||
Seed data lives in `convex/seed.ts` for local dev.
|
||||
|
||||
## Versioning + tags
|
||||
- Each upload is a new `SkillVersion`.
|
||||
- `latest` tag always points to most recent version unless user re-tags.
|
||||
@@ -101,7 +147,7 @@ From SKILL.md frontmatter + AgentSkills + Clawdis extensions:
|
||||
- Changelog is optional.
|
||||
|
||||
## Search
|
||||
- Vector search over: SKILL.md + other text files + metadata summary.
|
||||
- Vector search over: SKILL.md + other text files + metadata summary (souls index SOUL.md).
|
||||
- Convex embeddings + vector index.
|
||||
- Filters: tag, owner, `redactionApproved` only, min stars, updatedAt.
|
||||
|
||||
@@ -111,7 +157,7 @@ From SKILL.md frontmatter + AgentSkills + Clawdis extensions:
|
||||
- Soft-delete versions; downloads remain for non-deleted versions only.
|
||||
|
||||
## UI (SPA)
|
||||
- Home: search + filters + trending/featured + “Highlighted” batch.
|
||||
- Home: search + filters + trending/featured + “Highlighted” badge.
|
||||
- Skill detail: README render, files list, version history, tags, stats, badges.
|
||||
- Upload/edit: file picker + version + tag + changelog.
|
||||
- Account settings: name + delete account (soft delete).
|
||||
|
||||
+8
-8
@@ -1,5 +1,5 @@
|
||||
---
|
||||
summary: 'Install telemetry collected via `clawdhub sync` + opt-out.'
|
||||
summary: 'Install telemetry collected via `clawhub sync` + opt-out.'
|
||||
read_when:
|
||||
- Working on telemetry / privacy controls
|
||||
- Questions about what data is collected
|
||||
@@ -7,22 +7,22 @@ read_when:
|
||||
|
||||
# Telemetry
|
||||
|
||||
ClawdHub uses **minimal telemetry** to compute **install counts** (what’s actually in use) and to power better sorting/filtering.
|
||||
This is based on the CLI `clawdhub sync` command.
|
||||
OpenClaw uses **minimal telemetry** to compute **install counts** (what’s actually in use) and to power better sorting/filtering.
|
||||
This is based on the CLI `clawhub sync` command.
|
||||
|
||||
## When telemetry is collected
|
||||
|
||||
Telemetry is only sent when:
|
||||
|
||||
- You are **logged in** in the CLI (we already require auth for sync/publish flows).
|
||||
- You run `clawdhub sync`.
|
||||
- You run `clawhub sync`.
|
||||
- Telemetry is **not disabled** (see “How to disable” below).
|
||||
|
||||
If you are not logged in, nothing is reported.
|
||||
|
||||
## What we collect
|
||||
|
||||
On each `clawdhub sync`, the CLI reports a **full snapshot** of what it found, grouped by scan root (“folder/root”).
|
||||
On each `clawhub sync`, the CLI reports a **full snapshot** of what it found, grouped by scan root (“folder/root”).
|
||||
|
||||
For each root we store:
|
||||
|
||||
@@ -70,7 +70,7 @@ This is evaluated lazily (on the next telemetry report) to avoid background jobs
|
||||
|
||||
## Transparency + user controls
|
||||
|
||||
ClawdHub provides a private “Installed” tab on your own profile:
|
||||
OpenClaw provides a private “Installed” tab on your own profile:
|
||||
|
||||
- Shows the exact roots + installed skills we store.
|
||||
- Includes a **JSON export** view.
|
||||
@@ -85,7 +85,7 @@ Deleting your account also deletes your telemetry data.
|
||||
Set the environment variable:
|
||||
|
||||
```bash
|
||||
export CLAWDHUB_DISABLE_TELEMETRY=1
|
||||
export CLAWHUB_DISABLE_TELEMETRY=1
|
||||
```
|
||||
|
||||
With this set, the CLI will not send telemetry during `clawdhub sync`.
|
||||
With this set, the CLI will not send telemetry during `clawhub sync`.
|
||||
|
||||
@@ -6,16 +6,16 @@ read_when:
|
||||
|
||||
# Troubleshooting
|
||||
|
||||
## `clawdhub login` opens browser but never completes
|
||||
## `clawhub login` opens browser but never completes
|
||||
|
||||
- Ensure your browser can reach `http://127.0.0.1:<port>/callback` (local firewalls/VPNs can interfere).
|
||||
- Use headless mode:
|
||||
- create a token in the web UI (Settings → API tokens)
|
||||
- `clawdhub login --token clh_...`
|
||||
- `clawhub login --token clh_...`
|
||||
|
||||
## `whoami` / `publish` returns `Unauthorized` (401)
|
||||
|
||||
- Token missing or revoked: check your config file (`CLAWDHUB_CONFIG_PATH` override?).
|
||||
- Token missing or revoked: check your config file (`CLAWHUB_CONFIG_PATH` override?).
|
||||
- Ensure requests include `Authorization: Bearer ...` (CLI does this automatically).
|
||||
|
||||
## `publish` fails with `OPENAI_API_KEY is not configured`
|
||||
@@ -32,7 +32,7 @@ read_when:
|
||||
- Provide explicit roots:
|
||||
|
||||
```bash
|
||||
clawdhub sync --root /path/to/skills
|
||||
clawhub sync --root /path/to/skills
|
||||
```
|
||||
|
||||
## `update` refuses due to “local changes (no match)”
|
||||
@@ -40,8 +40,8 @@ clawdhub sync --root /path/to/skills
|
||||
- Your local files don’t match any published fingerprint.
|
||||
- Options:
|
||||
- keep local edits; skip updating
|
||||
- overwrite: `clawdhub update <slug> --force`
|
||||
- publish as fork: copy to new folder/slug then `clawdhub publish ... --fork-of upstream@version`
|
||||
- overwrite: `clawhub update <slug> --force`
|
||||
- publish as fork: copy to new folder/slug then `clawhub publish ... --fork-of upstream@version`
|
||||
|
||||
## `GET /api/*` works locally but not on Vercel
|
||||
|
||||
|
||||
+4
-4
@@ -6,7 +6,7 @@ read_when:
|
||||
|
||||
# Webhooks (Discord)
|
||||
|
||||
ClawdHub can post Discord embeds when skills are published or highlighted.
|
||||
OpenClaw can post Discord embeds when skills are published or highlighted.
|
||||
|
||||
## Setup
|
||||
|
||||
@@ -14,7 +14,7 @@ Set the webhook URL in the Convex environment:
|
||||
|
||||
- `DISCORD_WEBHOOK_URL` (required): Discord webhook URL.
|
||||
- `DISCORD_WEBHOOK_HIGHLIGHTED_ONLY` (optional): `true` to only send for highlighted skills.
|
||||
- `SITE_URL` (optional): Base site URL for links (default `https://clawdhub.com`).
|
||||
- `SITE_URL` (optional): Base site URL for links (default `https://clawhub.ai`).
|
||||
|
||||
## Events
|
||||
|
||||
@@ -38,13 +38,13 @@ Discord receives a JSON payload with a single embed:
|
||||
{
|
||||
"title": "Demo Skill",
|
||||
"description": "Nice skill",
|
||||
"url": "https://clawdhub.com/owner/demo-skill",
|
||||
"url": "https://clawhub.ai/owner/demo-skill",
|
||||
"fields": [
|
||||
{ "name": "Version", "value": "v1.2.3", "inline": true },
|
||||
{ "name": "Owner", "value": "@owner", "inline": true },
|
||||
{ "name": "Tags", "value": "latest, discord", "inline": false }
|
||||
],
|
||||
"footer": { "text": "ClawdHub" }
|
||||
"footer": { "text": "OpenClaw" }
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
+146
-51
@@ -9,19 +9,47 @@ import {
|
||||
ApiV1SearchResponseSchema,
|
||||
ApiV1WhoamiResponseSchema,
|
||||
parseArk,
|
||||
} from 'clawdhub-schema'
|
||||
} from 'clawhub-schema'
|
||||
import { unzipSync } from 'fflate'
|
||||
import { Agent, setGlobalDispatcher } from 'undici'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { readGlobalConfig } from '../packages/clawdhub/src/config'
|
||||
|
||||
const REQUEST_TIMEOUT_MS = 15_000
|
||||
|
||||
try {
|
||||
setGlobalDispatcher(
|
||||
new Agent({
|
||||
allowH2: true,
|
||||
connect: { timeout: REQUEST_TIMEOUT_MS },
|
||||
}),
|
||||
)
|
||||
} catch {
|
||||
// ignore dispatcher setup failures
|
||||
}
|
||||
|
||||
function mustGetToken() {
|
||||
const fromEnv = process.env.CLAWDHUB_E2E_TOKEN?.trim()
|
||||
const fromEnv = process.env.CLAWHUB_E2E_TOKEN?.trim() || process.env.CLAWDHUB_E2E_TOKEN?.trim()
|
||||
if (fromEnv) return fromEnv
|
||||
return null
|
||||
}
|
||||
|
||||
function getRegistry() {
|
||||
return (
|
||||
process.env.CLAWHUB_REGISTRY?.trim() ||
|
||||
process.env.CLAWDHUB_REGISTRY?.trim() ||
|
||||
'https://clawhub.ai'
|
||||
)
|
||||
}
|
||||
|
||||
function getSite() {
|
||||
return (
|
||||
process.env.CLAWHUB_SITE?.trim() || process.env.CLAWDHUB_SITE?.trim() || 'https://clawhub.ai'
|
||||
)
|
||||
}
|
||||
|
||||
async function makeTempConfig(registry: string, token: string | null) {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'clawdhub-e2e-'))
|
||||
const dir = await mkdtemp(join(tmpdir(), 'clawhub-e2e-'))
|
||||
const path = join(dir, 'config.json')
|
||||
await writeFile(
|
||||
path,
|
||||
@@ -31,9 +59,19 @@ async function makeTempConfig(registry: string, token: string | null) {
|
||||
return { dir, path }
|
||||
}
|
||||
|
||||
describe('clawdhub e2e', () => {
|
||||
async function fetchWithTimeout(input: RequestInfo | URL, init?: RequestInit) {
|
||||
const controller = new AbortController()
|
||||
const timeout = setTimeout(() => controller.abort('Timeout'), REQUEST_TIMEOUT_MS)
|
||||
try {
|
||||
return await fetch(input, { ...init, signal: controller.signal })
|
||||
} finally {
|
||||
clearTimeout(timeout)
|
||||
}
|
||||
}
|
||||
|
||||
describe('clawhub e2e', () => {
|
||||
it('prints CLI version via --cli-version', async () => {
|
||||
const result = spawnSync('bun', ['clawdhub', '--cli-version'], {
|
||||
const result = spawnSync('bun', ['clawhub', '--cli-version'], {
|
||||
cwd: process.cwd(),
|
||||
encoding: 'utf8',
|
||||
})
|
||||
@@ -42,12 +80,14 @@ describe('clawdhub e2e', () => {
|
||||
})
|
||||
|
||||
it('search endpoint returns a results array (schema parse)', async () => {
|
||||
const registry = process.env.CLAWDHUB_REGISTRY?.trim() || 'https://clawdhub.com'
|
||||
const registry = getRegistry()
|
||||
const url = new URL(ApiRoutes.search, registry)
|
||||
url.searchParams.set('q', 'gif')
|
||||
url.searchParams.set('limit', '5')
|
||||
|
||||
const response = await fetch(url.toString(), { headers: { Accept: 'application/json' } })
|
||||
const response = await fetchWithTimeout(url.toString(), {
|
||||
headers: { Accept: 'application/json' },
|
||||
})
|
||||
expect(response.ok).toBe(true)
|
||||
const json = (await response.json()) as unknown
|
||||
const parsed = parseArk(ApiV1SearchResponseSchema, json, 'API response')
|
||||
@@ -55,17 +95,17 @@ describe('clawdhub e2e', () => {
|
||||
})
|
||||
|
||||
it('cli search does not error on multi-result responses', async () => {
|
||||
const registry = process.env.CLAWDHUB_REGISTRY?.trim() || 'https://clawdhub.com'
|
||||
const site = process.env.CLAWDHUB_SITE?.trim() || 'https://clawdhub.com'
|
||||
const registry = getRegistry()
|
||||
const site = getSite()
|
||||
const token = mustGetToken() ?? (await readGlobalConfig())?.token ?? null
|
||||
|
||||
const cfg = await makeTempConfig(registry, token)
|
||||
try {
|
||||
const workdir = await mkdtemp(join(tmpdir(), 'clawdhub-e2e-workdir-'))
|
||||
const workdir = await mkdtemp(join(tmpdir(), 'clawhub-e2e-workdir-'))
|
||||
const result = spawnSync(
|
||||
'bun',
|
||||
[
|
||||
'clawdhub',
|
||||
'clawhub',
|
||||
'search',
|
||||
'gif',
|
||||
'--limit',
|
||||
@@ -79,7 +119,7 @@ describe('clawdhub e2e', () => {
|
||||
],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
env: { ...process.env, CLAWDHUB_CONFIG_PATH: cfg.path, CLAWDHUB_DISABLE_TELEMETRY: '1' },
|
||||
env: { ...process.env, CLAWHUB_CONFIG_PATH: cfg.path, CLAWHUB_DISABLE_TELEMETRY: '1' },
|
||||
encoding: 'utf8',
|
||||
},
|
||||
)
|
||||
@@ -93,17 +133,17 @@ describe('clawdhub e2e', () => {
|
||||
})
|
||||
|
||||
it('assumes a logged-in user (whoami succeeds)', async () => {
|
||||
const registry = process.env.CLAWDHUB_REGISTRY?.trim() || 'https://clawdhub.com'
|
||||
const site = process.env.CLAWDHUB_SITE?.trim() || 'https://clawdhub.com'
|
||||
const registry = getRegistry()
|
||||
const site = getSite()
|
||||
const token = mustGetToken() ?? (await readGlobalConfig())?.token ?? null
|
||||
if (!token) {
|
||||
throw new Error('Missing token. Set CLAWDHUB_E2E_TOKEN or run: bun clawdhub auth login')
|
||||
throw new Error('Missing token. Set CLAWHUB_E2E_TOKEN or run: bun clawhub auth login')
|
||||
}
|
||||
|
||||
const cfg = await makeTempConfig(registry, token)
|
||||
try {
|
||||
const whoamiUrl = new URL(ApiRoutes.whoami, registry)
|
||||
const whoamiRes = await fetch(whoamiUrl.toString(), {
|
||||
const whoamiRes = await fetchWithTimeout(whoamiUrl.toString(), {
|
||||
headers: { Accept: 'application/json', Authorization: `Bearer ${token}` },
|
||||
})
|
||||
expect(whoamiRes.ok).toBe(true)
|
||||
@@ -116,10 +156,10 @@ describe('clawdhub e2e', () => {
|
||||
|
||||
const result = spawnSync(
|
||||
'bun',
|
||||
['clawdhub', 'whoami', '--site', site, '--registry', registry],
|
||||
['clawhub', 'whoami', '--site', site, '--registry', registry],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
env: { ...process.env, CLAWDHUB_CONFIG_PATH: cfg.path, CLAWDHUB_DISABLE_TELEMETRY: '1' },
|
||||
env: { ...process.env, CLAWHUB_CONFIG_PATH: cfg.path, CLAWHUB_DISABLE_TELEMETRY: '1' },
|
||||
encoding: 'utf8',
|
||||
},
|
||||
)
|
||||
@@ -131,15 +171,15 @@ describe('clawdhub e2e', () => {
|
||||
})
|
||||
|
||||
it('sync dry-run finds skills from an explicit root', async () => {
|
||||
const registry = process.env.CLAWDHUB_REGISTRY?.trim() || 'https://clawdhub.com'
|
||||
const site = process.env.CLAWDHUB_SITE?.trim() || 'https://clawdhub.com'
|
||||
const registry = getRegistry()
|
||||
const site = getSite()
|
||||
const token = mustGetToken() ?? (await readGlobalConfig())?.token ?? null
|
||||
if (!token) {
|
||||
throw new Error('Missing token. Set CLAWDHUB_E2E_TOKEN or run: bun clawdhub auth login')
|
||||
throw new Error('Missing token. Set CLAWHUB_E2E_TOKEN or run: bun clawhub auth login')
|
||||
}
|
||||
|
||||
const cfg = await makeTempConfig(registry, token)
|
||||
const root = await mkdtemp(join(tmpdir(), 'clawdhub-e2e-sync-'))
|
||||
const root = await mkdtemp(join(tmpdir(), 'clawhub-e2e-sync-'))
|
||||
try {
|
||||
const skillDir = join(root, 'cool-skill')
|
||||
await mkdir(skillDir, { recursive: true })
|
||||
@@ -148,7 +188,7 @@ describe('clawdhub e2e', () => {
|
||||
const result = spawnSync(
|
||||
'bun',
|
||||
[
|
||||
'clawdhub',
|
||||
'clawhub',
|
||||
'sync',
|
||||
'--dry-run',
|
||||
'--all',
|
||||
@@ -161,7 +201,7 @@ describe('clawdhub e2e', () => {
|
||||
],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
env: { ...process.env, CLAWDHUB_CONFIG_PATH: cfg.path, CLAWDHUB_DISABLE_TELEMETRY: '1' },
|
||||
env: { ...process.env, CLAWHUB_CONFIG_PATH: cfg.path, CLAWHUB_DISABLE_TELEMETRY: '1' },
|
||||
encoding: 'utf8',
|
||||
},
|
||||
)
|
||||
@@ -174,17 +214,72 @@ describe('clawdhub e2e', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('publishes, deletes, and undeletes a skill (logged-in)', async () => {
|
||||
const registry = process.env.CLAWDHUB_REGISTRY?.trim() || 'https://clawdhub.com'
|
||||
const site = process.env.CLAWDHUB_SITE?.trim() || 'https://clawdhub.com'
|
||||
it('sync dry-run finds skills from clawdbot.json roots', async () => {
|
||||
const registry = getRegistry()
|
||||
const site = getSite()
|
||||
const token = mustGetToken() ?? (await readGlobalConfig())?.token ?? null
|
||||
if (!token) {
|
||||
throw new Error('Missing token. Set CLAWDHUB_E2E_TOKEN or run: bun clawdhub auth login')
|
||||
throw new Error('Missing token. Set CLAWHUB_E2E_TOKEN or run: bun clawhub auth login')
|
||||
}
|
||||
|
||||
const cfg = await makeTempConfig(registry, token)
|
||||
const workdir = await mkdtemp(join(tmpdir(), 'clawdhub-e2e-publish-'))
|
||||
const installWorkdir = await mkdtemp(join(tmpdir(), 'clawdhub-e2e-install-'))
|
||||
const root = await mkdtemp(join(tmpdir(), 'clawhub-e2e-clawdbot-'))
|
||||
const stateDir = join(root, 'state')
|
||||
const configPath = join(root, 'clawdbot.json')
|
||||
const workspace = join(root, 'clawd-work')
|
||||
const skillsRoot = join(workspace, 'skills')
|
||||
const skillDir = join(skillsRoot, 'auto-skill')
|
||||
|
||||
try {
|
||||
await mkdir(skillDir, { recursive: true })
|
||||
await writeFile(join(skillDir, 'SKILL.md'), '# Skill\n', 'utf8')
|
||||
|
||||
const config = `{
|
||||
// JSON5-style comments + trailing commas
|
||||
routing: {
|
||||
agents: {
|
||||
work: { name: 'Work', workspace: '${workspace}', },
|
||||
},
|
||||
},
|
||||
}`
|
||||
await writeFile(configPath, config, 'utf8')
|
||||
|
||||
const result = spawnSync(
|
||||
'bun',
|
||||
['clawhub', 'sync', '--dry-run', '--all', '--site', site, '--registry', registry],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
env: {
|
||||
...process.env,
|
||||
CLAWHUB_CONFIG_PATH: cfg.path,
|
||||
CLAWHUB_DISABLE_TELEMETRY: '1',
|
||||
CLAWDBOT_CONFIG_PATH: configPath,
|
||||
CLAWDBOT_STATE_DIR: stateDir,
|
||||
},
|
||||
encoding: 'utf8',
|
||||
},
|
||||
)
|
||||
expect(result.status).toBe(0)
|
||||
expect(result.stderr).not.toMatch(/error:/i)
|
||||
expect(result.stdout).toMatch(/Dry run/i)
|
||||
expect(result.stdout).toMatch(/auto-skill/i)
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true })
|
||||
await rm(cfg.dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('publishes, deletes, and undeletes a skill (logged-in)', async () => {
|
||||
const registry = getRegistry()
|
||||
const site = getSite()
|
||||
const token = mustGetToken() ?? (await readGlobalConfig())?.token ?? null
|
||||
if (!token) {
|
||||
throw new Error('Missing token. Set CLAWHUB_E2E_TOKEN or run: bun clawhub auth login')
|
||||
}
|
||||
|
||||
const cfg = await makeTempConfig(registry, token)
|
||||
const workdir = await mkdtemp(join(tmpdir(), 'clawhub-e2e-publish-'))
|
||||
const installWorkdir = await mkdtemp(join(tmpdir(), 'clawhub-e2e-install-'))
|
||||
const slug = `e2e-${Date.now()}`
|
||||
const skillDir = join(workdir, slug)
|
||||
|
||||
@@ -195,7 +290,7 @@ describe('clawdhub e2e', () => {
|
||||
const publish1 = spawnSync(
|
||||
'bun',
|
||||
[
|
||||
'clawdhub',
|
||||
'clawhub',
|
||||
'publish',
|
||||
skillDir,
|
||||
'--slug',
|
||||
@@ -215,7 +310,7 @@ describe('clawdhub e2e', () => {
|
||||
],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
env: { ...process.env, CLAWDHUB_CONFIG_PATH: cfg.path, CLAWDHUB_DISABLE_TELEMETRY: '1' },
|
||||
env: { ...process.env, CLAWHUB_CONFIG_PATH: cfg.path, CLAWHUB_DISABLE_TELEMETRY: '1' },
|
||||
encoding: 'utf8',
|
||||
},
|
||||
)
|
||||
@@ -225,7 +320,7 @@ describe('clawdhub e2e', () => {
|
||||
const publish2 = spawnSync(
|
||||
'bun',
|
||||
[
|
||||
'clawdhub',
|
||||
'clawhub',
|
||||
'publish',
|
||||
skillDir,
|
||||
'--slug',
|
||||
@@ -245,7 +340,7 @@ describe('clawdhub e2e', () => {
|
||||
],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
env: { ...process.env, CLAWDHUB_CONFIG_PATH: cfg.path, CLAWDHUB_DISABLE_TELEMETRY: '1' },
|
||||
env: { ...process.env, CLAWHUB_CONFIG_PATH: cfg.path, CLAWHUB_DISABLE_TELEMETRY: '1' },
|
||||
encoding: 'utf8',
|
||||
},
|
||||
)
|
||||
@@ -255,7 +350,7 @@ describe('clawdhub e2e', () => {
|
||||
const downloadUrl = new URL(ApiRoutes.download, registry)
|
||||
downloadUrl.searchParams.set('slug', slug)
|
||||
downloadUrl.searchParams.set('version', '1.0.1')
|
||||
const zipRes = await fetch(downloadUrl.toString())
|
||||
const zipRes = await fetchWithTimeout(downloadUrl.toString())
|
||||
expect(zipRes.ok).toBe(true)
|
||||
const zipBytes = new Uint8Array(await zipRes.arrayBuffer())
|
||||
const unzipped = unzipSync(zipBytes)
|
||||
@@ -264,7 +359,7 @@ describe('clawdhub e2e', () => {
|
||||
const install = spawnSync(
|
||||
'bun',
|
||||
[
|
||||
'clawdhub',
|
||||
'clawhub',
|
||||
'install',
|
||||
slug,
|
||||
'--version',
|
||||
@@ -279,7 +374,7 @@ describe('clawdhub e2e', () => {
|
||||
],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
env: { ...process.env, CLAWDHUB_CONFIG_PATH: cfg.path, CLAWDHUB_DISABLE_TELEMETRY: '1' },
|
||||
env: { ...process.env, CLAWHUB_CONFIG_PATH: cfg.path, CLAWHUB_DISABLE_TELEMETRY: '1' },
|
||||
encoding: 'utf8',
|
||||
},
|
||||
)
|
||||
@@ -287,10 +382,10 @@ describe('clawdhub e2e', () => {
|
||||
|
||||
const list = spawnSync(
|
||||
'bun',
|
||||
['clawdhub', 'list', '--site', site, '--registry', registry, '--workdir', installWorkdir],
|
||||
['clawhub', 'list', '--site', site, '--registry', registry, '--workdir', installWorkdir],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
env: { ...process.env, CLAWDHUB_CONFIG_PATH: cfg.path, CLAWDHUB_DISABLE_TELEMETRY: '1' },
|
||||
env: { ...process.env, CLAWHUB_CONFIG_PATH: cfg.path, CLAWHUB_DISABLE_TELEMETRY: '1' },
|
||||
encoding: 'utf8',
|
||||
},
|
||||
)
|
||||
@@ -300,7 +395,7 @@ describe('clawdhub e2e', () => {
|
||||
const update = spawnSync(
|
||||
'bun',
|
||||
[
|
||||
'clawdhub',
|
||||
'clawhub',
|
||||
'update',
|
||||
slug,
|
||||
'--force',
|
||||
@@ -313,14 +408,14 @@ describe('clawdhub e2e', () => {
|
||||
],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
env: { ...process.env, CLAWDHUB_CONFIG_PATH: cfg.path, CLAWDHUB_DISABLE_TELEMETRY: '1' },
|
||||
env: { ...process.env, CLAWHUB_CONFIG_PATH: cfg.path, CLAWHUB_DISABLE_TELEMETRY: '1' },
|
||||
encoding: 'utf8',
|
||||
},
|
||||
)
|
||||
expect(update.status).toBe(0)
|
||||
|
||||
const metaUrl = new URL(`${ApiRoutes.skills}/${slug}`, registry)
|
||||
const metaRes = await fetch(metaUrl.toString(), {
|
||||
const metaRes = await fetchWithTimeout(metaUrl.toString(), {
|
||||
headers: { Accept: 'application/json' },
|
||||
})
|
||||
expect(metaRes.status).toBe(200)
|
||||
@@ -328,7 +423,7 @@ describe('clawdhub e2e', () => {
|
||||
const del = spawnSync(
|
||||
'bun',
|
||||
[
|
||||
'clawdhub',
|
||||
'clawhub',
|
||||
'delete',
|
||||
slug,
|
||||
'--yes',
|
||||
@@ -341,24 +436,24 @@ describe('clawdhub e2e', () => {
|
||||
],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
env: { ...process.env, CLAWDHUB_CONFIG_PATH: cfg.path, CLAWDHUB_DISABLE_TELEMETRY: '1' },
|
||||
env: { ...process.env, CLAWHUB_CONFIG_PATH: cfg.path, CLAWHUB_DISABLE_TELEMETRY: '1' },
|
||||
encoding: 'utf8',
|
||||
},
|
||||
)
|
||||
expect(del.status).toBe(0)
|
||||
|
||||
const metaAfterDelete = await fetch(metaUrl.toString(), {
|
||||
const metaAfterDelete = await fetchWithTimeout(metaUrl.toString(), {
|
||||
headers: { Accept: 'application/json' },
|
||||
})
|
||||
expect(metaAfterDelete.status).toBe(404)
|
||||
|
||||
const downloadAfterDelete = await fetch(downloadUrl.toString())
|
||||
const downloadAfterDelete = await fetchWithTimeout(downloadUrl.toString())
|
||||
expect(downloadAfterDelete.status).toBe(404)
|
||||
|
||||
const undelete = spawnSync(
|
||||
'bun',
|
||||
[
|
||||
'clawdhub',
|
||||
'clawhub',
|
||||
'undelete',
|
||||
slug,
|
||||
'--yes',
|
||||
@@ -371,13 +466,13 @@ describe('clawdhub e2e', () => {
|
||||
],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
env: { ...process.env, CLAWDHUB_CONFIG_PATH: cfg.path, CLAWDHUB_DISABLE_TELEMETRY: '1' },
|
||||
env: { ...process.env, CLAWHUB_CONFIG_PATH: cfg.path, CLAWHUB_DISABLE_TELEMETRY: '1' },
|
||||
encoding: 'utf8',
|
||||
},
|
||||
)
|
||||
expect(undelete.status).toBe(0)
|
||||
|
||||
const metaAfterUndelete = await fetch(metaUrl.toString(), {
|
||||
const metaAfterUndelete = await fetchWithTimeout(metaUrl.toString(), {
|
||||
headers: { Accept: 'application/json' },
|
||||
})
|
||||
expect(metaAfterUndelete.status).toBe(200)
|
||||
@@ -385,7 +480,7 @@ describe('clawdhub e2e', () => {
|
||||
const cleanup = spawnSync(
|
||||
'bun',
|
||||
[
|
||||
'clawdhub',
|
||||
'clawhub',
|
||||
'delete',
|
||||
slug,
|
||||
'--yes',
|
||||
@@ -398,7 +493,7 @@ describe('clawdhub e2e', () => {
|
||||
],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
env: { ...process.env, CLAWDHUB_CONFIG_PATH: cfg.path, CLAWDHUB_DISABLE_TELEMETRY: '1' },
|
||||
env: { ...process.env, CLAWHUB_CONFIG_PATH: cfg.path, CLAWHUB_DISABLE_TELEMETRY: '1' },
|
||||
encoding: 'utf8',
|
||||
},
|
||||
)
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
import { expect, test } from '@playwright/test'
|
||||
|
||||
const navLabels = ['Skills', 'Upload', 'Import', 'Search']
|
||||
|
||||
test('skills loads without error', async ({ page }) => {
|
||||
await page.goto('/skills', { waitUntil: 'domcontentloaded' })
|
||||
await expect(page.locator('text=Something went wrong!')).toHaveCount(0)
|
||||
await expect(page.locator('h1', { hasText: 'Skills' })).toBeVisible()
|
||||
})
|
||||
|
||||
test('souls loads without error', async ({ page }) => {
|
||||
await page.goto('/souls', { waitUntil: 'domcontentloaded' })
|
||||
await expect(page.locator('text=Something went wrong!')).toHaveCount(0)
|
||||
await expect(page.locator('h1', { hasText: 'Souls' })).toBeVisible()
|
||||
})
|
||||
|
||||
test('header menu routes render', async ({ page }) => {
|
||||
await page.goto('/', { waitUntil: 'domcontentloaded' })
|
||||
|
||||
for (const label of navLabels) {
|
||||
const link = page.getByRole('link', { name: label }).first()
|
||||
await expect(link).toBeVisible()
|
||||
await link.click()
|
||||
|
||||
if (label === 'Skills') {
|
||||
await expect(page).toHaveURL(/\/skills/)
|
||||
await expect(page.locator('h1', { hasText: 'Skills' })).toBeVisible()
|
||||
}
|
||||
|
||||
if (label === 'Upload') {
|
||||
await expect(page).toHaveURL(/\/upload/)
|
||||
const heading = page.locator('h1.section-title', { hasText: /^Publish a /i })
|
||||
const signInCard = page.locator('text=Sign in to upload')
|
||||
await expect(heading.or(signInCard)).toBeVisible()
|
||||
}
|
||||
|
||||
if (label === 'Import') {
|
||||
await expect(page).toHaveURL(/\/import/)
|
||||
const heading = page.getByRole('heading', { name: 'Import from GitHub' })
|
||||
const signInCard = page.locator('text=Sign in to import and publish skills.')
|
||||
await expect(heading.or(signInCard)).toBeVisible()
|
||||
}
|
||||
|
||||
if (label === 'Search') {
|
||||
await expect(page).toHaveURL(/\/?(\?|$)/)
|
||||
await expect(page.locator('h1', { hasText: 'OpenClaw' })).toBeVisible()
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,97 @@
|
||||
import { expect, test } from '@playwright/test'
|
||||
|
||||
test('skills search paginates exact results', async ({ page }) => {
|
||||
await page.addInitScript(() => {
|
||||
const makeSearchResults = (count: number) =>
|
||||
Array.from({ length: count }, (_, index) => ({
|
||||
score: 0.9,
|
||||
skill: {
|
||||
_id: `skill_${index}`,
|
||||
slug: `skill-${index}`,
|
||||
displayName: `Skill ${index}`,
|
||||
summary: `Summary ${index}`,
|
||||
tags: {},
|
||||
stats: {
|
||||
downloads: 0,
|
||||
installsCurrent: 0,
|
||||
installsAllTime: 0,
|
||||
stars: 0,
|
||||
versions: 1,
|
||||
comments: 0,
|
||||
},
|
||||
createdAt: 0,
|
||||
updatedAt: 0,
|
||||
},
|
||||
version: null,
|
||||
}))
|
||||
|
||||
class MockWebSocket {
|
||||
url: string
|
||||
readyState = 0
|
||||
onopen?: () => void
|
||||
onmessage?: (event: { data: string }) => void
|
||||
onclose?: (event: { code: number; reason: string }) => void
|
||||
onerror?: () => void
|
||||
|
||||
constructor(url: string) {
|
||||
this.url = url
|
||||
window.setTimeout(() => {
|
||||
this.readyState = 1
|
||||
this.onopen?.()
|
||||
}, 0)
|
||||
}
|
||||
|
||||
send(data: string) {
|
||||
try {
|
||||
const message = JSON.parse(data) as {
|
||||
type?: string
|
||||
requestId?: number
|
||||
udfPath?: string
|
||||
args?: Array<Record<string, unknown>>
|
||||
}
|
||||
if (message.type === 'Action' && message.udfPath?.includes('searchSkills')) {
|
||||
const [args] = message.args ?? []
|
||||
const limit = typeof args?.limit === 'number' ? args.limit : 10
|
||||
const limits = (window as typeof window & { __searchLimits: number[] }).__searchLimits
|
||||
limits.push(limit)
|
||||
const response = {
|
||||
type: 'ActionResponse',
|
||||
requestId: message.requestId,
|
||||
success: true,
|
||||
result: makeSearchResults(limit),
|
||||
logLines: [],
|
||||
}
|
||||
window.setTimeout(() => {
|
||||
this.onmessage?.({ data: JSON.stringify(response) })
|
||||
}, 0)
|
||||
}
|
||||
} catch {
|
||||
this.onerror?.()
|
||||
}
|
||||
}
|
||||
|
||||
close(code = 1000, reason = 'closed') {
|
||||
this.readyState = 3
|
||||
this.onclose?.({ code, reason })
|
||||
}
|
||||
}
|
||||
|
||||
;(window as typeof window & { __searchLimits: number[] }).__searchLimits = []
|
||||
window.WebSocket = MockWebSocket as unknown as typeof WebSocket
|
||||
})
|
||||
|
||||
await page.goto('/skills', { waitUntil: 'domcontentloaded' })
|
||||
await expect(page.getByRole('heading', { name: 'Skills' })).toBeVisible()
|
||||
|
||||
const input = page.getByPlaceholder('Filter by name, slug, or summary…')
|
||||
await input.fill('remind')
|
||||
await expect(page.getByText('Skill 0')).toBeVisible()
|
||||
await expect(page.getByText('Scroll to load more')).toBeVisible()
|
||||
|
||||
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight))
|
||||
await expect(page.getByText('Skill 75')).toBeVisible()
|
||||
const limits = await page.evaluate(
|
||||
() => (window as typeof window & { __searchLimits: number[] }).__searchLimits,
|
||||
)
|
||||
expect(limits).toEqual([50, 100])
|
||||
})
|
||||
+27
-18
@@ -1,18 +1,22 @@
|
||||
{
|
||||
"name": "clawdhub",
|
||||
"name": "clawhub",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"workspaces": [
|
||||
"packages/*"
|
||||
],
|
||||
"scripts": {
|
||||
"preinstall": "bunx only-allow bun",
|
||||
"dev": "bun --bun vite dev --port 3000",
|
||||
"build": "bun --bun vite build",
|
||||
"preview": "bun --bun vite preview",
|
||||
"docs:list": "bun scripts/docs-list.ts",
|
||||
"check:peers": "bun scripts/check-peer-deps.ts",
|
||||
"test": "vitest run",
|
||||
"test:watch": "vitest",
|
||||
"test:e2e": "vitest run -c vitest.e2e.config.ts",
|
||||
"test:e2e:local": "bash scripts/run-playwright-local.sh",
|
||||
"test:pw": "playwright test",
|
||||
"coverage": "vitest run --coverage",
|
||||
"convex:deploy": "bunx convex deploy --typecheck=disable --yes",
|
||||
"lint": "bun run lint:biome && bun run lint:oxlint",
|
||||
@@ -21,7 +25,7 @@
|
||||
"format": "biome format --write ."
|
||||
},
|
||||
"dependencies": {
|
||||
"@auth/core": "^0.41.1",
|
||||
"@auth/core": "^0.37.4",
|
||||
"@convex-dev/auth": "^0.0.90",
|
||||
"@fontsource/bricolage-grotesque": "^5.2.10",
|
||||
"@fontsource/ibm-plex-mono": "^5.2.7",
|
||||
@@ -29,17 +33,20 @@
|
||||
"@monaco-editor/react": "^4.7.0",
|
||||
"@radix-ui/react-dropdown-menu": "^2.1.16",
|
||||
"@radix-ui/react-toggle-group": "^1.1.11",
|
||||
"@resvg/resvg-wasm": "^2.6.2",
|
||||
"@tailwindcss/vite": "^4.1.18",
|
||||
"@tanstack/react-devtools": "^0.9.0",
|
||||
"@tanstack/react-router": "^1.144.0",
|
||||
"@tanstack/react-router-devtools": "^1.144.0",
|
||||
"@tanstack/react-start": "^1.145.3",
|
||||
"@tanstack/router-plugin": "^1.145.2",
|
||||
"@tanstack/react-devtools": "^0.9.2",
|
||||
"@tanstack/react-router": "^1.151.6",
|
||||
"@tanstack/react-router-devtools": "^1.151.6",
|
||||
"@tanstack/react-start": "^1.152.0",
|
||||
"@tanstack/router-plugin": "^1.151.6",
|
||||
"@vercel/analytics": "^1.6.1",
|
||||
"clawdhub-schema": "^0.0.2",
|
||||
"clawhub-schema": "workspace:*",
|
||||
"clsx": "^2.1.1",
|
||||
"convex": "^1.31.2",
|
||||
"convex": "^1.31.6",
|
||||
"convex-helpers": "^0.1.111",
|
||||
"fflate": "^0.8.2",
|
||||
"h3": "2.0.1-rc.8",
|
||||
"lucide-react": "^0.562.0",
|
||||
"monaco-editor": "^0.55.1",
|
||||
"nitro": "^3.0.1-alpha.1",
|
||||
@@ -50,25 +57,27 @@
|
||||
"semver": "^7.7.3",
|
||||
"tailwind-merge": "^3.4.0",
|
||||
"tailwindcss": "^4.1.18",
|
||||
"vite-tsconfig-paths": "^6.0.3",
|
||||
"vite-tsconfig-paths": "^6.0.4",
|
||||
"yaml": "^2.8.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "^2.3.11",
|
||||
"@tanstack/devtools-vite": "^0.4.0",
|
||||
"@playwright/test": "^1.57.0",
|
||||
"@tanstack/devtools-vite": "^0.4.1",
|
||||
"@testing-library/dom": "^10.4.1",
|
||||
"@testing-library/react": "^16.3.1",
|
||||
"@types/node": "^25.0.3",
|
||||
"@types/react": "^19.2.7",
|
||||
"@types/node": "^25.0.9",
|
||||
"@types/react": "^19.2.8",
|
||||
"@types/react-dom": "^19.2.3",
|
||||
"@types/semver": "^7.7.1",
|
||||
"@vitejs/plugin-react": "^5.1.2",
|
||||
"@vitest/coverage-v8": "^4.0.16",
|
||||
"@vitest/coverage-v8": "^4.0.17",
|
||||
"jsdom": "^27.4.0",
|
||||
"oxlint": "^1.36.0",
|
||||
"oxlint-tsgolint": "^0.10.1",
|
||||
"only-allow": "^1.2.1",
|
||||
"oxlint": "^1.39.0",
|
||||
"oxlint-tsgolint": "^0.11.1",
|
||||
"typescript": "^5.9.3",
|
||||
"vite": "^7.3.0",
|
||||
"vitest": "^4.0.16"
|
||||
"vite": "^7.3.1",
|
||||
"vitest": "^4.0.17"
|
||||
}
|
||||
}
|
||||
|
||||
+20
-20
@@ -1,57 +1,57 @@
|
||||
# `clawdhub`
|
||||
# `clawhub`
|
||||
|
||||
ClawdHub CLI — install, update, search, and publish agent skills as folders.
|
||||
OpenClaw CLI — install, update, search, and publish agent skills as folders.
|
||||
|
||||
## Install
|
||||
|
||||
```bash
|
||||
# From this repo (shortcut script at repo root)
|
||||
bun clawdhub --help
|
||||
bun clawhub --help
|
||||
|
||||
# Once published to npm
|
||||
# npm i -g clawdhub
|
||||
# npm i -g clawhub
|
||||
```
|
||||
|
||||
## Auth (publish)
|
||||
|
||||
```bash
|
||||
clawdhub login
|
||||
clawhub login
|
||||
# or
|
||||
clawdhub auth login
|
||||
clawhub auth login
|
||||
|
||||
# Headless / token paste
|
||||
# or (token paste / headless)
|
||||
clawdhub login --token clh_...
|
||||
clawhub login --token clh_...
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- Browser login opens `https://clawdhub.com/cli/auth` and completes via a loopback callback.
|
||||
- Token stored in `~/Library/Application Support/clawdhub/config.json` on macOS (override via `CLAWDHUB_CONFIG_PATH`).
|
||||
- Browser login opens `https://clawhub.ai/cli/auth` and completes via a loopback callback.
|
||||
- Token stored in `~/Library/Application Support/clawhub/config.json` on macOS (override via `CLAWHUB_CONFIG_PATH`, legacy `CLAWDHUB_CONFIG_PATH`).
|
||||
|
||||
## Examples
|
||||
|
||||
```bash
|
||||
clawdhub search "postgres backups"
|
||||
clawdhub install my-skill-pack
|
||||
clawdhub update --all
|
||||
clawdhub update --all --no-input --force
|
||||
clawdhub publish ./my-skill-pack --slug my-skill-pack --name "My Skill Pack" --version 1.2.0 --changelog "Fixes + docs"
|
||||
clawhub search "postgres backups"
|
||||
clawhub install my-skill-pack
|
||||
clawhub update --all
|
||||
clawhub update --all --no-input --force
|
||||
clawhub publish ./my-skill-pack --slug my-skill-pack --name "My Skill Pack" --version 1.2.0 --changelog "Fixes + docs"
|
||||
```
|
||||
|
||||
## Sync (upload local skills)
|
||||
|
||||
```bash
|
||||
# Start anywhere; scans workdir first, then legacy Clawdis/Clawd locations.
|
||||
clawdhub sync
|
||||
# Start anywhere; scans workdir first, then legacy Clawdis/Clawd/OpenClaw/Moltbot locations.
|
||||
clawhub sync
|
||||
|
||||
# Explicit roots + non-interactive dry-run
|
||||
clawdhub sync --root ../clawdis/skills --all --dry-run
|
||||
clawhub sync --root ../clawdis/skills --all --dry-run
|
||||
```
|
||||
|
||||
## Defaults
|
||||
|
||||
- Site: `https://clawdhub.com` (override via `--site` or `CLAWDHUB_SITE`)
|
||||
- Registry: discovered from `/.well-known/clawdhub.json` on the site (override via `--registry` or `CLAWDHUB_REGISTRY`)
|
||||
- Workdir: current directory (override via `--workdir`)
|
||||
- Site: `https://clawhub.ai` (override via `--site` or `CLAWHUB_SITE`, legacy `CLAWDHUB_SITE`)
|
||||
- Registry: discovered from `/.well-known/clawhub.json` on the site (legacy `/.well-known/clawdhub.json`; override via `--registry` or `CLAWHUB_REGISTRY`)
|
||||
- Workdir: current directory (falls back to Clawdbot workspace if configured; override via `--workdir` or `CLAWHUB_WORKDIR`)
|
||||
- Install dir: `./skills` under workdir (override via `--dir`)
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
{
|
||||
"name": "clawdhub",
|
||||
"version": "0.1.0",
|
||||
"description": "ClawdHub CLI \u2014 install, update, search, and publish agent skills.",
|
||||
"name": "clawhub",
|
||||
"version": "0.4.0",
|
||||
"description": "OpenClaw CLI \\u2014 install, update, search, and publish agent skills.",
|
||||
"license": "MIT",
|
||||
"type": "module",
|
||||
"bin": {
|
||||
"clawhub": "bin/clawdhub.js",
|
||||
"clawdhub": "bin/clawdhub.js"
|
||||
},
|
||||
"files": [
|
||||
@@ -24,13 +25,15 @@
|
||||
"commander": "^14.0.2",
|
||||
"fflate": "^0.8.2",
|
||||
"ignore": "^7.0.5",
|
||||
"json5": "^2.2.3",
|
||||
"mime": "^4.1.0",
|
||||
"ora": "^9.0.0",
|
||||
"p-retry": "^7.1.1",
|
||||
"semver": "^7.7.3"
|
||||
"semver": "^7.7.3",
|
||||
"undici": "^7.16.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^25.0.3",
|
||||
"@types/node": "^25.0.9",
|
||||
"typescript": "^5.9.3"
|
||||
},
|
||||
"engines": {
|
||||
|
||||
@@ -21,12 +21,23 @@ describe('browserAuth', () => {
|
||||
expect(url).toContain('state=')
|
||||
})
|
||||
|
||||
it('builds auth url without label', () => {
|
||||
const url = buildCliAuthUrl({
|
||||
siteUrl: 'https://example.com',
|
||||
redirectUri: 'http://127.0.0.1:1234/callback',
|
||||
state: 'state123',
|
||||
})
|
||||
expect(url).toContain('https://example.com/cli/auth?')
|
||||
expect(url).not.toContain('label_b64=')
|
||||
})
|
||||
|
||||
it('accepts only loopback http redirect uris', () => {
|
||||
expect(isAllowedLoopbackRedirectUri('http://127.0.0.1:1234/callback')).toBe(true)
|
||||
expect(isAllowedLoopbackRedirectUri('http://localhost:1234/callback')).toBe(true)
|
||||
expect(isAllowedLoopbackRedirectUri('http://[::1]:1234/callback')).toBe(true)
|
||||
expect(isAllowedLoopbackRedirectUri('https://127.0.0.1:1234/callback')).toBe(false)
|
||||
expect(isAllowedLoopbackRedirectUri('http://evil.com/callback')).toBe(false)
|
||||
expect(isAllowedLoopbackRedirectUri('not a url')).toBe(false)
|
||||
})
|
||||
|
||||
it('receives token via loopback server', async () => {
|
||||
@@ -43,4 +54,43 @@ describe('browserAuth', () => {
|
||||
})
|
||||
await expect(server.waitForResult()).resolves.toEqual(payload)
|
||||
})
|
||||
|
||||
it('serves callback html', async () => {
|
||||
const server = await startLoopbackAuthServer({ timeoutMs: 2000 })
|
||||
const response = await fetch(server.redirectUri)
|
||||
expect(response.status).toBe(200)
|
||||
const text = await response.text()
|
||||
expect(text).toContain('OpenClaw CLI Login')
|
||||
server.close()
|
||||
})
|
||||
|
||||
it('returns 404 for unknown routes', async () => {
|
||||
const server = await startLoopbackAuthServer({ timeoutMs: 2000 })
|
||||
const response = await fetch(server.redirectUri.replace('/callback', '/nope'))
|
||||
expect(response.status).toBe(404)
|
||||
server.close()
|
||||
})
|
||||
|
||||
it('rejects invalid json payloads', async () => {
|
||||
const server = await startLoopbackAuthServer({ timeoutMs: 2000 })
|
||||
const tokenUrl = server.redirectUri.replace('/callback', '/token')
|
||||
const response = await fetch(tokenUrl, { method: 'POST', body: '{' })
|
||||
expect(response.status).toBe(400)
|
||||
await expect(server.waitForResult()).rejects.toThrow()
|
||||
})
|
||||
|
||||
it('rejects state mismatches', async () => {
|
||||
const server = await startLoopbackAuthServer({ timeoutMs: 2000 })
|
||||
await fetch(server.redirectUri.replace('/callback', '/token'), {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ token: 'clh_test', registry: 'https://example.com', state: 'nope' }),
|
||||
})
|
||||
await expect(server.waitForResult()).rejects.toThrow(/state mismatch/i)
|
||||
})
|
||||
|
||||
it('times out waiting for login', async () => {
|
||||
const server = await startLoopbackAuthServer({ timeoutMs: 25 })
|
||||
await expect(server.waitForResult()).rejects.toThrow(/timed out waiting for browser login/i)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -127,7 +127,7 @@ const CALLBACK_HTML = `<!doctype html>
|
||||
<html lang="en">
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>ClawdHub CLI Login</title>
|
||||
<title>OpenClaw CLI Login</title>
|
||||
<style>
|
||||
:root { color-scheme: light dark; }
|
||||
body { font-family: ui-sans-serif, system-ui, -apple-system, Segoe UI, sans-serif; padding: 24px; }
|
||||
|
||||
+117
-25
@@ -1,12 +1,16 @@
|
||||
#!/usr/bin/env node
|
||||
import { resolve } from 'node:path'
|
||||
import { stat } from 'node:fs/promises'
|
||||
import { join, resolve } from 'node:path'
|
||||
import { Command } from 'commander'
|
||||
import { getCliBuildLabel, getCliVersion } from './cli/buildInfo.js'
|
||||
import { resolveClawdbotDefaultWorkspace } from './cli/clawdbotConfig.js'
|
||||
import { cmdLoginFlow, cmdLogout, cmdWhoami } from './cli/commands/auth.js'
|
||||
import { cmdDeleteSkill, cmdUndeleteSkill } from './cli/commands/delete.js'
|
||||
import { cmdPublish } from './cli/commands/publish.js'
|
||||
import { cmdInstall, cmdList, cmdSearch, cmdUpdate } from './cli/commands/skills.js'
|
||||
import { cmdExplore, cmdInstall, cmdList, cmdSearch, cmdUpdate } from './cli/commands/skills.js'
|
||||
import { cmdStarSkill } from './cli/commands/star.js'
|
||||
import { cmdSync } from './cli/commands/sync.js'
|
||||
import { cmdUnstarSkill } from './cli/commands/unstar.js'
|
||||
import { configureCommanderHelp, styleEnvBlock, styleTitle } from './cli/helpStyle.js'
|
||||
import { DEFAULT_REGISTRY, DEFAULT_SITE } from './cli/registry.js'
|
||||
import type { GlobalOpts } from './cli/types.js'
|
||||
@@ -14,9 +18,9 @@ import { fail } from './cli/ui.js'
|
||||
import { readGlobalConfig } from './config.js'
|
||||
|
||||
const program = new Command()
|
||||
.name('clawdhub')
|
||||
.name('clawhub')
|
||||
.description(
|
||||
`${styleTitle(`ClawdHub CLI ${getCliBuildLabel()}`)}\n${styleEnvBlock(
|
||||
`${styleTitle(`OpenClaw CLI ${getCliBuildLabel()}`)}\n${styleEnvBlock(
|
||||
'install, update, search, and publish agent skills.',
|
||||
)}`,
|
||||
)
|
||||
@@ -28,17 +32,30 @@ const program = new Command()
|
||||
.option('--no-input', 'Disable prompts')
|
||||
.showHelpAfterError()
|
||||
.showSuggestionAfterError()
|
||||
.addHelpText('after', styleEnvBlock('\nEnv:\n CLAWDHUB_SITE\n CLAWDHUB_REGISTRY\n'))
|
||||
.addHelpText(
|
||||
'after',
|
||||
styleEnvBlock(
|
||||
'\nEnv:\n CLAWHUB_SITE\n CLAWHUB_REGISTRY\n CLAWHUB_WORKDIR\n (CLAWDHUB_* supported)\n',
|
||||
),
|
||||
)
|
||||
|
||||
configureCommanderHelp(program)
|
||||
|
||||
function resolveGlobalOpts(): GlobalOpts {
|
||||
async function resolveGlobalOpts(): Promise<GlobalOpts> {
|
||||
const raw = program.opts<{ workdir?: string; dir?: string; site?: string; registry?: string }>()
|
||||
const workdir = resolve(raw.workdir ?? process.cwd())
|
||||
const workdir = await resolveWorkdir(raw.workdir)
|
||||
const dir = resolve(workdir, raw.dir ?? 'skills')
|
||||
const site = raw.site ?? process.env.CLAWDHUB_SITE ?? DEFAULT_SITE
|
||||
const registrySource = raw.registry ? 'cli' : process.env.CLAWDHUB_REGISTRY ? 'env' : 'default'
|
||||
const registry = raw.registry ?? process.env.CLAWDHUB_REGISTRY ?? DEFAULT_REGISTRY
|
||||
const site = raw.site ?? process.env.CLAWHUB_SITE ?? process.env.CLAWDHUB_SITE ?? DEFAULT_SITE
|
||||
const registrySource = raw.registry
|
||||
? 'cli'
|
||||
: process.env.CLAWHUB_REGISTRY || process.env.CLAWDHUB_REGISTRY
|
||||
? 'env'
|
||||
: 'default'
|
||||
const registry =
|
||||
raw.registry ??
|
||||
process.env.CLAWHUB_REGISTRY ??
|
||||
process.env.CLAWDHUB_REGISTRY ??
|
||||
DEFAULT_REGISTRY
|
||||
return { workdir, dir, site, registry, registrySource }
|
||||
}
|
||||
|
||||
@@ -47,6 +64,39 @@ function isInputAllowed() {
|
||||
return globalFlags.input !== false
|
||||
}
|
||||
|
||||
async function resolveWorkdir(explicit?: string) {
|
||||
if (explicit?.trim()) return resolve(explicit.trim())
|
||||
const envWorkdir = process.env.CLAWHUB_WORKDIR?.trim() ?? process.env.CLAWDHUB_WORKDIR?.trim()
|
||||
if (envWorkdir) return resolve(envWorkdir)
|
||||
|
||||
const cwd = resolve(process.cwd())
|
||||
const hasMarker = await hasClawhubMarker(cwd)
|
||||
if (hasMarker) return cwd
|
||||
|
||||
const clawdbotWorkspace = await resolveClawdbotDefaultWorkspace()
|
||||
return clawdbotWorkspace ? resolve(clawdbotWorkspace) : cwd
|
||||
}
|
||||
|
||||
async function hasClawhubMarker(workdir: string) {
|
||||
const lockfile = join(workdir, '.clawhub', 'lock.json')
|
||||
if (await pathExists(lockfile)) return true
|
||||
const markerDir = join(workdir, '.clawhub')
|
||||
if (await pathExists(markerDir)) return true
|
||||
const legacyLockfile = join(workdir, '.clawdhub', 'lock.json')
|
||||
if (await pathExists(legacyLockfile)) return true
|
||||
const legacyMarkerDir = join(workdir, '.clawdhub')
|
||||
return pathExists(legacyMarkerDir)
|
||||
}
|
||||
|
||||
async function pathExists(path: string) {
|
||||
try {
|
||||
await stat(path)
|
||||
return true
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
program
|
||||
.command('login')
|
||||
.description('Log in (opens browser or stores token)')
|
||||
@@ -54,7 +104,7 @@ program
|
||||
.option('--label <label>', 'Token label (browser flow only)', 'CLI token')
|
||||
.option('--no-browser', 'Do not open browser (requires --token)')
|
||||
.action(async (options) => {
|
||||
const opts = resolveGlobalOpts()
|
||||
const opts = await resolveGlobalOpts()
|
||||
await cmdLoginFlow(opts, options, isInputAllowed())
|
||||
})
|
||||
|
||||
@@ -62,7 +112,7 @@ program
|
||||
.command('logout')
|
||||
.description('Remove stored token')
|
||||
.action(async () => {
|
||||
const opts = resolveGlobalOpts()
|
||||
const opts = await resolveGlobalOpts()
|
||||
await cmdLogout(opts)
|
||||
})
|
||||
|
||||
@@ -70,7 +120,7 @@ program
|
||||
.command('whoami')
|
||||
.description('Validate token')
|
||||
.action(async () => {
|
||||
const opts = resolveGlobalOpts()
|
||||
const opts = await resolveGlobalOpts()
|
||||
await cmdWhoami(opts)
|
||||
})
|
||||
|
||||
@@ -87,7 +137,7 @@ auth
|
||||
.option('--label <label>', 'Token label (browser flow only)', 'CLI token')
|
||||
.option('--no-browser', 'Do not open browser (requires --token)')
|
||||
.action(async (options) => {
|
||||
const opts = resolveGlobalOpts()
|
||||
const opts = await resolveGlobalOpts()
|
||||
await cmdLoginFlow(opts, options, isInputAllowed())
|
||||
})
|
||||
|
||||
@@ -95,7 +145,7 @@ auth
|
||||
.command('logout')
|
||||
.description('Remove stored token')
|
||||
.action(async () => {
|
||||
const opts = resolveGlobalOpts()
|
||||
const opts = await resolveGlobalOpts()
|
||||
await cmdLogout(opts)
|
||||
})
|
||||
|
||||
@@ -103,7 +153,7 @@ auth
|
||||
.command('whoami')
|
||||
.description('Validate token')
|
||||
.action(async () => {
|
||||
const opts = resolveGlobalOpts()
|
||||
const opts = await resolveGlobalOpts()
|
||||
await cmdWhoami(opts)
|
||||
})
|
||||
|
||||
@@ -113,7 +163,7 @@ program
|
||||
.argument('<query...>', 'Query string')
|
||||
.option('--limit <n>', 'Max results', (value) => Number.parseInt(value, 10))
|
||||
.action(async (queryParts, options) => {
|
||||
const opts = resolveGlobalOpts()
|
||||
const opts = await resolveGlobalOpts()
|
||||
const query = queryParts.join(' ').trim()
|
||||
await cmdSearch(opts, query, options.limit)
|
||||
})
|
||||
@@ -125,7 +175,7 @@ program
|
||||
.option('--version <version>', 'Version to install')
|
||||
.option('--force', 'Overwrite existing folder')
|
||||
.action(async (slug, options) => {
|
||||
const opts = resolveGlobalOpts()
|
||||
const opts = await resolveGlobalOpts()
|
||||
await cmdInstall(opts, slug, options.version, options.force)
|
||||
})
|
||||
|
||||
@@ -137,7 +187,7 @@ program
|
||||
.option('--version <version>', 'Update to specific version (single slug only)')
|
||||
.option('--force', 'Overwrite when local files do not match any version')
|
||||
.action(async (slug, options) => {
|
||||
const opts = resolveGlobalOpts()
|
||||
const opts = await resolveGlobalOpts()
|
||||
await cmdUpdate(opts, slug, options, isInputAllowed())
|
||||
})
|
||||
|
||||
@@ -145,10 +195,32 @@ program
|
||||
.command('list')
|
||||
.description('List installed skills (from lockfile)')
|
||||
.action(async () => {
|
||||
const opts = resolveGlobalOpts()
|
||||
const opts = await resolveGlobalOpts()
|
||||
await cmdList(opts)
|
||||
})
|
||||
|
||||
program
|
||||
.command('explore')
|
||||
.description('Browse latest updated skills from the registry')
|
||||
.option(
|
||||
'--limit <n>',
|
||||
'Number of skills to show (max 200)',
|
||||
(value) => Number.parseInt(value, 10),
|
||||
25,
|
||||
)
|
||||
.option(
|
||||
'--sort <order>',
|
||||
'Sort by newest, downloads, rating, installs, installsAllTime, or trending',
|
||||
'newest',
|
||||
)
|
||||
.option('--json', 'Output JSON')
|
||||
.action(async (options) => {
|
||||
const opts = await resolveGlobalOpts()
|
||||
const limit =
|
||||
typeof options.limit === 'number' && Number.isFinite(options.limit) ? options.limit : 25
|
||||
await cmdExplore(opts, { limit, sort: options.sort, json: options.json })
|
||||
})
|
||||
|
||||
program
|
||||
.command('publish')
|
||||
.description('Publish skill from folder')
|
||||
@@ -160,7 +232,7 @@ program
|
||||
.option('--changelog <text>', 'Changelog text')
|
||||
.option('--tags <tags>', 'Comma-separated tags', 'latest')
|
||||
.action(async (folder, options) => {
|
||||
const opts = resolveGlobalOpts()
|
||||
const opts = await resolveGlobalOpts()
|
||||
await cmdPublish(opts, folder, options)
|
||||
})
|
||||
|
||||
@@ -170,7 +242,7 @@ program
|
||||
.argument('<slug>', 'Skill slug')
|
||||
.option('--yes', 'Skip confirmation')
|
||||
.action(async (slug, options) => {
|
||||
const opts = resolveGlobalOpts()
|
||||
const opts = await resolveGlobalOpts()
|
||||
await cmdDeleteSkill(opts, slug, options, isInputAllowed())
|
||||
})
|
||||
|
||||
@@ -180,10 +252,30 @@ program
|
||||
.argument('<slug>', 'Skill slug')
|
||||
.option('--yes', 'Skip confirmation')
|
||||
.action(async (slug, options) => {
|
||||
const opts = resolveGlobalOpts()
|
||||
const opts = await resolveGlobalOpts()
|
||||
await cmdUndeleteSkill(opts, slug, options, isInputAllowed())
|
||||
})
|
||||
|
||||
program
|
||||
.command('star')
|
||||
.description('Add a skill to your highlights')
|
||||
.argument('<slug>', 'Skill slug')
|
||||
.option('--yes', 'Skip confirmation')
|
||||
.action(async (slug, options) => {
|
||||
const opts = await resolveGlobalOpts()
|
||||
await cmdStarSkill(opts, slug, options, isInputAllowed())
|
||||
})
|
||||
|
||||
program
|
||||
.command('unstar')
|
||||
.description('Remove a skill from your highlights')
|
||||
.argument('<slug>', 'Skill slug')
|
||||
.option('--yes', 'Skip confirmation')
|
||||
.action(async (slug, options) => {
|
||||
const opts = await resolveGlobalOpts()
|
||||
await cmdUnstarSkill(opts, slug, options, isInputAllowed())
|
||||
})
|
||||
|
||||
program
|
||||
.command('sync')
|
||||
.description('Scan local skills and publish new/updated ones')
|
||||
@@ -195,7 +287,7 @@ program
|
||||
.option('--tags <tags>', 'Comma-separated tags', 'latest')
|
||||
.option('--concurrency <n>', 'Concurrent registry checks (default: 4)', '4')
|
||||
.action(async (options) => {
|
||||
const opts = resolveGlobalOpts()
|
||||
const opts = await resolveGlobalOpts()
|
||||
const bump = String(options.bump ?? 'patch') as 'patch' | 'minor' | 'major'
|
||||
if (!['patch', 'minor', 'major'].includes(bump)) fail('--bump must be patch|minor|major')
|
||||
const concurrencyRaw = Number(options.concurrency ?? 4)
|
||||
@@ -217,7 +309,7 @@ program
|
||||
})
|
||||
|
||||
program.action(async () => {
|
||||
const opts = resolveGlobalOpts()
|
||||
const opts = await resolveGlobalOpts()
|
||||
const cfg = await readGlobalConfig()
|
||||
if (cfg?.token) {
|
||||
await cmdSync(opts, {}, isInputAllowed())
|
||||
|
||||
@@ -24,6 +24,7 @@ function shortCommit(value: string) {
|
||||
|
||||
export function getCliCommit() {
|
||||
const candidates = [
|
||||
process.env.CLAWHUB_COMMIT,
|
||||
process.env.CLAWDHUB_COMMIT,
|
||||
process.env.VERCEL_GIT_COMMIT_SHA,
|
||||
process.env.GITHUB_SHA,
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
/* @vitest-environment node */
|
||||
import { mkdir, mkdtemp, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join, resolve } from 'node:path'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { resolveClawdbotDefaultWorkspace, resolveClawdbotSkillRoots } from './clawdbotConfig.js'
|
||||
|
||||
const originalEnv = { ...process.env }
|
||||
|
||||
afterEach(() => {
|
||||
process.env = { ...originalEnv }
|
||||
})
|
||||
|
||||
describe('resolveClawdbotSkillRoots', () => {
|
||||
it('reads JSON5 config and resolves per-agent + shared skill roots', async () => {
|
||||
const base = await mkdtemp(join(tmpdir(), 'clawhub-clawdbot-'))
|
||||
const home = join(base, 'home')
|
||||
const stateDir = join(base, 'state')
|
||||
const configPath = join(base, 'clawdbot.json')
|
||||
const openclawStateDir = join(base, 'openclaw-state')
|
||||
|
||||
process.env.HOME = home
|
||||
process.env.CLAWDBOT_STATE_DIR = stateDir
|
||||
process.env.CLAWDBOT_CONFIG_PATH = configPath
|
||||
process.env.OPENCLAW_STATE_DIR = openclawStateDir
|
||||
process.env.OPENCLAW_CONFIG_PATH = join(openclawStateDir, 'openclaw.json')
|
||||
|
||||
const config = `{
|
||||
// JSON5 comments + trailing commas supported
|
||||
agents: {
|
||||
defaults: { workspace: '~/clawd-main', },
|
||||
list: [
|
||||
{ id: 'work', name: 'Work Bot', workspace: '~/clawd-work', },
|
||||
{ id: 'family', workspace: '~/clawd-family', },
|
||||
],
|
||||
},
|
||||
// legacy entries still supported
|
||||
agent: { workspace: '~/clawd-legacy', },
|
||||
routing: {
|
||||
agents: {
|
||||
work: { name: 'Work Bot', workspace: '~/clawd-work', },
|
||||
family: { workspace: '~/clawd-family' },
|
||||
},
|
||||
},
|
||||
skills: {
|
||||
load: { extraDirs: ['~/shared/skills', '/opt/skills',], },
|
||||
},
|
||||
}`
|
||||
await writeFile(configPath, config, 'utf8')
|
||||
|
||||
const { roots, labels } = await resolveClawdbotSkillRoots()
|
||||
|
||||
const expectedRoots = [
|
||||
resolve(stateDir, 'skills'),
|
||||
resolve(openclawStateDir, 'skills'),
|
||||
resolve(home, 'clawd-main', 'skills'),
|
||||
resolve(home, 'clawd-work', 'skills'),
|
||||
resolve(home, 'clawd-family', 'skills'),
|
||||
resolve(home, 'shared', 'skills'),
|
||||
resolve('/opt/skills'),
|
||||
]
|
||||
|
||||
expect(roots).toEqual(expect.arrayContaining(expectedRoots))
|
||||
expect(labels[resolve(stateDir, 'skills')]).toBe('Shared skills')
|
||||
expect(labels[resolve(openclawStateDir, 'skills')]).toBe('OpenClaw: Shared skills')
|
||||
expect(labels[resolve(home, 'clawd-main', 'skills')]).toBe('Agent: main')
|
||||
expect(labels[resolve(home, 'clawd-work', 'skills')]).toBe('Agent: Work Bot')
|
||||
expect(labels[resolve(home, 'clawd-family', 'skills')]).toBe('Agent: family')
|
||||
expect(labels[resolve(home, 'shared', 'skills')]).toBe('Extra: skills')
|
||||
expect(labels[resolve('/opt/skills')]).toBe('Extra: skills')
|
||||
})
|
||||
|
||||
it('resolves default workspace from agents.defaults and agents.list', async () => {
|
||||
const base = await mkdtemp(join(tmpdir(), 'clawhub-clawdbot-default-'))
|
||||
const home = join(base, 'home')
|
||||
const stateDir = join(base, 'state')
|
||||
const configPath = join(base, 'clawdbot.json')
|
||||
const workspaceMain = join(base, 'workspace-main')
|
||||
const workspaceList = join(base, 'workspace-list')
|
||||
const openclawStateDir = join(base, 'openclaw-state')
|
||||
|
||||
process.env.HOME = home
|
||||
process.env.CLAWDBOT_STATE_DIR = stateDir
|
||||
process.env.CLAWDBOT_CONFIG_PATH = configPath
|
||||
process.env.OPENCLAW_STATE_DIR = openclawStateDir
|
||||
process.env.OPENCLAW_CONFIG_PATH = join(openclawStateDir, 'openclaw.json')
|
||||
|
||||
const config = `{
|
||||
agents: {
|
||||
defaults: { workspace: "${workspaceMain}", },
|
||||
list: [
|
||||
{ id: 'main', workspace: "${workspaceList}", default: true },
|
||||
],
|
||||
},
|
||||
}`
|
||||
await writeFile(configPath, config, 'utf8')
|
||||
|
||||
const workspace = await resolveClawdbotDefaultWorkspace()
|
||||
expect(workspace).toBe(resolve(workspaceMain))
|
||||
})
|
||||
|
||||
it('falls back to default agent in agents.list when defaults missing', async () => {
|
||||
const base = await mkdtemp(join(tmpdir(), 'clawhub-clawdbot-list-'))
|
||||
const home = join(base, 'home')
|
||||
const configPath = join(base, 'clawdbot.json')
|
||||
const workspaceMain = join(base, 'workspace-main')
|
||||
const workspaceWork = join(base, 'workspace-work')
|
||||
const openclawStateDir = join(base, 'openclaw-state')
|
||||
|
||||
process.env.HOME = home
|
||||
process.env.CLAWDBOT_CONFIG_PATH = configPath
|
||||
process.env.OPENCLAW_STATE_DIR = openclawStateDir
|
||||
process.env.OPENCLAW_CONFIG_PATH = join(openclawStateDir, 'openclaw.json')
|
||||
|
||||
const config = `{
|
||||
agents: {
|
||||
list: [
|
||||
{ id: 'main', workspace: "${workspaceMain}", default: true },
|
||||
{ id: 'work', workspace: "${workspaceWork}" },
|
||||
],
|
||||
},
|
||||
}`
|
||||
await writeFile(configPath, config, 'utf8')
|
||||
|
||||
const workspace = await resolveClawdbotDefaultWorkspace()
|
||||
expect(workspace).toBe(resolve(workspaceMain))
|
||||
})
|
||||
|
||||
it('respects CLAWDBOT_STATE_DIR and CLAWDBOT_CONFIG_PATH overrides', async () => {
|
||||
const base = await mkdtemp(join(tmpdir(), 'clawhub-clawdbot-override-'))
|
||||
const home = join(base, 'home')
|
||||
const stateDir = join(base, 'custom-state')
|
||||
const configPath = join(base, 'config', 'clawdbot.json')
|
||||
const openclawStateDir = join(base, 'openclaw-state')
|
||||
|
||||
process.env.HOME = home
|
||||
process.env.CLAWDBOT_STATE_DIR = stateDir
|
||||
process.env.CLAWDBOT_CONFIG_PATH = configPath
|
||||
process.env.OPENCLAW_STATE_DIR = openclawStateDir
|
||||
process.env.OPENCLAW_CONFIG_PATH = join(openclawStateDir, 'openclaw.json')
|
||||
|
||||
const config = `{
|
||||
agent: { workspace: "${join(base, 'workspace-main')}" },
|
||||
}`
|
||||
await mkdir(join(base, 'config'), { recursive: true })
|
||||
await writeFile(configPath, config, 'utf8')
|
||||
|
||||
const { roots, labels } = await resolveClawdbotSkillRoots()
|
||||
|
||||
expect(roots).toEqual(
|
||||
expect.arrayContaining([
|
||||
resolve(stateDir, 'skills'),
|
||||
resolve(openclawStateDir, 'skills'),
|
||||
resolve(join(base, 'workspace-main'), 'skills'),
|
||||
]),
|
||||
)
|
||||
expect(labels[resolve(stateDir, 'skills')]).toBe('Shared skills')
|
||||
expect(labels[resolve(openclawStateDir, 'skills')]).toBe('OpenClaw: Shared skills')
|
||||
expect(labels[resolve(join(base, 'workspace-main'), 'skills')]).toBe('Agent: main')
|
||||
})
|
||||
|
||||
it('returns shared skills root when config is missing', async () => {
|
||||
const base = await mkdtemp(join(tmpdir(), 'clawhub-clawdbot-missing-'))
|
||||
const stateDir = join(base, 'state')
|
||||
const configPath = join(base, 'missing', 'clawdbot.json')
|
||||
const openclawStateDir = join(base, 'openclaw-state')
|
||||
|
||||
process.env.CLAWDBOT_STATE_DIR = stateDir
|
||||
process.env.CLAWDBOT_CONFIG_PATH = configPath
|
||||
process.env.OPENCLAW_STATE_DIR = openclawStateDir
|
||||
process.env.OPENCLAW_CONFIG_PATH = join(openclawStateDir, 'openclaw.json')
|
||||
|
||||
const { roots, labels } = await resolveClawdbotSkillRoots()
|
||||
|
||||
expect(roots).toEqual([resolve(stateDir, 'skills'), resolve(openclawStateDir, 'skills')])
|
||||
expect(labels[resolve(stateDir, 'skills')]).toBe('Shared skills')
|
||||
expect(labels[resolve(openclawStateDir, 'skills')]).toBe('OpenClaw: Shared skills')
|
||||
})
|
||||
|
||||
it('supports OpenClaw configuration files', async () => {
|
||||
const base = await mkdtemp(join(tmpdir(), 'clawhub-openclaw-'))
|
||||
const stateDir = join(base, 'openclaw-state')
|
||||
const workspace = join(base, 'openclaw-main')
|
||||
const configPath = join(stateDir, 'openclaw.json')
|
||||
|
||||
process.env.OPENCLAW_STATE_DIR = stateDir
|
||||
|
||||
await mkdir(stateDir, { recursive: true })
|
||||
const config = `{
|
||||
agents: {
|
||||
defaults: { workspace: "${workspace}", },
|
||||
},
|
||||
}`
|
||||
await writeFile(configPath, config, 'utf8')
|
||||
|
||||
const { roots, labels } = await resolveClawdbotSkillRoots()
|
||||
expect(roots).toEqual(
|
||||
expect.arrayContaining([resolve(stateDir, 'skills'), resolve(workspace, 'skills')]),
|
||||
)
|
||||
expect(labels[resolve(stateDir, 'skills')]).toBe('OpenClaw: Shared skills')
|
||||
expect(labels[resolve(workspace, 'skills')]).toBe('OpenClaw: Agent: main')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,203 @@
|
||||
import { readFile } from 'node:fs/promises'
|
||||
import { homedir } from 'node:os'
|
||||
import { basename, join, resolve } from 'node:path'
|
||||
import JSON5 from 'json5'
|
||||
|
||||
type ClawdbotConfig = {
|
||||
agent?: { workspace?: string }
|
||||
agents?: {
|
||||
defaults?: { workspace?: string }
|
||||
list?: Array<{
|
||||
id?: string
|
||||
name?: string
|
||||
workspace?: string
|
||||
default?: boolean
|
||||
}>
|
||||
}
|
||||
routing?: {
|
||||
agents?: Record<
|
||||
string,
|
||||
{
|
||||
name?: string
|
||||
workspace?: string
|
||||
}
|
||||
>
|
||||
}
|
||||
skills?: {
|
||||
load?: {
|
||||
extraDirs?: string[]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export type ClawdbotSkillRoots = {
|
||||
roots: string[]
|
||||
labels: Record<string, string>
|
||||
}
|
||||
|
||||
export async function resolveClawdbotSkillRoots(): Promise<ClawdbotSkillRoots> {
|
||||
const roots: string[] = []
|
||||
const labels: Record<string, string> = {}
|
||||
|
||||
const clawdbotStateDir = resolveClawdbotStateDir()
|
||||
const sharedSkills = resolveUserPath(join(clawdbotStateDir, 'skills'))
|
||||
pushRoot(roots, labels, sharedSkills, 'Shared skills')
|
||||
|
||||
const openclawStateDir = resolveOpenclawStateDir()
|
||||
const openclawShared = resolveUserPath(join(openclawStateDir, 'skills'))
|
||||
pushRoot(roots, labels, openclawShared, 'OpenClaw: Shared skills')
|
||||
|
||||
const [clawdbotConfig, openclawConfig] = await Promise.all([
|
||||
readClawdbotConfig(),
|
||||
readOpenclawConfig(),
|
||||
])
|
||||
if (!clawdbotConfig && !openclawConfig) return { roots, labels }
|
||||
|
||||
if (clawdbotConfig) {
|
||||
addConfigRoots(clawdbotConfig, roots, labels)
|
||||
}
|
||||
if (openclawConfig) {
|
||||
addConfigRoots(openclawConfig, roots, labels, 'OpenClaw')
|
||||
}
|
||||
|
||||
return { roots, labels }
|
||||
}
|
||||
|
||||
export async function resolveClawdbotDefaultWorkspace(): Promise<string | null> {
|
||||
const config = await readClawdbotConfig()
|
||||
const openclawConfig = await readOpenclawConfig()
|
||||
if (!config && !openclawConfig) return null
|
||||
|
||||
const defaultsWorkspace = resolveUserPath(
|
||||
config?.agents?.defaults?.workspace ?? config?.agent?.workspace ?? '',
|
||||
)
|
||||
if (defaultsWorkspace) return defaultsWorkspace
|
||||
|
||||
const listedAgents = config?.agents?.list ?? []
|
||||
const defaultAgent =
|
||||
listedAgents.find((entry) => entry.default) ?? listedAgents.find((entry) => entry.id === 'main')
|
||||
const listWorkspace = resolveUserPath(defaultAgent?.workspace ?? '')
|
||||
if (listWorkspace) return listWorkspace
|
||||
|
||||
if (!openclawConfig) return null
|
||||
const openclawDefaults = resolveUserPath(
|
||||
openclawConfig.agents?.defaults?.workspace ?? openclawConfig.agent?.workspace ?? '',
|
||||
)
|
||||
if (openclawDefaults) return openclawDefaults
|
||||
const openclawAgents = openclawConfig.agents?.list ?? []
|
||||
const openclawDefaultAgent =
|
||||
openclawAgents.find((entry) => entry.default) ??
|
||||
openclawAgents.find((entry) => entry.id === 'main')
|
||||
const openclawWorkspace = resolveUserPath(openclawDefaultAgent?.workspace ?? '')
|
||||
return openclawWorkspace || null
|
||||
}
|
||||
|
||||
function resolveClawdbotStateDir() {
|
||||
const override = process.env.CLAWDBOT_STATE_DIR?.trim()
|
||||
if (override) return resolveUserPath(override)
|
||||
return join(homedir(), '.clawdbot')
|
||||
}
|
||||
|
||||
function resolveClawdbotConfigPath() {
|
||||
const override = process.env.CLAWDBOT_CONFIG_PATH?.trim()
|
||||
if (override) return resolveUserPath(override)
|
||||
return join(resolveClawdbotStateDir(), 'clawdbot.json')
|
||||
}
|
||||
|
||||
function resolveOpenclawStateDir() {
|
||||
const override = process.env.OPENCLAW_STATE_DIR?.trim()
|
||||
if (override) return resolveUserPath(override)
|
||||
return join(homedir(), '.openclaw')
|
||||
}
|
||||
|
||||
function resolveOpenclawConfigPath() {
|
||||
const override = process.env.OPENCLAW_CONFIG_PATH?.trim()
|
||||
if (override) return resolveUserPath(override)
|
||||
return join(resolveOpenclawStateDir(), 'openclaw.json')
|
||||
}
|
||||
|
||||
function resolveUserPath(input: string) {
|
||||
const trimmed = input.trim()
|
||||
if (!trimmed) return ''
|
||||
if (trimmed.startsWith('~')) {
|
||||
return resolve(trimmed.replace(/^~(?=$|[\\/])/, homedir()))
|
||||
}
|
||||
return resolve(trimmed)
|
||||
}
|
||||
|
||||
async function readClawdbotConfig(): Promise<ClawdbotConfig | null> {
|
||||
return readConfigFile(resolveClawdbotConfigPath())
|
||||
}
|
||||
|
||||
async function readOpenclawConfig(): Promise<ClawdbotConfig | null> {
|
||||
return readConfigFile(resolveOpenclawConfigPath())
|
||||
}
|
||||
|
||||
async function readConfigFile(path: string): Promise<ClawdbotConfig | null> {
|
||||
try {
|
||||
const raw = await readFile(path, 'utf8')
|
||||
const parsed = JSON5.parse(raw)
|
||||
if (!parsed || typeof parsed !== 'object') return null
|
||||
return parsed as ClawdbotConfig
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function addConfigRoots(
|
||||
config: ClawdbotConfig,
|
||||
roots: string[],
|
||||
labels: Record<string, string>,
|
||||
labelPrefix?: string,
|
||||
) {
|
||||
const prefix = labelPrefix ? `${labelPrefix}: ` : ''
|
||||
|
||||
const mainWorkspace = resolveUserPath(
|
||||
config.agents?.defaults?.workspace ?? config.agent?.workspace ?? '',
|
||||
)
|
||||
if (mainWorkspace) {
|
||||
pushRoot(roots, labels, join(mainWorkspace, 'skills'), `${prefix}Agent: main`)
|
||||
}
|
||||
|
||||
const listedAgents = config.agents?.list ?? []
|
||||
for (const entry of listedAgents) {
|
||||
const workspace = resolveUserPath(entry?.workspace ?? '')
|
||||
if (!workspace) continue
|
||||
const name = entry?.name?.trim() || entry?.id?.trim() || 'agent'
|
||||
pushRoot(roots, labels, join(workspace, 'skills'), `${prefix}Agent: ${name}`)
|
||||
}
|
||||
|
||||
const agents = config.routing?.agents ?? {}
|
||||
for (const [agentId, entry] of Object.entries(agents)) {
|
||||
const workspace = resolveUserPath(entry?.workspace ?? '')
|
||||
if (!workspace) continue
|
||||
const name = entry?.name?.trim() || agentId
|
||||
pushRoot(roots, labels, join(workspace, 'skills'), `${prefix}Agent: ${name}`)
|
||||
}
|
||||
|
||||
const extraDirs = config.skills?.load?.extraDirs ?? []
|
||||
for (const dir of extraDirs) {
|
||||
const resolved = resolveUserPath(String(dir))
|
||||
if (!resolved) continue
|
||||
const label = `${prefix}Extra: ${basename(resolved) || resolved}`
|
||||
pushRoot(roots, labels, resolved, label)
|
||||
}
|
||||
}
|
||||
|
||||
function pushRoot(roots: string[], labels: Record<string, string>, root: string, label?: string) {
|
||||
const resolved = resolveUserPath(root)
|
||||
if (!resolved) return
|
||||
if (!roots.includes(resolved)) roots.push(resolved)
|
||||
if (!label) return
|
||||
const existing = labels[resolved]
|
||||
if (!existing) {
|
||||
labels[resolved] = label
|
||||
return
|
||||
}
|
||||
const parts = existing
|
||||
.split(', ')
|
||||
.map((part) => part.trim())
|
||||
.filter(Boolean)
|
||||
if (parts.includes(label)) return
|
||||
labels[resolved] = `${existing}, ${label}`
|
||||
}
|
||||
@@ -47,7 +47,7 @@ export async function cmdLogin(
|
||||
) {
|
||||
if (!tokenFlag && !inputAllowed) fail('Token required (use --token or remove --no-input)')
|
||||
|
||||
const token = tokenFlag || (await promptHidden('ClawdHub token: '))
|
||||
const token = tokenFlag || (await promptHidden('OpenClaw token: '))
|
||||
if (!token) fail('Token required')
|
||||
|
||||
const registry = await getRegistry(opts, { cache: true })
|
||||
@@ -79,7 +79,7 @@ export async function cmdLogout(opts: GlobalOpts) {
|
||||
export async function cmdWhoami(opts: GlobalOpts) {
|
||||
const cfg = await readGlobalConfig()
|
||||
const token = cfg?.token
|
||||
if (!token) fail('Not logged in. Run: clawdhub login')
|
||||
if (!token) fail('Not logged in. Run: clawhub login')
|
||||
const registry = await getRegistry(opts, { cache: true })
|
||||
|
||||
const spinner = createSpinner('Checking token')
|
||||
|
||||
@@ -4,11 +4,11 @@ import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { GlobalOpts } from '../types'
|
||||
|
||||
vi.mock('../../config.js', () => ({
|
||||
readGlobalConfig: vi.fn(async () => ({ registry: 'https://clawdhub.com', token: 'tkn' })),
|
||||
readGlobalConfig: vi.fn(async () => ({ registry: 'https://clawhub.ai', token: 'tkn' })),
|
||||
}))
|
||||
|
||||
vi.mock('../registry.js', () => ({
|
||||
getRegistry: vi.fn(async () => 'https://clawdhub.com'),
|
||||
getRegistry: vi.fn(async () => 'https://clawhub.ai'),
|
||||
}))
|
||||
|
||||
const mockApiRequest = vi.fn()
|
||||
@@ -35,8 +35,8 @@ function makeOpts(): GlobalOpts {
|
||||
return {
|
||||
workdir: '/work',
|
||||
dir: '/work/skills',
|
||||
site: 'https://clawdhub.com',
|
||||
registry: 'https://clawdhub.com',
|
||||
site: 'https://clawhub.ai',
|
||||
registry: 'https://clawhub.ai',
|
||||
registrySource: 'default',
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@ import { createSpinner, fail, formatError, isInteractive, promptConfirm } from '
|
||||
async function requireToken() {
|
||||
const cfg = await readGlobalConfig()
|
||||
const token = cfg?.token
|
||||
if (!token) fail('Not logged in. Run: clawdhub login')
|
||||
if (!token) fail('Not logged in. Run: clawhub login')
|
||||
return token
|
||||
}
|
||||
|
||||
|
||||
@@ -7,10 +7,10 @@ import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { GlobalOpts } from '../types'
|
||||
|
||||
vi.mock('../../config.js', () => ({
|
||||
readGlobalConfig: vi.fn(async () => ({ registry: 'https://clawdhub.com', token: 'tkn' })),
|
||||
readGlobalConfig: vi.fn(async () => ({ registry: 'https://clawhub.ai', token: 'tkn' })),
|
||||
}))
|
||||
|
||||
const mockGetRegistry = vi.fn(async (_opts: unknown, _params?: unknown) => 'https://clawdhub.com')
|
||||
const mockGetRegistry = vi.fn(async (_opts: unknown, _params?: unknown) => 'https://clawhub.ai')
|
||||
vi.mock('../registry.js', () => ({
|
||||
getRegistry: (opts: unknown, params?: unknown) => mockGetRegistry(opts, params),
|
||||
}))
|
||||
@@ -34,7 +34,7 @@ vi.mock('../ui.js', () => ({
|
||||
const { cmdPublish } = await import('./publish')
|
||||
|
||||
async function makeTmpWorkdir() {
|
||||
const root = await mkdtemp(join(tmpdir(), 'clawdhub-publish-'))
|
||||
const root = await mkdtemp(join(tmpdir(), 'clawhub-publish-'))
|
||||
return root
|
||||
}
|
||||
|
||||
@@ -42,8 +42,8 @@ function makeOpts(workdir: string): GlobalOpts {
|
||||
return {
|
||||
workdir,
|
||||
dir: join(workdir, 'skills'),
|
||||
site: 'https://clawdhub.com',
|
||||
registry: 'https://clawdhub.com',
|
||||
site: 'https://clawhub.ai',
|
||||
registry: 'https://clawhub.ai',
|
||||
registrySource: 'default',
|
||||
}
|
||||
}
|
||||
@@ -80,9 +80,9 @@ describe('cmdPublish', () => {
|
||||
})
|
||||
if (!publishCall) throw new Error('Missing publish call')
|
||||
const publishForm = (publishCall[1] as { form?: FormData }).form as FormData
|
||||
const payloadRaw = publishForm.get('payload')
|
||||
expect(typeof payloadRaw).toBe('string')
|
||||
const payload = JSON.parse(payloadRaw as string)
|
||||
const payloadEntry = publishForm.get('payload')
|
||||
if (typeof payloadEntry !== 'string') throw new Error('Missing publish payload')
|
||||
const payload = JSON.parse(payloadEntry)
|
||||
expect(payload.slug).toBe('my-skill')
|
||||
expect(payload.displayName).toBe('My Skill')
|
||||
expect(payload.version).toBe('1.0.0')
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user