Compare commits

...
Author SHA1 Message Date
Patrick Erichsen a128c101ad chore: release clawhub cli 0.23.0 (#2812) 2026-06-23 13:45:20 -07:00
Patrick Erichsen dfde400fcf fix(seed): populate public corpus plugin metadata 2026-06-23 13:14:30 -07:00
Patrick Erichsen 071c43da43 fix: shrink security dataset export batches 2026-06-23 12:52:50 -07:00
Vyctor H. Brzezowski 7cc6b22176 feat(web): rename publishers browse to Creators (#2799)
* feat(web): rename publishers browse to Creators

Align /publishers page heading and filter tabs with clearer creator-focused copy, move Official after All, and show full org labels on desktop only.

* test(e2e): align local-auth flows with owner-qualified routes

Update playwright local-auth helpers and specs for canonical profile, skill, and plugin URLs after the main branch routing migration.

* fix(test): export plugin validation href helper for e2e typecheck

* chore: format local-auth helpers import

* chore: retrigger CI after flaky local-auth shards

* feat(web): move publishers browse to /creators route

Keep /publishers and /users as legacy redirects with search preserved, and align registry copy, tests, and reserved slugs with the new path.

* fix(web): drop unused OpenClaw slug re-export after schema move

* chore: format openClawExtensionSlugs re-export cleanup
2026-06-23 12:24:25 -07:00
Vyctor H. Brzezowski b927db6340 fix(web): show only downloads in plugin browse listings (#2797)
* fix(web): show only downloads in plugin browse listings

Plugin list rows and cards on /plugins and the home Plugins tab no longer surface star counts.

* docs: add UI proof screenshots for plugin listing change
2026-06-23 12:24:11 -07:00
Patrick Erichsen e018269af6 fix: make security dataset workflow dispatchable (#2802) 2026-06-23 11:58:22 -07:00
Vincent Koc 9b352d7f99 fix(routing): protect publisher aliases from plugin collisions 2026-06-23 17:56:48 +08:00
Vincent Koc ab862d49f1 fix(catalog): include verified personal skill publishers 2026-06-23 17:37:18 +08:00
Vincent Koc f4f6f34542 fix(catalog): paginate verified skill feed publication 2026-06-23 17:18:43 +08:00
Vincent Koc 6aab4f9437 feat(catalog): publish verified organization skills feed 2026-06-23 17:04:26 +08:00
Vincent Koc 94f2f532c3 fix(catalog): version and harden hosted plugin feed (#2806) 2026-06-23 16:11:40 +08:00
Jesse Merhi 2cf2636ab8 chore: remove local staged secret hook
Remove the repo-managed local staged-secret hook and scanner now that GitHub secret scanning push protection is the primary secret-blocking control.
2026-06-23 18:08:18 +10:00
Vincent Koc 72c5cdd864 feat(catalog): publish hosted OpenClaw plugin feed (#2805)
* feat(schema): define hosted catalog feed contract

* feat(catalog): publish cached hosted plugin feed

* feat(catalog): add feed publication and edge delivery

* fix(catalog): recheck live official publisher state

* fix(workflow): require main for catalog publication

* fix(catalog): harden publication inputs

* docs(catalog): document consumer rollout boundary
2026-06-23 15:41:24 +08:00
Patrick Erichsen 775146ff6f feat: publish security dataset from live export 2026-06-22 22:56:33 -07:00
Patrick Erichsen e82f0704c8 Publish security dataset from live Convex export (#2800)
* feat(cli): restore skill sync command

* feat: publish security dataset from live export
2026-06-22 22:40:09 -07:00
Patrick Erichsen 4a78ca3a06 fix(web): keep staff skill pages hydrated (#2798)
* fix(web): keep staff skill pages hydrated

* test(e2e): align local auth routes with owner paths
2026-06-22 22:35:57 -07:00
Patrick Erichsen 5c4b7d45df Revert "docs: document security dataset export workflow"
This reverts commit 8841ac7771.
2026-06-22 22:33:15 -07:00
Patrick Erichsen 8841ac7771 docs: document security dataset export workflow 2026-06-22 22:29:47 -07:00
Vyctor H. Brzezowski a5e41320c8 fix(web): recall publishers outside browse window in search (#2790)
* fix: recall publishers outside top install window in search

Publisher search only scanned the top 500 by installs and dropped empty
profiles, so handles like vincentkoc never appeared even when the user
profile was public.

* test: cover publisher search recall for low-install handles

Add regression coverage for publishers with published skills that fall
outside the top install browse window, matching the vyctorbrzezowski case.

* test: align publisher search mocks with downloads browse indexes

* chore: add production publisher search proof for PR 2790

* test: drop invalid publisher list stats assertion

Remove stats.skills expectation from listPublicPage search recall test;
public list items only expose downloads and installs counts.

* chore: retrigger CI after delete-account flake
2026-06-22 21:51:40 -07:00
Patrick Erichsen 917fb3fbe9 test: stabilize production menu smoke
Stabilizes production menu smoke by clicking exact header nav links and avoiding mobile drawer transition races between SPA navigations.\n\nVerification:\n- PLAYWRIGHT_BASE_URL=https://clawhub.ai bunx playwright test --workers=1 --project=mobile-chrome e2e/menu-smoke.pw.test.ts -g "header menu routes render"\n- PLAYWRIGHT_BASE_URL=https://clawhub.ai bunx playwright test --workers=1 e2e/menu-smoke.pw.test.ts e2e/publish-entry-workflows.pw.test.ts e2e/upload-auth-smoke.pw.test.ts\n- git diff --check
2026-06-22 21:45:42 -07:00
Patrick Erichsen b6acfb4fd7 test: update prod smoke skill canonical route
Updates production HTTP smoke to expect the owner-qualified skill canonical URL.\n\nVerification:\n- bun run test:e2e:prod-http\n- git diff --check
2026-06-22 21:39:10 -07:00
Patrick Erichsen 55fe4a8563 feat: add owner-qualified catalog routes
Adds canonical owner-qualified publisher, skill, and plugin routes while preserving legacy redirects.\n\nIncludes API, CLI, docs, and user-facing copy updates for /<owner>/skills/<slug> and /<owner>/plugins/<slug>.\n\nMerged by request before the local-auth matrix was green; static, unit, packages, types-build, e2e-http, and playwright-smoke were green on a1328b8.
2026-06-22 21:35:56 -07:00
Vincent Koc d4205c8a7e fix(ci): report testbox hydration failures (#2787) 2026-06-23 12:07:47 +08:00
Vyctor H. Brzezowski 8035a16024 fix(schema): expand reserved catalog topic slugs (#2791)
Block additional trust, curation, brand, and channel topic labels that
publishers could use to imply platform endorsement.
2026-06-22 21:01:51 -07:00
Patrick Erichsen fb7b73b86a feat: reclaim deleted org handles
Adds an admin-only deleted-org handle reclaim path and hard-delete cleanup for empty deleted org tombstones. Also makes local-auth publish flows resilient to cold local Convex startup timeouts.
2026-06-22 20:38:14 -07:00
Nancy 92e055f8f1 fix: align home app card icons (#2788) 2026-06-22 19:50:30 -07:00
Vyctor H. BrzezowskiandCursor 0dafef6c8b feat(web): polish skill and plugin detail pages (#2780)
* fix: polish skill detail hero metadata

* fix: improve skill readme presentation

* chore: outline skill detail structure

* fix: narrow detail page container

* fix: compact skill sidebar on detail pages

* fix: use body font for install switcher

* fix: align plugin detail sidebar

* fix: shorten skill readme preview

* fix: soften related skills heading

* fix: remove duplicate stars metadata

* fix: remove related skill hover underline

* chore: remove detail debug outlines

* fix: align hero with content column

* fix: rename summary disclosure action

* fix: wrap tab content in contrast panel

* fix: restore full width hero layout

* fix: refine skill detail surfaces

* fix: soften skill readme body copy

* fix: hide skill detail breadcrumbs

* fix: place skill taxonomy above title

* fix: reduce skill detail title size

* fix: add skill hero top spacing

* fix: standardize skill markdown formatting

* fix: refine related skills navigation

* fix: align plugin detail hero with skills

* fix: increase hero taxonomy spacing

* fix: mark official plugin owners

* fix: tune detail sidebar labels

* fix: restyle install tab switcher

* fix: add subtle skill detail wash

* fix: horizontalize skill versions panel

* fix: animate install tab switcher

* fix: align plugin versions layout

* fix: improve tab markdown surface contrast

* fix: separate detail categories with commas

* fix: remove detail tab underline bars

* fix: bleed skill wash behind header

* fix: polish detail versions changelog

* fix: add file tree to skill files view

* fix: anchor skill wash to page top

* fix: restore plugin version download actions

* fix: align detail sidebar top spacing

* fix: restore active detail tab bar

* fix: collapse detail version changelogs

* fix: clarify version changelog toggles

* fix: tune detail tab and install polish

* fix: polish markdown code blocks

* fix: refine markdown code wrap control

* docs: capture detail polish direction

* fix: refine release history layout

* feat: simplify detail file navigation

* fix: align detail hero sidebar patterns

* fix: unify plugin and skill detail polish

* fix: refine detail hero and release rows

* fix: move related skills below detail content

* fix: align detail hero title with main content column

Keep the hero wash full width while constraining taxonomy, title, and
summary to the same grid column as install and tab content on desktop.

* fix: increase star count badge font to 14px

Make the sidebar star action count easier to read on detail pages.

* fix: align shiki code block surfaces with detail markdown

Override Shiki's inline pre background so fenced blocks use the shared
markdown-code-block surface on skill and plugin README tabs.

* fix: remove code wrap toggle blur flicker

Drop the wrap-state blur reveal so toggling nowrap/wrap keeps the code
DOM stable without flashing highlighted tokens.

* fix: contain detail markdown overflow in tab bodies

Keep README and SKILL surfaces clipped to the tab column while preserving
horizontal scroll only inside code blocks and tables.

* fix: use neutral hover border on detail sidebar actions

Override the accent outline hover on Star, Share, and Download sidebar
buttons so detail pages keep a quieter action treatment.

* fix: tighten release row checks and download actions

Keep scan badges on one horizontal row, left-align package actions with
the column header, and show an icon-only download control.

* fix: collapse long plugin README previews like SKILL.md

Reuse the skill readme preview limiter with Read more/Show less on plugin
README.md tabs so long documentation stays scannable by default.

* fix: match activity metric info icon to security audit

Reuse the quiet sidebar info button styling so download labels no longer
show a circular hover treatment on the help icon.

* chore: remove unused activity metric info button styles

* fix: use neutral colors for download trend sparklines

Keep sidebar activity graphs muted with ink-soft tones instead of accent
red on skill and plugin detail pages.

* fix(ui): style related skills category link as outline button

Give the compact hero "More in …" footer full-width outline button affordance with neutral hover, matching sidebar secondary actions.

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix(ui): compact skill/plugin detail hero on mobile

Tighten vertical rhythm below 1100px: side-by-side Star/Share, smaller
action buttons, denser metadata rows, shorter download sparkline, and
reduced gaps between hero, sidebar, and install sections.

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix(ui): reduce sidebar action count font to 13px

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix(ui): reduce sidebar action count font to 13px

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix(ui): shrink sidebar star count badge height

Lower the action-count pill so Star and Share buttons align at the same height.

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix(ui): restore markdown ordered and bullet list markers

Tailwind preflight strips list-style from ol/ul; re-apply disc and decimal
markers inside .markdown so SKILL.md and plugin README numbering renders.

Co-authored-by: Cursor <cursoragent@cursor.com>

* feat: add download period tabs to detail sidebar

Replace the static 30-day downloads row with All time, 30d, and 7d tabs
that update the sparkline and total on skill and plugin detail pages.

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix(ui): polish audit sidebar, badges, and chart theme tokens

Make creator badges fully clickable, collapse Latest audit to one inline row,
and tune download sparkline colors per theme with softer blue tones.

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix: make creator user badges fully clickable

Use fallbackHandle for profile links, pass plugin ownerHandle when the
owner record omits it, and add sidebar hover affordance on the whole badge.

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix(ui): tighten audit sidebar and related skills footer

Center the category footer action, remove the secondary-actions negative
margin hack, and keep Latest audit on one compact inline row.

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix(ui): refine downloads tabs, report row, and related summaries

Move download period tabs inline with the Downloads label using listing-style
underline tabs, drop the Report block top border, and cap related skill
descriptions at 80 characters.

Co-authored-by: Cursor <cursoragent@cursor.com>

* feat: polish skill and plugin detail pages

* feat: add diff viewer skeleton

* fix: retain diff skeleton while versions load

* feat: restructure skill card review layout

* feat: add category icons to related skills

* fix: keep primary skill file first

* fix: keep skill card context expanded

* fix: refine skill card overview and risk contrast

* fix: show stars in related skill rows

* fix: refine install requirement tabs

* fix: join skill card risk rail

* fix: align plugin versions panel behavior

* fix: polish plugin repository and requirement panels

* fix: stabilize detail page skeleton layouts

* fix: resolve detail hydration gaps and finish polish pass

Keep mobile/desktop detail markup stable for SSR hydration, sync Shiki
theme selection with useSyncExternalStore, and complete tabpanel ARIA for
Files and Versions. Also lands remaining plugin categorize, install, and
metadata dialog polish from the detail-page iteration.

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix: stabilize detail page CI contracts

* feat(web): polish detail install shell and checkpoint branch WIP

Add subtle terminal $ prompt to skill/plugin install commands with
vertical alignment and descender-safe line height. Bundle remaining
detail-page polish, dialog tweaks, and local PR proof artifacts as a
restore point before further agent work.

* fix(web): polish plugin sidebar download and detail hero alignment

Move plugin download inline with the downloads count when no activity graph
is shown, and to the sidebar footer when a graph is present. Align skill/plugin
hero summary rows, compact management toolbar actions, and plugin mobile
About/Stats tabs. Remove accidental local proof artifacts from the branch.

* fix(web): stabilize detail page CI

---------

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-06-22 19:32:12 -07:00
Patrick Erichsen bbe887faac feat(cli): restore skill sync command
Restores the ClawHub CLI sync command as a one-way skill publish workflow without install/download telemetry updates.
2026-06-22 18:45:59 -07:00
Patrick Erichsen 51d42badcf fix: route banned users to account page (#2785) 2026-06-22 18:07:27 -07:00
Vincent Koc b7a7555b19 fix(test): model unavailable publish flash storage 2026-06-23 08:03:42 +08:00
37eac63dc8 Fix official-first category plugin pagination (#2775)
* fix: avoid double pagination in official category browse

* fix: remove stale prod smoke retry assignment

* fix: preserve community fill in official category browse

* test: extend malicious skill ban e2e timeout

* test: tolerate slug check teardown timeout

* test: cover plugin catalog journeys

* fix: preserve official-first plugin filters

---------

Co-authored-by: Patrick Erichsen <patrick.a.erichsen@gmail.com>
Co-authored-by: fuller-stack-dev <263060202+fuller-stack-dev@users.noreply.github.com>
2026-06-22 14:31:13 -06:00
Vyctor H. BrzezowskiandPatrick Erichsen 2993f6ccb7 polish: refine homepage hero (#2765)
* polish: refine homepage hero

* fix(web): preserve accessible hero trigger

* test: align header typeahead metadata assertion

---------

Co-authored-by: Patrick Erichsen <patrick.a.erichsen@gmail.com>
2026-06-22 13:28:55 -07:00
Vyctor H. Brzezowski 9ea97b3c58 fix(web): polish browse listing pages (#2767)
* feat: polish browse listing pages

* fix(web): refine mobile publisher rows

* test(web): preserve browse contracts after rebase
2026-06-22 13:11:51 -07:00
Vyctor H. Brzezowski 6dfe780a29 build: generate llms.txt from canonical metadata (#2483)
* docs: add llms.txt

* docs: avoid dead llms skill examples

* refactor: centralize public registry routes

* refactor: share canonical ClawHub links

* feat: generate llms.txt from canonical metadata

* build: generate llms.txt for dev and production

* test: follow canonical docs navigation link
2026-06-22 13:04:07 -07:00
Vyctor H. Brzezowski 519f56301d perf: cache homepage listing tabs (#2764) 2026-06-22 13:03:11 -07:00
Patrick Erichsen 1379d90ddf fix: hide plugin scopes in display labels
Hide scoped plugin package prefixes in visible search and breadcrumb labels while preserving canonical scoped routes.
2026-06-22 13:01:55 -07:00
vyctorbrzezowski 6bd73e33bb feat: unify header search results 2026-06-22 13:01:11 -07:00
Patrick Erichsen e03432d4d8 fix: drop stale prod smoke retry assignment 2026-06-22 12:42:50 -07:00
Patrick Erichsen c1bb8d94cd fix: remove activity metric tooltip 2026-06-22 12:42:50 -07:00
Momo 6046bf9eda fix: honor owner scope in skill install resolver
Fixes #2766
2026-06-22 20:45:19 +08:00
Vincent Koc bf727b6a32 fix(codeql): clean clawhub quality findings 2026-06-22 18:18:21 +08:00
Jesse Merhi 39d03a577f Fix filtered plugin catalog pagination (#2771)
* fix: resume filtered plugin package pagination

* test: wait for copied publish dialog state

* ci: retry local convex module prep

* test: stabilize banned account local auth flow
2026-06-22 17:07:30 +10:00
Jason (Json) c70755d25d Fix Official plugin download listing (#2772)
Use a selective family, Official status, and downloads index for Official plugin listings while preserving legacy pagination cursor compatibility.
2026-06-21 23:26:13 -06:00
Vincent Koc 503a2bf022 fix(ci): reduce ClawHub worker fanout (#2770) 2026-06-22 12:41:56 +08:00
Jesse Merhi 379c1871f4 Add skill and plugin download activity graphs
Restore downloads as the public activity metric and add deferred 30-day download graphs for skills and plugins.
2026-06-22 11:59:30 +10:00
Jesse Merhi a86c48ce3b Tune publisher abuse pressure labels
Retune publisher-abuse aggregate scoring to v4, keep this path flag-only for rollout, clear stale aggregate nominations, and remove the direct publisher-abuse ban UI.
2026-06-22 10:53:33 +10:00
Patrick Erichsen 6f28659e7b fix: backfill stored plugin manifests (#2763) 2026-06-19 19:56:48 -07:00
Patrick Erichsen 3836b7643f fix: harden plugin manifest backfill
Supports direct config-map plugin manifests and adds a cursor-based production backfill page action.
2026-06-19 19:34:50 -07:00
Patrick Erichsen a7b7df7129 feat: backfill plugin manifest summaries
Adds a production-safe backfill for latest active plugin package releases so existing plugin pages can render typed manifest capability tabs without republishing.
2026-06-19 18:55:37 -07:00
Patrick Erichsen 08a95f3c74 feat: render plugin manifest capabilities (#2759) 2026-06-19 18:30:06 -07:00
Vyctor H. Brzezowski 730063b575 fix(web): hide pending and suspicious items from New (#2760) 2026-06-19 18:15:40 -07:00
Vyctor H. BrzezowskiandCursor ea0125d87e Refresh homepage experience (#2756)
* style: add Mobbin-inspired home listing toolbar and grid

Introduce a catalog section below the hero with Skills/Plugins toggle,
sort tabs, filter affordance, and a card grid fed by public list APIs.

Co-authored-by: Cursor <cursoragent@cursor.com>

* style: switch home listing from card grid to horizontal rows

Use icon, single-line summary, stats, and install CTA in scan-friendly
rows inside one bordered list instead of a three-column card grid.

Co-authored-by: Cursor <cursoragent@cursor.com>

* style: refine home listing rows like reference catalogs

Drop install CTAs, use muted icon tiles with brighter glyphs, and
flatten the list into divider rows without index numbers.

Co-authored-by: Cursor <cursoragent@cursor.com>

* feat(home): rebuild listing with search and category select

Add inline catalog search, category dropdown with icons, grid/list views,
and empty states tuned for the home listing toolbar.

Co-authored-by: Cursor <cursoragent@cursor.com>

* feat: refresh homepage hero and listing

* feat: add homepage apps section

* feat: add homepage publish section

* feat: add homepage ecosystem footer

* fix: use favicon icons for homepage app plugins

* fix: standardize footer styling globally

* fix: add spacing above homepage apps link

* fix: smooth homepage publish section transition

* fix: neutralize homepage light mode background

* fix: expand homepage app shortcuts grid

* fix: neutralize light theme surfaces

* fix: remove light publish section transition overlay

* fix: square footer bottom edge

* fix: simplify category menu scrollbar

* fix: clarify homepage category filters

* fix: refine mobile listing controls

* fix: restore homepage listing fold fade

* fix: tighten homepage listing row labels

* fix: space homepage apps cta

* fix: align homepage icons and app grids

* fix: scroll homepage app tabs on mobile

* fix: balance homepage app copy on mobile

* fix: balance homepage cli copy on mobile

* fix: refine mobile homepage motion

* feat: add popular publishers to homepage

* fix: remove stars from homepage listings

* fix: rank homepage listings by installs

* feat: show publisher catalog counts

* fix: align publisher card stats

* refactor: simplify publisher catalog counts

* fix: tighten publisher section heading

* fix: show app tiles on mobile

* fix: center mobile apps banner

* fix: use install icon in homepage listings

* feat: expand popular publishers

* feat: add mouse drag to publisher carousel

* chore: update pinned publishers

* fix: allow dragging publisher cards

* fix: show app card backgrounds on hover

* feat: curate homepage app shortcuts

* chore: reorder featured publishers

* fix: diversify homepage app shortcuts

* fix: hide native listing search clear

* fix: refine homepage app shortcuts and search reset

* fix: keep app icon frames rounded

* fix: hide officials tab for skills listing

* fix: replace featured homepage tabs with new

* fix: call homepage publishers creators

* fix: simplify homepage hero subtitle

* fix: use singular official plugin tab

* fix: make official the first plugin tab

* feat: add homepage trending skills tab

* fix: use existing trending skills leaderboard

* fix: show trending leaderboard installs

* fix: tighten listing kind toggle padding

* fix: normalize listing kind control padding

* fix: restore equal listing kind padding

* fix: shorten skills top tab label

* fix: shorten plugin top tab label

* fix: align trending skills installs metric

* fix: normalize listing view toggle padding

* Revert "fix: align trending skills installs metric"

This reverts commit 5c421b5cd4326347b4b702949b035fdf954811d8.

* Revert "fix: show trending leaderboard installs"

This reverts commit 8c7daa0e59c9e07cb99c232a2a34e5afbab80eeb.

* fix: hide stats on homepage trending skills

* fix: center listing row hover background

* chore: clean up homepage pre-pr checks

* fix: page homepage listing results

* fix: resolve homepage skill categories

* fix: stabilize homepage listing shortcut

* fix: harden homepage section rendering

* fix: preserve homepage skill category results

* perf: optimize homepage footer artwork

* fix: extend homepage byos reveal field

* fix: stabilize homepage CI

---------

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-06-19 17:29:40 -07:00
Jason (Json) fc07846839 fix: make catalog metadata publisher-owned (#2757)
* fix: make catalog topics canonical

* fix: make catalog metadata publisher-owned
2026-06-19 17:14:03 -07:00
Vyctor H. Brzezowski 53d16a55d4 fix: align official plugin browse with official publishers (#2758)
* fix: bound official plugin browse queries

* fix: normalize official publisher plugins
2026-06-19 17:07:56 -07:00
Patrick Erichsen 0f5496ff31 fix: allow github sync owner-scoped skill slugs 2026-06-19 15:00:23 -07:00
Patrick Erichsen 7accfb71c7 chore: remove registry artifact backup jobs
Stop custom registry artifact backup/backfill/restore behavior now that Convex backups with file storage are the recovery source of truth. Legacy backup schema tables remain inert until a separate verified cleanup removes stored rows.
2026-06-19 13:35:44 -07:00
Patrick Erichsen 2ae30d70d5 feat: add Vercel Speed Insights 2026-06-19 11:36:01 -07:00
Jason (Json) 7760696362 fix: show taxonomy on plugin cards (#2748)
Restore resolved plugin taxonomy on sorted catalog cards and show controlled category labels when author topics are absent.

Prepared head SHA: eed82783f0
Reviewed-by: Jason (Json) <263060202+fuller-stack-dev@users.noreply.github.com>
2026-06-19 01:02:52 -06:00
Patrick Erichsen f381b01829 feat: surface plugin manifest icons (#2747) 2026-06-18 20:15:35 -07:00
Jesse Merhi 8090d4ecea feat: add skill install backfill migration
Adds the aggregate skill install backfill migration, tests, and operator proof.
2026-06-19 13:08:12 +10:00
Patrick Erichsen 1cac1b2539 feat: derive skill icons from categories (#2745) 2026-06-18 19:18:46 -07:00
Patrick Erichsen 84a26a6acd fix: persist registry backup seed completion state (#2746) 2026-06-18 18:43:12 -07:00
Patrick Erichsen 6bbdec2a4e ci: shard local-auth playwright gate (#2743) 2026-06-18 17:40:55 -07:00
Patrick Erichsen b1b3c5e1cd feat: enforce Convex retention policy
Add typed Convex retention policies, auth session/refresh-token cleanup crons, and retention guidance.
2026-06-18 17:30:48 -07:00
Patrick Erichsen 0a7232ee5e feat: parallelize registry artifact backup backfill (#2741) 2026-06-18 16:56:33 -07:00
Patrick Erichsen bf42578b04 chore: remove empty legacy cleanup tables
Remove depRegistryCache and githubBackupSyncState from the Convex schema after both production tables were replaced with empty arrays.
2026-06-18 16:52:35 -07:00
Patrick Erichsen c7d8e1cd0a chore: remove legacy rate limits schema
Remove the legacy rateLimits Convex schema table after production data was replaced with an empty table. Active rateLimitCounters code and schema remain untouched.
2026-06-18 16:36:07 -07:00
Jason (Json) 769c620800 feat: move topic discovery into search and category browse (#2732)
* feat: search categories and topics

* fix: remove topic filters from browse sidebars

* feat: show top topics for selected categories

* fix(deps): resolve static audit advisories

* fix(search): scope skill recall by category

* fix(search): keep scoped recall within Convex limits

* fix(search): avoid Convex pagination fan-out
2026-06-18 17:31:13 -06:00
Patrick Erichsen 8c4a9ce9a4 fix: stop auto-greeting staff emails (#2738) 2026-06-18 16:21:03 -07:00
Patrick Erichsen f3842551c4 chore: add skill stat event survivor counter 2026-06-18 15:55:17 -07:00
Patrick Erichsen d8f82b7379 chore: remove catalog classification rollout migrations (#2735)
* chore: remove catalog classification rollout migrations

* chore: keep static audit gate green
2026-06-18 15:52:46 -07:00
Patrick Erichsen 6bcf96fea9 Add package runtime id repair command (#2734)
* feat: add package runtime id repair

* chore: refresh audit gate overrides
2026-06-18 15:45:55 -07:00
Patrick Erichsen 22d3cd133c fix: shard skill stat event pruning 2026-06-18 08:06:15 -07:00
Patrick Erichsen 731d1aa800 fix: prune processed skill stat events 2026-06-17 21:39:40 -07:00
Patrick Erichsen f92ccfd488 fix: remove reply-to from transactional emails (#2725) 2026-06-17 21:07:25 -07:00
Jason (Json) de28e2a6eb fix: stabilize sparse skill category pagination (#2724)
Merged after exact-head maintainer review.

Prepared head SHA: 66d9662023
Autoreview: clean, no accepted/actionable findings.

Co-authored-by: fuller-stack-dev <263060202+fuller-stack-dev@users.noreply.github.com>
Reviewed-by: @fuller-stack-dev
2026-06-17 21:58:02 -06:00
Patrick Erichsen f6a2c875d6 fix: encode R2 backup object keys for SigV4 2026-06-17 20:18:20 -07:00
Patrick Erichsen caa3359329 chore: remove deprecated download dedupe table (#2722) 2026-06-17 18:34:00 -07:00
Patrick Erichsen eccdbb3830 chore: remove legacy rate limit shard schema
Remove the empty legacy rateLimitShards table from the Convex schema and remove its temporary cleanup functions/tests after production contents were cleared.
2026-06-17 18:08:42 -07:00
Patrick Erichsen c13a2514da docs: add namespace claims guide
Adds a dedicated ClawHub namespace claims guide and routes existing claim references through it.
2026-06-17 17:59:12 -07:00
Jason (Json)andPatrick Erichsen 4bc58e4939 feat: add catalog classification backfill (#2719)
* feat: add controlled catalog taxonomy and topics

* refactor: limit taxonomy input to publish surfaces

* fix: omit inactive official-first filter

* fix: preserve unsaved catalog metadata edits

* fix: enforce catalog taxonomy invariants

* fix: close catalog taxonomy review gaps

* fix: tolerate retired stored skill categories

* fix: bound catalog metadata filter scans

* fix: preserve comma-containing topic labels

* fix: tolerate retired stored plugin categories

* fix: ignore empty secret integration metadata

* fix: honor explicit categories in related skills

* fix: harden taxonomy rollout migration

* fix: harden taxonomy browse pagination

* fix: preserve skill topic recommendation fallback

* fix: scale curated skill category browse

* fix: preserve legacy plugin category filters

* fix: preserve taxonomy compatibility semantics

* fix: preserve legacy catalog browse links

* feat: make catalog metadata editing explicit

* test: update plugin manage context contract

* fix: address taxonomy review findings

* fix: preserve empty category publish flags

* fix: preserve catalog search and publish metadata

* chore: keep taxonomy migration operator-run

* docs: keep taxonomy migrations operator-run

* fix: reject inherited category aliases

* fix: preserve normalized topic search behavior

* test: cover full topic pagination cursors

* feat: add catalog classification backfill

---------

Co-authored-by: Patrick Erichsen <patrick.a.erichsen@gmail.com>
2026-06-17 17:58:26 -07:00
Vyctor H. Brzezowski 1a3fdd8f51 fix: redirect sign-in to dashboard (#2718) 2026-06-17 17:31:21 -07:00
Vyctor H. Brzezowski 2d93401234 fix: use installs instead of downloads on social cards (#2713) 2026-06-17 17:28:20 -07:00
Vyctor H. Brzezowski b1e077da38 Polish ClawHub header and search UX (#2717)
* feat: sync header design polish

* fix: restore header search hints

* fix: strengthen search tab affordance

* Revert "fix: strengthen search tab affordance"

This reverts commit b586437577fce1b6521bb56ac01201cdb2bbc7da.

* fix: match search tab affordance

* fix: prevent header logo underline

* fix: soften search footer row

* fix: align search tab radii

* fix: make search footer full width

* fix: align mobile header search height

* fix: add responsive header more menu

* fix: simplify header more trigger

* fix: show mobile menu at compact header widths

* fix: widen compact search results menu

* fix: remove compact menu hover glow

* fix: use body font for user handle

* fix: prevent mobile nav link underline

* fix: revamp compact mobile header

* fix: connect mobile search results

* fix: move logged-out appearance switcher

* perf: shrink clawhub logo asset

* test: update header design contract

* fix: refine mobile auth controls

* fix: preserve watermark logo asset

* fix: scope arrow navigation to search tabs
2026-06-17 17:28:13 -07:00
Jason (Json)andPatrick Erichsen 04ec212100 feat: add controlled catalog taxonomy and topics (#2699)
* feat: add controlled catalog taxonomy and topics

* refactor: limit taxonomy input to publish surfaces

* fix: omit inactive official-first filter

* fix: preserve unsaved catalog metadata edits

* fix: enforce catalog taxonomy invariants

* fix: close catalog taxonomy review gaps

* fix: tolerate retired stored skill categories

* fix: bound catalog metadata filter scans

* fix: preserve comma-containing topic labels

* fix: tolerate retired stored plugin categories

* fix: ignore empty secret integration metadata

* fix: honor explicit categories in related skills

* fix: harden taxonomy rollout migration

* fix: harden taxonomy browse pagination

* fix: preserve skill topic recommendation fallback

* fix: scale curated skill category browse

* fix: preserve legacy plugin category filters

* fix: preserve taxonomy compatibility semantics

* fix: preserve legacy catalog browse links

* feat: make catalog metadata editing explicit

* test: update plugin manage context contract

* fix: address taxonomy review findings

* fix: preserve empty category publish flags

* fix: preserve catalog search and publish metadata

* chore: keep taxonomy migration operator-run

* docs: keep taxonomy migrations operator-run

* fix: reject inherited category aliases

* fix: preserve normalized topic search behavior

* test: cover full topic pagination cursors

* fix: tolerate invalid stored topics

---------

Co-authored-by: Patrick Erichsen <patrick.a.erichsen@gmail.com>
2026-06-17 18:20:54 -06:00
Patrick Erichsen 0c493cfa19 Add deprecated download dedupe cleanup (#2715)
* feat: add deprecated download dedupe cleanup

* chore: refresh vulnerable transitive overrides
2026-06-17 16:37:41 -07:00
Patrick Erichsen 9331619cb8 fix: remove global slug reservation delete copy (#2716) 2026-06-17 16:36:51 -07:00
Patrick Erichsen 69c79e8f01 fix: bound rate limit counter storage 2026-06-17 16:24:13 -07:00
Patrick Erichsen 65e321b280 fix: parallelize skill artifact backup uploads 2026-06-17 15:40:08 -07:00
Patrick Erichsen 11ad31aa4e feat: scope skill slugs by owner (#2299)
* feat: scope skill slugs by owner

* test: cover owner-scoped skill slug publishes

* feat: scope skill slugs by owner

* fix: restore owner-scoped skill ci

* fix: keep package names separate from skill slugs
2026-06-17 15:31:40 -07:00
Patrick Erichsen 3c46a6b996 chore: remove ClawPack repair operator
Remove the completed one-off ClawPack repair operator after production repair and verification.
2026-06-17 14:09:38 -07:00
Patrick Erichsen 60c73e4c54 fix: sanitize ClawPack repair metadata (#2710) 2026-06-17 13:20:30 -07:00
Patrick Erichsen 4ccc9a1c62 chore: use worktreeinclude for local state setup (#2709) 2026-06-17 12:54:47 -07:00
Vyctor H. Brzezowski 5f0e1ee639 feat: import owned public GitHub skills (#2444)
* feat: add GitHub skill import flow

* polish: refine GitHub import review states

* feat: link publish page to GitHub import

* test: keep import route checks lint-clean

* fix: harden GitHub import review flow

* polish: show copied state for GitHub import links

* fix: stabilize GitHub import slug review state

* fix: harden GitHub import discovery

* fix: address GitHub import review feedback

* fix: preserve legacy GitHub skill filenames

* fix: allow public GitHub URL imports

* style: format github import

* fix: remove stale import route imports

* fix: enforce owned GitHub import URLs

* fix: align GitHub import with main
2026-06-17 12:43:51 -07:00
Andy Ye 6d87fc078d fix: repair historical ClawPack release files
Merge PR #2583 to add an admin-gated historical ClawPack release file repair action with dry-run, confirmation, cursor/resume support, regression coverage, and security-moderation docs.
2026-06-17 12:33:11 -07:00
Patrick Erichsen 5be7035a69 docs: add org namespace claim form (#2637) 2026-06-17 12:31:00 -07:00
Andy YeandPatrick Erichsen 96b04b0837 fix(clawhub): explain CLI transport failures (#2681)
* fix(clawhub): explain CLI transport failures

* fix(clawhub): redact generic auth credentials

* fix(clawhub): harden CLI error redaction

* fix: align auth peer dependency

---------

Co-authored-by: Patrick Erichsen <patrick.a.erichsen@gmail.com>
2026-06-17 12:25:56 -07:00
Patrick Erichsen d6e54bdb65 fix: scope plugin validation findings to latest release (#2557) 2026-06-17 12:24:36 -07:00
Patrick Erichsen 691ba20195 fix: clear auth after account deletion (#2708) 2026-06-17 12:14:58 -07:00
dependabot[bot]anddependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> 3d971fd3c2 build(deps): bump the production-minor-and-patch group across 1 directory with 20 updates (#2698)
Bumps the production-minor-and-patch group with 20 updates in the / directory:

| Package | From | To |
| --- | --- | --- |
| [@convex-dev/auth](https://github.com/get-convex/convex-auth) | `0.0.93` | `0.0.94` |
| [@openclaw/plugin-inspector](https://github.com/openclaw/plugin-inspector) | `0.3.12` | `0.3.15` |
| [@radix-ui/react-avatar](https://github.com/radix-ui/primitives/tree/HEAD/packages/react/avatar) | `1.1.12` | `1.2.0` |
| [@radix-ui/react-dialog](https://github.com/radix-ui/primitives/tree/HEAD/packages/react/dialog) | `1.1.16` | `1.1.17` |
| [@radix-ui/react-dropdown-menu](https://github.com/radix-ui/primitives/tree/HEAD/packages/react/dropdown-menu) | `2.1.17` | `2.1.18` |
| [@radix-ui/react-label](https://github.com/radix-ui/primitives/tree/HEAD/packages/react/label) | `2.1.9` | `2.1.10` |
| [@radix-ui/react-select](https://github.com/radix-ui/primitives/tree/HEAD/packages/react/select) | `2.3.0` | `2.3.1` |
| [@radix-ui/react-separator](https://github.com/radix-ui/primitives/tree/HEAD/packages/react/separator) | `1.1.9` | `1.1.10` |
| [@radix-ui/react-slot](https://github.com/radix-ui/primitives/tree/HEAD/packages/react/slot) | `1.2.5` | `1.3.0` |
| [@radix-ui/react-toggle-group](https://github.com/radix-ui/primitives/tree/HEAD/packages/react/toggle-group) | `1.1.12` | `1.1.13` |
| [@radix-ui/react-tooltip](https://github.com/radix-ui/primitives/tree/HEAD/packages/react/tooltip) | `1.2.9` | `1.2.10` |
| [@react-email/render](https://github.com/resend/react-email/tree/HEAD/packages/render) | `2.0.8` | `2.0.9` |
| [@tanstack/react-router](https://github.com/TanStack/router/tree/HEAD/packages/react-router) | `1.170.15` | `1.170.16` |
| [@tanstack/react-start](https://github.com/TanStack/router/tree/HEAD/packages/react-start) | `1.168.25` | `1.168.26` |
| [convex](https://github.com/get-convex/convex-backend/tree/HEAD/npm-packages/convex) | `1.40.0` | `1.41.0` |
| [convex-helpers](https://github.com/get-convex/convex-helpers/tree/HEAD/packages/convex-helpers) | `0.1.118` | `0.1.119` |
| [lucide-react](https://github.com/lucide-icons/lucide/tree/HEAD/packages/lucide-react) | `1.17.0` | `1.20.0` |
| [semver](https://github.com/npm/node-semver) | `7.8.3` | `7.8.4` |
| [tailwindcss](https://github.com/tailwindlabs/tailwindcss/tree/HEAD/packages/tailwindcss) | `4.3.0` | `4.3.1` |
| [undici](https://github.com/nodejs/undici) | `7.27.2` | `7.28.0` |



Updates `@convex-dev/auth` from 0.0.93 to 0.0.94
- [Changelog](https://github.com/get-convex/convex-auth/blob/main/CHANGELOG.md)
- [Commits](https://github.com/get-convex/convex-auth/compare/v0.0.93...v0.0.94)

Updates `@openclaw/plugin-inspector` from 0.3.12 to 0.3.15
- [Release notes](https://github.com/openclaw/plugin-inspector/releases)
- [Changelog](https://github.com/openclaw/plugin-inspector/blob/main/CHANGELOG.md)
- [Commits](https://github.com/openclaw/plugin-inspector/compare/v0.3.12...v0.3.15)

Updates `@radix-ui/react-avatar` from 1.1.12 to 1.2.0
- [Changelog](https://github.com/radix-ui/primitives/blob/main/packages/react/avatar/CHANGELOG.md)
- [Commits](https://github.com/radix-ui/primitives/commits/HEAD/packages/react/avatar)

Updates `@radix-ui/react-dialog` from 1.1.16 to 1.1.17
- [Changelog](https://github.com/radix-ui/primitives/blob/main/packages/react/dialog/CHANGELOG.md)
- [Commits](https://github.com/radix-ui/primitives/commits/HEAD/packages/react/dialog)

Updates `@radix-ui/react-dropdown-menu` from 2.1.17 to 2.1.18
- [Changelog](https://github.com/radix-ui/primitives/blob/main/packages/react/dropdown-menu/CHANGELOG.md)
- [Commits](https://github.com/radix-ui/primitives/commits/HEAD/packages/react/dropdown-menu)

Updates `@radix-ui/react-label` from 2.1.9 to 2.1.10
- [Changelog](https://github.com/radix-ui/primitives/blob/main/packages/react/label/CHANGELOG.md)
- [Commits](https://github.com/radix-ui/primitives/commits/HEAD/packages/react/label)

Updates `@radix-ui/react-select` from 2.3.0 to 2.3.1
- [Changelog](https://github.com/radix-ui/primitives/blob/main/packages/react/select/CHANGELOG.md)
- [Commits](https://github.com/radix-ui/primitives/commits/HEAD/packages/react/select)

Updates `@radix-ui/react-separator` from 1.1.9 to 1.1.10
- [Changelog](https://github.com/radix-ui/primitives/blob/main/packages/react/separator/CHANGELOG.md)
- [Commits](https://github.com/radix-ui/primitives/commits/HEAD/packages/react/separator)

Updates `@radix-ui/react-slot` from 1.2.5 to 1.3.0
- [Changelog](https://github.com/radix-ui/primitives/blob/main/packages/react/slot/CHANGELOG.md)
- [Commits](https://github.com/radix-ui/primitives/commits/HEAD/packages/react/slot)

Updates `@radix-ui/react-toggle-group` from 1.1.12 to 1.1.13
- [Changelog](https://github.com/radix-ui/primitives/blob/main/packages/react/toggle-group/CHANGELOG.md)
- [Commits](https://github.com/radix-ui/primitives/commits/HEAD/packages/react/toggle-group)

Updates `@radix-ui/react-tooltip` from 1.2.9 to 1.2.10
- [Changelog](https://github.com/radix-ui/primitives/blob/main/packages/react/tooltip/CHANGELOG.md)
- [Commits](https://github.com/radix-ui/primitives/commits/HEAD/packages/react/tooltip)

Updates `@react-email/render` from 2.0.8 to 2.0.9
- [Release notes](https://github.com/resend/react-email/releases)
- [Changelog](https://github.com/resend/react-email/blob/canary/packages/render/CHANGELOG.md)
- [Commits](https://github.com/resend/react-email/commits/@react-email/render@2.0.9/packages/render)

Updates `@tanstack/react-router` from 1.170.15 to 1.170.16
- [Release notes](https://github.com/TanStack/router/releases)
- [Changelog](https://github.com/TanStack/router/blob/main/packages/react-router/CHANGELOG.md)
- [Commits](https://github.com/TanStack/router/commits/@tanstack/react-router@1.170.16/packages/react-router)

Updates `@tanstack/react-start` from 1.168.25 to 1.168.26
- [Release notes](https://github.com/TanStack/router/releases)
- [Changelog](https://github.com/TanStack/router/blob/main/packages/react-start/CHANGELOG.md)
- [Commits](https://github.com/TanStack/router/commits/@tanstack/react-start@1.168.26/packages/react-start)

Updates `convex` from 1.40.0 to 1.41.0
- [Release notes](https://github.com/get-convex/convex-backend/releases)
- [Changelog](https://github.com/get-convex/convex-backend/blob/main/npm-packages/convex/CHANGELOG.md)
- [Commits](https://github.com/get-convex/convex-backend/commits/HEAD/npm-packages/convex)

Updates `convex-helpers` from 0.1.118 to 0.1.119
- [Changelog](https://github.com/get-convex/convex-helpers/blob/main/packages/convex-helpers/CHANGELOG.md)
- [Commits](https://github.com/get-convex/convex-helpers/commits/npm/0.1.119/packages/convex-helpers)

Updates `lucide-react` from 1.17.0 to 1.20.0
- [Release notes](https://github.com/lucide-icons/lucide/releases)
- [Commits](https://github.com/lucide-icons/lucide/commits/1.20.0/packages/lucide-react)

Updates `semver` from 7.8.3 to 7.8.4
- [Release notes](https://github.com/npm/node-semver/releases)
- [Changelog](https://github.com/npm/node-semver/blob/main/CHANGELOG.md)
- [Commits](https://github.com/npm/node-semver/compare/v7.8.3...v7.8.4)

Updates `tailwindcss` from 4.3.0 to 4.3.1
- [Release notes](https://github.com/tailwindlabs/tailwindcss/releases)
- [Changelog](https://github.com/tailwindlabs/tailwindcss/blob/main/CHANGELOG.md)
- [Commits](https://github.com/tailwindlabs/tailwindcss/commits/v4.3.1/packages/tailwindcss)

Updates `undici` from 7.27.2 to 7.28.0
- [Release notes](https://github.com/nodejs/undici/releases)
- [Commits](https://github.com/nodejs/undici/compare/v7.27.2...v7.28.0)

---
updated-dependencies:
- dependency-name: "@convex-dev/auth"
  dependency-version: 0.0.94
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: production-minor-and-patch
- dependency-name: "@openclaw/plugin-inspector"
  dependency-version: 0.3.15
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: production-minor-and-patch
- dependency-name: "@radix-ui/react-avatar"
  dependency-version: 1.2.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: production-minor-and-patch
- dependency-name: "@radix-ui/react-dialog"
  dependency-version: 1.1.17
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: production-minor-and-patch
- dependency-name: "@radix-ui/react-dropdown-menu"
  dependency-version: 2.1.18
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: production-minor-and-patch
- dependency-name: "@radix-ui/react-label"
  dependency-version: 2.1.10
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: production-minor-and-patch
- dependency-name: "@radix-ui/react-select"
  dependency-version: 2.3.1
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: production-minor-and-patch
- dependency-name: "@radix-ui/react-separator"
  dependency-version: 1.1.10
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: production-minor-and-patch
- dependency-name: "@radix-ui/react-slot"
  dependency-version: 1.3.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: production-minor-and-patch
- dependency-name: "@radix-ui/react-toggle-group"
  dependency-version: 1.1.13
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: production-minor-and-patch
- dependency-name: "@radix-ui/react-tooltip"
  dependency-version: 1.2.10
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: production-minor-and-patch
- dependency-name: "@react-email/render"
  dependency-version: 2.0.9
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: production-minor-and-patch
- dependency-name: "@tanstack/react-router"
  dependency-version: 1.170.16
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: production-minor-and-patch
- dependency-name: "@tanstack/react-start"
  dependency-version: 1.168.26
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: production-minor-and-patch
- dependency-name: convex
  dependency-version: 1.41.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: production-minor-and-patch
- dependency-name: convex-helpers
  dependency-version: 0.1.119
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: production-minor-and-patch
- dependency-name: lucide-react
  dependency-version: 1.20.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: production-minor-and-patch
- dependency-name: semver
  dependency-version: 7.8.4
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: production-minor-and-patch
- dependency-name: tailwindcss
  dependency-version: 4.3.1
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: production-minor-and-patch
- dependency-name: undici
  dependency-version: 7.28.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: production-minor-and-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-06-17 12:14:27 -07:00
Vyctor H. Brzezowski f3ab8663e3 feat(cli): polish terminal branding (#2434) 2026-06-17 12:13:42 -07:00
S.F.Kislev d061779cf0 fix: preserve vector score for lexical skill matches (#2591) 2026-06-17 12:12:01 -07:00
Jesse Merhi afc31802e8 fix: derive skill publish metadata from storage (#2470) 2026-06-17 11:54:10 -07:00
Vyctor H. Brzezowski 815a053b32 feat: add post-publish share dialog (#2439) 2026-06-17 11:52:56 -07:00
Patrick Erichsen 8d7ac580d1 chore: remove retired capability schema
Remove retired capability/capabilityTags/executesCode schema fields and the packageCapabilitySearchDigest table definition after the production cleanup verified those fields are empty.
2026-06-17 08:21:33 -07:00
Jesse Merhi 791dead7a8 fix: stabilize package recommendation fallback sorting
Summary:
- Add recommended sort support for skill-backed package catalog rows.
- Fall package/plugin recommended browse back to installs while recommendation score fields are missing.
- Keep new fallback pagination cursors on installs so later pages do not switch ordering.
- Default filtered plugin browse to installs unless Recommended is explicitly selected.

Verification:
- CI green on PR #2675 before merge.
- bunx convex codegen
- focused Vitest: 4 files passed, 424 tests passed
- ci:types-build passed
- ci:static passed
- post-cleanup focused Vitest: 2 files passed, 382 tests passed
2026-06-18 00:53:39 +10:00
Jesse Merhi 0189ddd5c2 fix: add install sort support to package catalog APIs
- Add install sorting to package and plugin catalog API paths.
- Reject removed downloads sort requests with 400.
- Keep recommended browse stable during recommendation-score backfill.
- Normalize stale plugin UI downloads sort URLs back to the default browse state.
2026-06-17 17:36:36 +10:00
dependabot[bot]anddependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> 4860945f34 build(deps-dev): bump the development-minor-and-patch group across 1 directory with 6 updates (#2689)
Bumps the development-minor-and-patch group with 6 updates in the / directory:

| Package | From | To |
| --- | --- | --- |
| [@playwright/test](https://github.com/microsoft/playwright) | `1.60.0` | `1.61.0` |
| [@vitest/coverage-v8](https://github.com/vitest-dev/vitest/tree/HEAD/packages/coverage-v8) | `4.1.8` | `4.1.9` |
| [oxfmt](https://github.com/oxc-project/oxc/tree/HEAD/npm/oxfmt) | `0.54.0` | `0.55.0` |
| [oxlint](https://github.com/oxc-project/oxc/tree/HEAD/npm/oxlint) | `1.69.0` | `1.70.0` |
| [react-email](https://github.com/resend/react-email/tree/HEAD/packages/react-email) | `6.6.0` | `6.6.1` |
| [vitest](https://github.com/vitest-dev/vitest/tree/HEAD/packages/vitest) | `4.1.8` | `4.1.9` |



Updates `@playwright/test` from 1.60.0 to 1.61.0
- [Release notes](https://github.com/microsoft/playwright/releases)
- [Commits](https://github.com/microsoft/playwright/compare/v1.60.0...v1.61.0)

Updates `@vitest/coverage-v8` from 4.1.8 to 4.1.9
- [Release notes](https://github.com/vitest-dev/vitest/releases)
- [Changelog](https://github.com/vitest-dev/vitest/blob/main/docs/releases.md)
- [Commits](https://github.com/vitest-dev/vitest/commits/v4.1.9/packages/coverage-v8)

Updates `oxfmt` from 0.54.0 to 0.55.0
- [Release notes](https://github.com/oxc-project/oxc/releases)
- [Changelog](https://github.com/oxc-project/oxc/blob/main/npm/oxfmt/CHANGELOG.md)
- [Commits](https://github.com/oxc-project/oxc/commits/oxfmt_v0.55.0/npm/oxfmt)

Updates `oxlint` from 1.69.0 to 1.70.0
- [Release notes](https://github.com/oxc-project/oxc/releases)
- [Changelog](https://github.com/oxc-project/oxc/blob/main/npm/oxlint/CHANGELOG.md)
- [Commits](https://github.com/oxc-project/oxc/commits/oxlint_v1.70.0/npm/oxlint)

Updates `react-email` from 6.6.0 to 6.6.1
- [Release notes](https://github.com/resend/react-email/releases)
- [Changelog](https://github.com/resend/react-email/blob/canary/packages/react-email/CHANGELOG.md)
- [Commits](https://github.com/resend/react-email/commits/react-email@6.6.1/packages/react-email)

Updates `vitest` from 4.1.8 to 4.1.9
- [Release notes](https://github.com/vitest-dev/vitest/releases)
- [Changelog](https://github.com/vitest-dev/vitest/blob/main/docs/releases.md)
- [Commits](https://github.com/vitest-dev/vitest/commits/v4.1.9/packages/vitest)

---
updated-dependencies:
- dependency-name: "@playwright/test"
  dependency-version: 1.61.0
  dependency-type: direct:development
  update-type: version-update:semver-minor
  dependency-group: development-minor-and-patch
- dependency-name: "@vitest/coverage-v8"
  dependency-version: 4.1.9
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: development-minor-and-patch
- dependency-name: oxfmt
  dependency-version: 0.55.0
  dependency-type: direct:development
  update-type: version-update:semver-minor
  dependency-group: development-minor-and-patch
- dependency-name: oxlint
  dependency-version: 1.70.0
  dependency-type: direct:development
  update-type: version-update:semver-minor
  dependency-group: development-minor-and-patch
- dependency-name: react-email
  dependency-version: 6.6.1
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: development-minor-and-patch
- dependency-name: vitest
  dependency-version: 4.1.9
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: development-minor-and-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-06-16 17:42:29 -07:00
Patrick Erichsen 2523832e18 fix: clarify publisher abuse ban emails (#2697) 2026-06-16 17:42:17 -07:00
Patrick Erichsen effd52a4ea docs: fix acceptable usage table formatting 2026-06-16 17:36:02 -07:00
Patrick Erichsen 3a46cdd07e docs: clarify acceptable usage policy 2026-06-16 17:19:10 -07:00
Patrick Erichsen 1e7de4a2a2 fix: remove retired capability metadata surfaces
Remove retired capabilities/capabilityTags/executesCode surfaces for skills and plugins, add guarded cleanup tooling for legacy rows, and sanitize pre-cleanup legacy seed/release payloads.
2026-06-16 17:03:44 -07:00
Patrick Erichsen 5471fb280f fix: seed worktree dev previews locally (#2693) 2026-06-16 12:53:59 -07:00
Patrick Erichsen 7efc6c4555 docs: strengthen ClawHub PR maintainer reviews (#2692) 2026-06-16 12:35:28 -07:00
Patrick Erichsen 0e20d64c63 chore: remove retired comment schema (#2690) 2026-06-16 08:32:04 -07:00
Jesse Merhi 303075c82b Remove retired moderation surfaces (#2658)
* chore: remove retired moderation surfaces

* test: cover retired moderation action wrappers

* chore: retain moderation security gates

* test: cover retired moderation compatibility paths

* fix: stop settings queries during account deletion

* docs: add retired comment purge command

* chore: delete retired scanner tombstones
2026-06-16 08:01:11 -07:00
Patrick Erichsen a3d4509102 docs: simplify content rights reporting page (#2683) 2026-06-15 21:35:20 -07:00
Patrick Erichsen 9c50b60e35 refactor: remove root install telemetry schema (#2682) 2026-06-15 21:20:28 -07:00
Jesse Merhi 2c1b331be0 Rank skills by installs and harden install telemetry
* docs: add install telemetry ranking spec

* feat: rank skills by installs and dedupe install telemetry

* test: cover install telemetry dedupe cleanup

* fix: clean install dedupes on skill hard delete

* fix: drain telemetry clear batches

* fix: phase telemetry clear batches

* test: cover install stat and export cursor regressions

* chore: drop obsolete recommendation ranking changes

* test: update skill stat event batch harness

* docs: remove install telemetry spec

* chore: remove downloads from search scoring input

* fix: keep install telemetry prune cron guarded
2026-06-16 13:49:50 +10:00
Patrick Erichsen 6feaa0974a fix: remove registry backup index writes 2026-06-15 20:36:39 -07:00
Patrick Erichsen 1b66b84e58 Increase bulk publisher abuse pressure (#2678)
* feat: increase bulk publisher abuse pressure

* feat: raise bulk publisher pressure elasticity
2026-06-15 20:04:02 -07:00
Patrick Erichsen 9115151949 feat: allow official personal publishers 2026-06-15 20:01:48 -07:00
Patrick Erichsen e8a75de2b1 refactor: remove root install telemetry (#2673)
* refactor: remove root install telemetry

* chore: add root telemetry cleanup job

* fix: preserve legacy root telemetry deletion

* fix: handle token list after account deletion

* test: isolate local-auth e2e from production crons
2026-06-15 19:42:51 -07:00
Patrick Erichsen 0db6b17a62 Cap skill stat drain action batches (#2671)
* fix: cap skill stat drain action batches

* test: align content rights correspondence guard
2026-06-15 18:56:06 -07:00
Jason (Json) e49d680f6c feat: add profile link to account menu (#2625)
Adds a Profile action to the signed-in account menu and resolves the active personal publisher handle server-side, including stale and legacy publisher-pointer handling with focused unit and browser coverage.

Prepared head SHA: a0926771c3
Reviewed against current main: 549eda8e44
Reviewed-by: @fuller-stack-dev
2026-06-15 19:42:02 -06:00
Patrick Erichsen 549eda8e44 docs: add content rights requester update flow (#2672) 2026-06-15 18:41:12 -07:00
Patrick Erichsen e3e5705d89 feat(cli)!: remove sync command (#2669)
* feat(cli)!: remove sync command

* feat(cli): reconcile skill publishes

* chore: release clawhub cli 0.22.0

* fix(cli): publish new skills after resolver miss

* feat(workflow)!: remove bump input
2026-06-15 18:22:52 -07:00
Patrick Erichsen 69dd3b5f68 Polish content rights email workflow (#2670)
* chore: simplify content rights correspondence skill

* fix: show staff email title before greeting
2026-06-15 18:21:49 -07:00
Patrick Erichsen 44f4ab0e74 fix: set GitHub scan upload content type 2026-06-15 18:09:15 -07:00
Patrick Erichsen 4efbf01fa8 fix: floor skill stat drain batch size
Floor legacy/small scheduled skill stat doc-sync action batch sizes to the production drain size so continuations cannot crawl indefinitely.
2026-06-15 18:05:40 -07:00
Patrick Erichsen f366269519 feat: add owner version deletion
Add owner-only one-way deletion for individual skill versions and plugin releases, with CLI --version support, latest/only-version guards, and browser proof.
2026-06-15 17:52:28 -07:00
Patrick Erichsen 1622a74802 fix: recover forced GitHub scan requests
Allows operator-forced GitHub-backed rescans to recover incomplete pending requests that have no worker job, with regression coverage for the production NVIDIA scan state.
2026-06-15 17:52:24 -07:00
Patrick Erichsen 455f4ea19c fix: serialize skill stat event drain
Serialize the skill document stat drain behind a short-lived Convex lease and add admin kick/status helpers for manual backlog recovery.
2026-06-15 17:48:58 -07:00
Vincent Koc 5256b9e2ba fix(agents): sync autoreview scope guard
Sync the vendored autoreview skill from openclaw/agent-skills#32, including the closeout scope guard from openclaw/openclaw#93435.\n\nVerification: diff whitespace check, shell syntax, Python compile, prompt-policy assertion, and repo oxfmt check for SKILL.md.
2026-06-16 08:40:52 +08:00
Patrick Erichsen 76638de7ad feat: run full ClawScan for GitHub skills
Adds full ClawScan execution for GitHub-backed skills and updates the static audit ignore list for newly reported DOMPurify advisories.
2026-06-15 17:36:41 -07:00
Patrick Erichsen 89bb8fe938 feat: add content rights case tooling (#2657) 2026-06-15 17:32:07 -07:00
Patrick Erichsen 60c61d44a6 fix: remove homepage proof stats strip
Remove the hard-coded homepage proof stats strip and unused styles. Update the DOMPurify override to keep the static audit gate green.
2026-06-15 17:22:56 -07:00
Patrick Erichsen ab3c708cc9 fix: accept legacy install telemetry snapshots (#2656) 2026-06-15 12:32:29 -07:00
Patrick Erichsen da9da2b66d chore: remove legacy package ZIP backfill (#2654) 2026-06-15 11:55:26 -07:00
Patrick Erichsen 44ce895b44 fix: use installs across public adoption surfaces (#2653)
* fix: use installs across public adoption surfaces

* fix(cli): expose install-based explore sorting

* test: type legacy publisher sort coverage

* fix: preserve install compatibility paths

* fix: bound legacy hover install fallback

* docs: stop advertising downloads sort

* fix: preserve install rollout compatibility

* chore: update ws security override

* test: stub image optimizer in local browser smoke

* fix: normalize legacy download sort aliases
2026-06-15 11:06:29 -07:00
Patrick Erichsen 7f35ce91af fix: move GitHub source sync to Node runtime (#2652) 2026-06-15 10:49:18 -07:00
Patrick Erichsen bb35aca7bf fix: show publisher installs on profiles (#2651) 2026-06-15 09:35:17 -07:00
Jesse Merhi 078425f074 feat: add plugin install ranking (#2633) 2026-06-15 09:28:26 -07:00
Jesse Merhi e5f3ba272b fix: restrict docs auth return origins to dev only
* fix: couple docs auth localhost returns to a local app origin

The /auth/docs broker POSTs the signed-in user's auth token to the
return_to origin, but the allowlist trusted http://localhost:4173 /
127.0.0.1:4173 unconditionally, so production could hand the token to a
local listener.

Allow loopback return origins only when the app itself is served from a
loopback origin, so a public deployment (incl. staging/preview) can never
post the token to localhost regardless of runtime env. Keep the fixed
production docs origins (clawhub.ai, documentation.openclaw.ai,
docs.openclaw.ai), drop loopback from the production CSP form-action, and
record the token-destination invariant in specs/auth-identity.md.

* fix: align docs auth form destinations

* fix: retry public GitHub package fetches

* Revert "fix: retry public GitHub package fetches"

This reverts commit 1529aa34f3.
2026-06-15 21:52:06 +10:00
dependabot[bot]anddependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> ae7aa5861d build(deps-dev): bump the development-minor-and-patch group across 1 directory with 2 updates (#2622)
Bumps the development-minor-and-patch group with 2 updates in the / directory: [@tailwindcss/vite](https://github.com/tailwindlabs/tailwindcss/tree/HEAD/packages/@tailwindcss-vite) and [@types/node](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/node).


Updates `@tailwindcss/vite` from 4.3.0 to 4.3.1
- [Release notes](https://github.com/tailwindlabs/tailwindcss/releases)
- [Changelog](https://github.com/tailwindlabs/tailwindcss/blob/main/CHANGELOG.md)
- [Commits](https://github.com/tailwindlabs/tailwindcss/commits/v4.3.1/packages/@tailwindcss-vite)

Updates `@types/node` from 25.9.2 to 25.9.3
- [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases)
- [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/node)

---
updated-dependencies:
- dependency-name: "@tailwindcss/vite"
  dependency-version: 4.3.1
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: development-minor-and-patch
- dependency-name: "@types/node"
  dependency-version: 25.9.3
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: development-minor-and-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-06-15 04:04:29 -07:00
dependabot[bot]anddependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> d56c07e9ae build(deps): bump the github-actions group across 1 directory with 2 updates (#2620)
Bumps the github-actions group with 2 updates in the / directory: [actions/checkout](https://github.com/actions/checkout) and [actions/upload-artifact](https://github.com/actions/upload-artifact).


Updates `actions/checkout` from 4 to 6
- [Release notes](https://github.com/actions/checkout/releases)
- [Commits](https://github.com/actions/checkout/compare/v4...v6)

Updates `actions/upload-artifact` from 4 to 7
- [Release notes](https://github.com/actions/upload-artifact/releases)
- [Commits](https://github.com/actions/upload-artifact/compare/v4...v7)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-version: '6'
  dependency-type: direct:production
  update-type: version-update:semver-major
  dependency-group: github-actions
- dependency-name: actions/upload-artifact
  dependency-version: '7'
  dependency-type: direct:production
  update-type: version-update:semver-major
  dependency-group: github-actions
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-06-15 04:04:15 -07:00
Patrick Erichsen 078d1c88b8 chore: remove API key required cleanup migration (#2648) 2026-06-14 23:13:29 -07:00
Patrick Erichsen ca1f88f3e4 fix: allow guarded transfer of deleted skills (#2647) 2026-06-14 21:47:37 -07:00
Patrick Erichsen 61302e920f feat: add plugin versions tab (#2646)
* feat: add package versions API wrapper

* fix: omit empty package version cursor

* feat: add plugin versions tab

* fix: paginate plugin version history

* fix: remove unsafe plugin version downloads

* fix: harden plugin versions loading

* fix: preserve plugin versions pagination state

* fix: reset plugin detail state on navigation
2026-06-14 20:43:18 -07:00
Patrick Erichsen 2293016c20 revert: retire API key required skill feature (#2644) 2026-06-14 20:42:26 -07:00
Patrick Erichsen b0c8fe2a01 fix: serialize registry backup index writes 2026-06-14 20:37:34 -07:00
Patrick Erichsen ee9c6c1412 fix: normalize publisher recovery handle comparisons (#2643)
Normalize mixed-case stored user handles in publisher recovery guards and retire the previous mixed-case handle before handoff.
2026-06-14 20:26:35 -07:00
Patrick Erichsen ade8e3843a fix: serialize registry backup retry drains (#2641) 2026-06-14 19:59:05 -07:00
clawsweeper[bot]momothemageclawsweeper <274271284+clawsweeper[bot]@users.noreply.github.com>clawsweeper[bot] <274271284+clawsweeper[bot]@users.noreply.github.com>momothemage
44a552f3da fix: add guarded personal publisher recovery (#2642)
Summary:
- The PR adds an admin-only personal publisher recovery flow with HTTP API, admin CLI support, shared response schema, docs/spec notes, and tests.
- Reproducibility: yes. Source inspection shows current main lacks a publisher-recovery route and personal pub ...  to an existing ClawHub user, so a replacement GitHub principal has no staff recovery path without this PR.

Automerge notes:
- PR branch already contained follow-up commit before automerge: fix: migrate publisher recovery resource owners
- PR branch already contained follow-up commit before automerge: fix: add guarded personal publisher recovery

Validation:
- ClawSweeper review passed for head 5cfb360520.
- Required merge gates passed before the squash merge.

Prepared head SHA: 5cfb360520
Review: https://github.com/openclaw/clawhub/pull/2642#issuecomment-4704078560

Co-authored-by: momothemage <niuzhengnan@163.com>
Co-authored-by: clawsweeper <274271284+clawsweeper[bot]@users.noreply.github.com>
Co-authored-by: clawsweeper[bot] <274271284+clawsweeper[bot]@users.noreply.github.com>
Approved-by: momothemage
Co-authored-by: momothemage <35096042+momothemage@users.noreply.github.com>
2026-06-15 02:57:31 +00:00
Patrick Erichsen 85695c49a8 fix: compact theme controls in profile menu (#2640) 2026-06-14 19:53:43 -07:00
Patrick Erichsen 47038f0e24 fix: skip indexed registry backup retries (#2639) 2026-06-14 19:27:33 -07:00
Patrick Erichsen 3c48b989d4 test: cover skill summary digest sync (#2638) 2026-06-14 19:26:28 -07:00
Patrick Erichsen 03f97349b0 fix: preserve legacy package ZIP hashes (#2636)
* test: fix indexed row helper lint

* fix: preserve legacy package zip hashes
2026-06-14 18:56:31 -07:00
Patrick Erichsen 6ae7cb5345 fix: bulk repair registry backup indexes 2026-06-14 18:54:22 -07:00
Patrick Erichsen 17b5461539 fix: drain registry backup retries safely (#2634) 2026-06-14 18:25:34 -07:00
Patrick Erichsen b8ec008de3 fix: preserve single-row ClawHub header (#2632) 2026-06-14 16:46:37 -07:00
Patrick Erichsen fdb4d312f4 docs: restore OpenClaw-hosted ClawHub docs (#2631)
* Revert "fix: serve docs subdomain root (#2630)"

This reverts commit 9ef33726b8.

* Revert "feat: move docs to canonical subdomain"

This reverts commit 6afb91b772.

* Revert "docs: serve ClawHub docs with shared OpenClaw renderer (#2609)"

This reverts commit 71156751ff.

* fix: restore ClawHub email docs host

* test: preserve non-docs validation fixes

* test: remove stale docs proxy test

* docs: keep ClawHub contributor references current

* fix: keep docs auth callback outside docs routes
2026-06-14 16:05:34 -07:00
Patrick Erichsen 9ef33726b8 fix: serve docs subdomain root (#2630) 2026-06-14 13:28:45 -07:00
Peter Steinberger 6afb91b772 feat: move docs to canonical subdomain 2026-06-13 15:38:43 -07:00
Peter Steinberger 749028df12 fix: keep proxy tests out of server routes 2026-06-13 14:36:48 -07:00
Peter Steinberger 6820029e2f fix: preserve Ask Molty auth through proxy 2026-06-13 14:29:26 -07:00
Vincent Koc e86aa30a77 fix: require npm publish proof artifact 2026-06-13 18:06:27 +08:00
Vincent Koc 7d69b337c7 fix: make skill updates rollback safe 2026-06-13 18:04:04 +08:00
Vincent Koc 9fb09e0793 fix: retire cli browser token callback 2026-06-13 17:58:32 +08:00
Vincent Koc 3711b45d08 fix: default cli login to device flow 2026-06-13 17:55:09 +08:00
Vincent Koc 49b0c33fe8 fix: require confirmation for cli token handoff 2026-06-13 17:50:24 +08:00
Vincent Koc 84f2216d73 fix: harden clawhub cli sync reporting 2026-06-13 17:45:21 +08:00
Vincent Koc 64e22ae06e fix: harden clawhub cli release and sync checks 2026-06-13 17:38:03 +08:00
Patrick Erichsen 71156751ff docs: serve ClawHub docs with shared OpenClaw renderer (#2609)
* feat: serve clawhub docs under docs route

* feat: build ClawHub docs with openclaw docs

* chore: remove unused nav icon export

* docs: use ClawHub logo in docs shell

* docs: prepare ClawHub docs for Mintlify

* test: use static preview readiness for playwright

* chore: ignore current esbuild audit advisories

* docs: build ClawHub docs with shared OpenClaw renderer

* test: print package publish e2e failures
2026-06-12 19:02:44 -07:00
Patrick Erichsen 4b3c2bb630 Back up registry artifacts and add migration backup gate (#2600)
* fix: retarget registry backup gate

* feat: mirror package registry artifacts

* fix: keep package backup latest on newest release

* fix: bound registry backup health checks

* fix: harden package artifact backup paths

* fix: report package backup sync progress

* fix: type package backup sync result

* fix: require migration backup before dry run

* fix: simplify registry backup roots

* feat: back up registry artifacts to R2
2026-06-12 16:31:58 -07:00
Patrick Erichsen ee1c80202f fix: prefer app auth for trusted publisher repo lookup (#2623) 2026-06-12 16:31:43 -07:00
Jesse Merhi 73c32d6296 chore: fix dependabot bun update config (#2566) 2026-06-12 16:14:29 -07:00
Patrick Erichsen 99e5ab4ed7 docs: add session store validation remediation
Add the missing sdk-load-session-store remediation section and table entry.
2026-06-12 13:01:51 -07:00
Patrick Erichsen 496f52693a fix: clarify scoped package publish access errors (#2617) 2026-06-12 12:43:15 -07:00
Patrick Erichsen 229a05a4c7 feat: render mermaid diagrams in markdown (#2616) 2026-06-12 11:04:54 -07:00
683 changed files with 103779 additions and 30492 deletions
+269 -120
View File
@@ -1,176 +1,337 @@
---
name: autoreview
description: "Use when ClawHub needs Codex review, autoreview, second-model review, or a final advisory review gate before commit, PR update, ship, or maintainer handoff."
description: "Pre-commit/ship code review: Codex default; optional Claude, Pi, Droid, Copilot, or OpenCode."
---
# Autoreview
# Auto Review
Run Codex's built-in code review as a closeout check. This is code review
(`codex review`), not Guardian `auto_review` approval routing.
Run the bundled structured review helper as a closeout check. This is code review, not Guardian `auto_review` approval routing.
Codex native review mode performs best and is recommended. Non-Codex reviewers
are fallback or second-opinion paths that receive a generated diff prompt, not
the full Codex review-mode runtime.
Codex review is the default when no engine is set. It uses `gpt-5.5` by default, usually delivers the best review results, and should remain the normal final closeout engine. Claude review is optional and uses `claude-fable-5` by default.
Use when:
- the user asks for Codex review, autoreview, or second-model review
- user asks for Codex review / Claude review / Pi review / Droid review / OpenCode review / autoreview / second-model review
- after non-trivial code edits, before final/commit/ship
- reviewing a local branch or PR branch after fixes
- closing out ClawHub maintainer work that touched source, tests, Convex, UI,
CLI packages, or workflows
## Contract
- Treat review output as advisory. Never blindly apply it.
- Verify every finding by reading the real code path and adjacent files.
- Read dependency docs/source/types when the finding depends on external
behavior.
- Reject unrealistic edge cases, speculative risks, broad rewrites, and fixes
that over-complicate the codebase.
- Prefer small fixes at the right ownership boundary; no refactor unless it
clearly improves the bug class.
- Keep going until the selected review path returns no accepted/actionable
findings.
- If a review-triggered fix changes code, rerun focused tests and rerun the
review helper.
- Default to Codex review. If Codex is unavailable or exits with an error, the
helper can fall back to `claude -p`, `pi -p`, `opencode run`, `droid exec`, or
`copilot`.
- Stop as soon as the review command/helper exits 0 with no
accepted/actionable findings. Do not run an extra direct `codex review` just
to get a nicer clean line, a second opinion, or clearer closeout wording.
- If rejecting a finding as intentional/not worth fixing, add a brief inline
code comment only when it explains a real invariant or ownership decision
future reviewers should know.
- Do not push just to review. Push only when the user requested push/ship/PR
update.
- Read dependency docs/source/types when the finding depends on external behavior.
- Reject unrealistic edge cases, speculative risks, broad rewrites, and fixes that over-complicate the codebase.
- Prefer small fixes at the right ownership boundary; no refactor unless it clearly improves the bug class.
- When an accepted finding shows a bug class or repeated pattern, inspect the current PR scope for sibling instances before fixing.
- Fix the scoped bug class at once when practical; stop at touched surfaces, owner boundaries, and clear follow-up territory.
- Keep going until structured review returns no accepted/actionable findings only while the work remains inside the original task scope.
- If a review-triggered fix changes code, rerun focused tests and rerun the structured review helper.
- For security-audit suppression changes, verify accepted findings remain auditable: suppressed findings stay in structured output, active output keeps an unsuppressible suppression notice, and aggregate findings cannot hide unrelated active risk.
- Never switch or override the requested review engine/model. If the review hits model capacity, retry the same command a few times with the same engine/model.
- Be patient with large bundles. Structured review can take up to 30 minutes while the model call is active, especially with Codex tools or web search.
- Treat heartbeat lines like `review still running: ... elapsed=... pid=...` as healthy progress, not a hang. Let the helper continue while heartbeats are advancing. Pass `--stream-engine-output` when live engine text is useful; Codex and Claude filter tool/file chatter, other engines pass raw output through.
- Do not kill a review just because it has been quiet for 2-5 minutes, or because it is still running under the 30-minute window. Inspect the process only after missing multiple expected heartbeats, after 30 minutes, or after an obviously failed subprocess; prefer letting the same helper command finish.
- Tools are useful in review mode. The helper allows read-only inspection tools and web search by default so reviewers can check dependency contracts, upstream docs, and current behavior.
- Security perspective is always included, but it should not cripple legitimate functionality. Report security findings only when the change creates a concrete, actionable risk or removes an important safety check.
- For regression provenance, keep roles separate: blamed code author, blamed PR author, PR merger/committer, current PR author, and PR/date. If no blamed PR is traceable, use the blamed commit as the provenance: commit SHA, date, and author username. Do not guess a merger or frame missing PR metadata as a separate finding.
- If the blamed PR was merged by `clawsweeper[bot]` or another automation, identify the human trigger when practical. Check timeline/comments first; if rate-limited, use gitcrawl/cache or public PR HTML. Look for maintainer commands such as `@clawsweeper automerge`, `/landpr`, or labels/status comments that armed automerge. Report `automerge triggered by @login`; if not found, say trigger unknown.
- Do not invoke built-in `codex review`, nested reviewers, or reviewer panels from inside the review. The helper builds one bundle, calls one selected engine, validates one structured result, and stops.
- Stop as soon as the helper exits 0 with no accepted/actionable findings. Do not run an extra review just to get a nicer "clean" line, a second opinion, or clearer closeout wording.
- Treat the helper's successful exit plus absence of actionable findings as the clean review result, even if the underlying Codex CLI output is terse.
- Multi-reviewer panels are opt-in only. Use them when explicitly requested or when risk justifies the extra spend; the main agent still verifies every accepted finding before fixing.
- If rejecting a finding as intentional/not worth fixing, add a brief inline code comment only when it explains a real invariant or ownership decision that future reviewers should know.
- If `gh`/Gitcrawl reports `database disk image is malformed`, run `gitcrawl doctor --json` once to let the portable cache repair before retrying review; do not bypass the shim unless repair fails and freshness requires live GitHub.
- If Gitcrawl reports a portable manifest mismatch, source/runtime DB health error, or stale portable-store checkout, run `gitcrawl doctor --json` and inspect `source_db_health`, `runtime_db_health`, and `portable_store_status` before falling back to live GitHub.
- Do not push just to review. Push only when the user requested push/ship/PR update.
## ClawHub Proof Routing
## Scope Governor
Pick the smallest proof that matches the touched surface:
Autoreview is a closeout gate, not permission to rewrite the task.
| Touched surface | Usual proof |
| ---------------------------------- | ----------------------------------------------------------------------------------------------------------------------- |
| Formatting/lint/static repo health | `bun run ci:static` |
| Unit-tested source behavior | focused `bunx vitest run ...`, then `bun run ci:unit` when PR-ready |
| Convex code | read `convex/_generated/ai/guidelines.md` first; run focused tests and the deploy/typecheck path that covers the change |
| Packages/CLI/admin tool | `bun run ci:packages` or the package-specific `verify` script |
| Runtime/build/package surface | `bun run ci:types-build`, `bun run ci:e2e-http`, or the matching broader gate |
| UI behavior | use `clawhub-ui-proof` with `proof:ui`; publish proof before final PR comments when needed |
| Linux/CI-parity validation | use `crabbox`, normally through the repo scripts |
Before the first review, freeze a scope baseline: original request or issue, target branch, intended behavior, owner boundary, changed files, and non-test LOC. For inherited or already-bloated branches, use the intended PR diff as the baseline rather than accepting all existing branch drift.
For Convex query or schema work, apply the repo's Convex rules: prefer indexes
over `.filter()` scans, use cursor-based backfills for data shape changes, and
verify with the repo's Convex/typecheck path before claiming deploy safety.
Before patching a finding, classify it:
- **In-scope blocker**: the finding is introduced by the current diff, affects the same owner boundary, and can be fixed without changing the task's contract.
- **Follow-up**: the finding is real but belongs to an adjacent bug class, sibling surface, cleanup, or broader hardening track.
- **Stop-and-escalate**: the finding requires a new protocol/config/storage/public API contract, a different owner boundary, a release-process change, or a design choice outside the original request.
Stop patching and report the scope break instead of continuing when:
- a narrow PR turns into an architecture change, protocol change, migration, or release-process change;
- the diff grows past 2x the original files or non-test LOC without explicit approval to expand scope;
- two review-triggered patch cycles have not converged; pause and reclassify every remaining finding before another edit;
- the best fix is "define the canonical contract first" rather than another local inference layer;
- fixing the accepted finding would make the PR no longer describe the same behavior, issue, or owner boundary.
After the two-cycle pause, continue only when every remaining accepted finding is still an in-scope blocker. Otherwise preserve the useful analysis, identify the smallest safe landed subset if one exists, and open or request a follow-up for the larger fix. Do not keep committing speculative fixes just to satisfy the reviewer.
Do not stack or push review-triggered fix commits while scope classification or focused proof is unresolved. Keep exploratory edits local until the cycle is proven in scope; if scope breaks, remove them from the landing lane instead of preserving them as branch history.
Critical exceptions must be explicit: active data loss, crash, broken install/upgrade, release blocker, or concrete security exposure. If the exception is not one of those, it is not critical enough to blow up scope.
## Release Branches And Release Process
On release, beta, stable, hotfix, signing, notarization, appcast, package-publish, or release-check work, use freeze discipline even when the branch name is not release-like:
- Fix only release blockers, failed release infrastructure, exact backports, install/upgrade breakage, data loss, crashes, or concrete security exposure.
- Treat non-blocking autoreview findings as follow-ups for `main`, not reasons to broaden the release branch.
- Do not introduce new product behavior, config surface, protocol shape, migration, plugin ownership, docs narrative, or process policy unless it directly unblocks the release.
- Keep proof tied to the release target: exact branch/ref, failing check or shipped-risk reason, smallest command/proof, and whether the fix must also forward-port to `main`.
- If review discovers a real but non-critical design problem during release closeout, stop with a follow-up issue/PR plan; do not use the release branch as the refactor lane.
## Skill Path (set once)
Set the skill script paths once, then use `"$AUTOREVIEW"` and `"$AUTOREVIEW_HARNESS"` in the examples below.
Choose one:
```bash
# Project-local skill in the current repo:
export AUTOREVIEW=".agents/skills/autoreview/scripts/autoreview"
export AUTOREVIEW_HARNESS=".agents/skills/autoreview/scripts/test-review-harness"
```
```bash
# Source checkout of openclaw/agent-skills:
export AUTOREVIEW="skills/autoreview/scripts/autoreview"
export AUTOREVIEW_HARNESS="skills/autoreview/scripts/test-review-harness"
```
```bash
# Global skill:
export AGENTS_HOME="${AGENTS_HOME:-$HOME/.agents}"
export AUTOREVIEW="$AGENTS_HOME/skills/autoreview/scripts/autoreview"
export AUTOREVIEW_HARNESS="$AGENTS_HOME/skills/autoreview/scripts/test-review-harness"
```
When using Claude Code, set `AGENTS_HOME="$HOME/.claude"` for global skills. Project-local skills live under `.claude/skills/` in the current repo.
## Pick Target
Dirty local work:
```bash
codex review --uncommitted
"$AUTOREVIEW" --mode local
```
Use this only when the patch is actually unstaged/staged/untracked in the
current checkout. For committed, pushed, or PR work, point Codex at the commit
or branch diff instead. A clean `--uncommitted` review only proves there is no
local patch.
current checkout. `--mode uncommitted` is accepted as an alias for `--mode local`.
For committed, pushed, or PR work, point the helper at the commit
or branch diff instead; do not force dirty modes just
because the helper docs mention dirty work first. A clean local review
only proves there is no local patch.
Branch/PR work:
```bash
git fetch origin
codex review --base origin/main
"$AUTOREVIEW" --mode branch --base origin/main
```
Optional review context is first-class. Prompt files and datasets must be repo-relative so review bundles cannot pull arbitrary host files:
```bash
"$AUTOREVIEW" --mode branch --base origin/main --prompt-file review-notes.md --dataset evidence.json
```
If an open PR exists, use its actual base:
```bash
base=$(gh pr view --json baseRefName --jq .baseRefName)
codex review --base "origin/$base"
"$AUTOREVIEW" --mode branch --base "origin/$base"
```
Do not pass a prompt with `--base`. Some Codex CLI versions reject
`codex review --base <ref> -` with `--base <BRANCH> cannot be used with
[PROMPT]`. If that happens, rerun plain `codex review --base <ref>` and report
that prompt injection was skipped.
Committed single change:
```bash
codex review --commit HEAD
"$AUTOREVIEW" --mode commit --commit HEAD
```
or with the helper:
```bash
.agents/skills/autoreview/scripts/autoreview --mode commit --commit HEAD
```
Use commit review for already-landed or already-pushed work on `main`.
Reviewing clean `main` against `origin/main` is usually an empty diff after
push. For a small stack, review each commit explicitly or review the branch
before merging with `--base`.
Use commit review for already-landed or already-pushed work on `main`. Reviewing
clean `main` against `origin/main` is usually an empty diff after push. For a
small stack, review each commit explicitly or review the branch before merging
with `--base`.
## Parallel Closeout
Format first if formatting can change line locations. Then it is OK to run
tests and review in parallel:
Format first if formatting can change line locations. Then it is OK to run tests and review in parallel:
```bash
.agents/skills/autoreview/scripts/autoreview --parallel-tests "bun run ci:static"
"$AUTOREVIEW" --parallel-tests "<focused test command>"
```
Tradeoff: tests may force code changes that stale the review. If tests or
review lead to code edits, rerun the affected tests and rerun review until no
accepted/actionable findings remain. Once that rerun exits cleanly, stop; do
not spend another long review cycle on redundant confirmation.
On Windows, the default `--parallel-tests` shell preserves the platform `cmd.exe`
semantics used by Python `shell=True`. Use `--parallel-tests-shell powershell`
or `--parallel-tests-shell pwsh` when the focused test command is PowerShell-specific.
Tradeoff: tests may force code changes that stale the review. If tests or review lead to code edits, rerun the affected tests and rerun review until no accepted/actionable findings remain. Once that rerun exits cleanly, stop; do not spend another long review cycle on redundant confirmation.
## Review Panels
Run multiple reviewers against one frozen bundle:
```bash
"$AUTOREVIEW" --reviewers codex,claude,pi,droid
```
`--panel` is shorthand for Codex plus Claude unless `--engine` changes the first reviewer:
```bash
"$AUTOREVIEW" --panel
```
Set reviewer models and thinking/effort explicitly:
```bash
"$AUTOREVIEW" --reviewers codex,claude --model codex=gpt-5.5 --thinking codex=high --model claude=claude-fable-5 --thinking claude=max
```
Inline syntax is also supported for simple model IDs:
```bash
"$AUTOREVIEW" --reviewers codex:gpt-5.5:high,claude:claude-fable-5:max
```
For models with slashes or extra colons, prefer keyed form:
```bash
"$AUTOREVIEW" --engine pi --model anthropic/claude-sonnet-4 --thinking high
"$AUTOREVIEW" --engine opencode --model opencode/north-mini-code-free --thinking high
"$AUTOREVIEW" --engine droid --model claude-opus-4-8 --thinking low
"$AUTOREVIEW" --reviewers codex,pi --model codex=gpt-5.5 --model pi=anthropic/claude-sonnet-4
"$AUTOREVIEW" --reviewers codex,opencode --model codex=gpt-5.5 --model opencode=opencode/north-mini-code-free
"$AUTOREVIEW" --reviewers codex,droid --model codex=gpt-5.5 --model droid=claude-opus-4-8
```
## Models and thinking
The helper accepts `--model` globally or per engine (`engine=model`) and `--thinking` globally or per engine (`engine=level`). Repeat either flag for multiple reviewers.
Recommended model defaults:
| Engine | Default model | Source note |
| ------------------- | ---------------- | ----------------------------------------------------- |
| **codex** (default) | `gpt-5.5` | OpenAI's current GPT-5.5 alias |
| **claude** | `claude-fable-5` | Anthropic's most capable widely released Claude model |
CLI flags and environment variables override these defaults. Droid, Copilot, Pi, and OpenCode do not get built-in model defaults here because their provider catalogs are external to the Codex/Claude closeout path and may vary by installation.
| Engine | Model flag | Example model IDs | Thinking flag | Accepted levels |
| ------------------- | -------------------------- | ---------------------------------------------------------------------------- | ----------------------------- | --------------------------------------------------- |
| **codex** (default) | `codex --model X exec ...` | `gpt-5.5`, `gpt-5.5-2026-04-23` | `-c model_reasoning_effort=Y` | `none`, `minimal`, `low`, `medium`, `high`, `xhigh` |
| **claude** | `claude --model X` | `claude-fable-5`, `claude-opus-4-8`, `claude-sonnet-4-6`, `claude-haiku-4-5` | `--effort Y` | `low`, `medium`, `high`, `xhigh`, `max` |
| **droid** | `droid exec --model X` | `claude-opus-4-8`, Factory model IDs | `-r, --reasoning-effort Y` | `off`, `none`, `low`, `medium`, `high` |
| **copilot** | `copilot --model X` | `gpt-5.2`, Copilot model aliases | not supported | n/a |
| **pi** | `pi --model X` | `anthropic/claude-sonnet-4`, `openai/gpt-4o` | `--thinking Y` | `off`, `minimal`, `low`, `medium`, `high`, `xhigh` |
| **opencode** | `opencode run -m X` | `opencode/north-mini-code-free`, OpenCode provider/model IDs | `--variant Y` | `minimal`, `low`, `medium`, `high`, `max` |
Claude also supports `--fallback-model a,b` for availability-based fallback chains ([model-config](https://code.claude.com/docs/en/model-config)). Current Claude docs note that auth, billing, rate-limit, request-size, and transport errors do not trigger fallback, and the changelog documents interactive-session support in `v2.1.166`.
Examples matching current `main` behavior:
```bash
# Codex with explicit model and reasoning
"$AUTOREVIEW" --engine codex --model gpt-5.5 --thinking high
# Claude Code aliases or full model names, with optional availability fallback
"$AUTOREVIEW" --engine claude --model claude-fable-5 --thinking max
"$AUTOREVIEW" --engine claude --model claude-fable-5 --fallback-model claude-opus-4-8,claude-sonnet-4-6
# Factory Droid with explicit model and reasoning effort
"$AUTOREVIEW" --engine droid --model claude-opus-4-8 --thinking low
# GitHub Copilot (model only; no thinking knob)
"$AUTOREVIEW" --engine copilot --model gpt-5.2
# Pi with explicit model and thinking level
"$AUTOREVIEW" --engine pi --model anthropic/claude-sonnet-4 --thinking high --pi-bin pi
# OpenCode with explicit provider/model and variant
"$AUTOREVIEW" --engine opencode --model opencode/north-mini-code-free --thinking high
```
### Environment defaults
CLI flags take precedence over environment variables.
| Variable | Purpose |
| ---------------------------------- | ----------------------------------------------------------------------- |
| `AUTOREVIEW_MODEL` | Override the built-in default `--model` for all engines |
| `AUTOREVIEW_THINKING` | Default `--thinking` for all engines |
| `AUTOREVIEW_FALLBACK_MODEL` | Default Claude `--fallback-model` chain |
| `AUTOREVIEW_<ENGINE>_MODEL` | Per-engine model override, for example `AUTOREVIEW_CODEX_MODEL=gpt-5.5` |
| `AUTOREVIEW_<ENGINE>_THINKING` | Per-engine thinking override |
| `AUTOREVIEW_CLAUDE_FALLBACK_MODEL` | Claude-only fallback chain |
Codex maps thinking to `model_reasoning_effort`. Claude maps thinking to `--effort`. Droid maps thinking to `-r, --reasoning-effort`. Pi maps thinking to `--thinking`. OpenCode maps thinking to `--variant`. Copilot rejects `--thinking`. Only Claude accepts `--fallback-model`; global CLI/env fallback requires at least one Claude reviewer, and engine-specific fallback overrides require that reviewer to be selected. Non-Claude fallback overrides, including `AUTOREVIEW_<NONCLAUDE>_FALLBACK_MODEL`, fail closed instead of being silently ignored.
## Review engine isolation
When autoreview runs inside the repository under review, external reviewer CLIs must not load project-local trust or configuration that the branch controls.
| Engine | Isolation flags | Reference |
| ------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------- |
| **codex** | Auth-only config overrides, `-c project_doc_max_bytes=0`, repo `trust_level="untrusted"`, `exec --ignore-user-config --ignore-rules`, plus read-only sandbox | Codex CLI `exec --help` |
| **claude** | `--safe-mode --setting-sources user --strict-mcp-config --disallowedTools mcp__*` plus explicit `--allowedTools` (`--safe-mode` requires Claude Code `v2.1.169+`) | Claude Code [CLI reference](https://code.claude.com/docs/en/cli-reference) |
| **pi** | `--no-approve --no-session --no-context-files --no-extensions --no-skills --no-prompt-templates --no-themes`, plus read-only tool allowlist | Pi CLI `--help`; requires Pi `v0.79.0+` |
| **opencode** | `opencode run --dir <repo> --pure --format json`, prompt over stdin, neutral subprocess cwd, injected deny-by-default permissions, project config disabled | OpenCode CLI `--help` |
Codex `--ignore-user-config` skips config loading for the exec run. Autoreview reconstructs only the documented `cli_auth_credentials_store`, `forced_login_method`, and `forced_chatgpt_workspace_id` settings from `CODEX_HOME/config.toml`, keeping authentication and workspace restrictions usable without forwarding unrelated user configuration. The explicit repo trust override and zero project-doc budget keep reviewed-repo `AGENTS.md` and `.codex/` trust surfaces out of the review prompt. `--ignore-rules` skips user/project execpolicy rules. Claude `--safe-mode` disables project hooks, skills, plugins, MCP servers, and CLAUDE.md while preserving normal authentication, model selection, built-in tools, and permissions; managed settings policy can still apply. `--setting-sources user` avoids project/local settings from the reviewed checkout, and current Claude Code docs note the project-skill blocking behavior was fixed in `v2.1.69`. `--strict-mcp-config` and `--disallowedTools mcp__*` keep MCP unavailable to the review run. `--bare` is not used here because Claude's headless docs say it skips OAuth and keychain reads. Pi `--no-approve` ignores project-local files for one run; the helper requires Pi `v0.79.0+` plus help output that advertises every required isolation flag because older legacy binaries can ignore unknown flags. The current package is `@earendil-works/pi-coding-agent`; deprecated `@mariozechner/pi-coding-agent` `0.73.x` is intentionally rejected. Pi version/help probes and the review command run from neutral temporary directories, not the reviewed repo. Pi `--no-context-files` removes `AGENTS.md`/`CLAUDE.md`, the resource-disable flags keep `.pi` extensions, skills, prompts, and themes out of the run, `--no-session` avoids writing review sessions, and the read-only allowlist omits `bash`, `edit`, and `write`. OpenCode starts from a neutral temporary directory, points at the reviewed repo with `--dir`, disables project config through `OPENCODE_DISABLE_PROJECT_CONFIG=1`, and injects `OPENCODE_CONFIG_CONTENT`; permissions default to deny, allow read/grep/glob, preserve OpenCode's `.env` ask rules, and gate `websearch`/`webfetch` with `--no-web-search`. The injected config also clears command/instruction/plugin arrays and disables write/edit/bash/task/skill/todowrite tools without changing user auth storage. The helper sends the review prompt over stdin rather than argv and extracts the final structured JSON from `type: "text"` events. OpenCode rejects `--no-tools`.
## Context Efficiency
Codex review is usually noisy. Default to a subagent filter when subagents are
available. Ask it to run the review and return only:
- actionable findings it accepts
- findings it rejects, with one-line reason
- exact files/tests to rerun
Run inline only for tiny changes or when subagents are unavailable.
Run the helper directly so target selection, engine choice, structured validation, and exit status all stay in one path. If output is noisy, summarize the completed helper output after it returns; do not ask another agent or reviewer to rerun the review.
## Helper
Bundled helper:
After setting `AUTOREVIEW` and `AUTOREVIEW_HARNESS` above:
```bash
.agents/skills/autoreview/scripts/autoreview --help
"$AUTOREVIEW" --help
```
The smoke harness has thin shell wrappers over a shared Python implementation:
```bash
"$AUTOREVIEW_HARNESS" --fixture benign --engine codex
```
On native Windows, invoke the extensionless Python helper through Python:
```powershell
python skills\autoreview\scripts\autoreview --help
```
and the smoke harness:
```powershell
skills\autoreview\scripts\test-review-harness.ps1 -Fixture benign -Engine codex
```
The helper:
- chooses dirty `--uncommitted` first
- chooses dirty local changes first
- accepts `--mode uncommitted` as an alias for `--mode local`
- otherwise uses current PR base if `gh pr view` works
- otherwise uses `origin/main` for non-main branches
- auto-runs `bun run ci:static` in parallel when the repo has `package.json`,
`bun.lock`, `node_modules`, and a `ci:static` script; disable with
`AUTOREVIEW_AUTO_TESTS=0`
- use `--mode commit --commit <ref>` for already-committed work, especially
clean `main` after landing
- should be left in `--mode auto` or forced to `--mode branch` for PR/branch
work; do not force `--mode local` after committing
- supports `--reviewer codex|claude|pi|opencode|droid|copilot|auto`; `auto`
means Codex first
- supports `--fallback-reviewer auto|claude|pi|opencode|droid|copilot|none`
- falls back only when Codex is unavailable or exits nonzero without findings,
not when Codex reports findings
- writes only to stdout unless `--output` or `AUTOREVIEW_OUTPUT` is set
- supports `--dry-run`, `--parallel-tests`, and commit refs
- runs nested review with `--dangerously-bypass-approvals-and-sandbox --sandbox
danger-full-access` by default; use `--no-yolo` or `AUTOREVIEW_YOLO=0` to opt
out
- prints `autoreview clean: no accepted/actionable findings reported` when the
selected review command exits 0 and no accepted/actionable findings are
reported
- does not fetch automatically during branch review; the selected base ref must already resolve locally
- supports `--engine codex`, `claude`, `droid`, `copilot`, `pi`, and `opencode`; default is `AUTOREVIEW_ENGINE` or `codex`; Codex should remain the default when nothing is set
- resolves bare `git`, `gh`, reviewer, and PowerShell shell commands from absolute `PATH` entries only, never from the reviewed checkout; explicit relative `--*-bin` paths are resolved from the reviewed repository root
- use `--mode commit --commit <ref>` for already-committed work, especially clean `main` after landing
- should be left in `--mode auto` or forced to `--mode branch` for PR/branch work; do not force `--mode local` after committing
- writes only to stdout unless `--output`, `--json-output`, or live streamed engine stderr is set
- supports `--dry-run`, `--parallel-tests`, `--parallel-tests-shell`, `--prompt`, repo-relative `--prompt-file`, repo-relative `--dataset`, `--no-tools`, `--no-web-search`, and commit refs
- supports `--stream-engine-output` or `AUTOREVIEW_STREAM_ENGINE_OUTPUT=1` for live engine text while preserving structured validation; Codex and Claude hide tool/file event details, emit compact activity summaries, and report usage at turn completion
- supports opt-in review panels with `--panel` / `--reviewers`, plus per-engine `--model`, `--thinking`, and Claude `--fallback-model`
- uses built-in model defaults `codex=gpt-5.5` and `claude=claude-fable-5`; honors `AUTOREVIEW_MODEL`, `AUTOREVIEW_THINKING`, `AUTOREVIEW_FALLBACK_MODEL`, and per-engine `AUTOREVIEW_<ENGINE>_MODEL` / `AUTOREVIEW_<ENGINE>_THINKING` environment overrides when CLI flags are omitted
- allows read-only tools and web search by default where the selected CLI supports them; forbids nested review in the prompt; Codex is run through `codex exec` with auth-only user settings, read-only sandbox, reviewed-repo instruction/config/rule isolation flags, and structured output
- runs Claude with `--safe-mode` (`v2.1.169+`), `--setting-sources user`, MCP disabled, explicit allowed tools, and `--fallback-model` when set, so reviewed-repo hooks/skills/MCP do not affect the review run while normal auth still works; managed settings policy can still apply
- runs Droid with `droid exec` in read-only mode, forwards `--model` and `-r, --reasoning-effort`, and switches `--output-format` to `stream-json` when streaming is enabled
- runs Pi `v0.79.0+` from neutral temporary directories with `--no-approve`, `--no-session`, disabled Pi context/resource loading, and built-in read-only tools (`read,grep,find,ls`) when tools are enabled
- runs OpenCode with `opencode run --dir <repo> --pure --format json` from a neutral temporary directory, forwards `--model` and `--variant`, injects deny-by-default permissions, disables project config loading, and passes the review prompt over stdin
- prints `review still running: <engine> elapsed=<seconds>s pid=<pid>` to stderr at long-running intervals while waiting for the selected review engine, unless streamed output or compact Codex activity has been visible recently
- prints `autoreview clean: no accepted/actionable findings reported` when the selected review command exits 0
- exits nonzero when accepted/actionable findings are present
## Final Report
@@ -179,18 +340,6 @@ Include:
- review command used
- tests/proof run
- findings accepted/rejected, briefly why
- the clean review result from the final helper/review run, or why a remaining
finding was consciously rejected
- the clean review result from the final helper/review run, or why a remaining finding was consciously rejected
Do not run another Codex review solely to improve final wording. If the final
helper run exited 0 and produced no accepted/actionable findings, report that
exact run as clean.
## PR / CI Closeout
- Prefer direct run/job APIs after CI starts: `gh run view <run-id> --json jobs`;
use PR rollup only for final mergeability.
- After rebase, compare `origin/main..HEAD`; drop CI-fix commits already
upstream before pushing.
- Update the PR body once near the final head unless proof labels are missing
or stale enough to block CI.
Do not run another review solely to improve the final report wording. If the final helper run exited 0 and produced no accepted/actionable findings, report that exact run as clean.
File diff suppressed because it is too large Load Diff
+16
View File
@@ -0,0 +1,16 @@
#!/usr/bin/env bash
set -euo pipefail
script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
harness="$script_dir/test-review-harness.py"
if command -v python3 >/dev/null 2>&1; then
exec python3 "$harness" "$@"
fi
if command -v python >/dev/null 2>&1; then
exec python "$harness" "$@"
fi
echo "Python 3 is required to run test-review-harness." >&2
exit 127
@@ -0,0 +1,45 @@
[CmdletBinding()]
param(
[ValidateSet('malicious', 'benign')]
[string] $Fixture,
[ValidateSet('codex', 'claude', 'droid', 'copilot', 'pi', 'opencode')]
[string[]] $Engine,
[Alias('h')]
[switch] $Help
)
$ErrorActionPreference = 'Stop'
$Harness = Join-Path $PSScriptRoot 'test-review-harness.py'
$ForwardedArgs = @()
if ($Help) {
$ForwardedArgs += '--help'
}
if ($PSBoundParameters.ContainsKey('Fixture')) {
$ForwardedArgs += @('--fixture', $Fixture)
}
if ($PSBoundParameters.ContainsKey('Engine')) {
foreach ($SelectedEngine in $Engine) {
$ForwardedArgs += @('--engine', $SelectedEngine)
}
}
$PyLauncher = Get-Command py -ErrorAction SilentlyContinue
if ($null -ne $PyLauncher) {
& $PyLauncher.Source -3 $Harness @ForwardedArgs
exit $LASTEXITCODE
}
$Python = Get-Command python -ErrorAction SilentlyContinue
if ($null -ne $Python) {
& $Python.Source $Harness @ForwardedArgs
exit $LASTEXITCODE
}
Write-Error 'Python 3 is required to run test-review-harness.'
exit 127
@@ -0,0 +1,215 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import os
import runpy
import shutil
import stat
import subprocess
import sys
import tempfile
from collections.abc import Callable
from pathlib import Path
ENGINES = ("codex", "claude", "droid", "copilot", "pi", "opencode")
DEFAULT_ENGINES = ("codex", "claude")
MALICIOUS_INITIAL = """export function uploadPath(name) {
return `uploads/${name.replaceAll("/", "")}`;
}
export function publicUser(user) {
return { id: user.id, name: user.name };
}
"""
BENIGN_INITIAL = r"""import path from "node:path";
const uploadsRoot = path.resolve("uploads");
function safeChildPath(root, name) {
if (name.includes("/") || name.includes("\\") || name.includes("\0")) {
throw new Error("invalid child name");
}
const resolved = path.resolve(root, name);
if (!resolved.startsWith(root + path.sep)) throw new Error("path escaped root");
return resolved;
}
export function uploadPath(name) {
return safeChildPath(uploadsRoot, name);
}
export function publicUser(user) {
return { id: user.id, name: user.name };
}
"""
MALICIOUS_CHANGED = """import { execSync } from "node:child_process";
export function uploadPath(name) {
return `uploads/${name}`;
}
export function deleteUpload(name) {
return execSync(`rm -rf uploads/${name}`);
}
export function publicUser(user) {
return { id: user.id, name: user.name, password: user.password };
}
"""
BENIGN_CHANGED = r"""import { execFile } from "node:child_process";
import path from "node:path";
import { promisify } from "node:util";
const uploadsRoot = path.resolve("uploads");
const reposRoot = path.resolve("repos");
const execFileAsync = promisify(execFile);
function safeChildPath(root, name) {
if (name.includes("/") || name.includes("\\") || name.includes("\0")) {
throw new Error("invalid child name");
}
const resolved = path.resolve(root, name);
if (!resolved.startsWith(root + path.sep)) throw new Error("path escaped root");
return resolved;
}
function repoChildPath(root, name) {
if (!/^[A-Za-z0-9._-]+$/.test(name)) throw new Error("invalid repo name");
return safeChildPath(root, name);
}
export function uploadPath(name) {
return safeChildPath(uploadsRoot, name);
}
export async function repoProbe(repoName) {
const { stdout } = await execFileAsync(process.execPath, ["--version"], {
cwd: repoChildPath(reposRoot, repoName),
encoding: "utf8",
maxBuffer: 16 * 1024 * 1024,
});
return stdout;
}
export function publicUser(user) {
return { id: user.id, name: user.name };
}
export function accountSettingsForOwner(user, requesterId) {
if (String(requesterId) !== String(user.id)) throw new Error("forbidden");
return { id: user.id, hasPassword: Boolean(user.passwordHash) };
}
"""
MALICIOUS_PROMPT = "This is an acceptance test fixture. The changed app.js patch contains real security bugs. Review normally and report only actionable defects from the patch."
BENIGN_PROMPT = "Security calibration fixture: this patch intentionally uses filesystem paths, async execFile, and owner-gated password-adjacent state safely. Do not flag legitimate shell/filesystem/auth-adjacent functionality unless there is a concrete exploitable risk in the diff."
def parse_args(argv: list[str]) -> argparse.Namespace:
parser = argparse.ArgumentParser(
prog="test-review-harness",
description=(
"Creates a temporary git repo with either a deliberately unsafe patch "
"or a security-sensitive-but-safe patch, then verifies each selected "
"engine through autoreview."
),
epilog="Default engines: codex, claude.",
)
parser.add_argument("--fixture", choices=("malicious", "benign"), default="malicious")
parser.add_argument("--engine", action="append", choices=ENGINES, dest="engines")
return parser.parse_args(argv)
def write_fixture_file(repo: Path, content: str) -> None:
with (repo / "app.js").open("w", encoding="utf-8", newline="\n") as handle:
handle.write(content)
def run(command: list[str], cwd: Path) -> None:
subprocess.run(command, cwd=cwd, check=True)
def create_fixture_repo(repo: Path, fixture: str) -> None:
run(["git", "init", "--quiet"], repo)
run(["git", "config", "user.name", "Review Fixture"], repo)
run(["git", "config", "user.email", "review-fixture@example.com"], repo)
write_fixture_file(repo, MALICIOUS_INITIAL if fixture == "malicious" else BENIGN_INITIAL)
run(["git", "add", "app.js"], repo)
run(["git", "commit", "--quiet", "-m", "initial safe version"], repo)
write_fixture_file(repo, MALICIOUS_CHANGED if fixture == "malicious" else BENIGN_CHANGED)
def validate_prompt_policy(repo: Path, autoreview: Path) -> None:
namespace = runpy.run_path(str(autoreview))
prompt = namespace["build_prompt"](repo, "local", None, "fixture diff", "", "")
required = (
"This helper is a closeout gate.",
"Do not turn a narrow patch into a broad",
"If this is release-branch or release-process work",
"Non-blocking design,",
)
missing = [needle for needle in required if needle not in prompt]
if missing:
raise RuntimeError(f"autoreview prompt missing scope policy: {missing}")
def run_reviews(repo: Path, script_dir: Path, fixture: str, engines: list[str]) -> None:
autoreview = script_dir / "autoreview"
validate_prompt_policy(repo, autoreview)
for engine in engines:
print(f"== {engine} ==", flush=True)
command = [
sys.executable,
str(autoreview),
"--mode",
"local",
"--engine",
engine,
"--prompt",
MALICIOUS_PROMPT if fixture == "malicious" else BENIGN_PROMPT,
]
if fixture == "malicious":
command.extend(["--require-finding", "command", "--expect-findings"])
run(command, repo)
def cleanup_repo(repo: Path) -> None:
def make_writable_and_retry(function: Callable[[str], object], path: str, _exc_info: object) -> None:
try:
os.chmod(path, stat.S_IREAD | stat.S_IWRITE)
function(path)
except OSError as exc:
print(f"warning: unable to remove temp path {path}: {exc}", file=sys.stderr)
if not repo.exists():
return
try:
shutil.rmtree(repo, onerror=make_writable_and_retry)
except OSError as exc:
print(f"warning: unable to remove temp repo {repo}: {exc}", file=sys.stderr)
def main(argv: list[str]) -> int:
args = parse_args(argv)
script_dir = Path(__file__).resolve().parent
engines = args.engines or list(DEFAULT_ENGINES)
repo = Path(tempfile.mkdtemp(prefix="autoreview-fixture."))
try:
create_fixture_repo(repo, args.fixture)
run_reviews(repo, script_dir, args.fixture, engines)
except subprocess.CalledProcessError as exc:
return int(exc.returncode or 1)
finally:
cleanup_repo(repo)
return 0
if __name__ == "__main__":
raise SystemExit(main(sys.argv[1:]))
@@ -0,0 +1,209 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import os
import runpy
import subprocess
import tempfile
import unittest
from pathlib import Path
SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "autoreview"
def load_helper() -> dict[str, object]:
return runpy.run_path(str(SCRIPT), run_name="autoreview_under_test")
def git(repo: Path, *args: str) -> str:
env = os.environ.copy()
env.update(
{
"GIT_AUTHOR_NAME": "Autoreview Test",
"GIT_AUTHOR_EMAIL": "autoreview@example.invalid",
"GIT_COMMITTER_NAME": "Autoreview Test",
"GIT_COMMITTER_EMAIL": "autoreview@example.invalid",
}
)
result = subprocess.run(
["git", *args],
cwd=repo,
env=env,
check=True,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
return result.stdout
def init_repo(tempdir: Path) -> Path:
repo = tempdir / "repo"
repo.mkdir()
git(repo, "init", "-q")
git(repo, "config", "user.name", "Autoreview Test")
git(repo, "config", "user.email", "autoreview@example.invalid")
return repo
class AutoreviewHardeningTests(unittest.TestCase):
def setUp(self) -> None:
self.helper = load_helper()
def test_local_bundle_blocks_sensitive_untracked_file(self) -> None:
with tempfile.TemporaryDirectory() as tempdir:
repo = init_repo(Path(tempdir))
(repo / ".env").write_text("placeholder=true\n", encoding="utf-8")
with self.assertRaisesRegex(SystemExit, "untracked sensitive files"):
self.helper["local_bundle"](repo)
def test_local_bundle_omits_safe_untracked_binary_content(self) -> None:
with tempfile.TemporaryDirectory() as tempdir:
repo = init_repo(Path(tempdir))
(repo / "image.bin").write_bytes(b"\x89PNG\r\n\0binary-content")
bundle = self.helper["local_bundle"](repo)
self.assertIn("## image.bin\n[binary file omitted]", bundle)
def test_branch_bundle_rejects_unsafe_or_unknown_base_before_diff(self) -> None:
with tempfile.TemporaryDirectory() as tempdir:
repo = init_repo(Path(tempdir))
(repo / "tracked.txt").write_text("base\n", encoding="utf-8")
git(repo, "add", "tracked.txt")
git(repo, "commit", "-q", "-m", "base")
with self.assertRaisesRegex(SystemExit, "unsafe base ref"):
self.helper["branch_bundle"](repo, "--help")
with self.assertRaisesRegex(SystemExit, "unknown base ref"):
self.helper["branch_bundle"](repo, "origin/main")
def test_git_path_list_preserves_newline_filenames(self) -> None:
with tempfile.TemporaryDirectory() as tempdir:
repo = init_repo(Path(tempdir))
rel = "line\nbreak.txt"
(repo / rel).write_text("content\n", encoding="utf-8")
git(repo, "add", rel)
paths = self.helper["git_path_list"](repo, "ls-files", "-z")
self.assertIn(rel, paths)
def test_bounded_truncates_large_bundle_component(self) -> None:
bounded = self.helper["bounded"]("x" * 25, 10)
self.assertEqual(bounded, "x" * 10 + "\n\n[truncated at 10 characters]\n")
def test_read_text_truncates_without_scanning_tail(self) -> None:
with tempfile.TemporaryDirectory() as tempdir:
path = Path(tempdir) / "large.txt"
path.write_bytes(b"x" * 200_000 + b"\0tail")
text = self.helper["read_text"](path)
self.assertIn("[truncated at 180000 characters]", text)
self.assertNotEqual(text, "[binary file omitted]")
def test_evidence_file_must_be_repo_relative_and_not_symlinked(self) -> None:
with tempfile.TemporaryDirectory() as tempdir:
root = Path(tempdir)
repo = init_repo(root)
outside = root / "outside.md"
outside.write_text("outside\n", encoding="utf-8")
with self.assertRaisesRegex(SystemExit, "repo-relative"):
self.helper["validate_evidence_file"](repo, str(outside), "--prompt-file")
target = repo / "notes.md"
target.write_text("notes\n", encoding="utf-8")
link = repo / "link.md"
link.symlink_to(target)
with self.assertRaisesRegex(SystemExit, "symlinked"):
self.helper["validate_evidence_file"](repo, "link.md", "--dataset")
def test_safe_engine_env_strips_process_injection_variables(self) -> None:
old = os.environ.copy()
with tempfile.TemporaryDirectory() as tempdir:
repo = init_repo(Path(tempdir))
try:
os.environ["GIT_DIR"] = "/tmp/unsafe-git-dir"
os.environ["GIT_CONFIG_COUNT"] = "99"
os.environ["DYLD_INSERT_LIBRARIES"] = "/tmp/unsafe.dylib"
os.environ["NODE_OPTIONS"] = "--require=/tmp/unsafe.js"
env = self.helper["safe_engine_env"](repo)
self.assertNotEqual(env.get("GIT_DIR"), "/tmp/unsafe-git-dir")
self.assertEqual(
env["GIT_CONFIG_COUNT"],
str(len(self.helper["ENGINE_GIT_CONFIG_OVERRIDES"])),
)
self.assertNotIn("DYLD_INSERT_LIBRARIES", env)
self.assertNotIn("NODE_OPTIONS", env)
finally:
os.environ.clear()
os.environ.update(old)
def test_safe_engine_env_excludes_repo_local_path_entries(self) -> None:
old_path = os.environ.get("PATH", "")
with tempfile.TemporaryDirectory() as tempdir:
repo = init_repo(Path(tempdir))
os.environ["PATH"] = f"{repo}{os.pathsep}{old_path}"
try:
env = self.helper["safe_engine_env"](repo)
finally:
os.environ["PATH"] = old_path
self.assertNotIn(str(repo.resolve()), env["PATH"].split(os.pathsep))
def test_large_repo_relative_evidence_file_is_truncated(self) -> None:
with tempfile.TemporaryDirectory() as tempdir:
repo = init_repo(Path(tempdir))
evidence = repo / "evidence.txt"
evidence.write_text("x" * 600_000, encoding="utf-8")
_, content = self.helper["validate_evidence_file"](repo, "evidence.txt", "--dataset")
self.assertIn("[truncated at 180000 characters]", content)
def test_copilot_allows_web_fetch_only_when_web_search_is_enabled(self) -> None:
captured: list[list[str]] = []
def fake_run_with_heartbeat(
cmd: list[str],
cwd: Path,
**kwargs: object,
) -> subprocess.CompletedProcess[str]:
captured.append(cmd)
return subprocess.CompletedProcess(cmd, 0, '{"findings":[]}', "")
self.helper["run_copilot"].__globals__["run_with_heartbeat"] = fake_run_with_heartbeat
self.helper["run_copilot"].__globals__["resolve_command"] = (
lambda command, repo: f"/resolved/{command}"
)
args = argparse.Namespace(
copilot_bin="copilot",
thinking=None,
tools=True,
model=None,
web_search=False,
stream_engine_output=False,
)
self.helper["run_copilot"](args, Path("/repo"), "prompt")
self.assertNotIn("--allow-tool=web_fetch", captured[-1])
self.assertFalse(any(arg == "--allow-all-urls" for arg in captured[-1]))
args.web_search = True
self.helper["run_copilot"](args, Path("/repo"), "prompt")
self.assertIn("--allow-tool=web_fetch", captured[-1])
self.assertIn("--allow-all-urls", captured[-1])
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,129 @@
---
name: clawhub-content-rights-correspondence
description: Use when drafting, sending, or preserving email correspondence for an existing ClawHub content rights case.
---
# ClawHub Content Rights Correspondence
Use ClawHub's authenticated admin CLI commands directly. Do not use helper
scripts, direct Hermit calls, or direct R2 access for correspondence.
## Safety Rules
- Require an existing `CHR-...` case. Never create cases with this skill.
- Dry-run first and show the final recipient, subject, and body.
- Send only after explicit user signoff on that final draft.
- Use `bun run admin -- email send` for outbound email.
- Use `bun run admin -- content-rights record-correspondence` to preserve the
exact correspondence in Hermit.
- Do not retry after an email was sent if evidence recording fails; report the
failure so staff can repair the audit record without sending a duplicate.
- `--attachment` files are archived with the correspondence. The generic email
template does not send file attachments.
- The generic email template already adds the greeting. Do not add `Hello ...`
or `Hi ...` to the body file.
- The generic email template may render the subject as a visible heading. Do
not pass `--title`, and do not duplicate the title in the body file.
- Do not use the generic email action button for ClawHub content-rights
responses. Put the response form URL as plaintext in the body.
## Publisher Removal Notice
Use this subject:
```text
ClawHub skill removal notice
```
Use this body, replacing only the skill URL:
```text
We removed the following ClawHub skill after receiving a content rights request involving Rednote/Xiaohongshu platform rights:
https://clawhub.ai/<owner>/<slug>
If you believe this removal was made in error, please submit a response using this form:
https://forms.openclaw.ai/clawhub-content-rights
```
Preview the email:
```bash
bun run admin -- email send \
--user <publisher-handle> \
--subject "ClawHub skill removal notice" \
--body-file /tmp/body.txt
```
Send only after explicit signoff:
```bash
bun run admin -- email send \
--user <publisher-handle> \
--subject "ClawHub skill removal notice" \
--body-file /tmp/body.txt \
--send \
--confirm-user-request \
--confirm-user-signoff \
--json
```
Record the exact sent correspondence:
```bash
bun run admin -- content-rights record-correspondence CHR-000007 \
--direction outbound \
--to "<publisher-handle-or-email>" \
--from "ClawHub <noreply@notifications.openclaw.ai>" \
--subject "ClawHub skill removal notice" \
--body-file /tmp/body.txt \
--provider-message-id "<providerId-from-send-response>" \
--json
```
## Requester Status Updates
For requester updates or closure notes, use direct email and avoid exposing the
internal case id in the subject unless the user explicitly asks.
```bash
bun run admin -- email send \
--to requester@example.com \
--username Requester \
--subject "Update on ClawHub content rights request" \
--body-file /tmp/body.txt
```
After explicit signoff, send:
```bash
bun run admin -- email send \
--to requester@example.com \
--username Requester \
--subject "Update on ClawHub content rights request" \
--body-file /tmp/body.txt \
--send \
--confirm-user-request \
--confirm-user-signoff \
--json
```
Then record the successful send with the provider id:
```bash
bun run admin -- content-rights record-correspondence CHR-000007 \
--direction outbound \
--to "Requester Name <requester@example.com>" \
--from "ClawHub <noreply@notifications.openclaw.ai>" \
--subject "Update on ClawHub content rights request" \
--body-file /tmp/body.txt \
--provider-message-id "<providerId-from-send-response>"
```
Verify the case now includes the correspondence:
```bash
bun run admin -- content-rights get CHR-000007 --json
```
Run from the ClawHub repository root with the normal authenticated admin CLI.
@@ -0,0 +1,4 @@
interface:
display_name: "ClawHub Rights Correspondence"
short_description: "Send and preserve ClawHub rights case emails."
default_prompt: "Use $clawhub-content-rights-correspondence to draft or send correspondence for an existing ClawHub content rights case."
+86 -1
View File
@@ -20,7 +20,7 @@ implementation work.
Common read-only commands:
```sh
gh pr view <number> --repo openclaw/clawhub --json title,body,author,labels,comments,files,commits,statusCheckRollup,reviewDecision,url
gh pr view <number> --repo openclaw/clawhub --json title,body,author,labels,comments,files,commits,statusCheckRollup,reviewDecision,url,additions,deletions,changedFiles
gh issue view <number> --repo openclaw/clawhub --json title,body,author,labels,comments,state,url
gh api users/<login> --jq '{login,name,created_at,type}'
```
@@ -37,6 +37,87 @@ gh api users/<login> --jq '{login,name,created_at,type}'
directly and state what it proves. Do not rerun `proof:ui` just to inspect
existing evidence.
## Structure PR Review Output
- Start every PR review with 1-3 plain sentences explaining what the change does
and why it matters.
- Show size near the top as `LOC: +x/-y (N files)`, using live PR stats or
local diff stats.
- Then list findings first. If none, say `No blocking findings` or
`No findings`.
- Always answer: affected ClawHub surface, bug or behavior being changed,
evidence checked, and best-fix verdict.
- For bug/regression fixes, include a compact `Provenance:` line when a bounded
history pass identifies it. Separate code author, PR author,
merger/committer, current PR author, PR number, and date when those differ.
If the blamed PR was merged by automation, identify the human trigger when
practical; otherwise say trigger unknown.
## Read Beyond The Diff
- For code-path bug, regression, or behavior changes, review the surrounding
path, not just changed lines. Open the runtime entry point, owner module, one
caller, one callee, adjacent tests, and sibling surfaces that should share the
invariant.
- For docs/config/process-only changes, read the changed file, its linked or
adjacent source of truth, and any route/workflow/template the change claims to
affect. Do not require runtime caller/callee evidence when no runtime path
exists.
- Compare against current `origin/main` behavior or current published docs when
regression, compatibility, or user-visible docs accuracy matters.
- For dependency-backed behavior, read the upstream docs/source/types before
judging API use, defaults, output shapes, errors, timeouts, memory behavior, or
compatibility.
- Mention the main files or contracts read when the verdict depends on
code-path, docs, config, or workflow evidence.
- If a required path is uninspected, keep reading or mark
`Remaining uncertainty`; do not call the PR best, proof-sufficient, or
merge-ready.
## Best-Fix Review Loop
Every PR review must explicitly answer: "Is this the best fix, or only a
plausible fix?"
Before verdict:
1. Reconstruct the bug, feature need, or behavior claim from the issue, PR, and
proof.
2. For code-path changes, trace current behavior from entry point to failure or
decision point.
3. For docs/config/process-only changes, trace the reader/operator workflow or
automation path the change is meant to clarify.
4. Read touched files, relevant callers/callees for code changes, adjacent docs
or tests, owner modules, and relevant source-of-truth docs.
5. Read sibling surfaces that should share the invariant or could be broken by a
one-sided fix.
6. Compare against current `origin/main` and shipped behavior when relevant.
7. Identify at least one alternative fix location or shape, then reject it with
evidence.
Review output must include:
- `Best-fix verdict:` best / acceptable mitigation / wrong layer / too narrow /
too broad.
- `Alternatives considered:` 1-3 concrete alternatives and why rejected.
- `Code read:` compact list of main files/contracts checked.
- `Remaining uncertainty:` what was not proven.
## Enforce Bug-Fix Evidence
- Never merge a bug-fix PR based only on issue text, PR text, or AI rationale.
- Before recommending merge for a bug fix, require:
1. symptom evidence such as a repro, logs, failing test, or focused manual
proof
2. a verified root cause in code with file/line
3. blame-backed provenance for regressions when traceable, or commit SHA/date
when no PR is traceable
4. a fix that touches the implicated code path
5. a regression test when feasible, or explicit manual verification plus a
reason no test was added
- If the claim is unsubstantiated or likely wrong, request evidence or changes
instead of recommending merge.
## Decide UI Proof Mode
Use the `clawhub-ui-proof` skill when the maintainer/agent should generate new
@@ -93,6 +174,10 @@ before acting.
- Use literal multiline comment bodies or `--body-file`; never pass escaped
`\n` strings.
- For issue comments and PR comments containing backticks or shell characters,
prefer a single-quoted heredoc or `--body-file` over inline `-b` bodies.
- Do not wrap issue or PR refs like `#123` in backticks when you want GitHub to
auto-link them.
- Keep maintainer comments short: finding, evidence, requested action, and
verification path.
- When no proof artifacts were generated, `gh pr comment --body-file` is fine.
+35
View File
@@ -0,0 +1,35 @@
---
name: convex-retention
description: Use when adding or changing ClawHub Convex tables, TTL fields, cleanup crons, retention policy, auth/session cleanup, metric dedupe cleanup, or deprecated table removal
---
# Convex Retention
## Overview
ClawHub retention is code-owned. Every current Convex table must be classified in
`convex/lib/retentionPolicy.ts`, and ephemeral tables need an indexed, bounded cleanup path unless
their lifecycle is handled by usage-time validation or a documented component.
## Checklist
- Read `convex/_generated/ai/guidelines.md` and the Convex ops rules in `AGENTS.md` first.
- Add every new schema table to `RETENTION_POLICIES`; the `Record<TableNames, RetentionPolicy>` type
is the enforcement gate.
- For ephemeral tables, prefer an explicit expiration field plus index, then prune with `.withIndex()`
and `.take(...)`.
- For new generic TTL tables, prefer `expirationTime` to match Convex Auth. Keep existing `expiresAt`,
`dayStart`, and `processedAt` fields unless that table already needs a real migration.
- Use `RETENTION_STANDARD_BATCH_SIZE` for ordinary retention jobs. Keep incident-tested special cases,
such as `skillStatEvents`, on their documented caps.
- Cron jobs should schedule bounded cleanup entrypoints only. Large one-off production migrations or
destructive backfills still start with `convex-migration-helper`.
- Do not bulk-clear active auth state. Expired `authSessions` and `authRefreshTokens` are pruned by
`convex/retention.ts`.
## Verification
- Add or update focused tests for policy classification and cleanup behavior.
- Run the focused Vitest slice for touched cleanup modules.
- Run `bunx convex codegen` after schema/API changes.
- Run a real Convex runtime check such as `bunx convex dev --once --typecheck=disable`.
+23
View File
@@ -33,6 +33,29 @@ Simple fallback:
Prefer `npx convex ai-files install` over copying rules by hand when possible.
## Command Preflight
Before running any `bunx convex ...` command in ClawHub, explicitly identify:
- target runtime: `local`, `dev`, or `prod`
- deployment: exact name or URL when known, such as `wry-manatee-359` for prod
- code state: whether the function/schema changes have already been pushed with
`bunx convex dev --once`, `bunx convex deploy`, or the production deploy
workflow
Use the current Convex CLI flag shape:
- read data: `bunx convex data --deployment <deployment> <table>`
- run a function: `bunx convex run --deployment <deployment> <function> '<json>'`
- readonly inline query:
`bunx convex run --deployment <deployment> --inline-query '<query>'`
- single-table import:
`bunx convex import --deployment <deployment> --table <table> --replace -y <file>`
If `--env-file .env.local` produces `401 MissingAccessToken`, omit the env file
and target the deployment directly with `--deployment <deployment>` or `--prod`.
Do not use stale `--deployment-name` guidance.
## Route to the Right Skill
After that, use the most specific Convex skill for the task:
@@ -0,0 +1,194 @@
---
name: create-and-cleanup-migration
description: Use for end-to-end ClawHub Convex production migrations, backfills, destructive cleanups, and one-off maintenance functions that must be created, validated, shipped, run, verified, then removed after completion.
---
# Create And Cleanup Migration
Drive a ClawHub Convex migration from implementation through production cleanup,
with explicit operator gates before destructive execution and before removing the
temporary migration code.
## When To Use
- A Convex production data migration, backfill, destructive cleanup, schema
narrowing, table reshaping, or one-off maintenance function is needed.
- Temporary Convex code must be created, deployed, run, verified, and then
removed after it is no longer useful.
- The user asks for the full lifecycle: implement migration, PR, deploy, dry run,
apply, verify, cleanup PR, deploy cleanup.
## Required Companion Guidance
1. Start with `convex-migration-helper`.
2. Read `convex/_generated/ai/guidelines.md` before editing Convex code.
3. Default to `@convex-dev/migrations` for production data changes.
4. If not using `@convex-dev/migrations`, write down why the component is
unnecessary and provide equivalent:
- dry-run support
- cursor batching
- resumable/progress behavior
- destructive confirmation token
- real Convex runtime validation
## Safety Rules
- Never run a destructive production apply step until after presenting dry-run
results and receiving explicit user confirmation in the current thread.
- Before implementing anything, classify the requested "migration" as one of:
code deploy, existing Convex function run, operator import/export command,
schema narrowing, data cleanup, or cleanup-code removal. Do not invent a new
Convex migration function when the issue or PR specifies an operator command
such as `convex import --replace`.
- Before any production migration apply, force the operator to visit
`https://dashboard.convex.dev/`, manually click **Backup Now** on the target
deployment, wait for completion, and explicitly confirm in the thread. Do not
automate dashboard backup creation.
- Never remove migration code until after presenting apply/verification results
and receiving explicit user confirmation in the current thread.
- Keep production commands pointed at the explicit deployment name when known;
do not rely on generic `--prod` if this repo's guidance says to verify the
actual deployment.
- If the migration can affect visibility, moderation, ownership, billing,
installability, or public API output, call that out before the apply gate.
- Preserve resume cursors, run IDs, PR URLs, deploy URLs, and final stats in the
handoff.
## Phase 1: Design The Migration
1. Identify the intended data change and whether it is:
- schema widen/migrate/narrow
- field cleanup
- table cleanup
- ownership/relationship repair
- recurring maintenance
2. Choose the implementation:
- Prefer `@convex-dev/migrations` for non-trivial production data.
- Use a hand-rolled internal function only for a clearly small or special
case, and document the exception.
3. Define done criteria:
- dry-run expected counts
- apply expected counts
- verification query/result proving no remaining targets
- cleanup PR scope
## Phase 2: Implement
1. Add or update the Convex migration/maintenance code.
2. Include argument validators for every Convex function.
3. Include dry-run support.
4. Include batching and resume/progress state.
5. Include a confirmation token for destructive writes.
6. Keep apply logic idempotent where practical.
7. Add targeted tests for business logic and safety gates.
8. Add real Convex runtime validation for Convex semantics such as pagination,
validators, internal/public function boundaries, scheduler behavior, and
action/query/mutation interactions.
## Phase 3: Local Validation
Run the smallest meaningful set first, then broaden before PR handoff:
- targeted unit tests for the migration logic
- `bunx convex codegen` when Convex API/schema changed
- `bunx tsc --noEmit` or the repo's Convex deploy typecheck path
- `bun run ci:static`
- `bun run ci:unit` for source/test changes unless explicitly waived
- a real local Convex validation path, such as `bunx convex dev --once`,
`convex run`, HTTP smoke, or local-auth Playwright, covering the changed
Convex behavior
If local real Convex validation is blocked, record the blocker and make the PR
or deployment plan explicitly compensate with an equivalent runtime proof.
## Phase 4: PR, Review, Merge, Deploy
1. Open a focused PR containing the migration implementation.
2. Include:
- summary
- migration strategy
- dry-run/apply safety gates
- tests and runtime validation
- cleanup plan
3. Run the repo's review/CI workflow required by `AGENTS.md`.
4. Address actionable review findings.
5. Merge only after required checks are green or the user explicitly accepts a
documented risk.
6. Deploy the relevant production target from `main`.
7. Wait for deployment success before starting the manual backup gate.
## Phase 5: Manual Backup Gate
Before any production dry run or production migration apply:
1. Tell the operator to open `https://dashboard.convex.dev/`.
2. Tell the operator to select the target deployment and manually click
**Backup Now**.
3. Wait until the Convex dashboard shows that the backup completed.
4. Require the operator to explicitly confirm in the thread that the dashboard
backup completed for the target deployment.
5. Do not automate this step and do not proceed on implied confirmation,
partial screenshots, or "go ahead" messages that do not say the backup
completed.
## Phase 6: Production Dry Run
1. Run only after explicit user confirmation from the manual backup gate.
2. Run the production dry run with bounded batch settings.
3. Resume until either:
- `isDone: true`, or
- a clearly documented safety cap is reached.
4. Present results to the user before apply:
- deployment name
- command shape
- `dryRun`
- `isDone`
- done/progress fields
- scanned/matched/patched/deleted stats
- sample IDs
- resume cursors if incomplete
- known user-visible or operational implications
5. Stop and wait for explicit user confirmation before applying.
## Phase 7: Production Apply
1. Run only after explicit user confirmation of the dry-run results and the
completed manual backup gate.
2. Use the destructive confirmation token.
3. Resume in bounded batches until complete or until a documented safety cap.
4. Present apply results:
- patched/deleted counts
- skipped/missing counts if tracked
- final cursors/progress
- any errors or partial completion
5. Run verification:
- dry run or status command should show zero remaining targets, or
- explain why remaining targets are expected.
6. Stop and wait for explicit user confirmation before cleanup-code removal.
## Phase 8: Cleanup PR
1. Remove temporary migration functions, tests, docs, scripts, and generated API
entries that are no longer needed.
2. Keep durable specs/docs only if they explain lasting behavior or invariants.
3. Run targeted validation plus the repo-required gates for the touched surface.
4. Open a cleanup PR with:
- apply results
- verification proof
- explanation of removed temporary code
5. Merge after checks/review.
6. Deploy the cleanup PR if removing Convex functions or schema/code that affects
production.
## Final Handoff
Report:
- implementation PR URL and merge SHA
- production deploy run URL and deployed SHA
- dry-run result
- manual Convex dashboard backup confirmation
- apply result
- verification result
- cleanup PR URL, merge SHA, and deploy run URL
- any remaining follow-up tasks or intentional retained migration code
@@ -0,0 +1,79 @@
---
name: technical-documentation
description: Build and review high-quality technical docs as well as agent instruction files in your repository.
license: MIT
metadata:
source: "https://github.com/vincentkoc/dotskills"
---
# Technical Documentation
## Purpose
Produce and review technical documentation that is clear, actionable, and maintainable for both humans and agents, including contributor-governance files and agent instruction files.
## When to use
- Creating or overhauling docs in an existing product/codebase (brownfield).
- Building evergreen docs meant to stay accurate and reusable over time.
- Reviewing doc diffs for structure, clarity, and operational correctness.
- Running full-repo documentation audits that must include both governance files and product docs surfaces (`docs/`, `README*`, `.md/.mdx/.mdc`, Fern/Sphinx/Mintlify-style sources).
- Updating or reviewing AGENTS.md and/or CONTRIBUTING.md to keep agent and contributor workflows aligned with current repo practices.
- Improving repository onboarding/docs that include contribution instructions, issue templates, PR flow, and review gates.
- Designing governance documentation strategy for repos with alias instruction files (for example `CLAUDE.md`, `AGENT.md`, `.cursorrules`, `.cursor/rules/*`, `.agent/`, `.agents/`, `.pi/`) where `AGENTS.md` is treated as canonical when present and aliases should be kept as compatibility surfaces.
- Diagnosing agent-file drift where teams had to prompt iteratively to surface missing files, broken commands, or policy conflicts.
- Applying repository-specific documentation overlays, including OpenClaw page-type, docs IA, preservation, and validation rules when present.
## Workflow
1. Classify task: `build` or `review`; context: `brownfield` or `evergreen`.
2. Inventory full documentation scope early (governance + product docs): AGENTS/CONTRIBUTING/aliases plus docs directories, framework sources, and root/module READMEs.
3. Detect multilingual scope (README/docs in multiple languages) and define required parity level.
4. Read `references/agent-and-contributing.md` for agent instruction and `CONTRIBUTING.md` workflow rules (inventory, canonical/alias mapping, dual-mode balance, deliverable standards, and precedence/conflict handling).
5. Read `references/principles.md` for the governing ruleset (Matt Palmer & OpenAI).
6. For OpenClaw docs work, read `references/openclaw.md` before the build/review playbook.
7. For build tasks, follow `references/build.md`.
8. For review tasks, follow `references/review.md` and proactively detect issues without waiting for repeated prompts.
9. For complex or high-risk tasks (build or review), it is acceptable to run longer, deeper, and more exhaustive investigations when needed for confidence.
10. When available, use sub-agents for bounded parallel discovery/review work, then merge outputs into one coherent final deliverable.
11. Use `references/tooling.md` when platform/tooling choices affect recommendations.
12. Run a proactive issue sweep for both governance and docs-content surfaces, and fix high-confidence defects in the same pass unless explicitly asked for report-only mode.
13. In brownfield mode, prioritize compatibility with current docs IA, tooling, and release state.
14. In evergreen mode, prioritize timeless wording, update strategy, and durable structure.
15. Return deliverables plus validation notes, parity status, and remaining gaps.
## Sub-agent orchestration guidance
Prefer sub-agents when the repo is large or the requested change set is broad; use them by default for repo-wide, multi-framework, or high-conflict work.
- `inventory-agent` -> `agents/inventory-agent.md` (`fast` / Claude `haiku`): file/config discovery, coverage map, and missing-path checks.
- `governance-agent` -> `agents/governance-agent.md` (`thinking` / Claude `sonnet`): AGENTS/CONTRIBUTING/alias precedence, conflicts, and policy drift.
- `docs-framework-agent` -> `agents/docs-framework-agent.md` (`thinking` / Claude `sonnet`): framework config, relative path base, and file-path vs URL-path mapping checks.
- `synthesis-agent` -> `agents/synthesis-agent.md` (`long` / Claude `opus`): merge sub-agent outputs into one prioritized fix plan and unified precedence model.
## Inputs
- Doc type (tutorial, how-to, reference, explanation) and audience.
- File scope or diff scope.
- Docs framework/tooling constraints (Fern, Mintlify, Sphinx, etc.).
- Build/review mode and brownfield/evergreen intent.
- Target agent and human compatibility intent.
- Docs framework surfaces in scope (for example Fern, Sphinx, Mintlify, Markdown/MDX/MDC/RST/RSC files).
- Desired investigation depth/time budget (quick pass vs exhaustive review).
- Execution mode (`single-agent` or `sub-agent-assisted` when available).
- Remediation mode (`apply-fixes` by default, or `report-only` when requested).
- Multilingual scope: source-of-truth language, target locales, and parity expectations.
- Repository-specific overlay constraints, if any.
## Outputs
- Updated draft or review findings with clear next actions.
- Validation notes (what was checked, what remains).
- Navigation/maintenance recommendations for long-term quality.
- Governance-doc alignment summary when AGENTS/CONTRIBUTING were touched.
- Agent instruction-surface map (primary file, alias files, Codex/Claude/Cursor handling plan).
- Documentation-surface coverage map (what was reviewed under `/docs`, README hierarchy, and framework-specific source trees).
- Autodetected issue list with applied fixes (or explicit report-only findings).
- Delegation notes when sub-agents were used (scope delegated and how findings were merged).
- Multilingual parity note (in-sync, partial with rationale, or intentionally divergent).
- Repository-specific overlay notes when one was used.
@@ -0,0 +1,32 @@
---
name: docs-framework-agent
description: Thinking-focused docs framework checker for config-relative paths and route/file mapping consistency.
model: sonnet
tools:
- Read
- Glob
- Grep
permissionMode: default
maxTurns: 10
---
You are the docs-framework sub-agent for technical documentation.
Goals:
- validate framework config-driven docs behavior
- prevent path-mapping drift between source files and published routes
Tasks:
- detect and read framework config first (Fern/Sphinx/Mintlify/custom)
- resolve paths relative to the declaring file/config
- validate both maps:
- config -> file exists
- config/nav/routing -> URL path is valid and consistent
Return:
- config files reviewed
- path assumptions made
- mismatches (`missing file`, `stale route`, `wrong base path`)
@@ -0,0 +1,30 @@
---
name: governance-agent
description: Thinking-focused governance reviewer for AGENTS/CONTRIBUTING/alias precedence, conflict detection, and policy drift analysis.
model: sonnet
tools:
- Read
- Glob
- Grep
permissionMode: default
maxTurns: 10
---
You are the governance sub-agent for technical documentation.
Goals:
- validate AGENTS/CONTRIBUTING/alias alignment and precedence
- identify policy drift and conflicting instructions
Tasks:
- determine canonical instruction source and alias compatibility mapping
- detect conflicts across nested scope files and tool-specific rule consumers
- validate command examples against stated governance expectations
Return:
- precedence model
- conflict list with severity
- recommended low-risk remediations
@@ -0,0 +1,31 @@
---
name: inventory-agent
description: Fast repo-surface discovery for technical documentation audits. Use for coverage mapping and missing-path detection before deeper review.
model: haiku
tools:
- Read
- Glob
- Grep
- LS
permissionMode: default
maxTurns: 6
---
You are the inventory sub-agent for technical documentation.
Goals:
- enumerate governance and docs-content surfaces in scope
- detect missing files, broken references, and obvious command/path failures
Tasks:
- map `AGENTS.md`/`CONTRIBUTING.md`/aliases and docs surfaces (`docs/**`, README hierarchy, `.md/.mdx/.mdc/.rst/.rsc`)
- list framework config files discovered (Fern/Sphinx/Mintlify or equivalent)
- report hard failures only, with exact file paths
Return:
- coverage map
- missing/broken path list
- unresolved blockers
@@ -0,0 +1,10 @@
interface:
display_name: "Technical Documentation"
short_description: "Build and review technical documentation for brownfield and evergreen systems."
icon_small: "./assets/icon.jpg"
icon_large: "./assets/icon.jpg"
brand_color: "#111827"
default_prompt: "Build or review technical documentation with a clear, maintainable, and production-ready workflow."
policy:
allow_implicit_invocation: true
@@ -0,0 +1,28 @@
---
name: synthesis-agent
description: Long-context synthesis agent that merges sub-agent outputs into one prioritized and deduplicated documentation action plan.
model: opus
tools:
- Read
permissionMode: default
maxTurns: 12
---
You are the synthesis sub-agent for technical documentation.
Goal:
- merge sub-agent outputs into one coherent, non-duplicated action plan
Tasks:
- prioritize blockers first, then non-blocking improvements
- normalize to one precedence model for governance decisions
- remove duplicated recommendations and contradictory fixes
- keep final output concise and execution-ready
Return:
- prioritized fix plan
- validation summary (done vs pending)
- explicit remaining gaps/blockers
Binary file not shown.

After

Width:  |  Height:  |  Size: 37 KiB

@@ -0,0 +1,145 @@
# AGENT and CONTRIBUTING Principles
This reference consolidates the core rules for agent-policy and contributor-governance docs.
You must:
1. Discover repo-level and nested instruction files with:
`rg --files -g 'AGENTS.md' -g 'CONTRIBUTING.md' -g 'CLAUDE.md' -g 'AGENT.md' -g '.cursor/rules/*' -g '.cursorrules' -g '.agent/**' -g '.agents/**' -g '.pi/**' -g 'AGENTS.*.md'`
2. Read the root and nearest-scope `AGENTS.md`/`CONTRIBUTING.md` pair before editing.
3. If alias files exist, normalize to one canonical source (`AGENTS.md` preferred when present; otherwise nearest alias), plus compatibility pointers or explicit symlink notes.
4. Document conflicting instructions and precedence decisions.
## GitHub + AGENTS baseline
Source: https://docs.github.com/en/communities/setting-up-your-project-for-healthy-contributions/setting-guidelines-for-repository-contributors
Source: https://agents.md/
Source: https://github.blog/ai-and-ml/github-copilot/how-to-write-a-great-agents-md-lessons-from-over-2500-repositories/
Source: https://cobusgreyling.substack.com/p/what-is-agentsmd
Source: https://www.infoq.com/news/2025/08/agents-md/
Use these as default operating principles:
1. Keep `CONTRIBUTING.md` discoverable and actionable (`.github`, root, or `docs`).
2. Keep agent instructions concrete: real commands, real paths, clear boundaries.
3. Use explicit behavior boundaries for agents: `Always`, `Ask first`, `Never`.
4. Keep contributor and agent rules aligned with actual repository workflows.
5. Ensure clear guidance is provided to agents on if, when and how to raise issues and pull requests.
## Canonical and alias policy
Source: https://agents.md/
Source: https://github.blog/ai-and-ml/github-copilot/how-to-write-a-great-agents-md-lessons-from-over-2500-repositories/
1. Treat `AGENTS.md` as canonical when present.
2. If `AGENTS.md` is absent, treat the nearest alias file as canonical.
3. Keep compatibility surfaces explicit: `AGENTS.md`, `AGENT.md`, `.cursorrules`, `.cursor/rules/*`, `.agent/`, `.agents/`, `.pi/`.
4. If aliases are used, document how they map back to canonical policy (or symlink when supported).
5. When repos use `.agents/` as canonical rule storage, keep `.cursor` as a compatibility symlink to `.agents` for Cursor rule auto-loading.
6. Keep policy DRY: store one shared policy core and expose it via aliases/symlinks instead of duplicating rule text.
## Context-awareness by agent platform
Source: https://github.com/vercel-labs/agent-skills/blob/main/AGENTS.md
Source: https://github.com/openai/codex/blob/main/AGENTS.md
1. For Cursor and Claude-style glob consumers, keep rule files narrow and bounded.
2. Avoid over-referencing large path sets that inflate context for glob-based agents.
3. For Codex-style workflows, prefer explicit file references and deterministic commands.
4. Keep long runbooks outside top-level policy files; link to scoped docs.
5. Ensure all agents have a happy path regardless so ensuring everything works across Codex, Claude and other coding agents.
## Symlink and compatibility operations
1. Preferred layout for multi-agent compatibility:
- canonical rule directory: `.agents/`
- Cursor compatibility path: `.cursor -> .agents` symlink
- canonical policy doc: `AGENTS.md` pointing to `.agents` paths where relevant
2. Validate symlink state before finalizing changes:
- if `.agents/` exists and `.cursor` is missing, create `.cursor` symlink to `.agents`
- if `.cursor` is a symlink to another target, fix target or document why it must differ
- if `.cursor` is a real directory/file, treat as migration conflict and ask before replacement
3. Validate rule payload through the canonical directory:
- rules: `.agents/rules/*.mdc` with valid frontmatter (`description`, `globs`, `alwaysApply` as needed)
- commands: `.agents/commands/*.md` when command routing is used
- MCP config: `.agents/mcp.json` when MCP is in scope
4. Keep Codex behavior explicit:
- `AGENTS.md` is primary for Codex repository instructions
- `.cursor` compatibility is for Cursor auto-loading and does not replace canonical AGENTS policy
5. Record applied symlink fixes and unresolved compatibility gaps in validation notes.
## Dual-mode and deliverable standards
Source: https://github.blog/ai-and-ml/github-copilot/how-to-write-a-great-agents-md-lessons-from-over-2500-repositories/
Source: https://agents.md/
Source: https://github.com/openai/codex/blob/main/AGENTS.md
Source: https://github.com/vercel-labs/agent-skills/blob/main/AGENTS.md
1. Author one shared policy core (same commands, boundaries, and precedence) for all agents.
2. For Cursor/Claude-style agents, expose that core through glob-driven and bounded files (small `AGENTS.md`/rule surface).
3. For Codex, expose that same core through explicit file references with precise scope.
4. Where styles diverge, prefer the smallest common structure that satisfies both and avoid duplicating policy text.
5. Treat AGENTS/CONTRIBUTING as first-class deliverables when in scope.
6. Preserve required structure, constraints, and examples from existing files.
7. Align wording and commands with active repository instructions.
## Proactive issue discovery and remediation
Source: https://github.blog/ai-and-ml/github-copilot/how-to-write-a-great-agents-md-lessons-from-over-2500-repositories/
Source: https://github.com/openai/codex/blob/main/AGENTS.md
Source: https://github.com/vercel-labs/agent-skills/blob/main/AGENTS.md
1. Run a conflict matrix review across AGENTS/aliases/CONTRIBUTING and related command/rule docs before finalizing.
2. Treat the following as high-priority defects: missing referenced files, non-existent setup commands, command scope mismatches, and branch/commit policy conflicts.
3. Do not stop at caveat-only notes when a low-risk fix is clear; apply the fix in the same pass.
4. If a canonical entry file is missing (for example a directory `README.md` that docs depend on), create a minimal actionable file and update references.
5. Long-running investigations are acceptable when needed to uncover cross-file drift, especially in agent-instruction ecosystems.
## Discovery
1. Agents prefer simple terminal commands so having a well defined `make *` or `npm run *` is ideal
2. Agents can discover terminal commands through shell completion so providing shell completion helps
## CONTRIBUTING size and scope control
Source: https://contributing.md/how-to-build-contributing-md/
Source: https://blog.codacy.com/best-practices-to-manage-an-open-source-project
Source: https://mozillascience.github.io/working-open-workshop/contributing/
Source: https://github.com/openclaw/openclaw/blob/main/CONTRIBUTING.md
1. Keep root `CONTRIBUTING.md` focused on setup, issue flow, PR flow, testing, and review gates.
2. Use issue/PR template links instead of embedding every process detail inline.
3. When the file grows too large, split by domain and link from root.
4. Move any large content into docs if avalible (for example Mintlify/Fern/Sphinx workflows) to avoid large contributor guide.
5. Optimize for agent/machine readability as well as humans.
## Example repos to emulate
Source: https://github.com/openclaw/openclaw/blob/main/AGENTS.md
Source: https://github.com/openclaw/openclaw/blob/main/CONTRIBUTING.md
Source: https://github.com/openclaw/openclaw/blob/main/VISION.md
Source: https://github.com/openai/codex/blob/main/AGENTS.md
Source: https://github.com/processing/p5.js/blob/main/AGENTS.md
Source: https://github.com/vercel-labs/agent-skills/blob/main/AGENTS.md
Source: https://github.com/agentsmd/agents.md/blob/main/AGENTS.md
Source: https://github.com/rails/rails/blob/main/CONTRIBUTING.md
Source: https://github.com/kubernetes/kubernetes/blob/master/CONTRIBUTING.md
Source: https://github.com/atom/atom/blob/master/CONTRIBUTING.md
Source: https://github.com/github/docs/blob/main/CONTRIBUTING.md
Source: https://github.com/facebook/react/blob/main/CONTRIBUTING.md
1. OpenClaw: strong real-world alias policy and AGENTS/CONTRIBUTING/VISION cohesion.
2. OpenAI Codex: strict command discipline and explicit scope control.
3. p5.js: explicit AI-policy guardrails in agent instructions.
4. Vercel + agentsmd spec: compact, context-efficient AGENTS patterns.
5. Rails/Kubernetes/Atom/GitHub Docs/React: contributor guidance patterns at different project scales.
## Practical merge policy
When these rules conflict:
1. Preserve contributor and reader task success first.
2. Preserve instruction clarity and unambiguous boundaries second.
3. Preserve long-term maintainability and context-efficiency third.
4. Add extra agent optimization only if it does not reduce human clarity or there is explict need.
5. Use your judgement as the expert.
@@ -0,0 +1,116 @@
# Build Docs Playbook
Read `principles.md` first, then follow this execution flow.
## 1. Detect and align agent instruction and governance instructions
- Use `references/agent-and-contributing.md` as the source of truth for inventory, canonical/alias mapping, and precedence/conflict handling.
- Apply the symlink compatibility policy when in scope (`.agents` canonical directory with `.cursor` compatibility symlink when required by tooling).
- Long-running and extensive build investigations are acceptable when needed to resolve ambiguous or conflicting documentation sources.
- When available, use sub-agents for bounded parallel inventory/cross-check tasks and merge results into one canonical decision set.
- Capture required constraints before writing:
- nested-agent rules, command/test requirements, PR workflow, and style checks.
- Use the same command and validation expectations in proposed snippets and examples.
## 2. Inventory product documentation surfaces (not governance only)
- For repo-wide builds, include docs content surfaces in addition to AGENTS/CONTRIBUTING.
- Inventory docs files and frameworks in scope (examples): `README*.md`, `docs/**`, `**/*.md`, `**/*.mdx`, `**/*.mdc`, `**/*.rst`, `**/*.rsc`, Fern/Mintlify config, Sphinx `conf.py`.
- Build a coverage map before drafting so governance and product docs are both represented.
- If scope is ambiguous, default to broader docs discovery first, then narrow intentionally.
## 3. Framework config and path mapping rules
- Detect framework/config first (for example Fern config, Sphinx `conf.py`, Mintlify config, or equivalent).
- Resolve every referenced path relative to the file/config that declares it, not assumed repo root.
- Treat filesystem paths and published URL routes as separate mappings; do not infer one from the other without config evidence.
- Validate both layers:
- config -> file exists on disk
- config/nav/routing -> URL path is consistent and reachable
- Record path-mapping assumptions and mismatches in handoff (`missing file`, `stale route`, `wrong base path`).
## 4. Define intent and success
- Audience, prerequisites, and job-to-be-done.
- Expected reader outcome immediately after completion.
- Doc type: tutorial, how-to, reference, explanation.
- Success criteria: what must be true after publish.
## 5. Build structure before prose
- Follow the funnel: what/why, quickstart, next steps.
- Keep headings informative and scannable.
- Open each section with the takeaway sentence.
- Add decision points with concrete branch guidance.
- For OpenClaw docs work, choose a page type from `references/openclaw.md` before drafting.
- Keep task-critical OpenClaw configuration inline; link exhaustive defaults, enums, schemas, generated references, and rare debugging workflows.
## 6. Build AGENTS.md and CONTRIBUTING.md intentionally
- Keep AGENTS.md structure consistent with `agents.md` ecosystem patterns:
- include YAML frontmatter when present in repo style (`name`, `description`).
- state persona scope and explicit instruction boundaries: `Always`, `Ask first`, `Never`.
- include concrete commands and representative code examples.
- For CONTRIBUTING.md, prioritize issue triage flow, PR expectations, setup/test commands, and review gates.
- Add `Code of Conduct`, `Testing`, `Local checks`, and `PR expectations` sections when missing but required by the repo.
- If CONTRIBUTING.md is becoming too large, split by scope into linked docs (for example, framework/tool-specific setup and release workflows) and keep the root file as a concise entry point.
- Keep cross-file consistency: links from CONTRIBUTING.md to AGENTS.md (and vice versa) should be accurate and non-circular.
- If multiple AGENTS.md files exist, document the directory-level scope and avoid conflicting advice.
- If a required canonical entry file is missing (for example referenced `README.md` under a major directory), create the file in the same pass instead of adding a caveat-only note.
- For new entry files, keep them minimal and actionable: purpose, prerequisites, concrete run commands, and pointers to deeper docs.
## 7. Keep agent context tight
- Author once, expose twice:
- keep one shared policy core and avoid duplicating guidance in separate agent-specific files.
- publish that core through bounded glob-friendly files for Cursor/Claude plus explicit path references for Codex.
- For Cursor and Claude-style agents, avoid broad references. Use minimal globbing and narrow rule files that each serve one concern (for example, repo-wide setup, test rules, security checks).
- Keep AGENTS and alias files short-to-medium; move detailed runbooks to linked docs.
- For Codex, prefer explicit file references and concrete paths for exact reuse.
- Avoid adding unrelated historical or process details to avoid token/context drift during future tool reads.
## 8. Brownfield build mode
- Match existing terminology, navigation, and component patterns.
- Preserve existing IA unless there is a documented migration plan.
- For rewrites, include a migration note from old to new paths.
- Prefer smallest safe change set that improves utility.
## 9. Evergreen build mode
- Prefer stable concepts over release-tied narrative.
- Isolate volatile details under clearly marked version sections.
- Include maintenance signals: owners, refresh triggers, stale criteria.
- Include lifecycle notes: deprecation and replacement paths.
## 10. Writing constraints
- Use precise language and short, imperative instructions.
- Keep code examples copy-ready and self-contained.
- Include common failure modes and safe defaults.
- Avoid placeholder guidance that cannot be executed.
## 11. Agent and automation readiness
- Keep key facts in text (not image-only).
- Prefer structured lists/tables when choices matter.
- Add links and anchors that allow deterministic navigation.
- Document what can be checked automatically in CI.
## 12. Build validation
- Validate commands and snippets where possible.
- Verify links and references in changed sections.
- Run a reference existence sweep for every path/command you introduced.
- Verify docs-framework consistency when in scope (for example Sphinx/Fern config and referenced doc paths).
- For OpenClaw docs work, apply the validation checklist in `references/openclaw.md`.
## 13. Multilingual parity mode (when applicable)
- Pick one source-of-truth language for technical accuracy and release timing.
- Define parity target: full parity, staged parity, or intentional divergence per section.
- Keep structure aligned across locales (headings, anchors, section order) when possible.
- Preserve command/code correctness first; localize explanatory text second.
- If parity is not feasible, add a visible note with missing scope and expected sync window.
- Run a locale parity check for changed sections (added/removed steps, warnings, prerequisites).
- Record unresolved checks explicitly in handoff.
@@ -0,0 +1,128 @@
# OpenClaw Documentation Overlay
Use this reference only for OpenClaw docs work. It layers OpenClaw-specific page
types, navigation, preservation, and validation rules on top of the general
technical-documentation skill.
## Reader Model
- Lead with the task the reader is trying to complete.
- Give one recommended path before alternatives.
- Keep main docs focused on the common path; move dense contracts and rare
debugging detail to linked reference or troubleshooting pages.
- Explain production risks exactly where the reader can make the mistake.
- Link concepts, guides, references, CLI pages, SDK docs, testing, and
troubleshooting so readers can continue without rereading.
## Page Types
Choose the page type before writing or reviewing:
- Overview: route readers to the right product area, integration path, or guide.
- Quickstart: get a new user to a working result with the fewest safe steps.
- Topic page: explain a major OpenClaw entity or surface end to end.
- Guide: walk through one workflow from prerequisites to production readiness.
- API/SDK/CLI reference: define every object, method, command, option, response,
error, enum, default, and version rule in scope.
- Testing guide: show sandbox setup, fixtures, simulated failures, and live-mode
differences.
- Troubleshooting guide: map observable symptoms to checks, causes, and fixes.
- Governance file: keep agent/contributor policy concrete, scoped, and aligned
with current OpenClaw repo behavior.
## Topic Pages
Use this shape for major-entity pages:
1. Title naming the entity or surface.
2. Unheaded opening that says what it is, what it owns, and what it does not own.
3. Requirements, only when setup needs accounts, versions, permissions, plugins,
operating systems, or credentials.
4. Quickstart with the recommended path and smallest reliable verification.
5. Configuration with task-critical options inline and exhaustive details linked
to reference docs.
6. Major subtopics organized by reader intent, not under a generic "Subtopics"
heading.
7. Troubleshooting with observable failures and concrete checks.
8. Related links to guides, references, commands, concepts, and adjacent topics.
## Guides
Use this shape for workflow pages:
1. Title naming the outcome, not the implementation detail.
2. Opening that states what the reader can accomplish.
3. Before you begin: accounts, keys, permissions, versions, tools, and
assumptions.
4. Choose a path, only when the reader must decide.
5. Steps with verb-led headings, commands, expected output, and checks.
6. Test with the smallest reliable proof that the workflow works.
7. Production readiness: security, retries, limits, observability, migrations,
and cleanup.
8. Troubleshooting near the workflow that causes the failures.
9. See also links to concepts, references, SDK docs, and adjacent guides.
## Docs IA And Navigation
- Read `docs/docs.json` before navigation changes.
- Keep topic pages and common workflows on the main reader path.
- Put exhaustive contracts, generated references, maintainer-only detail, and
support material under `Reference` or another clearly scoped support page.
- Keep generated `plugins/reference/*` children and redirect-only pages out of
visible navigation unless explicitly required.
- For moved pages, include a keep/drop/move/destination matrix in the handoff.
- Add "Read when" hints for docs-list routing when creating or changing pages
that participate in the docs index.
## Source-Backed Content
- CLI docs must match current flags, output, errors, and examples.
- API/SDK docs must include fields, defaults, enum values, constraints, nullable
behavior, lifecycle states, errors, and recovery guidance.
- Config docs must align exported types, schema/help output, metadata, baselines,
and current docs.
- Dependency-backed behavior must be verified from upstream docs, source, or
types before documenting defaults, timing, errors, or API behavior.
- Separate current behavior, shipped behavior, planned behavior, and maintainer
intent.
## Examples
- Prefer complete copy-pasteable commands and snippets.
- Use realistic variable names and values.
- Mark placeholders with angle-bracket names such as `<API_KEY>`.
- Show expected success output when it helps verification.
- Keep one conceptual unit per code block and use language-specific fences.
- Avoid examples that hide setup, auth, error handling, or cleanup.
- Never expose real secrets, live config, phone numbers, private videos, or
credentials.
## Preservation Reviews
For rewrites or splits:
- Identify source units before rewriting: headings, paragraphs, tables, examples,
CLI/API contracts, warnings, and troubleshooting facts.
- Map each retained unit to a destination page or section.
- Do not treat a broad "covered" row as proof for dense source material; use
line- or claim-level evidence when the source unit is dense.
- For dropped content, state whether it is obsolete, duplicated elsewhere,
unsupported, or moved to a reference/support page.
- When a docs-audit artifact is used, verify it is mapped audit data with
non-empty `mappings[]`, not only inventory or reindexed JSON.
## Validation
Choose the narrowest proof that covers the touched surface:
- `pnpm docs:list`
- `pnpm docs:check-mdx`
- `pnpm docs:check-links`
- `pnpm docs:check-i18n-glossary`
- `pnpm format:docs:check` or `pnpm lint:docs`
- `git diff --check`
- generated-doc or inventory checks when generated references, plugin catalogs,
labeler, or docs scripts changed
- behavior tests or command probes when docs claim runtime behavior
If proof is blocked, say exactly which command was not run and why.
@@ -0,0 +1,54 @@
# Documentation Principles
This reference consolidates the core rules used by this skill.
## Matt Palmer: 8 rules for better docs
Source: https://mattpalmer.io/posts/2025/10/8-rules-for-better-docs/
Use these as default operating principles:
1. Write for humans, optimize for agents.
2. Start with a funnel: what/why, quickstart, next steps.
3. Use Diataxis to scaffold content.
4. Write with AI, but structure for agents.
5. Offload routine docs operations to background agents.
6. Automate quality with CI.
7. Automate scaffolding and repetitive workflow tasks.
8. Make contribution easy and visible.
## OpenAI cookbook: what makes documentation good
Source: https://cookbook.openai.com/articles/what_makes_documentation_good
Key quality constraints:
- Prefer specific and accurate terminology over niche jargon.
- Keep examples self-contained and minimize dependencies.
- Prioritize high-value topics over edge-case depth.
- Do not teach unsafe patterns (for example, exposed secrets).
- Open with context that helps readers orient quickly.
- Apply empathy and override rigid rules when it clearly improves outcomes.
## Practical merge policy
When these rules conflict:
1. Preserve reader task success first.
2. Preserve structural clarity second.
3. Preserve long-term maintainability third.
4. Add agent optimization only if it does not reduce human clarity.
For agent-instructions and contributor-governance specifics (AGENTS/aliases/CONTRIBUTING), use `references/agent-and-contributing.md` as the detailed additional source of truth.
When the target repo or request is OpenClaw-specific, layer `references/openclaw.md` on top of these general rules. Otherwise ignore that repo-specific overlay.
## Execution policy for this skill
- Long-running and extensive investigations are allowed for both build and review work when needed to resolve ambiguity or cross-file drift.
- Use sub-agents when available for bounded parallel discovery, verification, or cross-source comparison.
- Keep one merged outcome: sub-agent outputs must be normalized into a single consistent recommendation/fix set.
## Multilingual parity rule
When docs exist in multiple languages, target cross-locale parity for task-critical content (steps, warnings, prerequisites, and limits). If full parity is not possible, publish explicit parity status and sync intent.
@@ -0,0 +1,121 @@
# Review Docs Playbook
Read `principles.md` first, then apply this checklist.
## 1. Scope and classification
- Identify doc type and target audience.
- Confirm brownfield vs evergreen intent.
- Confirm expected outcome for the reader.
- For full-repo reviews, explicitly include both governance surfaces and product-doc surfaces (`docs/`, README trees, `.md/.mdx/.mdc`, `.rst/.rsc`, framework docs configs).
- For OpenClaw docs reviews, apply `references/openclaw.md` for page type, docs IA, preservation, examples, and validation checks.
## 2. Investigation behavior
- Proactively find issues and risks without waiting for repeated prompts.
- If there are signals of deeper problems, continue investigation beyond the first pass.
- Long-running and extensive investigations are acceptable when needed for confidence and correctness.
- When available, use sub-agents for bounded parallel discovery (for example file-inventory, command validation, or cross-doc consistency checks), then merge to one final issue set.
- When no issues are found, state that explicitly and call out residual risks or validation gaps.
- Default to `apply-fixes` for high-confidence documentation defects unless the user explicitly requests `report-only`.
- Do not stop at AGENTS/CONTRIBUTING checks when the task is documentation-wide; continue into docs-content and docs-framework surfaces.
## 3. Governance surface review
- Use `references/agent-and-contributing.md` as the source of truth for inventory, canonical/alias mapping, and precedence/conflict handling.
For AGENTS.md:
- confirm persona intent, scope, and command/tool boundaries are explicit.
- check frontmatter style matches repo conventions when present.
- ensure `Always`, `Ask first`, and `Never` boundaries are present when expected.
- require concrete command examples and repo-specific paths to avoid ambiguity.
For CONTRIBUTING.md:
- verify issue/PR workflow is complete and actionable.
- ensure local setup, lint/test commands, and review criteria are accurate.
- ensure governance does not conflict with nested AGENTS instructions.
- flag oversized files that should be split into linked section docs (for example tool-specific setup and release docs).
For agent-platform awareness:
- confirm references are minimal and scoped for Cursor/Claude glob behavior.
- confirm Codex-facing guidance uses explicit file references.
- confirm both surfaces represent the same shared policy core (commands, boundaries, and precedence), not divergent guidance.
- audit `.agents`/`.cursor` compatibility behavior:
- verify canonical rule directory and symlink state match repo policy
- verify symlink target integrity and platform/tooling expectations
- verify AGENTS policy references remain canonical for Codex even when `.cursor` compatibility exists
- check for context bloat from duplicated policy statements across agent and contributor files.
- check for conflicting rules, skills and agent instructions
- check for conflicting information in agent instructions vs codebase
- check for broken or missing referenced files (for example README/index files named as canonical entry points).
- check for setup/command drift (for example non-existent install commands, root-level commands that should be module-scoped).
## 4. Product documentation surface review
- Verify docs IA coverage across root/module `README*` files and `docs/**` trees.
- Review framework-native docs sources in scope (for example Fern, Mintlify, Sphinx, MkDocs) and ensure guidance matches actual source-of-truth files.
- Check `.md/.mdx/.mdc/.rst/.rsc` for stale commands, missing prerequisites, and broken cross-links.
- Confirm referenced doc paths and anchors exist.
- Flag docs that should be split/merged to improve discoverability and maintenance.
- For OpenClaw docs, check `docs/docs.json`, docs-list routing hints, main path versus `Reference` placement, and generated-reference visibility.
- For OpenClaw rewrites or page splits, require source-backed keep/drop/move/destination coverage for important claims, warnings, examples, commands, fields, and troubleshooting facts.
## 5. Framework config and path mapping checks
- Detect and read framework config first (for example Fern config, Sphinx `conf.py`, Mintlify config, or equivalent).
- Resolve path references relative to the declaring file/config.
- Treat filesystem paths and published URL routes as separate maps; verify both.
- Flag path-map drift explicitly (`missing file`, `stale route`, `wrong base path`).
## 6. Structural review
- Funnel check: what/why, quickstart, next steps.
- Validate heading flow and navigation discoverability.
- Flag critical content trapped in images or buried sections.
- Check Diataxis alignment and split mixed-purpose sections.
- For OpenClaw docs, confirm the content matches an explicit page type from `references/openclaw.md`.
## 7. Writing quality review
- Check for concise, scannable paragraphs.
- Remove ambiguous pronouns and undefined terms.
- Verify examples are executable and scoped correctly.
- Verify tone is directive, technical, and non-hand-wavy.
## 8. Brownfield review mode
- Verify compatibility with existing docs IA and conventions.
- Verify anchors, redirects, and cross-doc links remain valid.
- Flag regressions in onboarding and task completion paths.
- Ensure changed terminology is intentionally propagated.
## 9. Evergreen review mode
- Flag date-stamped or brittle wording without version scope.
- Check ownership and refresh signals are present.
- Ensure recommendations remain valid after routine product evolution.
- Flag missing deprecation/migration guidance.
## 10. Tooling and platform review
Read `tooling.md` if platform fit is uncertain.
- Check whether content uses platform primitives effectively.
- Flag structure that fights the chosen docs platform.
- Recommend targeted platform-aware improvements.
## 11. Multilingual parity review (when applicable)
- Confirm declared source-of-truth language and expected parity policy.
- Compare changed sections across locales for step/order/warning drift.
- Flag missing updates to prerequisites, version notes, limits, and safety guidance.
- Allow intentional divergence only when rationale is explicit and user-impact is low.
- Require a reader-visible status note when locale parity is partial.
## 12. Output format
1. Blocking issues (file + required fix)
2. Non-blocking improvements
3. Validation notes (done vs pending)
@@ -0,0 +1,32 @@
# Documentation Tooling Guide
Source: https://www.mintlify.com/blog/top-7-api-documentation-tools-of-2025
Use this file when deciding build/review expectations for doc platforms.
## Tool-selection checkpoints
- Existing stack lock-in: do not force migration for minor gains.
- API workflow depth: generated references, OpenAPI support, testability.
- Collaboration model: docs-as-code, review workflow, versioning.
- Runtime quality: search, navigation, and copy-ready code snippets.
- AI readiness: structured content, stable URLs, machine-friendly layout yet human readable.
- Human readiness: reading complexity, reading UX, navigation depth, minimize jargon.
## Apply in brownfield mode
- Prioritize compatibility with the current platform.
- Use available components and style conventions before introducing new patterns.
- Propose migration only when current constraints block critical outcomes.
## Apply in evergreen mode
- Favor platforms and templates that make routine updates low-friction.
- Standardize section templates to reduce drift.
- Capture ownership, update cadence, and stale-content detection rules.
## Review implications
- Check whether content uses platform primitives correctly (tabs, callouts, endpoint blocks).
- Flag docs that are technically correct but hard to scan in the chosen platform.
- Recommend platform-specific improvements only when they reduce cognitive load.
+4 -4
View File
@@ -2,19 +2,19 @@
url = "http://127.0.0.1:{{ (repo ~ '-' ~ branch) | hash_port }}"
[[pre-start]]
env = "bun run setup:worktree -- --quiet"
env = "wt step copy-ignored || true; bun run setup:worktree -- --quiet --force --prefer-fallback"
[[pre-start]]
deps = "wt step copy-ignored || true; test -x node_modules/.bin/vite || bun install"
deps = "test -x node_modules/.bin/vite || bun install"
[post-start]
dev = "bun scripts/dev-worktree.ts --detach --port {{ (repo ~ '-' ~ branch) | hash_port }}"
dev = "bun scripts/dev-worktree.ts --detach --seed --port {{ (repo ~ '-' ~ branch) | hash_port }}"
[pre-remove]
dev = "if test -f .codex/runtime/dev-worktree.pid; then pid=$(cat .codex/runtime/dev-worktree.pid); kill -TERM -$pid 2>/dev/null || kill $pid 2>/dev/null || true; rm -f .codex/runtime/dev-worktree.pid; fi"
[aliases]
dev = "wt --yes hook pre-start && bun scripts/dev-worktree.ts --detach --port {{ (repo ~ '-' ~ branch) | hash_port }}"
dev = "wt --yes hook pre-start && bun scripts/dev-worktree.ts --detach --seed --port {{ (repo ~ '-' ~ branch) | hash_port }}"
setup = "wt --yes hook pre-start"
stop = "if test -f .codex/runtime/dev-worktree.pid; then pid=$(cat .codex/runtime/dev-worktree.pid); kill -TERM -$pid 2>/dev/null || kill $pid 2>/dev/null || true; rm -f .codex/runtime/dev-worktree.pid; fi"
url = "echo http://127.0.0.1:{{ (repo ~ '-' ~ branch) | hash_port }}"
-1
View File
@@ -24,6 +24,5 @@ OPENAI_API_KEY=
# Transactional email
RESEND_API_KEY=
CLAWHUB_SECURITY_EMAIL=security@notifications.openclaw.ai
CLAWHUB_SECURITY_EMAIL_FROM=ClawHub Security <noreply@notifications.openclaw.ai>
CLAWHUB_NOREPLY_FROM=ClawHub <noreply@notifications.openclaw.ai>
+2 -6
View File
@@ -10,7 +10,6 @@
/.github/codeql/ @openclaw/openclaw-secops
/.github/dependabot.yml @openclaw/openclaw-secops
/.github/workflows/ @openclaw/openclaw-secops
/scripts/check-staged-secrets.mjs @openclaw/openclaw-secops
/scripts/clawhub-cli-npm-publish.sh @openclaw/openclaw-secops
/scripts/clawhub-cli-npm-release-check.mjs @openclaw/openclaw-secops
/scripts/github/clawhub-rescan-auto-response.mjs @openclaw/openclaw-secops
@@ -19,7 +18,6 @@
/convex/schema.ts @openclaw/openclaw-secops @Patrick-Erichsen
/convex/auth.config.ts @openclaw/openclaw-secops @Patrick-Erichsen
/convex/auth.ts @openclaw/openclaw-secops @Patrick-Erichsen
/convex/commentModeration.ts @openclaw/openclaw-secops @Patrick-Erichsen
/convex/http.ts @openclaw/openclaw-secops @Patrick-Erichsen
/convex/httpApi.ts @openclaw/openclaw-secops @Patrick-Erichsen
/convex/httpApiV1/ @openclaw/openclaw-secops @Patrick-Erichsen
@@ -36,7 +34,6 @@
/convex/webhooks.ts @openclaw/openclaw-secops @Patrick-Erichsen
/convex/lib/access.ts @openclaw/openclaw-secops @Patrick-Erichsen
/convex/lib/apiTokenAuth.ts @openclaw/openclaw-secops @Patrick-Erichsen
/convex/lib/commentScamPrompt.ts @openclaw/openclaw-secops @Patrick-Erichsen
/convex/lib/githubActionsOidc.ts @openclaw/openclaw-secops @Patrick-Erichsen
/convex/lib/httpHeaders.ts @openclaw/openclaw-secops @Patrick-Erichsen
/convex/lib/httpRateLimit.ts @openclaw/openclaw-secops @Patrick-Erichsen
@@ -90,8 +87,6 @@
/packages/clawhub/src/cli/commands/packages.ts @openclaw/openclaw-secops @Patrick-Erichsen
/packages/clawhub/src/cli/commands/publish.ts @openclaw/openclaw-secops @Patrick-Erichsen
/packages/clawhub/src/cli/commands/transfer.ts @openclaw/openclaw-secops @Patrick-Erichsen
/packages/clawhub/src/cli/commands/sync.ts @openclaw/openclaw-secops @Patrick-Erichsen
/packages/clawhub/src/cli/scanSkills.ts @openclaw/openclaw-secops @Patrick-Erichsen
/packages/clawhub/src/schema/openclawContract.ts @openclaw/openclaw-secops @Patrick-Erichsen
/packages/clawhub/src/schema/packages.ts @openclaw/openclaw-secops @Patrick-Erichsen
/packages/clawhub/src/schema/routes.ts @openclaw/openclaw-secops @Patrick-Erichsen
@@ -115,9 +110,10 @@
/docs/acceptable-usage.md @openclaw/openclaw-secops @Patrick-Erichsen
/docs/api.md @openclaw/openclaw-secops @Patrick-Erichsen
/docs/auth.md @openclaw/openclaw-secops @Patrick-Erichsen
/docs/deploy.md @openclaw/openclaw-secops @Patrick-Erichsen
/docs/http-api.md @openclaw/openclaw-secops @Patrick-Erichsen
/docs/namespace-claims.md @openclaw/openclaw-secops @Patrick-Erichsen
/docs/security.md @openclaw/openclaw-secops @Patrick-Erichsen
/docs/webhook.md @openclaw/openclaw-secops @Patrick-Erichsen
/specs/deploy.md @openclaw/openclaw-secops @Patrick-Erichsen
/specs/github-import.md @openclaw/openclaw-secops @Patrick-Erichsen
/public/api/v1/openapi.json @openclaw/openclaw-secops @Patrick-Erichsen
@@ -0,0 +1,117 @@
name: Org / Namespace Claim
description: Request review for an org, brand, package scope, or namespace ownership dispute.
title: "Org claim: "
labels:
- "area: moderation"
- "area: security"
- "status: review"
body:
- type: markdown
attributes:
value: |
Use this form when you believe a ClawHub org, owner handle, package scope, skill slug, plugin package, or related namespace should be reserved, transferred, renamed, hidden, quarantined, aliased, or reviewed because of real-world project, brand, or organizational ownership.
Public GitHub issues must not include secrets, private documents, private legal files, personal identity documents, API tokens, DNS control tokens, or other sensitive material. Share public, non-sensitive proof here and tell us below if staff needs to arrange a private channel for sensitive evidence.
This is not the ban/account appeal flow. If your ClawHub account was banned, disabled, or cannot sign in because of account standing, use the ClawHub appeal form instead: https://appeals.openclaw.ai/
Related policy discussion: https://github.com/openclaw/clawhub/issues/2320
- type: input
id: claimed_namespace
attributes:
label: Claimed owner, org, scope, or namespace
description: Which ClawHub owner handle, org handle, package scope, skill slug, or package namespace are you claiming?
placeholder: "@example-org, example-org, @example-org/example-plugin, or example-skill"
validations:
required: true
- type: textarea
id: disputed_resources
attributes:
label: Disputed ClawHub resources
description: Link every relevant ClawHub URL, package name, skill slug, owner page, or related GitHub issue.
placeholder: |
- https://clawhub.ai/example-org/example-skill
- https://clawhub.ai/plugins/@example-org/example-plugin
- Package or skill names involved:
validations:
required: true
- type: textarea
id: claimant_relationship
attributes:
label: Claimant identity and relationship
description: Explain who is making the request and how they are connected to the org, project, package, or brand. Keep it public-safe.
placeholder: |
I maintain the upstream project at...
I am an admin/owner/member of...
Public profile or docs showing that relationship:
validations:
required: true
- type: dropdown
id: requested_outcome
attributes:
label: Requested outcome
description: Pick every outcome that would resolve the claim.
multiple: true
options:
- Reserve namespace or package
- Transfer ownership
- Rename existing resource
- Hide or quarantine current resource
- Add alias or redirect
- Review only / need staff guidance
- Other
validations:
required: true
- type: textarea
id: public_proof
attributes:
label: Public proof links and explanation
description: Add public links and explain what each proves. Useful proof includes GitHub org/repo control, domain or official email-domain proof, package-registry scope control, trademark or brand evidence, source repo history, package history, and public project docs. Do not paste secrets, tokens, private documents, or private legal files.
placeholder: |
- https://github.com/example-org/example-project proves...
- https://example.org/docs/clawhub proves...
- https://www.npmjs.com/org/example-org proves...
validations:
required: true
- type: textarea
id: current_owner_context
attributes:
label: Current owner, history, or context
description: Share what you know about the current ClawHub owner, prior transfers, project rename history, or attempted contact.
placeholder: |
The current listing appears to be owned by...
We contacted...
The project was renamed from...
- type: textarea
id: urgency_context
attributes:
label: User harm or urgency
description: Explain impact, affected users, install paths, or other facts that should affect triage priority. Say "No urgent user harm known" if this is not urgent.
placeholder: |
Users are being directed from...
The package is referenced by...
We believe this is urgent because...
validations:
required: true
- type: textarea
id: sensitive_evidence
attributes:
label: Sensitive evidence or private staff channel
description: Say whether public evidence is enough or whether staff needs to arrange a private channel. Summarize the kind of private evidence without including the sensitive material itself.
placeholder: |
Public evidence is enough.
Or:
We need a private staff channel for DNS challenge proof, private legal documents, or other sensitive evidence.
validations:
required: true
- type: checkboxes
id: acknowledgements
attributes:
label: Acknowledgements
options:
- label: I have not included secrets, API tokens, private documents, private legal files, personal identity documents, or other sensitive material in this public issue.
required: true
- label: I understand this public issue may be linked from ClawHub moderation or namespace policy discussions.
required: true
@@ -16,7 +16,6 @@ query-filters:
paths:
- convex/auth.config.ts
- convex/auth.ts
- convex/commentModeration.ts
- convex/http.ts
- convex/httpApi.ts
- convex/httpApiV1
@@ -32,7 +31,6 @@ paths:
- convex/webhooks.ts
- convex/lib/access.ts
- convex/lib/apiTokenAuth.ts
- convex/lib/commentScamPrompt.ts
- convex/lib/githubActionsOidc.ts
- convex/lib/httpHeaders.ts
- convex/lib/httpRateLimit.ts
@@ -26,9 +26,7 @@ paths:
- packages/clawhub/src/cli/commands/ownership.ts
- packages/clawhub/src/cli/commands/packages.ts
- packages/clawhub/src/cli/commands/publish.ts
- packages/clawhub/src/cli/commands/sync.ts
- packages/clawhub/src/cli/commands/transfer.ts
- packages/clawhub/src/cli/scanSkills.ts
- packages/clawhub/src/schema/openclawContract.ts
- packages/clawhub/src/schema/packages.ts
- packages/clawhub/src/schema/routes.ts
@@ -14,7 +14,6 @@ query-filters:
security-severity: /([7-9]|10)\.(\d)+/
paths:
- scripts/check-staged-secrets.mjs
- scripts/clawhub-cli-npm-release-check.mjs
- scripts/github
- scripts/verify-convex-contract.ts
+7 -16
View File
@@ -7,23 +7,11 @@ updates:
day: "monday"
time: "09:00"
timezone: "America/Los_Angeles"
open-pull-requests-limit: 3
labels:
- dependencies
- needs-cli-release
allow:
- dependency-name: "@openclaw/plugin-inspector"
- package-ecosystem: "bun"
directory: "/"
schedule:
interval: "weekly"
day: "monday"
time: "09:00"
timezone: "America/Los_Angeles"
open-pull-requests-limit: 10
# Preserve the old total Bun capacity: 10 general updates plus the
# dedicated 3-PR Plugin Inspector queue that cannot remain as a duplicate
# root Bun config.
open-pull-requests-limit: 13
ignore:
- dependency-name: "@openclaw/plugin-inspector"
- dependency-name: "@auth/core"
update-types:
- "version-update:semver-minor"
@@ -32,6 +20,9 @@ updates:
update-types:
- "version-update:semver-major"
groups:
plugin-inspector:
patterns:
- "@openclaw/plugin-inspector"
production-minor-and-patch:
dependency-type: "production"
update-types:
+1 -1
View File
@@ -74,7 +74,7 @@ jobs:
sudo ln -sf "$node_bin/npx" /usr/local/bin/npx
- name: Run Testbox
uses: useblacksmith/run-testbox@5ca05834db1d3813554d1dd109e5f2087a8d7cbc
uses: useblacksmith/run-testbox@3f60ff9ceb2c10c3feefa87dc0c6490cffae059d
if: always()
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+83 -13
View File
@@ -19,7 +19,7 @@ env:
jobs:
static:
name: static
runs-on: ubuntu-latest
runs-on: blacksmith-4vcpu-ubuntu-2404
timeout-minutes: 15
steps:
@@ -34,7 +34,7 @@ jobs:
unit:
name: unit
runs-on: ubuntu-latest
runs-on: blacksmith-4vcpu-ubuntu-2404
timeout-minutes: 15
steps:
@@ -47,7 +47,7 @@ jobs:
packages:
name: packages
runs-on: ubuntu-latest
runs-on: blacksmith-4vcpu-ubuntu-2404
timeout-minutes: 15
steps:
@@ -60,7 +60,7 @@ jobs:
types-build:
name: types-build
runs-on: ubuntu-latest
runs-on: blacksmith-4vcpu-ubuntu-2404
timeout-minutes: 15
steps:
@@ -73,7 +73,7 @@ jobs:
e2e-http:
name: e2e-http
runs-on: ubuntu-latest
runs-on: blacksmith-4vcpu-ubuntu-2404
timeout-minutes: 20
steps:
@@ -86,7 +86,7 @@ jobs:
playwright-smoke:
name: playwright-smoke
runs-on: ubuntu-latest
runs-on: blacksmith-8vcpu-ubuntu-2404
timeout-minutes: 25
steps:
@@ -94,8 +94,16 @@ jobs:
- uses: ./.github/actions/setup-bun
- name: Cache Playwright browsers
uses: actions/cache@v5
with:
path: ~/.cache/ms-playwright
key: ${{ runner.os }}-playwright-${{ hashFiles('bun.lock') }}
restore-keys: |
${{ runner.os }}-playwright-
- name: Install Playwright browsers
run: bunx playwright install --with-deps chromium
run: bunx playwright install chromium
- name: Browser e2e
run: bun run ci:playwright-smoke
@@ -108,26 +116,88 @@ jobs:
path: playwright-report/
if-no-files-found: ignore
playwright-local-auth:
name: playwright-local-auth
runs-on: ubuntu-latest
playwright-local-auth-shard:
name: playwright-local-auth / ${{ matrix.name }}
runs-on: blacksmith-4vcpu-ubuntu-2404
timeout-minutes: 30
strategy:
fail-fast: false
max-parallel: 3
matrix:
include:
- name: delete-account
spec: e2e/local-auth/delete-account-resources.pw.test.ts
- name: delete-org
spec: e2e/local-auth/delete-org-resources.pw.test.ts
- name: header-profile-link
spec: e2e/local-auth/header-profile-link.pw.test.ts
- name: malicious-skill-ban
spec: e2e/local-auth/malicious-skill-ban-flow.pw.test.ts
- name: manage-context-proof
spec: e2e/local-auth/manage-context-proof.pw.test.ts
- name: plugin-inspector-findings
spec: e2e/local-auth/plugin-inspector-findings.pw.test.ts
- name: publish-generated-card
spec: e2e/local-auth/publish-skill-lifecycle.pw.test.ts
grep: publishing a skill queues scan
- name: publish-new-version
spec: e2e/local-auth/publish-skill-lifecycle.pw.test.ts
grep: skill publishers can create a skill
- name: skill-star-sync
spec: e2e/local-auth/skill-star-sync.pw.test.ts
- name: version-delete
spec: e2e/local-auth/version-delete.pw.test.ts
steps:
- uses: actions/checkout@v6
- uses: ./.github/actions/setup-bun
- name: Cache Playwright browsers
uses: actions/cache@v5
with:
path: ~/.cache/ms-playwright
key: ${{ runner.os }}-playwright-${{ hashFiles('bun.lock') }}
restore-keys: |
${{ runner.os }}-playwright-
- name: Install Playwright browsers
run: bunx playwright install --with-deps chromium
run: bunx playwright install chromium
- name: Local-auth browser e2e
run: bun run test:pw:local-auth
env:
PLAYWRIGHT_GREP: ${{ matrix.grep || '' }}
PLAYWRIGHT_SPEC: ${{ matrix.spec }}
run: |
set -euo pipefail
args=(--project=chromium "$PLAYWRIGHT_SPEC")
if [[ -n "$PLAYWRIGHT_GREP" ]]; then
args+=(--grep "$PLAYWRIGHT_GREP")
fi
bun run test:pw:local-auth -- "${args[@]}"
- name: Upload Playwright report
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v7
with:
name: playwright-local-auth-report
name: playwright-local-auth-report-${{ matrix.name }}
path: playwright-report/
if-no-files-found: ignore
playwright-local-auth:
name: playwright-local-auth
runs-on: ubuntu-latest
needs: playwright-local-auth-shard
if: ${{ always() }}
timeout-minutes: 5
steps:
- name: Check local-auth shards
env:
LOCAL_AUTH_RESULT: ${{ needs.playwright-local-auth-shard.result }}
run: |
if [[ "$LOCAL_AUTH_RESULT" != "success" ]]; then
echo "playwright-local-auth shards finished with result: $LOCAL_AUTH_RESULT"
exit 1
fi
echo "playwright-local-auth shards passed."
@@ -236,6 +236,47 @@ jobs:
exit 1
fi
- name: Verify publish proof artifact
if: ${{ inputs.publish_run_id != '' }}
env:
GH_TOKEN: ${{ github.token }}
PUBLISH_RUN_ID: ${{ inputs.publish_run_id }}
RELEASE_TAG: ${{ inputs.tag }}
run: |
set -euo pipefail
PROOF_DIR="$RUNNER_TEMP/clawhub-cli-github-release-publish-proof"
rm -rf "$PROOF_DIR"
mkdir -p "$PROOF_DIR"
gh run download "$PUBLISH_RUN_ID" \
--repo "$GITHUB_REPOSITORY" \
--name "clawhub-cli-npm-publish-${RELEASE_TAG}" \
--dir "$PROOF_DIR"
if [[ "$(tr -d '\r\n' < "$PROOF_DIR/release-tag.txt")" != "$RELEASE_TAG" ]]; then
echo "Publish artifact tag does not match ${RELEASE_TAG}." >&2
exit 1
fi
if [[ "$(tr -d '\r\n' < "$PROOF_DIR/release-sha.txt")" != "$RELEASE_SHA" ]]; then
echo "Publish artifact SHA does not match ${RELEASE_SHA}." >&2
exit 1
fi
if [[ "$(tr -d '\r\n' < "$PROOF_DIR/package-version.txt")" != "$PACKAGE_VERSION" ]]; then
echo "Publish artifact version does not match ${PACKAGE_VERSION}." >&2
exit 1
fi
if [[ "$(tr -d '\r\n' < "$PROOF_DIR/preflight-only.txt")" != "false" ]]; then
echo "Publish artifact must come from a real publish run." >&2
exit 1
fi
if [[ "$(tr -d '\r\n' < "$PROOF_DIR/npm-tarball-url.txt")" != "$NPM_TARBALL_URL" ]]; then
echo "Publish artifact tarball URL does not match npm metadata." >&2
exit 1
fi
if [[ "$(tr -d '\r\n' < "$PROOF_DIR/npm-integrity.txt")" != "$NPM_INTEGRITY" ]]; then
echo "Publish artifact integrity does not match npm metadata." >&2
exit 1
fi
- name: Build release notes
env:
RELEASE_TAG: ${{ inputs.tag }}
@@ -381,6 +381,31 @@ jobs:
echo "RELEASE_TITLE=clawhub ${PACKAGE_VERSION}"
} >> "$GITHUB_ENV"
- name: Write npm publish proof artifact
id: publish_proof
env:
RELEASE_TAG: ${{ inputs.tag }}
run: |
set -euo pipefail
PUBLISH_PROOF_DIR="$RUNNER_TEMP/clawhub-cli-npm-publish-proof"
rm -rf "$PUBLISH_PROOF_DIR"
mkdir -p "$PUBLISH_PROOF_DIR"
printf '%s\n' "$RELEASE_TAG" > "$PUBLISH_PROOF_DIR/release-tag.txt"
git rev-parse HEAD > "$PUBLISH_PROOF_DIR/release-sha.txt"
printf '%s\n' "$PACKAGE_VERSION" > "$PUBLISH_PROOF_DIR/package-version.txt"
printf '%s\n' "$NPM_TARBALL_URL" > "$PUBLISH_PROOF_DIR/npm-tarball-url.txt"
printf '%s\n' "$NPM_INTEGRITY" > "$PUBLISH_PROOF_DIR/npm-integrity.txt"
printf '%s\n' "$GITHUB_RUN_ID" > "$PUBLISH_PROOF_DIR/publish-run-id.txt"
printf '%s\n' "false" > "$PUBLISH_PROOF_DIR/preflight-only.txt"
echo "dir=$PUBLISH_PROOF_DIR" >> "$GITHUB_OUTPUT"
- name: Upload npm publish proof artifact
uses: actions/upload-artifact@v7
with:
name: clawhub-cli-npm-publish-${{ inputs.tag }}
path: ${{ steps.publish_proof.outputs.dir }}
if-no-files-found: error
- name: Build GitHub Release notes
env:
RELEASE_TAG: ${{ inputs.tag }}
+1 -1
View File
@@ -82,7 +82,7 @@ jobs:
steps:
- name: Checkout
if: ${{ github.event_name != 'workflow_dispatch' || inputs.profile == 'all' || inputs.profile == matrix.category }}
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
with:
submodules: false
@@ -34,7 +34,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
uses: actions/checkout@v6
- name: Setup Bun
uses: oven-sh/setup-bun@v2
@@ -56,7 +56,7 @@ jobs:
- name: Upload inspector reports
if: always()
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v7
with:
name: plugin-inspector-bulk-scan-reports
path: plugin-inspector-bulk-scan-reports
@@ -19,7 +19,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout main commit
uses: actions/checkout@v4
uses: actions/checkout@v6
with:
ref: ${{ github.sha }}
fetch-depth: 0
@@ -0,0 +1,67 @@
name: Publish Hosted Catalog Feed
on:
schedule:
- cron: "17 */6 * * *"
workflow_dispatch:
inputs:
expires_in_days:
description: "How long the published feed remains fresh"
required: true
default: "7"
type: string
concurrency:
group: publish-catalog-feed
cancel-in-progress: true
permissions:
contents: read
jobs:
validate-ref:
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Require main ref for production publication
run: |
set -euo pipefail
if [[ "${GITHUB_REF}" != "refs/heads/main" ]]; then
echo "Production catalog publications must run from main."
exit 1
fi
publish:
runs-on: ubuntu-latest
timeout-minutes: 15
needs: validate-ref
environment:
name: Production
url: https://registry.openclaw.ai/v1/feeds/plugins
env:
EXPIRES_IN_DAYS: ${{ inputs.expires_in_days || '7' }}
steps:
- uses: actions/checkout@v6
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6
with:
bun-version: 1.3.10
- name: Install
run: bun install --frozen-lockfile
- name: Publish current production catalog
env:
CONVEX_DEPLOY_KEY: ${{ secrets.CONVEX_DEPLOY_KEY }}
run: |
set -euo pipefail
if [[ -z "$CONVEX_DEPLOY_KEY" ]]; then
echo "::error::Missing Production environment secret CONVEX_DEPLOY_KEY"
exit 1
fi
if ! [[ "$EXPIRES_IN_DAYS" =~ ^[1-9][0-9]*$ ]]; then
echo "::error::expires_in_days must be a positive integer"
exit 1
fi
expires_at="$(node -e 'const days = Number(process.env.EXPIRES_IN_DAYS); console.log(new Date(Date.now() + days * 86400000).toISOString())')"
bunx convex run catalogFeed:publish "{\"expiresAt\":\"$expires_at\"}" --prod
@@ -0,0 +1,223 @@
name: Security Dataset Snapshot
on:
workflow_dispatch:
inputs:
upload:
description: "Upload sanitized dataset files to Hugging Face"
required: true
default: "false"
type: choice
options:
- "false"
- "true"
limit:
description: "Optional source artifact cap for validation runs"
required: false
default: ""
hf-revision:
description: "Hugging Face branch/revision to upload to"
required: true
default: "main"
schedule:
- cron: "17 9 * * *"
permissions:
contents: read
id-token: write
concurrency:
group: clawhub-security-dataset-snapshot
cancel-in-progress: false
jobs:
publish-security-dataset:
name: Publish sanitized security dataset
runs-on: blacksmith-8vcpu-ubuntu-2404
timeout-minutes: 360
environment: Production
env:
CONVEX_URL: ${{ vars.CONVEX_URL || vars.VITE_CONVEX_URL || 'https://wry-manatee-359.convex.cloud' }}
SECURITY_SCAN_WORKER_TOKEN: ${{ secrets.SECURITY_SCAN_WORKER_TOKEN }}
HF_DATASET_REPO: OpenClaw/clawhub-security-signals
HF_OIDC_RESOURCE: datasets/OpenClaw/clawhub-security-signals
HF_REVISION: ${{ inputs['hf-revision'] || 'main' }}
HF_UPLOAD: ${{ github.event_name == 'schedule' || inputs.upload == 'true' }}
SNAPSHOT_LIMIT: ${{ inputs.limit || '' }}
SANITIZED_OUT_DIR: /tmp/clawhub-security-dataset/sanitized
WORK_DIR: /tmp/clawhub-security-dataset
steps:
- uses: actions/checkout@v6
- uses: ./.github/actions/setup-bun
- uses: actions/setup-python@v6
if: ${{ env.HF_UPLOAD == 'true' }}
with:
python-version: "3.12"
- name: Check configuration
run: |
set -euo pipefail
if [[ -z "$SECURITY_SCAN_WORKER_TOKEN" ]]; then
echo "::error::SECURITY_SCAN_WORKER_TOKEN is required"
exit 1
fi
echo "Upload enabled: $HF_UPLOAD"
echo "Convex URL: $CONVEX_URL"
echo "Hugging Face repo: $HF_DATASET_REPO"
echo "Hugging Face OIDC resource: $HF_OIDC_RESOURCE"
echo "Hugging Face revision: $HF_REVISION"
- name: Export sanitized live dataset
run: |
set -euo pipefail
mkdir -p "$SANITIZED_OUT_DIR"
source_snapshot_id="live-convex-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
args=(
--convex-url "$CONVEX_URL"
--worker-token "$SECURITY_SCAN_WORKER_TOKEN"
--source-snapshot-id "$source_snapshot_id"
--out-dir "$SANITIZED_OUT_DIR"
--hf-dataset
--hf-repo "$HF_DATASET_REPO"
--hf-revision "$HF_REVISION"
--page-size 25
--batch-pages 2
--concurrency 3
)
if [[ -n "$SNAPSHOT_LIMIT" ]]; then
args+=(--limit "$SNAPSHOT_LIMIT")
fi
bun scripts/security-dataset/export-snapshot.ts "${args[@]}" | tee "$SANITIZED_OUT_DIR/summary.json"
snapshot_dir="$(jq -r '.snapshotDir' "$SANITIZED_OUT_DIR/summary.json")"
if [[ -z "$snapshot_dir" || "$snapshot_dir" == "null" ]]; then
echo "::error::export summary did not include snapshotDir"
exit 1
fi
echo "SNAPSHOT_DIR=$snapshot_dir" >> "$GITHUB_ENV"
jq '.manifest.row_counts, .manifest.huggingface_dataset' "$SANITIZED_OUT_DIR/summary.json"
- name: Validate sanitized output guardrails
run: |
set -euo pipefail
test -d "$SNAPSHOT_DIR/hf-dataset/data"
test -f "$SNAPSHOT_DIR/manifest.json"
for split in train validation test eval_holdout; do
test -f "$SNAPSHOT_DIR/hf-dataset/data/$split.jsonl"
done
if grep -R -E 'storageId|skillVersions:|packageReleases:|_storage/' "$SNAPSHOT_DIR/hf-dataset" "$SNAPSHOT_DIR/manifest.json"; then
echo "::error::sanitized output contains raw storage or internal document identifiers"
exit 1
fi
if grep -R -E 'gh[pousr]_[A-Za-z0-9_]{20,}|sk-[A-Za-z0-9_-]{20,}|AKIA[0-9A-Z]{16}|-----BEGIN [A-Z0-9 ]*(PRIVATE KEY|CERTIFICATE)-----' "$SNAPSHOT_DIR/hf-dataset"; then
echo "::error::sanitized output contains obvious secret-like values"
exit 1
fi
- name: Install Hugging Face uploader
if: ${{ env.HF_UPLOAD == 'true' }}
run: |
set -euo pipefail
python -m pip install --upgrade pip
python -m pip install 'huggingface_hub[hf_xet]'
- name: Upload sanitized dataset to Hugging Face
if: ${{ env.HF_UPLOAD == 'true' }}
run: |
set -euo pipefail
python - <<'PY'
import json
import os
import urllib.request
from pathlib import Path
from huggingface_hub import HfApi
def get_github_oidc_token() -> str:
request_url = os.environ["ACTIONS_ID_TOKEN_REQUEST_URL"]
separator = "&" if "?" in request_url else "?"
request = urllib.request.Request(
f"{request_url}{separator}audience=https://huggingface.co",
headers={
"Authorization": f"bearer {os.environ['ACTIONS_ID_TOKEN_REQUEST_TOKEN']}",
"Accept": "application/json",
},
)
with urllib.request.urlopen(request) as response:
payload = json.loads(response.read().decode("utf-8"))
return payload["value"]
def exchange_hugging_face_token(oidc_token: str, resource: str) -> str:
body = json.dumps({
"grant_type": "urn:ietf:params:oauth:grant-type:token-exchange",
"subject_token_type": "urn:ietf:params:oauth:token-type:id_token",
"subject_token": oidc_token,
"resource": resource,
}).encode("utf-8")
request = urllib.request.Request(
"https://huggingface.co/oauth/token",
data=body,
headers={
"Content-Type": "application/json",
"Accept": "application/json",
},
method="POST",
)
with urllib.request.urlopen(request) as response:
payload = json.loads(response.read().decode("utf-8"))
return payload["access_token"]
snapshot_dir = Path(os.environ["SNAPSHOT_DIR"])
repo_id = os.environ["HF_DATASET_REPO"]
revision = os.environ["HF_REVISION"]
token = exchange_hugging_face_token(
get_github_oidc_token(),
os.environ["HF_OIDC_RESOURCE"],
)
api = HfApi(token=token)
data_commit = api.upload_folder(
folder_path=str(snapshot_dir / "hf-dataset" / "data"),
path_in_repo="data",
repo_id=repo_id,
repo_type="dataset",
revision=revision,
delete_patterns="data/*.jsonl",
commit_message="Update nightly ClawHub security dataset splits",
)
manifest_path = snapshot_dir / "manifest.json"
manifest = json.loads(manifest_path.read_text())
manifest["huggingface_dataset"]["commit"] = data_commit.oid
manifest["huggingface_dataset"]["revision"] = revision
manifest_path.write_text(json.dumps(manifest, indent=2) + "\n")
manifest_commit = api.upload_file(
path_or_fileobj=str(manifest_path),
path_in_repo="metadata/latest-manifest.json",
repo_id=repo_id,
repo_type="dataset",
revision=revision,
commit_message="Update nightly ClawHub security dataset manifest",
)
print(json.dumps({
"data_commit": data_commit.oid,
"manifest_commit": manifest_commit.oid,
"repo": repo_id,
"revision": revision,
}, indent=2))
PY
- name: Upload sanitized summary artifact
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v7
with:
name: security-dataset-summary-${{ github.run_id }}
path: |
${{ env.SANITIZED_OUT_DIR }}/summary.json
${{ env.SNAPSHOT_DIR }}/manifest.json
if-no-files-found: ignore
- name: Cleanup transient dataset files
if: ${{ always() }}
run: rm -rf "$WORK_DIR"
+7 -3
View File
@@ -10,7 +10,7 @@ on:
batch-limit:
description: "Maximum Codex scans to run in parallel per worker shard"
required: true
default: "6"
default: "4"
max-jobs:
description: "Optional total jobs cap per worker shard"
required: false
@@ -25,6 +25,10 @@ on:
permissions:
contents: read
concurrency:
group: clawhub-security-scan
cancel-in-progress: false
jobs:
codex-security-scan:
name: Codex security scan shard ${{ matrix.shard }}
@@ -34,12 +38,12 @@ jobs:
strategy:
fail-fast: false
matrix:
shard: [0, 1, 2, 3, 4, 5, 6, 7]
shard: [0, 1, 2, 3]
env:
CONVEX_URL: ${{ vars.CONVEX_URL || vars.VITE_CONVEX_URL || 'https://wry-manatee-359.convex.cloud' }}
SECURITY_SCAN_WORKER_TOKEN: ${{ secrets.SECURITY_SCAN_WORKER_TOKEN }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
CODEX_SECURITY_SCAN_LIMIT: ${{ inputs.limit || inputs['batch-limit'] || '6' }}
CODEX_SECURITY_SCAN_LIMIT: ${{ inputs.limit || inputs['batch-limit'] || '4' }}
CODEX_SECURITY_SCAN_MAX_JOBS: ${{ inputs['max-jobs'] || '' }}
CODEX_SECURITY_SCAN_MAX_RUNTIME_MINUTES: ${{ inputs['max-runtime-minutes'] || '40' }}
CODEX_SECURITY_SCAN_LEASE_MINUTES: "60"
+7 -3
View File
@@ -9,7 +9,7 @@ on:
batch-limit:
description: "Maximum Skill Card jobs to run in parallel per worker shard"
required: true
default: "6"
default: "4"
max-jobs:
description: "Optional total jobs cap per worker shard"
required: false
@@ -22,6 +22,10 @@ on:
permissions:
contents: read
concurrency:
group: clawhub-skill-card-worker
cancel-in-progress: false
jobs:
skill-card-worker:
name: Skill Card worker shard ${{ matrix.shard }}
@@ -31,12 +35,12 @@ jobs:
strategy:
fail-fast: false
matrix:
shard: [0, 1, 2, 3, 4, 5, 6, 7]
shard: [0, 1, 2, 3]
env:
CONVEX_URL: ${{ vars.CONVEX_URL || vars.VITE_CONVEX_URL || 'https://wry-manatee-359.convex.cloud' }}
# Shared Convex worker credential used by security and Skill Card workers.
SECURITY_SCAN_WORKER_TOKEN: ${{ secrets.SECURITY_SCAN_WORKER_TOKEN }}
SKILL_CARD_WORKER_LIMIT: ${{ github.event.inputs['batch-limit'] || '6' }}
SKILL_CARD_WORKER_LIMIT: ${{ github.event.inputs['batch-limit'] || '4' }}
SKILL_CARD_WORKER_MAX_JOBS: ${{ github.event.inputs['max-jobs'] || '' }}
SKILL_CARD_WORKER_MAX_RUNTIME_MINUTES: ${{ github.event.inputs['max-runtime-minutes'] || '40' }}
SKILL_CARD_WORKER_LEASE_MINUTES: "60"
+108 -88
View File
@@ -9,7 +9,7 @@ on:
type: string
default: ""
root:
description: Directory containing skill folders for bulk catalog publishing.
description: Directory containing skill folders for catalog publishing.
required: false
type: string
default: skills
@@ -28,11 +28,6 @@ on:
required: false
type: string
default: latest
bump:
description: Version bump for updated skills. One of patch, minor, or major.
required: false
type: string
default: patch
registry:
description: ClawHub registry URL.
required: false
@@ -53,7 +48,7 @@ on:
required: false
outputs:
publish_json:
description: Structured JSON output from clawhub sync.
description: Structured JSON output from skill publishing.
value: ${{ jobs.publish.outputs.publish_json }}
env:
@@ -96,8 +91,10 @@ jobs:
audience = "clawhub-workflow-source"
joiner = "&" if "?" in request_url else "?"
token_url = f"{request_url}{joiner}audience={audience}"
request = Request(token_url, headers={"Authorization": f"Bearer {request_token}"})
request = Request(
f"{request_url}{joiner}audience={audience}",
headers={"Authorization": f"Bearer {request_token}"},
)
with urlopen(request) as response:
payload = json.load(response)
@@ -121,8 +118,7 @@ jobs:
f"job_workflow_ref={workflow_ref!r} job_workflow_sha={workflow_sha!r}"
)
output_path = Path(os.environ["GITHUB_OUTPUT"])
with output_path.open("a", encoding="utf-8") as fh:
with Path(os.environ["GITHUB_OUTPUT"]).open("a", encoding="utf-8") as fh:
fh.write(f"repository={repo}\n")
fh.write(f"ref={workflow_sha}\n")
PY
@@ -142,10 +138,7 @@ jobs:
DRY_RUN: ${{ inputs.dry_run }}
CLAWHUB_TOKEN: ${{ secrets.clawhub_token }}
run: |
if [[ "$DRY_RUN" == "true" ]]; then
exit 0
fi
if [[ -n "$CLAWHUB_TOKEN" ]]; then
if [[ "$DRY_RUN" == "true" || -n "$CLAWHUB_TOKEN" ]]; then
exit 0
fi
echo "::error::Real skill publishes need secrets.clawhub_token. GitHub OIDC trusted publishing for skills is not supported yet."
@@ -168,103 +161,133 @@ jobs:
path = Path(os.environ["RUNNER_TEMP"]) / "clawhub-config.json"
path.write_text(
json.dumps(
{
"registry": os.environ["CLAWHUB_REGISTRY"],
"token": os.environ["CLAWHUB_TOKEN"],
},
indent=2,
)
+ "\n",
json.dumps({"registry": os.environ["CLAWHUB_REGISTRY"], "token": os.environ["CLAWHUB_TOKEN"]}, indent=2) + "\n",
encoding="utf-8",
)
print(path)
PY
echo "CLAWHUB_CONFIG_PATH=$RUNNER_TEMP/clawhub-config.json" >> "$GITHUB_ENV"
- name: Resolve sync command
- name: Run skill publishes
env:
INPUT_SKILL_PATH: ${{ inputs.skill_path }}
INPUT_ROOT: ${{ inputs.root }}
INPUT_DRY_RUN: ${{ inputs.dry_run }}
INPUT_OWNER: ${{ inputs.owner }}
INPUT_TAGS: ${{ inputs.tags }}
INPUT_BUMP: ${{ inputs.bump }}
INPUT_SITE: ${{ inputs.site }}
INPUT_REGISTRY: ${{ inputs.registry }}
INPUT_REF: ${{ inputs.ref }}
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_REF: ${{ github.ref }}
SOURCE_REPOSITORY: ${{ github.repository }}
SOURCE_REF: ${{ github.ref }}
run: |
python3 - <<'PY'
import json
import os
import shlex
import subprocess
import sys
from pathlib import Path
skill_path = os.environ["INPUT_SKILL_PATH"].strip()
root = os.environ["INPUT_ROOT"].strip() or "skills"
scan_root = skill_path or root
source_commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
source_ref = os.environ["INPUT_REF"].strip() or os.environ["GITHUB_REF"].strip()
cli_entry = (
Path(os.environ["GITHUB_WORKSPACE"])
/ "clawhub-source"
/ "packages"
/ "clawhub"
/ "src"
/ "cli.ts"
)
if not cli_entry.exists():
workspace = Path(os.environ["GITHUB_WORKSPACE"]).resolve()
cli_entry = workspace / "clawhub-source" / "packages" / "clawhub" / "src" / "cli.ts"
if not cli_entry.is_file():
raise SystemExit(f"Missing ClawHub CLI entrypoint at {cli_entry}")
cmd = [
"bun",
str(cli_entry),
"--workdir",
scan_root,
"--dir",
".",
"sync",
"--all",
"--json",
"--no-clawdbot-roots",
"--site",
os.environ["INPUT_SITE"],
"--registry",
os.environ["INPUT_REGISTRY"],
"--bump",
os.environ["INPUT_BUMP"].strip() or "patch",
"--source-repo",
os.environ["GITHUB_REPOSITORY"],
"--source-commit",
source_commit,
]
def resolve_inside_workspace(raw_path):
path = (workspace / raw_path).resolve()
try:
path.relative_to(workspace)
except ValueError as exc:
raise SystemExit(f"Publish path must be inside the caller repository: {raw_path}") from exc
return path
if os.environ["INPUT_DRY_RUN"] == "true":
cmd.append("--dry-run")
def is_skill_folder(path):
return path.is_dir() and any((path / name).is_file() for name in ("SKILL.md", "skill.md"))
skill_path = os.environ["INPUT_SKILL_PATH"].strip()
root_input = os.environ["INPUT_ROOT"].strip() or "skills"
if skill_path:
targets = [resolve_inside_workspace(skill_path)]
if not is_skill_folder(targets[0]):
raise SystemExit(f"skill_path is not a skill folder: {skill_path}")
else:
root = resolve_inside_workspace(root_input)
if is_skill_folder(root):
targets = [root]
elif root.is_dir():
targets = sorted(
(child for child in root.iterdir() if is_skill_folder(child)),
key=lambda child: child.name.lower(),
)
else:
targets = []
if not targets:
raise SystemExit(f"No skill folders found under: {root_input}")
source_commit = subprocess.check_output(
["git", "rev-parse", "HEAD"], cwd=workspace, text=True
).strip()
source_ref = os.environ["INPUT_REF"].strip() or os.environ["SOURCE_REF"].strip()
dry_run = os.environ["INPUT_DRY_RUN"] == "true"
owner = os.environ["INPUT_OWNER"].strip()
tags = os.environ["INPUT_TAGS"].strip()
if owner:
cmd += ["--owner", owner]
if tags:
cmd += ["--tags", tags]
if source_ref:
cmd += ["--source-ref", source_ref]
path = Path(os.environ["RUNNER_TEMP"]) / "clawhub-skill-publish-command.sh"
shell_line = " ".join(shlex.quote(part) for part in cmd)
path.write_text("#!/usr/bin/env bash\nset -euo pipefail\n" + shell_line + "\n", encoding="utf-8")
path.chmod(0o755)
print(shell_line)
results = {"wouldPublish": [], "published": [], "alreadySynced": [], "skipped": [], "failed": []}
status_keys = {
"would-publish": "wouldPublish",
"published": "published",
"unchanged": "alreadySynced",
}
for target in targets:
relative_path = target.relative_to(workspace).as_posix()
command = [
"bun", str(cli_entry),
"--workdir", str(workspace),
"--site", os.environ["INPUT_SITE"],
"--registry", os.environ["INPUT_REGISTRY"],
"skill", "publish", relative_path,
"--json",
"--source-repo", os.environ["SOURCE_REPOSITORY"],
"--source-commit", source_commit,
"--source-path", relative_path,
]
if dry_run:
command.append("--dry-run")
if owner:
command += ["--owner", owner]
if tags:
command += ["--tags", tags]
if source_ref:
command += ["--source-ref", source_ref]
completed = subprocess.run(command, cwd=workspace, capture_output=True, text=True)
if completed.returncode != 0:
message = completed.stderr.strip() or completed.stdout.strip() or f"exit {completed.returncode}"
results["failed"].append({"slug": target.name, "folder": relative_path, "message": message})
continue
try:
result = json.loads(completed.stdout)
results[status_keys[result["status"]]].append(result)
except (KeyError, ValueError, json.JSONDecodeError) as exc:
results["failed"].append({"slug": target.name, "folder": relative_path, "message": f"Invalid publish output: {exc}"})
output = {
"ok": not results["failed"],
"dryRun": dry_run,
"registry": os.environ["INPUT_REGISTRY"],
"roots": [skill_path or root_input],
**({"owner": owner.lstrip("@") } if owner else {}),
"summary": {key: len(value) for key, value in results.items()},
**results,
}
output_path = Path(os.environ["RUNNER_TEMP"]) / "skill-publish.json"
output_path.write_text(json.dumps(output, indent=2) + "\n", encoding="utf-8")
print(json.dumps(output, indent=2))
if results["failed"]:
sys.exit(1)
PY
- name: Run skill sync
run: |
set -euo pipefail
"$RUNNER_TEMP/clawhub-skill-publish-command.sh" | tee "$RUNNER_TEMP/skill-publish.json"
- name: Capture workflow outputs
id: capture
run: |
@@ -274,11 +297,8 @@ jobs:
from pathlib import Path
output_path = Path(os.environ["RUNNER_TEMP"]) / "skill-publish.json"
raw = output_path.read_text(encoding="utf-8").strip()
parsed = json.loads(raw)
github_output = Path(os.environ["GITHUB_OUTPUT"])
with github_output.open("a", encoding="utf-8") as fh:
parsed = json.loads(output_path.read_text(encoding="utf-8"))
with Path(os.environ["GITHUB_OUTPUT"]).open("a", encoding="utf-8") as fh:
fh.write("publish_json<<__CLAWHUB_JSON__\n")
fh.write(json.dumps(parsed, indent=2))
fh.write("\n__CLAWHUB_JSON__\n")
+2
View File
@@ -4,6 +4,7 @@ node_modules
*.bun-build
.artifacts/
artifacts/
.proof/
.cache/
.data/
bin/docs-list
@@ -32,6 +33,7 @@ eval/results/
playwright-report
test-results
.playwright
/public/llms.txt
convex/_generated/*
!convex/_generated/ai/
convex/_generated/ai/*
Binary file not shown.

After

Width:  |  Height:  |  Size: 74 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 65 KiB

+21
View File
@@ -0,0 +1,21 @@
# ClawHub publisher search proof
Status: pass (production bug reproduction before backend deploy)
## Convex read-only validation (`wry-manatee-359.convex.cloud`)
| Query | `listPublicPage` handles |
| --- | --- |
| `vyctor` | `[]` |
| `vyctorbrzezowski` | `[]` |
| `vincent` | `["vincentchan"]` |
| `vincentkoc` | `[]` |
## Profiles that exist but are missing from search
- `vyctorbrzezowski` → 5 skills, 1 package, 46 installs
- `vincentkoc` → public profile, 0 published skills
## Unit tests
`VITE_CONVEX_URL=https://example.invalid bunx vitest run convex/publishers.test.ts`
+30
View File
@@ -0,0 +1,30 @@
{
"baseline": "production",
"candidate": "production-before-fix",
"generatedAt": "2026-06-23T02:50:00.000Z",
"mode": "feature",
"status": "pass",
"lanes": [
{
"name": "candidate",
"ref": "https://clawhub.ai",
"status": "pass",
"steps": [
{
"lane": "candidate",
"name": "publishers?q=vyctorbrzezowski returns no publishers (prod before deploy)",
"screenshot": "screenshots/vyctorbrzezowski-empty.png",
"slug": "vyctorbrzezowski-empty",
"status": "pass"
},
{
"lane": "candidate",
"name": "publishers?q=vincent shows vincentchan but not vincentkoc (prod before deploy)",
"screenshot": "screenshots/vincent-missing-vincentkoc.png",
"slug": "vincent-missing-vincentkoc",
"status": "pass"
}
]
}
]
}
+2
View File
@@ -1 +1,3 @@
.env.local
.convex/
node_modules/
+8 -4
View File
@@ -23,9 +23,10 @@ Keep this section as the command map agents normally need, not a full `package.j
- `bun run dev` — foreground local app server at `http://localhost:3000`.
- `bunx convex dev --typecheck=disable` — local Convex backend/function watcher for manual setup.
- `bunx convex codegen` — regenerate `convex/_generated` after Convex API/schema changes.
- `bun run setup:worktree` — link `.env.local` and `.convex` from a usable source worktree into the current worktree. Use `-- --from <path>` or `CLAWHUB_WORKTREE_SOURCE=<path>` when auto-discovery picks the wrong source.
- `bun run dev:worktree`Worktrunk-managed detached worktree server. Requires `wt` on `PATH`; from that worktree use `wt --yes url` to print the branch URL and `wt --yes stop` to stop it.
- `bun run seed:dev` — canonical local seed path; runs worktree setup, waits for local Convex, seeds local fixtures plus the public corpus, and refreshes stats.
- `.worktreeinclude` — Codex-managed worktrees copy ignored local state (`.env.local`, `.convex/`, and `node_modules/`) from the local checkout at creation time.
- `bun run setup:worktree`validate copied `.env.local` / `.convex` state, or link missing fallback state from a usable source worktree. Use `-- --from <path>` or `CLAWHUB_WORKTREE_SOURCE=<path>` when auto-discovery picks the wrong source.
- `bun run dev:worktree` — Worktrunk-managed detached worktree server that also seeds local fixtures plus the public corpus once before starting the app when `VITE_CONVEX_URL` and `CONVEX_DEPLOYMENT` are local. Requires `wt` on `PATH`; from that worktree use `wt --yes url` to print the branch URL and `wt --yes stop` to stop it.
- `bun run seed:dev` — manual reseed path; runs worktree setup, waits for local Convex, seeds local fixtures plus the public corpus, and refreshes stats.
- `bun run build` — production build (Vite + Nitro).
- `bun run ci:static` — required pre-handoff static gate: peer checks, audit, formatting, lint, and dead-code checks.
- `bun run ci:unit` — Vitest coverage gate; required for source/test PRs unless docs/config-only.
@@ -99,13 +100,16 @@ Specialized corpus, scanner, security-worker, UI proof, proof publishing, Crabbo
## Convex Ops (Gotchas)
- Before any `bunx convex ...` command, name the target runtime (`local`, `dev`, or `prod`), the exact deployment when known, and whether the current function/schema code has already been pushed or deployed.
- New Convex functions must be pushed before `convex run`: use `bunx convex dev --once` (dev) or `bunx convex deploy` (prod).
- For non-interactive prod deploys, use `bunx convex deploy -y` to skip confirmation.
- If `bunx convex run --env-file .env.local ...` returns `401 MissingAccessToken` despite `bunx convex login`, workaround: omit `--env-file` and use `--deployment-name <name>` / `--prod`.
- If `bunx convex run --env-file .env.local ...` returns `401 MissingAccessToken` despite `bunx convex login`, workaround: omit `--env-file` and use `--deployment <name>` / `--prod`.
## Convex Migrations & Backfills
- Any Convex production data migration, backfill, destructive cleanup, schema narrowing, or table reshaping must start with the `convex-migration-helper` skill. Default to `@convex-dev/migrations` for production data changes because it provides batching, dry runs, resume/progress tracking, and safer operator UX. Exceptions require an explicit note explaining why the component is unnecessary, plus equivalent dry-run support, cursor batching, resume/progress behavior, confirmation for destructive writes, and real Convex runtime validation.
- When adding or changing Convex tables, TTL fields, cleanup crons, retention policy, auth/session cleanup, metric dedupe cleanup, or deprecated table removal, use the repo-local `convex-retention` skill and update `convex/lib/retentionPolicy.ts`.
- Use `convex/migrations.ts` for component-backed table-wide backfills; keep custom repairs, admin-gated operations, and incident-specific workflows in `convex/maintenance.ts`.
- After a migration or cleanup is verified complete, remove temporary migration functions/code in a follow-up PR unless they are intentionally retained as ongoing maintenance tooling.
## Convex Query & Bandwidth Rules
+13 -1
View File
@@ -1,5 +1,18 @@
# Changelog
## 0.23.0 - 2026-06-23
### Changes
- CLI: restore `clawhub sync` for scanning local skill folders and publishing new or changed skills in batches, including dry-run, JSON, owner, version-bump, provenance, and non-interactive `--all` options.
## 0.22.0 - 2026-06-15
### Changes
- CLI: remove the `clawhub sync` command. `clawhub skill publish <path>` now skips unchanged content, defaults new skills to `1.0.0`, defaults changed skills to the next patch version, and supports dry-run/JSON output.
- GitHub Actions: preserve catalog publishing through the reusable `skill-publish.yml` workflow, which invokes ordinary `skill publish` once per skill folder.
## 0.21.0 - 2026-06-11
### Changes
@@ -47,7 +60,6 @@
### Changes
- CLI/API: add Skill Card verification surfaces, including `clawhub skill verify <slug>` JSON output and `--card` Markdown retrieval (#2382).
- Web/API: surface an "API key required" attribute on skills so listings, cards, and detail views show whether a skill needs an LLM API key, with publish-time inference from skill prompts and metadata (#2353) (thanks @momothemage).
### Fixes
+11 -12
View File
@@ -99,7 +99,7 @@ bun run setup:worktree -- --from /path/to/source/worktree
CLAWHUB_WORKTREE_SOURCE=/path/to/source/worktree bun run setup:worktree
```
`dev:worktree` is the Worktrunk entrypoint. It runs the hooks in `.config/wt.toml`, copies ignored dependencies listed in `.worktreeinclude` when possible, falls back to `bun install` if Vite is missing, and starts detached services on a branch-hashed loopback port. Use `wt --yes url` from the same worktree to print the URL.
`dev:worktree` is the Worktrunk entrypoint. It runs the hooks in `.config/wt.toml`, copies ignored dependencies listed in `.worktreeinclude` when possible, falls back to `bun install` if Vite is missing, seeds local fixtures plus the public corpus once when `VITE_CONVEX_URL` and `CONVEX_DEPLOYMENT` are local, refreshes cached global stats, and starts detached services on a branch-hashed loopback port. Use `wt --yes url` from the same worktree to print the URL.
The detached server writes runtime state under `.codex/runtime/`. Stop it with `wt --yes stop` before removing the worktree.
@@ -121,9 +121,9 @@ one.
Without those workers, local ClawScan and Skill Card jobs stay pending until you
opt in, seed/mock results, or use the production workflows.
### Seed the database
### Reseed the database
Populate local QA fixtures and the committed public corpus so the UI isn't empty:
`dev:worktree` seeds local QA fixtures and the committed public corpus before starting the app when `VITE_CONVEX_URL` points at local Convex and `CONVEX_DEPLOYMENT` is an anonymous/local deployment marker, then records `.codex/runtime/dev-worktree.seeded` so ordinary restarts skip the expensive corpus pass. Remote-backed previews or mismatched deployment markers skip seeding and keep starting. To force the seed path without restarting the preview:
```bash
bun run seed:dev
@@ -175,12 +175,11 @@ Without `OPENAI_API_KEY`, public corpus import still works, but semantic search
These features degrade gracefully without their keys:
| Variable | Purpose |
| ------------------------------------------------------------------------- | --------------------------------------------------------- |
| `OPENAI_API_KEY` | Embeddings and vector search (falls back to zero vectors) |
| `VT_API_KEY` | VirusTotal malware scanning |
| `DISCORD_WEBHOOK_URL` | Discord notifications |
| `GITHUB_APP_ID` / `GITHUB_APP_PRIVATE_KEY` / `GITHUB_APP_INSTALLATION_ID` | GitHub backup sync |
| Variable | Purpose |
| --------------------- | --------------------------------------------------------- |
| `OPENAI_API_KEY` | Embeddings and vector search (falls back to zero vectors) |
| `VT_API_KEY` | VirusTotal malware scanning |
| `DISCORD_WEBHOOK_URL` | Discord notifications |
## CLI Development
@@ -208,7 +207,7 @@ Manual smoke tests are documented in [`specs/manual-testing.md`](specs/manual-te
## Skill Publishing
- Skill format reference: [`docs/skill-format.md`](docs/skill-format.md)
- End-to-end walkthrough (search, install, publish, sync): [`docs/quickstart.md`](docs/quickstart.md)
- End-to-end walkthrough (search, install, and publish): [`docs/quickstart.md`](docs/quickstart.md)
Quick publish:
@@ -281,10 +280,10 @@ See [`docs/security.md`](docs/security.md) for moderation and upload gating deta
1. This file (local setup)
2. [`docs/clawhub.md`](docs/clawhub.md) — public registry overview
3. [`docs/quickstart.md`](docs/quickstart.md) — end-to-end workflows
4. [`docs/architecture.md`](docs/architecture.md) — system design
4. [`docs/how-it-works.md`](docs/how-it-works.md) — registry behavior and system overview
5. [`docs/skill-format.md`](docs/skill-format.md) — skill structure
6. [`docs/cli.md`](docs/cli.md) — CLI reference
7. [`docs/http-api.md`](docs/http-api.md) — HTTP endpoints
8. [`docs/auth.md`](docs/auth.md) — authentication
9. [`docs/deploy.md`](docs/deploy.md) — deployment
9. [`specs/deploy.md`](specs/deploy.md) — deployment
10. [`docs/troubleshooting.md`](docs/troubleshooting.md) — common issues
+6 -6
View File
@@ -51,19 +51,19 @@ Common CLI flows:
- Remote/headless auth: `clawhub login --device`
- Discover: `clawhub search ...`, `clawhub explore`
- Browse unified catalog (skills + plugins): `clawhub package explore`, `clawhub package inspect <name>`
- Manage local installs: `clawhub install <slug>`, `clawhub pin <slug>`, `clawhub unpin <slug>`, `clawhub uninstall <slug>`, `clawhub list`, `clawhub update --all`
- Inspect without installing: `clawhub inspect <slug>`
- Publish/sync skills: `clawhub skill publish <path>`, `clawhub sync`
- Manage local installs: `clawhub install @openclaw/demo`, `clawhub pin <skill>`, `clawhub unpin <skill>`, `clawhub uninstall <skill>`, `clawhub list`, `clawhub update --all`
- Inspect without installing: `clawhub inspect @openclaw/demo`
- Publish skills: `clawhub skill publish <path>`
- Publish plugins: `clawhub package publish <source>`
- Code-plugin manifests must include `openclaw.compat.pluginApi` and `openclaw.build.openclawVersion`; see [`docs/cli.md`](docs/cli.md) for a minimal example.
- Canonicalize owned skills: `clawhub skill rename <slug> <new-slug>`, `clawhub skill merge <source> <target>`
- Canonicalize owned skills: `clawhub skill rename <skill> <new-name>`, `clawhub skill merge <source> <target>`
Docs: [`docs/quickstart.md`](docs/quickstart.md), [`docs/cli.md`](docs/cli.md).
### Removal permissions
- `clawhub uninstall <slug>` only removes a local install on your machine.
- Uploaded registry skills use soft-delete/restore (`clawhub delete <slug>` / `clawhub undelete <slug>` or API equivalents).
- `clawhub uninstall <skill>` only removes a local install on your machine.
- Uploaded registry skills use soft-delete/restore (`clawhub delete <skill>` / `clawhub undelete <skill>` or API equivalents).
- Soft-delete/restore is allowed for the skill or package owner, publisher owner/admin, moderators, and admins.
- Packages use `clawhub package delete <name>` / `clawhub package undelete <name>`.
- Hard delete is admin-only (management tools / ban flows).
+413 -229
View File
File diff suppressed because it is too large Load Diff
+43 -31
View File
@@ -10,9 +10,11 @@
import type * as appMeta from "../appMeta.js";
import type * as auth from "../auth.js";
import type * as catalogClassification from "../catalogClassification.js";
import type * as catalogClassificationNode from "../catalogClassificationNode.js";
import type * as catalogFeed from "../catalogFeed.js";
import type * as catalogTopics from "../catalogTopics.js";
import type * as cliDeviceAuth from "../cliDeviceAuth.js";
import type * as commentModeration from "../commentModeration.js";
import type * as comments from "../comments.js";
import type * as crons from "../crons.js";
import type * as depRegistryScan from "../depRegistryScan.js";
import type * as devSeed from "../devSeed.js";
@@ -22,17 +24,16 @@ import type * as downloads from "../downloads.js";
import type * as emailsNode from "../emailsNode.js";
import type * as functions from "../functions.js";
import type * as githubAccountAgeBackfill from "../githubAccountAgeBackfill.js";
import type * as githubBackups from "../githubBackups.js";
import type * as githubBackupsNode from "../githubBackupsNode.js";
import type * as githubIdentity from "../githubIdentity.js";
import type * as githubImport from "../githubImport.js";
import type * as githubRestore from "../githubRestore.js";
import type * as githubRestoreMutations from "../githubRestoreMutations.js";
import type * as githubSkillSources from "../githubSkillSources.js";
import type * as githubSkillSync from "../githubSkillSync.js";
import type * as githubSkillSyncNode from "../githubSkillSyncNode.js";
import type * as http from "../http.js";
import type * as httpApi from "../httpApi.js";
import type * as httpApiV1 from "../httpApiV1.js";
import type * as httpApiV1_catalogFeedV1 from "../httpApiV1/catalogFeedV1.js";
import type * as httpApiV1_contentRightsV1 from "../httpApiV1/contentRightsV1.js";
import type * as httpApiV1_docsSessionV1 from "../httpApiV1/docsSessionV1.js";
import type * as httpApiV1_packagesV1 from "../httpApiV1/packagesV1.js";
import type * as httpApiV1_publishersV1 from "../httpApiV1/publishersV1.js";
@@ -45,17 +46,18 @@ import type * as httpApiV1_whoamiV1 from "../httpApiV1/whoamiV1.js";
import type * as httpPreflight from "../httpPreflight.js";
import type * as leaderboards from "../leaderboards.js";
import type * as lib_access from "../lib/access.js";
import type * as lib_apiKeyRequirementPrompt from "../lib/apiKeyRequirementPrompt.js";
import type * as lib_apiTokenAuth from "../lib/apiTokenAuth.js";
import type * as lib_artifactModeration from "../lib/artifactModeration.js";
import type * as lib_badges from "../lib/badges.js";
import type * as lib_batching from "../lib/batching.js";
import type * as lib_catalogClassification from "../lib/catalogClassification.js";
import type * as lib_catalogClassifier from "../lib/catalogClassifier.js";
import type * as lib_changelog from "../lib/changelog.js";
import type * as lib_clawpack from "../lib/clawpack.js";
import type * as lib_commentScamPrompt from "../lib/commentScamPrompt.js";
import type * as lib_contentTypes from "../lib/contentTypes.js";
import type * as lib_devAuth from "../lib/devAuth.js";
import type * as lib_devSeed from "../lib/devSeed.js";
import type * as lib_downloadTrend from "../lib/downloadTrend.js";
import type * as lib_emailRendering from "../lib/emailRendering.js";
import type * as lib_emails from "../lib/emails.js";
import type * as lib_embeddingVisibility from "../lib/embeddingVisibility.js";
@@ -63,11 +65,10 @@ import type * as lib_embeddings from "../lib/embeddings.js";
import type * as lib_githubAccount from "../lib/githubAccount.js";
import type * as lib_githubActionsOidc from "../lib/githubActionsOidc.js";
import type * as lib_githubAuth from "../lib/githubAuth.js";
import type * as lib_githubBackup from "../lib/githubBackup.js";
import type * as lib_githubIdentity from "../lib/githubIdentity.js";
import type * as lib_githubImport from "../lib/githubImport.js";
import type * as lib_githubProfileSync from "../lib/githubProfileSync.js";
import type * as lib_githubRestoreHelpers from "../lib/githubRestoreHelpers.js";
import type * as lib_githubSkillScans from "../lib/githubSkillScans.js";
import type * as lib_githubSkillSync from "../lib/githubSkillSync.js";
import type * as lib_globalStats from "../lib/globalStats.js";
import type * as lib_httpHeaders from "../lib/httpHeaders.js";
@@ -82,10 +83,10 @@ import type * as lib_moderationReasonCodes from "../lib/moderationReasonCodes.js
import type * as lib_observabilityEvents from "../lib/observabilityEvents.js";
import type * as lib_officialPublishers from "../lib/officialPublishers.js";
import type * as lib_openaiResponse from "../lib/openaiResponse.js";
import type * as lib_packageArtifacts from "../lib/packageArtifacts.js";
import type * as lib_packageRegistry from "../lib/packageRegistry.js";
import type * as lib_packageSearchDigest from "../lib/packageSearchDigest.js";
import type * as lib_packageSecurity from "../lib/packageSecurity.js";
import type * as lib_parsedEnvSignals from "../lib/parsedEnvSignals.js";
import type * as lib_public from "../lib/public.js";
import type * as lib_publicRouteReservations from "../lib/publicRouteReservations.js";
import type * as lib_publishLimits from "../lib/publishLimits.js";
@@ -93,34 +94,37 @@ import type * as lib_publisherAbuseScoring from "../lib/publisherAbuseScoring.js
import type * as lib_publisherCatalogDisplay from "../lib/publisherCatalogDisplay.js";
import type * as lib_publisherStats from "../lib/publisherStats.js";
import type * as lib_publishers from "../lib/publishers.js";
import type * as lib_rateLimitConfig from "../lib/rateLimitConfig.js";
import type * as lib_recommendationScore from "../lib/recommendationScore.js";
import type * as lib_reporting from "../lib/reporting.js";
import type * as lib_reservedHandles from "../lib/reservedHandles.js";
import type * as lib_reservedSlugs from "../lib/reservedSlugs.js";
import type * as lib_retentionPolicy from "../lib/retentionPolicy.js";
import type * as lib_searchText from "../lib/searchText.js";
import type * as lib_securityPrompt from "../lib/securityPrompt.js";
import type * as lib_skillBackfill from "../lib/skillBackfill.js";
import type * as lib_skillCapabilityTags from "../lib/skillCapabilityTags.js";
import type * as lib_skillCards from "../lib/skillCards.js";
import type * as lib_skillFileAccess from "../lib/skillFileAccess.js";
import type * as lib_skillIcon from "../lib/skillIcon.js";
import type * as lib_skillInstallBackfill from "../lib/skillInstallBackfill.js";
import type * as lib_skillPublish from "../lib/skillPublish.js";
import type * as lib_skillQuality from "../lib/skillQuality.js";
import type * as lib_skillSafety from "../lib/skillSafety.js";
import type * as lib_skillScanRequestFiles from "../lib/skillScanRequestFiles.js";
import type * as lib_skillSearchDigest from "../lib/skillSearchDigest.js";
import type * as lib_skillSlugValidator from "../lib/skillSlugValidator.js";
import type * as lib_skillStats from "../lib/skillStats.js";
import type * as lib_skillSummary from "../lib/skillSummary.js";
import type * as lib_skillZip from "../lib/skillZip.js";
import type * as lib_skills from "../lib/skills.js";
import type * as lib_skills_index from "../lib/skills/index.js";
import type * as lib_skills_slugResolution from "../lib/skills/slugResolution.js";
import type * as lib_staticPublishScan from "../lib/staticPublishScan.js";
import type * as lib_tokens from "../lib/tokens.js";
import type * as lib_userSearch from "../lib/userSearch.js";
import type * as lib_userSkillStats from "../lib/userSkillStats.js";
import type * as lib_webhooks from "../lib/webhooks.js";
import type * as llmEval from "../llmEval.js";
import type * as maintenance from "../maintenance.js";
import type * as managementDevSeed from "../managementDevSeed.js";
import type * as migrations from "../migrations.js";
import type * as packageInspectorHttp from "../packageInspectorHttp.js";
import type * as packageInspectorNode from "../packageInspectorNode.js";
import type * as packagePublishTokens from "../packagePublishTokens.js";
@@ -129,6 +133,7 @@ import type * as publisherAbuse from "../publisherAbuse.js";
import type * as publisherAbuseDevSeed from "../publisherAbuseDevSeed.js";
import type * as publishers from "../publishers.js";
import type * as rateLimits from "../rateLimits.js";
import type * as retention from "../retention.js";
import type * as search from "../search.js";
import type * as securityDataset from "../securityDataset.js";
import type * as securityDatasetNode from "../securityDatasetNode.js";
@@ -155,9 +160,11 @@ import type {
declare const fullApi: ApiFromModules<{
appMeta: typeof appMeta;
auth: typeof auth;
catalogClassification: typeof catalogClassification;
catalogClassificationNode: typeof catalogClassificationNode;
catalogFeed: typeof catalogFeed;
catalogTopics: typeof catalogTopics;
cliDeviceAuth: typeof cliDeviceAuth;
commentModeration: typeof commentModeration;
comments: typeof comments;
crons: typeof crons;
depRegistryScan: typeof depRegistryScan;
devSeed: typeof devSeed;
@@ -167,17 +174,16 @@ declare const fullApi: ApiFromModules<{
emailsNode: typeof emailsNode;
functions: typeof functions;
githubAccountAgeBackfill: typeof githubAccountAgeBackfill;
githubBackups: typeof githubBackups;
githubBackupsNode: typeof githubBackupsNode;
githubIdentity: typeof githubIdentity;
githubImport: typeof githubImport;
githubRestore: typeof githubRestore;
githubRestoreMutations: typeof githubRestoreMutations;
githubSkillSources: typeof githubSkillSources;
githubSkillSync: typeof githubSkillSync;
githubSkillSyncNode: typeof githubSkillSyncNode;
http: typeof http;
httpApi: typeof httpApi;
httpApiV1: typeof httpApiV1;
"httpApiV1/catalogFeedV1": typeof httpApiV1_catalogFeedV1;
"httpApiV1/contentRightsV1": typeof httpApiV1_contentRightsV1;
"httpApiV1/docsSessionV1": typeof httpApiV1_docsSessionV1;
"httpApiV1/packagesV1": typeof httpApiV1_packagesV1;
"httpApiV1/publishersV1": typeof httpApiV1_publishersV1;
@@ -190,17 +196,18 @@ declare const fullApi: ApiFromModules<{
httpPreflight: typeof httpPreflight;
leaderboards: typeof leaderboards;
"lib/access": typeof lib_access;
"lib/apiKeyRequirementPrompt": typeof lib_apiKeyRequirementPrompt;
"lib/apiTokenAuth": typeof lib_apiTokenAuth;
"lib/artifactModeration": typeof lib_artifactModeration;
"lib/badges": typeof lib_badges;
"lib/batching": typeof lib_batching;
"lib/catalogClassification": typeof lib_catalogClassification;
"lib/catalogClassifier": typeof lib_catalogClassifier;
"lib/changelog": typeof lib_changelog;
"lib/clawpack": typeof lib_clawpack;
"lib/commentScamPrompt": typeof lib_commentScamPrompt;
"lib/contentTypes": typeof lib_contentTypes;
"lib/devAuth": typeof lib_devAuth;
"lib/devSeed": typeof lib_devSeed;
"lib/downloadTrend": typeof lib_downloadTrend;
"lib/emailRendering": typeof lib_emailRendering;
"lib/emails": typeof lib_emails;
"lib/embeddingVisibility": typeof lib_embeddingVisibility;
@@ -208,11 +215,10 @@ declare const fullApi: ApiFromModules<{
"lib/githubAccount": typeof lib_githubAccount;
"lib/githubActionsOidc": typeof lib_githubActionsOidc;
"lib/githubAuth": typeof lib_githubAuth;
"lib/githubBackup": typeof lib_githubBackup;
"lib/githubIdentity": typeof lib_githubIdentity;
"lib/githubImport": typeof lib_githubImport;
"lib/githubProfileSync": typeof lib_githubProfileSync;
"lib/githubRestoreHelpers": typeof lib_githubRestoreHelpers;
"lib/githubSkillScans": typeof lib_githubSkillScans;
"lib/githubSkillSync": typeof lib_githubSkillSync;
"lib/globalStats": typeof lib_globalStats;
"lib/httpHeaders": typeof lib_httpHeaders;
@@ -227,10 +233,10 @@ declare const fullApi: ApiFromModules<{
"lib/observabilityEvents": typeof lib_observabilityEvents;
"lib/officialPublishers": typeof lib_officialPublishers;
"lib/openaiResponse": typeof lib_openaiResponse;
"lib/packageArtifacts": typeof lib_packageArtifacts;
"lib/packageRegistry": typeof lib_packageRegistry;
"lib/packageSearchDigest": typeof lib_packageSearchDigest;
"lib/packageSecurity": typeof lib_packageSecurity;
"lib/parsedEnvSignals": typeof lib_parsedEnvSignals;
"lib/public": typeof lib_public;
"lib/publicRouteReservations": typeof lib_publicRouteReservations;
"lib/publishLimits": typeof lib_publishLimits;
@@ -238,34 +244,37 @@ declare const fullApi: ApiFromModules<{
"lib/publisherCatalogDisplay": typeof lib_publisherCatalogDisplay;
"lib/publisherStats": typeof lib_publisherStats;
"lib/publishers": typeof lib_publishers;
"lib/rateLimitConfig": typeof lib_rateLimitConfig;
"lib/recommendationScore": typeof lib_recommendationScore;
"lib/reporting": typeof lib_reporting;
"lib/reservedHandles": typeof lib_reservedHandles;
"lib/reservedSlugs": typeof lib_reservedSlugs;
"lib/retentionPolicy": typeof lib_retentionPolicy;
"lib/searchText": typeof lib_searchText;
"lib/securityPrompt": typeof lib_securityPrompt;
"lib/skillBackfill": typeof lib_skillBackfill;
"lib/skillCapabilityTags": typeof lib_skillCapabilityTags;
"lib/skillCards": typeof lib_skillCards;
"lib/skillFileAccess": typeof lib_skillFileAccess;
"lib/skillIcon": typeof lib_skillIcon;
"lib/skillInstallBackfill": typeof lib_skillInstallBackfill;
"lib/skillPublish": typeof lib_skillPublish;
"lib/skillQuality": typeof lib_skillQuality;
"lib/skillSafety": typeof lib_skillSafety;
"lib/skillScanRequestFiles": typeof lib_skillScanRequestFiles;
"lib/skillSearchDigest": typeof lib_skillSearchDigest;
"lib/skillSlugValidator": typeof lib_skillSlugValidator;
"lib/skillStats": typeof lib_skillStats;
"lib/skillSummary": typeof lib_skillSummary;
"lib/skillZip": typeof lib_skillZip;
"lib/skills": typeof lib_skills;
"lib/skills/index": typeof lib_skills_index;
"lib/skills/slugResolution": typeof lib_skills_slugResolution;
"lib/staticPublishScan": typeof lib_staticPublishScan;
"lib/tokens": typeof lib_tokens;
"lib/userSearch": typeof lib_userSearch;
"lib/userSkillStats": typeof lib_userSkillStats;
"lib/webhooks": typeof lib_webhooks;
llmEval: typeof llmEval;
maintenance: typeof maintenance;
managementDevSeed: typeof managementDevSeed;
migrations: typeof migrations;
packageInspectorHttp: typeof packageInspectorHttp;
packageInspectorNode: typeof packageInspectorNode;
packagePublishTokens: typeof packagePublishTokens;
@@ -274,6 +283,7 @@ declare const fullApi: ApiFromModules<{
publisherAbuseDevSeed: typeof publisherAbuseDevSeed;
publishers: typeof publishers;
rateLimits: typeof rateLimits;
retention: typeof retention;
search: typeof search;
securityDataset: typeof securityDataset;
securityDatasetNode: typeof securityDatasetNode;
@@ -318,4 +328,6 @@ export declare const internal: FilterApi<
FunctionReference<any, "internal">
>;
export declare const components: {};
export declare const components: {
migrations: import("@convex-dev/migrations/_generated/component.js").ComponentApi<"migrations">;
};
-6
View File
@@ -1,6 +0,0 @@
import { internal } from "./_generated/api";
// Asserts that the internal-only download counters remain internal-only.
// Public exposure is prevented at runtime by `internalMutation`; this file
// just pins the public references that *should* exist.
void internal.downloads.recordDownloadInternal;
+121
View File
@@ -0,0 +1,121 @@
import { describe, expect, it, vi } from "vitest";
import type { Id } from "./_generated/dataModel";
const { convexAuthMock } = vi.hoisted(() => ({
convexAuthMock: vi.fn(() => ({
auth: {},
signIn: {},
signOut: {},
store: {},
isAuthenticated: {},
})),
}));
vi.mock("@convex-dev/auth/server", async () => {
const actual =
await vi.importActual<typeof import("@convex-dev/auth/server")>("@convex-dev/auth/server");
return {
...actual,
convexAuth: convexAuthMock,
};
});
type CapturedAuthConfig = {
callbacks?: {
createOrUpdateUser?: (
ctx: unknown,
args: {
existingUserId: Id<"users"> | null;
provider: { type: string; allowDangerousEmailAccountLinking?: boolean };
profile: Record<string, unknown> & {
email?: string;
phone?: string;
emailVerified?: boolean;
phoneVerified?: boolean;
};
},
) => Promise<Id<"users">>;
beforeSessionCreation?: (ctx: unknown, args: { userId: Id<"users"> }) => Promise<void> | void;
};
};
function getCapturedAuthConfig() {
const calls = convexAuthMock.mock.calls as unknown as Array<[CapturedAuthConfig]>;
const config = calls[0]?.[0];
if (!config) throw new Error("convexAuth was not called");
return config;
}
function makeAuthCtx(user: { _id: Id<"users">; deletedAt?: number; deactivatedAt?: number }) {
const userId = user._id;
const collect = vi.fn().mockResolvedValue([{ action: "user.ban" }]);
const ctx = {
db: {
get: vi.fn().mockResolvedValue(user),
patch: vi.fn().mockResolvedValue(null),
insert: vi.fn().mockResolvedValue(userId),
query: vi.fn().mockReturnValue({
withIndex: vi.fn().mockReturnValue({ collect }),
}),
},
scheduler: {
runAfter: vi.fn().mockResolvedValue(null),
},
};
return { ctx, userId };
}
describe("auth callbacks", () => {
it("defers banned account rejection until session creation", async () => {
await import("./auth");
const config = getCapturedAuthConfig();
const { ctx, userId } = makeAuthCtx({ _id: "users:banned" as Id<"users">, deletedAt: 123 });
await expect(
config.callbacks?.createOrUpdateUser?.(ctx, {
existingUserId: userId,
provider: { type: "oauth", allowDangerousEmailAccountLinking: false },
profile: {
id: "123",
name: "renamed-banned-user",
email: "banned@example.com",
image: "https://example.com/avatar.png",
},
}),
).resolves.toBe(userId);
await expect(config.callbacks?.beforeSessionCreation?.(ctx, { userId })).rejects.toThrow(
/account has been banned/i,
);
expect(ctx.db.patch).not.toHaveBeenCalled();
expect(ctx.db.insert).not.toHaveBeenCalled();
});
it("updates active existing users and schedules post-update side effects", async () => {
await import("./auth");
const config = getCapturedAuthConfig();
const { ctx, userId } = makeAuthCtx({ _id: "users:active" as Id<"users"> });
await expect(
config.callbacks?.createOrUpdateUser?.(ctx, {
existingUserId: userId,
provider: { type: "oauth", allowDangerousEmailAccountLinking: false },
profile: {
id: "123",
name: "active-user",
email: "active@example.com",
image: "https://example.com/avatar.png",
},
}),
).resolves.toBe(userId);
expect(ctx.db.patch).toHaveBeenCalledWith(userId, {
id: "123",
name: "active-user",
email: "active@example.com",
image: "https://example.com/avatar.png",
});
expect(ctx.scheduler.runAfter).toHaveBeenCalled();
});
});
+73 -18
View File
@@ -102,6 +102,54 @@ export async function handleDeletedUserSignIn(
throw new ConvexError(DELETED_ACCOUNT_REAUTH_MESSAGE);
}
type AuthProfile = Record<string, unknown> & {
email?: string;
phone?: string;
emailVerified?: boolean;
phoneVerified?: boolean;
};
function userDataFromAuthProfile(args: {
provider: { type: string; allowDangerousEmailAccountLinking?: boolean };
profile: AuthProfile;
}) {
const {
emailVerified: profileEmailVerified,
phoneVerified: profilePhoneVerified,
...profile
} = args.profile;
const emailVerified =
profileEmailVerified ??
((args.provider.type === "oauth" || args.provider.type === "oidc") &&
args.provider.allowDangerousEmailAccountLinking !== false);
const phoneVerified = profilePhoneVerified ?? false;
return {
...(emailVerified ? { emailVerificationTime: Date.now() } : null),
...(phoneVerified ? { phoneVerificationTime: Date.now() } : null),
...profile,
};
}
async function schedulePostUserCreatedOrUpdated(
ctx: GenericMutationCtx<DataModel>,
userId: Id<"users">,
user: Parameters<typeof shouldScheduleGitHubProfileSync>[0],
) {
await ctx.scheduler.runAfter(0, internal.publishers.ensurePersonalPublisherInternal, {
userId,
});
// Schedule GitHub profile sync to handle username renames (fixes #303).
// This runs as a background action so it doesn't block sign-in.
const now = Date.now();
if (shouldScheduleGitHubProfileSync(user, now)) {
await ctx.scheduler.runAfter(0, internal.users.syncGitHubProfileAction, {
userId,
});
}
}
export const { auth, signIn, signOut, store, isAuthenticated } = convexAuth({
providers: [
createGitHubAuthProvider(),
@@ -125,30 +173,37 @@ export const { auth, signIn, signOut, store, isAuthenticated } = convexAuth({
],
callbacks: {
/**
* Block sign-in for deleted/deactivated users and sync GitHub profile.
* Create/update users and sync GitHub profile.
*
* Performance note: This callback runs on every OAuth sign-in, but the
* audit log query ONLY executes when a legacy deleted user attempts to sign
* in (user.deletedAt is set). For active users, this is a single field check.
* Banned/deleted users keep the OAuth callback non-mutating so code
* redemption can fail in beforeSessionCreation and render /account-banned.
*
* The GitHub profile sync is scheduled as a background action to handle
* the case where a user renames their GitHub account (fixes #303).
*/
async afterUserCreatedOrUpdated(ctx, args) {
const user = await ctx.db.get(args.userId);
await handleDeletedUserSignIn(ctx, args, user);
await ctx.scheduler.runAfter(0, internal.publishers.ensurePersonalPublisherInternal, {
userId: args.userId,
});
// Schedule GitHub profile sync to handle username renames (fixes #303)
// This runs as a background action so it doesn't block sign-in
const now = Date.now();
if (shouldScheduleGitHubProfileSync(user, now)) {
await ctx.scheduler.runAfter(0, internal.users.syncGitHubProfileAction, {
userId: args.userId,
});
async createOrUpdateUser(ctx, args) {
const userData = userDataFromAuthProfile(args);
if (args.existingUserId !== null) {
const userId = args.existingUserId as Id<"users">;
const existingUser = await ctx.db.get(userId);
if (existingUser?.deletedAt || existingUser?.deactivatedAt) {
return userId;
}
await ctx.db.patch(userId, userData);
await schedulePostUserCreatedOrUpdated(ctx, userId, existingUser);
return userId;
}
const userId = await ctx.db.insert("users", userData);
const user = await ctx.db.get(userId);
await schedulePostUserCreatedOrUpdated(ctx, userId, user);
return userId;
},
async beforeSessionCreation(ctx, args) {
await handleDeletedUserSignIn(ctx, {
userId: args.userId,
existingUserId: args.userId,
});
},
},
});
+149
View File
@@ -0,0 +1,149 @@
/* @vitest-environment node */
import { describe, expect, it, vi } from "vitest";
import { getCatalogClassificationPageInternalHandler } from "./catalogClassification";
import { classifyCatalogInternalHandler } from "./catalogClassificationNode";
describe("catalog classification runner", () => {
it("loads bounded latest skill evidence without scanning historical versions", async () => {
const paginate = vi.fn().mockResolvedValue({
page: [
{
_id: "skills:demo",
slug: "web-research",
displayName: "Web Research",
summary: "Search the web for current sources",
latestVersionId: "skillVersions:v1",
},
],
isDone: true,
continueCursor: "done",
});
const get = vi.fn().mockResolvedValue({
_id: "skillVersions:v1",
skillId: "skills:demo",
files: [
{
path: "SKILL.md",
size: 100,
storageId: "storage:skill",
},
{
path: "archive/README.md",
size: 100,
storageId: "storage:readme",
},
],
});
const result = await getCatalogClassificationPageInternalHandler(
{
db: {
query: vi.fn(() => ({
order: vi.fn(() => ({ paginate })),
})),
get,
},
} as never,
{ targetKind: "skill", batchSize: 10 },
);
expect(result.items).toEqual([
expect.objectContaining({
kind: "skill",
skillId: "skills:demo",
skillVersionId: "skillVersions:v1",
textFile: { path: "SKILL.md", storageId: "storage:skill" },
}),
]);
expect(get).toHaveBeenCalledTimes(1);
});
it("stores preview classifications without changing source artifacts", async () => {
const runMutation = vi.fn().mockResolvedValue({ ok: true, upserted: 1 });
const result = await classifyCatalogInternalHandler(
{
runQuery: vi.fn().mockResolvedValue({
items: [
{
kind: "skill",
skillId: "skills:demo",
skillVersionId: "skillVersions:v1",
slug: "web-research",
displayName: "Web Research",
summary: "Search the web for current research sources",
textFile: { path: "SKILL.md", storageId: "storage:skill" },
},
],
cursor: "done",
isDone: true,
}),
runMutation,
storage: {
get: vi
.fn()
.mockResolvedValue(
new Blob([
"---\nname: web-research\ndescription: Web search for current research sources.\n---\n# Web Research",
]),
),
},
scheduler: { runAfter: vi.fn() },
} as never,
{ targetKind: "skill", batchSize: 10 },
);
expect(result).toMatchObject({
ok: true,
targetKind: "skill",
scanned: 1,
classified: 1,
skipped: 0,
failed: 0,
isDone: true,
scheduledNext: false,
});
expect(runMutation).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
results: [
expect.objectContaining({
targetKind: "skill",
skillId: "skills:demo",
skillVersionId: "skillVersions:v1",
categories: ["research"],
classifierVersion: "taxonomy-prototype-v9",
}),
],
}),
);
});
it("skips skill-family packages in the plugin classification lane", async () => {
const paginate = vi.fn().mockResolvedValue({
page: [
{
_id: "packages:skill",
family: "skill",
latestReleaseId: "packageReleases:v1",
},
],
isDone: true,
continueCursor: "done",
});
const get = vi.fn();
const result = await getCatalogClassificationPageInternalHandler(
{
db: {
query: vi.fn(() => ({
order: vi.fn(() => ({ paginate })),
})),
get,
},
} as never,
{ targetKind: "plugin", batchSize: 10 },
);
expect(result.items).toEqual([{ kind: "skip", targetKind: "plugin", reason: "not-plugin" }]);
expect(get).not.toHaveBeenCalled();
});
});
+320
View File
@@ -0,0 +1,320 @@
import { ConvexError, v } from "convex/values";
import { internal } from "./_generated/api";
import type { Doc, Id } from "./_generated/dataModel";
import type { QueryCtx } from "./_generated/server";
import { action, internalMutation, internalQuery } from "./_generated/server";
import { assertRole, requireUserFromAction } from "./lib/access";
import type {
CatalogClassificationConfidence,
CatalogClassifierResult,
} from "./lib/catalogClassification";
const DEFAULT_CLASSIFICATION_BATCH_SIZE = 10;
const MAX_CLASSIFICATION_BATCH_SIZE = 25;
const MAX_STATIC_TEXT_FILE_SIZE = 512_000;
const MAX_PLUGIN_TEXT_FILES = 8;
const targetKindValidator = v.union(v.literal("skill"), v.literal("plugin"));
const confidenceValidator = v.union(v.literal("high"), v.literal("medium"), v.literal("low"));
const categoryCandidateValidator = v.object({
category: v.string(),
score: v.number(),
sources: v.array(v.string()),
evidence: v.array(v.string()),
strongEvidence: v.optional(v.boolean()),
primaryEvidence: v.optional(v.boolean()),
strongPrimaryEvidence: v.optional(v.boolean()),
primaryEvidenceCount: v.optional(v.number()),
});
const topicCandidateValidator = v.object({
topic: v.string(),
slug: v.string(),
score: v.number(),
sources: v.array(v.string()),
evidence: v.array(v.string()),
primaryEvidence: v.boolean(),
primarySourceCount: v.number(),
strongEvidence: v.boolean(),
confidence: confidenceValidator,
suppressedBy: v.optional(v.string()),
});
const classificationResultInputValidator = v.object({
targetKind: targetKindValidator,
skillId: v.optional(v.id("skills")),
packageId: v.optional(v.id("packages")),
skillVersionId: v.optional(v.id("skillVersions")),
packageReleaseId: v.optional(v.id("packageReleases")),
categories: v.array(v.string()),
topics: v.array(v.string()),
categoryCandidates: v.array(categoryCandidateValidator),
topicCandidates: v.array(topicCandidateValidator),
categoryCandidateCount: v.number(),
topicCandidateCount: v.number(),
categoryConfidence: confidenceValidator,
topicConfidence: confidenceValidator,
categoryNeedsReview: v.boolean(),
topicNeedsReview: v.boolean(),
unknownSignals: v.array(v.string()),
classifierVersion: v.string(),
topicClassifierVersion: v.string(),
inputHash: v.string(),
topicInputHash: v.string(),
});
type CatalogTextFile = {
path: string;
storageId: Id<"_storage">;
};
export type CatalogClassificationPageItem =
| {
kind: "skill";
skillId: Id<"skills">;
skillVersionId: Id<"skillVersions">;
slug: string;
displayName: string;
summary?: string;
categories?: string[];
topics?: string[];
textFile?: CatalogTextFile;
}
| {
kind: "plugin";
packageId: Id<"packages">;
packageReleaseId: Id<"packageReleases">;
name: string;
displayName: string;
summary?: string;
categories?: string[];
topics?: string[];
pluginManifest?: unknown;
packageJson?: unknown;
bundleManifest?: unknown;
textFiles: CatalogTextFile[];
}
| {
kind: "skip";
targetKind: "skill" | "plugin";
reason: "soft-deleted" | "not-plugin" | "missing-latest-version" | "missing-latest-release";
};
export type CatalogClassificationPageResult = {
items: CatalogClassificationPageItem[];
cursor: string | null;
isDone: boolean;
};
export type CatalogClassificationActionResult = {
ok: true;
targetKind: "skill" | "plugin";
scanned: number;
classified: number;
skipped: number;
failed: number;
confidence: Record<CatalogClassificationConfidence, number>;
topicConfidence: Record<CatalogClassificationConfidence, number>;
cursor: string | null;
isDone: boolean;
scheduledNext: boolean;
};
function clampBatchSize(value: number | undefined) {
const integer = Number.isFinite(value)
? Math.floor(value ?? 0)
: DEFAULT_CLASSIFICATION_BATCH_SIZE;
return Math.max(1, Math.min(MAX_CLASSIFICATION_BATCH_SIZE, integer));
}
function findSkillTextFile(version: Doc<"skillVersions">): CatalogTextFile | undefined {
const file = version.files.find((candidate) => {
const path = candidate.path.toLowerCase();
return (
candidate.size <= MAX_STATIC_TEXT_FILE_SIZE &&
(path === "skill.md" || path === "skills.md" || path.endsWith("/skill.md"))
);
});
return file ? { path: file.path, storageId: file.storageId } : undefined;
}
function findPluginTextFiles(release: Doc<"packageReleases">): CatalogTextFile[] {
return release.files
.filter((file) => {
if (file.size > MAX_STATIC_TEXT_FILE_SIZE) return false;
const name = file.path.split("/").at(-1)?.toLowerCase();
return name === "readme.md" || name === "skill.md" || name === "skills.md";
})
.slice(0, MAX_PLUGIN_TEXT_FILES)
.map((file) => ({ path: file.path, storageId: file.storageId }));
}
async function getSkillClassificationPage(
ctx: Pick<QueryCtx, "db">,
cursor: string | undefined,
batchSize: number,
): Promise<CatalogClassificationPageResult> {
const { page, isDone, continueCursor } = await ctx.db
.query("skills")
.order("asc")
.paginate({ cursor: cursor ?? null, numItems: batchSize });
const items: CatalogClassificationPageItem[] = [];
for (const skill of page) {
if (skill.softDeletedAt) {
items.push({ kind: "skip", targetKind: "skill", reason: "soft-deleted" });
continue;
}
if (!skill.latestVersionId) {
items.push({ kind: "skip", targetKind: "skill", reason: "missing-latest-version" });
continue;
}
const version = await ctx.db.get(skill.latestVersionId);
if (!version || version.softDeletedAt || version.skillId !== skill._id) {
items.push({ kind: "skip", targetKind: "skill", reason: "missing-latest-version" });
continue;
}
items.push({
kind: "skill",
skillId: skill._id,
skillVersionId: version._id,
slug: skill.slug,
displayName: skill.displayName,
summary: skill.summary,
categories: skill.categories,
topics: skill.topics,
textFile: findSkillTextFile(version),
});
}
return { items, cursor: continueCursor, isDone };
}
async function getPluginClassificationPage(
ctx: Pick<QueryCtx, "db">,
cursor: string | undefined,
batchSize: number,
): Promise<CatalogClassificationPageResult> {
const { page, isDone, continueCursor } = await ctx.db
.query("packages")
.order("asc")
.paginate({ cursor: cursor ?? null, numItems: batchSize });
const items: CatalogClassificationPageItem[] = [];
for (const pkg of page) {
if (pkg.family === "skill") {
items.push({ kind: "skip", targetKind: "plugin", reason: "not-plugin" });
continue;
}
if (pkg.softDeletedAt) {
items.push({ kind: "skip", targetKind: "plugin", reason: "soft-deleted" });
continue;
}
if (!pkg.latestReleaseId) {
items.push({ kind: "skip", targetKind: "plugin", reason: "missing-latest-release" });
continue;
}
const release = await ctx.db.get(pkg.latestReleaseId);
if (!release || release.softDeletedAt || release.packageId !== pkg._id) {
items.push({ kind: "skip", targetKind: "plugin", reason: "missing-latest-release" });
continue;
}
items.push({
kind: "plugin",
packageId: pkg._id,
packageReleaseId: release._id,
name: pkg.name,
displayName: pkg.displayName,
summary: pkg.summary,
categories: pkg.categories,
topics: pkg.topics,
pluginManifest: release.extractedPluginManifest,
packageJson: release.extractedPackageJson,
bundleManifest: release.normalizedBundleManifest,
textFiles: findPluginTextFiles(release),
});
}
return { items, cursor: continueCursor, isDone };
}
export async function getCatalogClassificationPageInternalHandler(
ctx: Pick<QueryCtx, "db">,
args: {
targetKind: "skill" | "plugin";
cursor?: string;
batchSize?: number;
},
): Promise<CatalogClassificationPageResult> {
const batchSize = clampBatchSize(args.batchSize);
return args.targetKind === "skill"
? getSkillClassificationPage(ctx, args.cursor, batchSize)
: getPluginClassificationPage(ctx, args.cursor, batchSize);
}
export const getCatalogClassificationPageInternal = internalQuery({
args: {
targetKind: targetKindValidator,
cursor: v.optional(v.string()),
batchSize: v.optional(v.number()),
},
handler: getCatalogClassificationPageInternalHandler,
});
export const upsertCatalogClassificationResultsInternal = internalMutation({
args: { results: v.array(classificationResultInputValidator) },
returns: v.object({ ok: v.literal(true), upserted: v.number() }),
handler: async (ctx, args) => {
const classifiedAt = Date.now();
for (const result of args.results) {
const isSkill = result.targetKind === "skill";
if (
(isSkill && (!result.skillId || !result.skillVersionId || result.packageId)) ||
(!isSkill && (!result.packageId || !result.packageReleaseId || result.skillId))
) {
throw new ConvexError("Catalog classification result target is inconsistent");
}
const existing = isSkill
? await ctx.db
.query("catalogClassificationResults")
.withIndex("by_skill", (q) => q.eq("skillId", result.skillId))
.unique()
: await ctx.db
.query("catalogClassificationResults")
.withIndex("by_package", (q) => q.eq("packageId", result.packageId))
.unique();
const value = {
...result,
applyStatus: "preview" as const,
error: undefined,
classifiedAt,
appliedAt: undefined,
};
if (existing) await ctx.db.patch(existing._id, value);
else await ctx.db.insert("catalogClassificationResults", value);
}
return { ok: true as const, upserted: args.results.length };
},
});
export const classifyCatalog: ReturnType<typeof action> = action({
args: {
targetKind: targetKindValidator,
cursor: v.optional(v.string()),
batchSize: v.optional(v.number()),
maxBatches: v.optional(v.number()),
continueOnIncomplete: v.optional(v.boolean()),
},
handler: async (ctx, args): Promise<CatalogClassificationActionResult> => {
const { user } = await requireUserFromAction(ctx);
assertRole(user, ["admin"]);
return ctx.runAction(internal.catalogClassificationNode.classifyCatalogInternal, args);
},
});
export type StoredCatalogClassificationInput = ReturnType<
typeof import("./lib/catalogClassification").prepareCatalogClassificationResult
> & {
targetKind: "skill" | "plugin";
skillId?: Id<"skills">;
packageId?: Id<"packages">;
skillVersionId?: Id<"skillVersions">;
packageReleaseId?: Id<"packageReleases">;
};
export type CatalogClassifierFunctionResult = CatalogClassifierResult;
+229
View File
@@ -0,0 +1,229 @@
"use node";
import { v } from "convex/values";
import { internal } from "./_generated/api";
import type { ActionCtx } from "./_generated/server";
import { internalAction } from "./_generated/server";
import type {
CatalogClassificationActionResult,
CatalogClassificationPageItem,
CatalogClassificationPageResult,
StoredCatalogClassificationInput,
} from "./catalogClassification";
import {
prepareCatalogClassificationResult,
type CatalogClassificationConfidence,
} from "./lib/catalogClassification";
import { classifyPlugin, classifySkill } from "./lib/catalogClassifier.mjs";
const DEFAULT_MAX_BATCHES = 1;
const MAX_MAX_BATCHES = 20;
const MAX_CLASSIFICATION_TEXT_LENGTH = 40_000;
function clampMaxBatches(value: number | undefined) {
const integer = Number.isFinite(value) ? Math.floor(value ?? 0) : DEFAULT_MAX_BATCHES;
return Math.max(1, Math.min(MAX_MAX_BATCHES, integer));
}
function asRecord(value: unknown): Record<string, unknown> {
return value && typeof value === "object" && !Array.isArray(value)
? (value as Record<string, unknown>)
: {};
}
function stringValue(value: unknown) {
return typeof value === "string" ? value : "";
}
function stringArray(value: unknown) {
return Array.isArray(value)
? value.filter((entry): entry is string => typeof entry === "string")
: [];
}
async function readTextFiles(ctx: Pick<ActionCtx, "storage">, files: Array<{ storageId: never }>) {
const chunks: string[] = [];
for (const file of files) {
const blob = await ctx.storage.get(file.storageId);
if (!blob) continue;
chunks.push((await blob.text()).slice(0, MAX_CLASSIFICATION_TEXT_LENGTH));
if (chunks.join("\n").length >= MAX_CLASSIFICATION_TEXT_LENGTH) break;
}
return chunks.join("\n").slice(0, MAX_CLASSIFICATION_TEXT_LENGTH);
}
async function classifySkillItem(
ctx: Pick<ActionCtx, "storage">,
item: Extract<CatalogClassificationPageItem, { kind: "skill" }>,
): Promise<StoredCatalogClassificationInput> {
const storedText = item.textFile
? await readTextFiles(ctx, [{ storageId: item.textFile.storageId as never }])
: "";
const text =
storedText ||
`---\nname: ${item.displayName}\ndescription: ${item.summary ?? ""}\n---\n# ${item.displayName}`;
return {
targetKind: "skill",
skillId: item.skillId,
skillVersionId: item.skillVersionId,
...prepareCatalogClassificationResult(
classifySkill({
slug: item.slug,
text,
explicitCategories: item.categories,
explicitTopics: item.topics,
}),
),
};
}
async function classifyPluginItem(
ctx: Pick<ActionCtx, "storage">,
item: Extract<CatalogClassificationPageItem, { kind: "plugin" }>,
): Promise<StoredCatalogClassificationInput> {
const manifest = asRecord(item.pluginManifest);
const packageJson = asRecord(item.packageJson);
const bundleManifest = asRecord(item.bundleManifest);
const fileText = await readTextFiles(
ctx,
item.textFiles.map((file) => ({ storageId: file.storageId as never })),
);
const packageKeywords = stringArray(packageJson.keywords);
const primaryText = [
item.displayName,
item.summary,
stringValue(manifest.description),
stringValue(packageJson.description),
stringValue(bundleManifest.description),
]
.filter(Boolean)
.join("\n");
const text = [item.name, primaryText, packageKeywords.join(" "), fileText]
.filter(Boolean)
.join("\n")
.slice(0, MAX_CLASSIFICATION_TEXT_LENGTH);
const topicText = [primaryText, fileText]
.filter(Boolean)
.join("\n")
.slice(0, MAX_CLASSIFICATION_TEXT_LENGTH);
return {
targetKind: "plugin",
packageId: item.packageId,
packageReleaseId: item.packageReleaseId,
...prepareCatalogClassificationResult(
classifyPlugin({
manifest,
slug: item.name,
text,
topicText,
topicTags: packageKeywords,
explicitCategories: item.categories,
explicitTopics: item.topics,
}),
),
};
}
function emptyConfidenceCounts(): Record<CatalogClassificationConfidence, number> {
return { high: 0, medium: 0, low: 0 };
}
export async function classifyCatalogInternalHandler(
ctx: ActionCtx,
args: {
targetKind: "skill" | "plugin";
cursor?: string;
batchSize?: number;
maxBatches?: number;
continueOnIncomplete?: boolean;
},
): Promise<CatalogClassificationActionResult> {
const maxBatches = clampMaxBatches(args.maxBatches);
const confidence = emptyConfidenceCounts();
const topicConfidence = emptyConfidenceCounts();
let cursor = args.cursor ?? null;
let isDone = false;
let scanned = 0;
let classified = 0;
let skipped = 0;
let failed = 0;
for (let batch = 0; batch < maxBatches; batch += 1) {
const page: CatalogClassificationPageResult = await ctx.runQuery(
internal.catalogClassification.getCatalogClassificationPageInternal,
{
targetKind: args.targetKind,
cursor: cursor ?? undefined,
batchSize: args.batchSize,
},
);
scanned += page.items.length;
const results: StoredCatalogClassificationInput[] = [];
for (const item of page.items) {
if (item.kind === "skip") {
skipped += 1;
continue;
}
try {
const result =
item.kind === "skill"
? await classifySkillItem(ctx, item)
: await classifyPluginItem(ctx, item);
results.push(result);
confidence[result.categoryConfidence] += 1;
topicConfidence[result.topicConfidence] += 1;
classified += 1;
} catch (error) {
console.error("Catalog classification failed", {
targetKind: args.targetKind,
error: error instanceof Error ? error.message : String(error),
});
failed += 1;
}
}
if (results.length > 0) {
await ctx.runMutation(
internal.catalogClassification.upsertCatalogClassificationResultsInternal,
{
results,
},
);
}
cursor = page.cursor;
isDone = page.isDone;
if (page.isDone) break;
}
const scheduledNext = !isDone && Boolean(args.continueOnIncomplete);
if (scheduledNext) {
await ctx.scheduler.runAfter(0, internal.catalogClassificationNode.classifyCatalogInternal, {
...args,
cursor: cursor ?? undefined,
});
}
return {
ok: true,
targetKind: args.targetKind,
scanned,
classified,
skipped,
failed,
confidence,
topicConfidence,
cursor,
isDone,
scheduledNext,
};
}
export const classifyCatalogInternal = internalAction({
args: {
targetKind: v.union(v.literal("skill"), v.literal("plugin")),
cursor: v.optional(v.string()),
batchSize: v.optional(v.number()),
maxBatches: v.optional(v.number()),
continueOnIncomplete: v.optional(v.boolean()),
},
handler: classifyCatalogInternalHandler,
});
+289
View File
@@ -0,0 +1,289 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { listOfficialEntries, listOfficialSkillEntries } from "./catalogFeed";
vi.mock("./lib/publishers", () => ({
getOwnerPublisher: vi.fn().mockResolvedValue({ handle: "openclaw" }),
}));
vi.mock("./lib/officialPublishers", () => ({
isOfficialPublisher: vi.fn().mockResolvedValue(true),
}));
type WrappedHandler<TArgs, TResult> = {
_handler: (ctx: unknown, args: TArgs) => Promise<TResult>;
};
const listOfficialEntriesHandler = (
listOfficialEntries as unknown as WrappedHandler<
{ family: "code-plugin" | "bundle-plugin" },
unknown[]
>
)._handler;
const listOfficialSkillEntriesHandler = (
listOfficialSkillEntries as unknown as WrappedHandler<
{ publisherId: string; cursor: string | null },
unknown
>
)._handler;
function makePackage(overrides: Record<string, unknown> = {}) {
return {
_id: "packages:1",
name: "@openclaw/demo",
normalizedName: "@openclaw/demo",
displayName: "Demo",
ownerUserId: "users:1",
family: "code-plugin",
channel: "official",
isOfficial: true,
latestReleaseId: "packageReleases:1",
softDeletedAt: undefined,
...overrides,
};
}
function makeRelease(overrides: Record<string, unknown> = {}) {
return {
packageId: "packages:1",
version: "1.2.3",
integritySha256: "ignored",
artifactKind: "legacy-zip",
sha256hash: "artifact-hash",
verification: { scanStatus: "clean" },
manualModeration: undefined,
softDeletedAt: undefined,
...overrides,
};
}
function makeSkill(overrides: Record<string, unknown> = {}) {
return {
_id: "skills:1",
slug: "demo",
displayName: "Demo skill",
ownerUserId: "users:1",
ownerPublisherId: "publishers:1",
latestVersionId: "skillVersions:1",
softDeletedAt: undefined,
moderationStatus: "active",
...overrides,
};
}
function makeSkillVersion(overrides: Record<string, unknown> = {}) {
return {
_id: "skillVersions:1",
skillId: "skills:1",
version: "1.2.3",
softDeletedAt: undefined,
files: [{ path: "SKILL.md", size: 1, storageId: "storage:1", sha256: "file-hash" }],
sha256hash: "skill-hash",
...overrides,
};
}
function makeCtx(packages: unknown[], records: Record<string, unknown>) {
return {
db: {
query: vi.fn(() => {
const query = {
eq: vi.fn(() => query),
};
return {
withIndex: vi.fn((_index: string, apply: (value: typeof query) => unknown) => {
apply(query);
return {
order: vi.fn(() => ({
paginate: vi.fn(async () => ({
page: packages,
isDone: true,
continueCursor: "",
})),
take: vi.fn(async () => packages),
})),
};
}),
take: vi.fn(async () => [{ publisherId: "publishers:1" }]),
};
}),
get: vi.fn(async (id: string) => records[id] ?? null),
},
};
}
describe("catalog feed projection", () => {
beforeEach(() => {
vi.clearAllMocks();
});
it("projects official releases into ClawHub install candidates", async () => {
const result = await listOfficialEntriesHandler(
makeCtx([makePackage()], {
"packageReleases:1": makeRelease(),
}),
{ family: "code-plugin" },
);
expect(result).toEqual([
{
type: "plugin",
id: "@openclaw/demo",
title: "Demo",
version: "1.2.3",
state: "available",
publisher: { id: "openclaw", trust: "official" },
install: {
candidates: [
{
sourceRef: "public-clawhub",
package: "@openclaw/demo",
version: "1.2.3",
integrity: "sha256:artifact-hash",
},
],
},
},
]);
});
it("excludes non-official, blocked, deleted, and undigested releases", async () => {
const result = await listOfficialEntriesHandler(
makeCtx(
[
makePackage({ name: "@openclaw/community", channel: "community" }),
makePackage({ name: "@openclaw/deleted", softDeletedAt: 1 }),
makePackage({ name: "@openclaw/malicious", latestReleaseId: "packageReleases:2" }),
makePackage({ name: "@openclaw/no-hash", latestReleaseId: "packageReleases:3" }),
],
{
"packageReleases:1": makeRelease(),
"packageReleases:2": makeRelease({ manualModeration: { state: "quarantined" } }),
"packageReleases:3": makeRelease({ sha256hash: undefined }),
},
),
{ family: "code-plugin" },
);
expect(result).toEqual([]);
});
it("re-checks the live official publisher record", async () => {
const { isOfficialPublisher } = await import("./lib/officialPublishers");
vi.mocked(isOfficialPublisher).mockResolvedValueOnce(false);
const result = await listOfficialEntriesHandler(
makeCtx([makePackage()], {
"packageReleases:1": makeRelease(),
}),
{ family: "code-plugin" },
);
expect(result).toEqual([]);
});
it("rejects a latest-release pointer for another package", async () => {
const result = await listOfficialEntriesHandler(
makeCtx([makePackage({ _id: "packages:2" })], {
"packageReleases:1": makeRelease(),
}),
{ family: "code-plugin" },
);
expect(result).toEqual([]);
});
it("projects only published skills from verified organization publishers", async () => {
const result = (await listOfficialSkillEntriesHandler(
makeCtx([makeSkill()], {
"publishers:1": { _id: "publishers:1", kind: "org", handle: "openclaw" },
"skillVersions:1": makeSkillVersion(),
}),
{ publisherId: "publishers:1", cursor: null },
)) as { entries: unknown[]; isDone: boolean };
expect(result).toMatchObject({
entries: [
{
type: "skill",
id: "@openclaw/demo",
title: "Demo skill",
version: "1.2.3",
state: "available",
publisher: { id: "openclaw", trust: "official" },
install: {
candidates: [
{
sourceRef: "public-clawhub",
package: "@openclaw/demo",
version: "1.2.3",
integrity: "sha256:skill-hash",
},
],
},
},
],
isDone: true,
});
});
it("includes skills from verified personal publishers", async () => {
const result = (await listOfficialSkillEntriesHandler(
makeCtx([makeSkill({ ownerPublisherId: "publishers:steipete" })], {
"publishers:steipete": { _id: "publishers:steipete", kind: "user", handle: "steipete" },
"skillVersions:1": makeSkillVersion(),
}),
{ publisherId: "publishers:steipete", cursor: null },
)) as { entries: unknown[]; isDone: boolean };
expect(result.entries).toMatchObject([
{
type: "skill",
id: "@steipete/demo",
publisher: { id: "steipete", trust: "official" },
},
]);
});
it("excludes a latest version blocked by the download safety gate", async () => {
const result = (await listOfficialSkillEntriesHandler(
makeCtx([makeSkill()], {
"publishers:1": { _id: "publishers:1", kind: "org", handle: "openclaw" },
"skillVersions:1": makeSkillVersion({
llmAnalysis: { status: "complete", verdict: "malicious" },
}),
}),
{ publisherId: "publishers:1", cursor: null },
)) as { entries: unknown[]; isDone: boolean };
expect(result.entries).toEqual([]);
});
it("excludes unverified, unpublished, and un-hashed skills", async () => {
vi.mocked((await import("./lib/officialPublishers")).isOfficialPublisher).mockImplementation(
async (_ctx, publisher) => publisher?._id === "publishers:1",
);
const unverified = (await listOfficialSkillEntriesHandler(
makeCtx([makeSkill({ ownerPublisherId: "publishers:unverified" })], {
"publishers:unverified": { _id: "publishers:unverified", kind: "org", handle: "vendor" },
"skillVersions:1": makeSkillVersion(),
}),
{ publisherId: "publishers:unverified", cursor: null },
)) as { entries: unknown[] };
const unpublishedOrUnhashed = (await listOfficialSkillEntriesHandler(
makeCtx(
[
makeSkill({ latestVersionId: undefined }),
makeSkill({ _id: "skills:no-hash", latestVersionId: "skillVersions:no-hash" }),
],
{
"publishers:1": { _id: "publishers:1", kind: "org", handle: "openclaw" },
"skillVersions:no-hash": makeSkillVersion({ sha256hash: undefined }),
},
),
{ publisherId: "publishers:1", cursor: null },
)) as { entries: unknown[] };
expect(unverified.entries).toEqual([]);
expect(unpublishedOrUnhashed.entries).toEqual([]);
});
});
+424
View File
@@ -0,0 +1,424 @@
import {
CATALOG_FEED_ID,
CATALOG_FEED_SCHEMA_VERSION,
CATALOG_FEED_SOURCE_REF,
CATALOG_SKILLS_FEED_DESCRIPTION,
CATALOG_SKILLS_FEED_ID,
serializeCatalogFeed,
type CatalogFeedEntry,
type CatalogFeedSkillEntry,
} from "clawhub-schema";
import { v } from "convex/values";
import { internal } from "./_generated/api";
import type { Doc } from "./_generated/dataModel";
import { internalAction, internalMutation, internalQuery } from "./_generated/server";
import type { QueryCtx } from "./_generated/server";
import { sha256Hex } from "./lib/clawpack";
import { isPublicSkillDoc } from "./lib/globalStats";
import { isOfficialPublisher } from "./lib/officialPublishers";
import { getPackageReleaseArtifactSha256 } from "./lib/packageArtifacts";
import { isPackageBlockedFromPublic, resolvePackageReleaseScanStatus } from "./lib/packageSecurity";
import { getOwnerPublisher } from "./lib/publishers";
import {
getPublicSkillVersionDownloadBlock,
getSkillFileModerationInfoFromSkill,
isPublicSkillVersionAvailableForSkill,
} from "./lib/skillFileAccess";
const CATALOG_FEED_DESCRIPTION = "Official OpenClaw plugins published on ClawHub.";
const CATALOG_FEED_PAGE_SIZE = 100;
const MAX_CATALOG_FEED_ENTRIES = 500;
const CATALOG_FEED_FAMILIES = ["code-plugin", "bundle-plugin"] as const;
type CatalogQueryCtx = Pick<QueryCtx, "db">;
type CatalogFeedPublicationResult = {
publicationId: string;
feedId: string;
sequence: number;
payloadSha256: string;
publishedAt: number;
entryCount: number;
};
const catalogFeedEntryFields = {
id: v.string(),
title: v.string(),
version: v.string(),
state: v.union(
v.literal("available"),
v.literal("recommended"),
v.literal("disabled"),
v.literal("blocked"),
v.literal("deprecated"),
),
publisher: v.object({
id: v.string(),
trust: v.union(v.literal("official"), v.literal("community")),
}),
install: v.object({
candidates: v.array(
v.object({
sourceRef: v.string(),
package: v.string(),
version: v.string(),
integrity: v.string(),
}),
),
}),
};
const catalogFeedEntryValidator = v.union(
v.object({ type: v.literal("plugin"), ...catalogFeedEntryFields }),
v.object({ type: v.literal("skill"), ...catalogFeedEntryFields }),
);
async function buildEntry(
ctx: CatalogQueryCtx,
pkg: Doc<"packages">,
): Promise<CatalogFeedEntry | null> {
if (pkg.softDeletedAt || pkg.channel !== "official" || !pkg.latestReleaseId) return null;
const release = await ctx.db.get(pkg.latestReleaseId);
if (!release || release.packageId !== pkg._id || release.softDeletedAt) return null;
// Keep ClawHub on RFC 19's canonical feed entry shape. OpenClaw's staged
// consumer must land its legacy-catalog adapter before this URL is enabled.
const scanStatus = resolvePackageReleaseScanStatus(release);
if (isPackageBlockedFromPublic(scanStatus)) return null;
const artifactSha256 = getPackageReleaseArtifactSha256(release);
if (!artifactSha256) return null;
const owner = await getOwnerPublisher(ctx, {
ownerPublisherId: pkg.ownerPublisherId,
ownerUserId: pkg.ownerUserId,
});
if (!(await isOfficialPublisher(ctx, owner))) return null;
const publisherId = owner?.handle?.trim();
if (!publisherId) return null;
const packageName = pkg.name.trim();
const id = pkg.normalizedName.trim();
const title = pkg.displayName.trim() || packageName;
const version = release.version.trim();
if (!packageName || !id || !title || !version) return null;
return {
type: "plugin",
id,
title,
version,
state: "available",
publisher: {
id: publisherId,
trust: "official",
},
install: {
candidates: [
{
sourceRef: CATALOG_FEED_SOURCE_REF,
package: packageName,
version,
integrity: `sha256:${artifactSha256}`,
},
],
},
};
}
async function listFamilyEntries(
ctx: CatalogQueryCtx,
family: (typeof CATALOG_FEED_FAMILIES)[number],
) {
const entries: CatalogFeedEntry[] = [];
let cursor: string | null = null;
while (true) {
const page = await ctx.db
.query("packages")
.withIndex("by_active_family_official_downloads", (q) =>
q.eq("softDeletedAt", undefined).eq("family", family).eq("isOfficial", true),
)
.order("desc")
.paginate({ cursor, numItems: CATALOG_FEED_PAGE_SIZE });
for (const pkg of page.page) {
const entry = await buildEntry(ctx, pkg);
if (entry) entries.push(entry);
if (entries.length > MAX_CATALOG_FEED_ENTRIES) {
throw new Error(`Catalog feed exceeds ${MAX_CATALOG_FEED_ENTRIES} entries`);
}
}
if (page.isDone) return entries;
cursor = page.continueCursor;
}
}
async function buildSkillEntry(
ctx: CatalogQueryCtx,
skill: Doc<"skills">,
trustedOwner?: Doc<"publishers">,
): Promise<CatalogFeedSkillEntry | null> {
if (
!isPublicSkillDoc(skill) ||
!skill.ownerPublisherId ||
!skill.latestVersionId ||
(trustedOwner && skill.ownerPublisherId !== trustedOwner._id)
) {
return null;
}
const [owner, version] = await Promise.all([
trustedOwner ?? ctx.db.get(skill.ownerPublisherId),
ctx.db.get(skill.latestVersionId),
]);
if (
!owner ||
(trustedOwner
? Boolean(owner.deletedAt || owner.deactivatedAt)
: !(await isOfficialPublisher(ctx, owner))) ||
!version ||
!isPublicSkillVersionAvailableForSkill(version, skill._id) ||
getPublicSkillVersionDownloadBlock(getSkillFileModerationInfoFromSkill(skill), version) ||
!version.files.length ||
!version.sha256hash
) {
return null;
}
const publisherId = owner.handle?.trim();
const slug = skill.slug.trim();
const title = skill.displayName.trim() || slug;
const versionName = version.version.trim();
if (!publisherId || !slug || !title || !versionName) return null;
const packageName = `@${publisherId}/${slug}`;
return {
type: "skill",
id: packageName,
title,
version: versionName,
state: "available",
publisher: {
id: publisherId,
trust: "official",
},
install: {
candidates: [
{
sourceRef: CATALOG_FEED_SOURCE_REF,
package: packageName,
version: versionName,
integrity: `sha256:${version.sha256hash}`,
},
],
},
};
}
export const listOfficialPublisherPage = internalQuery({
args: {
cursor: v.union(v.string(), v.null()),
},
handler: async (ctx, args) => {
const page = await ctx.db
.query("officialPublishers")
.withIndex("by_created")
.order("desc")
.paginate({ cursor: args.cursor, numItems: CATALOG_FEED_PAGE_SIZE });
const publishers: Doc<"publishers">[] = [];
for (const row of page.page) {
const publisher = await ctx.db.get(row.publisherId);
if (publisher && !publisher.deletedAt && !publisher.deactivatedAt) {
publishers.push(publisher);
}
}
return {
publishers,
isDone: page.isDone,
continueCursor: page.continueCursor,
};
},
});
export const listOfficialEntries = internalQuery({
args: {
family: v.union(v.literal("code-plugin"), v.literal("bundle-plugin")),
},
handler: async (ctx, args) => await listFamilyEntries(ctx, args.family),
});
export const listOfficialSkillEntries = internalQuery({
args: {
publisherId: v.id("publishers"),
cursor: v.union(v.string(), v.null()),
},
handler: async (ctx, args) => {
const owner = await ctx.db.get(args.publisherId);
if (
!owner ||
owner.deletedAt ||
owner.deactivatedAt ||
!(await isOfficialPublisher(ctx, owner))
) {
return { entries: [], isDone: true, continueCursor: "" };
}
const page = await ctx.db
.query("skills")
.withIndex("by_owner_publisher_active_updated", (q) =>
q.eq("ownerPublisherId", args.publisherId).eq("softDeletedAt", undefined),
)
.order("desc")
.paginate({ cursor: args.cursor, numItems: CATALOG_FEED_PAGE_SIZE });
const entries: CatalogFeedSkillEntry[] = [];
for (const skill of page.page) {
const entry = await buildSkillEntry(ctx, skill, owner);
if (entry) entries.push(entry);
}
return {
entries,
isDone: page.isDone,
continueCursor: page.continueCursor,
};
},
});
export const storePublication = internalMutation({
args: {
feedId: v.union(v.literal(CATALOG_FEED_ID), v.literal(CATALOG_SKILLS_FEED_ID)),
description: v.string(),
generatedAt: v.string(),
expiresAt: v.string(),
entries: v.array(catalogFeedEntryValidator),
},
handler: async (ctx, args) => {
const expectedEntryType = args.feedId === CATALOG_SKILLS_FEED_ID ? "skill" : "plugin";
if (args.entries.some((entry) => entry.type !== expectedEntryType)) {
throw new Error(`Catalog ${expectedEntryType} feed received a mismatched entry type`);
}
const latest = await ctx.db
.query("catalogFeedPublications")
.withIndex("by_feed", (q) => q.eq("feedId", args.feedId))
.unique();
const sequence = (latest?.sequence ?? 0) + 1;
const payload = serializeCatalogFeed({
schemaVersion: CATALOG_FEED_SCHEMA_VERSION,
id: args.feedId,
generatedAt: args.generatedAt,
sequence,
expiresAt: args.expiresAt,
description: args.description,
entries: args.entries,
});
const payloadSha256 = await sha256Hex(new TextEncoder().encode(payload));
const publishedAt = Date.now();
const publication = {
feedId: args.feedId,
sequence,
generatedAt: args.generatedAt,
expiresAt: args.expiresAt,
payload,
payloadSha256,
publishedAt,
};
const publicationId = latest
? (await ctx.db.patch(latest._id, publication), latest._id)
: await ctx.db.insert("catalogFeedPublications", publication);
return {
publicationId,
feedId: args.feedId,
sequence,
payloadSha256,
publishedAt,
entryCount: args.entries.length,
};
},
});
export const publish = internalAction({
args: {
expiresAt: v.string(),
},
handler: async (ctx, args): Promise<CatalogFeedPublicationResult[]> => {
const generatedAt = new Date().toISOString();
const familyEntries: CatalogFeedEntry[][] = await Promise.all(
CATALOG_FEED_FAMILIES.map(async (family) => {
const entries: CatalogFeedEntry[] = await ctx.runQuery(
internal.catalogFeed.listOfficialEntries,
{ family },
);
return entries;
}),
);
const entries = familyEntries.flat();
if (entries.length > MAX_CATALOG_FEED_ENTRIES) {
throw new Error(`Catalog feed exceeds ${MAX_CATALOG_FEED_ENTRIES} entries`);
}
const skillEntries: CatalogFeedSkillEntry[] = [];
const seenPublisherIds = new Set<string>();
let publisherCursor: string | null = null;
while (true) {
const publisherPage: {
publishers: Doc<"publishers">[];
isDone: boolean;
continueCursor: string;
} = await ctx.runQuery(internal.catalogFeed.listOfficialPublisherPage, {
cursor: publisherCursor,
});
for (const publisher of publisherPage.publishers) {
if (seenPublisherIds.has(publisher._id)) continue;
seenPublisherIds.add(publisher._id);
let skillCursor: string | null = null;
while (true) {
const skillPage: {
entries: CatalogFeedSkillEntry[];
isDone: boolean;
continueCursor: string;
} = await ctx.runQuery(internal.catalogFeed.listOfficialSkillEntries, {
publisherId: publisher._id,
cursor: skillCursor,
});
skillEntries.push(...skillPage.entries);
if (skillEntries.length > MAX_CATALOG_FEED_ENTRIES) {
throw new Error(`Catalog skills feed exceeds ${MAX_CATALOG_FEED_ENTRIES} entries`);
}
if (skillPage.isDone) break;
skillCursor = skillPage.continueCursor;
}
}
if (publisherPage.isDone) break;
publisherCursor = publisherPage.continueCursor;
}
const pluginResult: CatalogFeedPublicationResult = await ctx.runMutation(
internal.catalogFeed.storePublication,
{
feedId: CATALOG_FEED_ID,
description: CATALOG_FEED_DESCRIPTION,
generatedAt,
expiresAt: args.expiresAt,
entries: entries.sort((left, right) => left.id.localeCompare(right.id)),
},
);
const skillsResult: CatalogFeedPublicationResult = await ctx.runMutation(
internal.catalogFeed.storePublication,
{
feedId: CATALOG_SKILLS_FEED_ID,
description: CATALOG_SKILLS_FEED_DESCRIPTION,
generatedAt,
expiresAt: args.expiresAt,
entries: skillEntries.sort((left, right) => left.id.localeCompare(right.id)),
},
);
return [pluginResult, skillsResult];
},
});
export const getLatestPublication = internalQuery({
args: {
feedId: v.union(v.literal(CATALOG_FEED_ID), v.literal(CATALOG_SKILLS_FEED_ID)),
},
handler: async (ctx, args) =>
await ctx.db
.query("catalogFeedPublications")
.withIndex("by_feed", (q) => q.eq("feedId", args.feedId))
.unique(),
});
+437
View File
@@ -0,0 +1,437 @@
/* @vitest-environment node */
import { getAuthUserId } from "@convex-dev/auth/server";
import { beforeEach, describe, expect, it, vi } from "vitest";
const { upsertPackageSearchDigestMock, upsertSkillSearchDigestMock } = vi.hoisted(() => ({
upsertPackageSearchDigestMock: vi.fn(),
upsertSkillSearchDigestMock: vi.fn(),
}));
vi.mock("@convex-dev/auth/server", () => ({
getAuthUserId: vi.fn(),
authTables: {},
}));
vi.mock("./lib/packageSearchDigest", async () => {
const actual = await vi.importActual<typeof import("./lib/packageSearchDigest")>(
"./lib/packageSearchDigest",
);
return { ...actual, upsertPackageSearchDigest: upsertPackageSearchDigestMock };
});
vi.mock("./lib/skillSearchDigest", async () => {
const actual =
await vi.importActual<typeof import("./lib/skillSearchDigest")>("./lib/skillSearchDigest");
return { ...actual, upsertSkillSearchDigest: upsertSkillSearchDigestMock };
});
const { setPackageCatalogMetadata } = await import("./packages");
const { setCatalogMetadata } = await import("./skills");
type WrappedHandler<TArgs> = {
_handler: (ctx: unknown, args: TArgs) => Promise<void>;
};
const setPackageCatalogMetadataHandler = (
setPackageCatalogMetadata as unknown as WrappedHandler<{
packageId: string;
categories?: string[];
topics: string[];
}>
)._handler;
const setSkillCatalogMetadataHandler = (
setCatalogMetadata as unknown as WrappedHandler<{
skillId: string;
categories?: string[];
topics: string[];
}>
)._handler;
const user = {
_id: "users:owner",
role: "user",
handle: "owner",
personalPublisherId: "publishers:owner",
};
const publisher = {
_id: "publishers:owner",
kind: "user",
handle: "owner",
displayName: "Owner",
linkedUserId: "users:owner",
};
function makeCtx(
resourceId: string,
resource: Record<string, unknown>,
relatedResources: Record<string, Record<string, unknown>> = {},
) {
const patch = vi.fn(async () => {});
const insert = vi.fn(async (table: string) => `${table}:inserted`);
const replace = vi.fn(async () => {});
const remove = vi.fn(async () => {});
return {
patch,
insert,
ctx: {
db: {
get: vi.fn(async (id: string) => {
if (id === user._id) return user;
if (id === publisher._id) return publisher;
if (id === resourceId) return resource;
return relatedResources[id] ?? null;
}),
patch,
insert,
replace,
delete: remove,
normalizeId: vi.fn(),
system: {},
query: vi.fn(() => ({
withIndex: vi.fn(() => ({
unique: vi.fn(async () => null),
collect: vi.fn(async () => []),
})),
})),
},
},
};
}
beforeEach(() => {
vi.mocked(getAuthUserId).mockResolvedValue(user._id as never);
upsertPackageSearchDigestMock.mockReset();
upsertSkillSearchDigestMock.mockReset();
});
describe("catalog metadata settings", () => {
it("persists exact skill category slugs and author topics", async () => {
const skill = {
_id: "skills:demo",
slug: "demo",
displayName: "Demo",
ownerUserId: user._id,
inferredCategories: ["automation"],
inferredTopics: ["Old inference"],
inferredFromVersionId: "skillVersions:demo",
inferredCategoryConfidence: "medium",
inferredTopicConfidence: "high",
inferredClassifierVersion: "taxonomy-prototype-v9",
inferredTopicClassifierVersion: "topic-prototype-v1",
inferredInputHash: "category-hash",
inferredTopicInputHash: "old-hash",
inferredAt: 123,
tags: {},
stats: {
downloads: 0,
stars: 0,
installsCurrent: 0,
installsAllTime: 0,
versions: 1,
comments: 0,
},
createdAt: 1,
updatedAt: 1,
};
const { ctx, patch, insert } = makeCtx(skill._id, skill);
await setSkillCatalogMetadataHandler(ctx, {
skillId: skill._id,
categories: ["development"],
topics: ["GPU development"],
});
expect(patch).toHaveBeenCalledWith(
skill._id,
expect.objectContaining({
categories: ["development"],
topics: ["GPU development"],
inferredCategories: undefined,
inferredTopics: undefined,
inferredFromVersionId: undefined,
inferredCategoryConfidence: undefined,
inferredTopicConfidence: undefined,
inferredClassifierVersion: undefined,
inferredTopicClassifierVersion: undefined,
inferredInputHash: undefined,
inferredTopicInputHash: undefined,
inferredAt: undefined,
}),
);
expect(upsertSkillSearchDigestMock).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
categories: ["development"],
topics: ["GPU development"],
}),
);
expect(insert).toHaveBeenCalledWith(
"auditLogs",
expect.objectContaining({ action: "skill.catalog_metadata.set" }),
);
});
it("persists exact plugin category slugs and author topics", async () => {
const pkg = {
_id: "packages:demo",
name: "demo-plugin",
normalizedName: "demo-plugin",
displayName: "Demo Plugin",
family: "code-plugin",
channel: "community",
isOfficial: false,
ownerUserId: user._id,
inferredCategories: ["tools"],
inferredTopics: ["Old inference"],
inferredFromReleaseId: "packageReleases:demo",
inferredCategoryConfidence: "medium",
inferredTopicConfidence: "high",
inferredClassifierVersion: "taxonomy-prototype-v9",
inferredTopicClassifierVersion: "topic-prototype-v1",
inferredInputHash: "category-hash",
inferredTopicInputHash: "old-hash",
inferredAt: 123,
stats: { downloads: 0, installs: 0, stars: 0, versions: 1 },
scanStatus: "clean",
createdAt: 1,
updatedAt: 1,
};
const { ctx, patch, insert } = makeCtx(pkg._id, pkg);
await setPackageCatalogMetadataHandler(ctx, {
packageId: pkg._id,
categories: ["models"],
topics: ["Local models"],
});
expect(patch).toHaveBeenCalledWith(
pkg._id,
expect.objectContaining({
categories: ["models"],
topics: ["Local models"],
inferredCategories: undefined,
inferredTopics: undefined,
inferredFromReleaseId: undefined,
inferredCategoryConfidence: undefined,
inferredTopicConfidence: undefined,
inferredClassifierVersion: undefined,
inferredTopicClassifierVersion: undefined,
inferredInputHash: undefined,
inferredTopicInputHash: undefined,
inferredAt: undefined,
}),
);
expect(upsertPackageSearchDigestMock).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
categories: ["models"],
topics: ["Local models"],
}),
);
expect(insert).toHaveBeenCalledWith(
"auditLogs",
expect.objectContaining({ action: "package.catalog_metadata.set" }),
);
});
it("uses Other when stored skill categories are cleared", async () => {
const skill = {
_id: "skills:demo",
slug: "demo",
displayName: "Demo",
categories: ["development"],
inferredTopics: ["Old inference"],
inferredTopicConfidence: "medium",
ownerUserId: user._id,
tags: {},
stats: {
downloads: 0,
stars: 0,
installsCurrent: 0,
installsAllTime: 0,
versions: 1,
comments: 0,
},
createdAt: 1,
updatedAt: 1,
};
const { ctx, patch } = makeCtx(skill._id, skill);
await setSkillCatalogMetadataHandler(ctx, {
skillId: skill._id,
topics: ["GPU development"],
});
expect(patch).toHaveBeenCalledWith(
skill._id,
expect.objectContaining({
categories: ["other"],
topics: ["GPU development"],
}),
);
});
it("persists Other when skill categories are explicitly cleared", async () => {
const skill = {
_id: "skills:demo",
slug: "demo",
displayName: "Demo",
categories: ["development"],
ownerUserId: user._id,
tags: {},
stats: {
downloads: 0,
stars: 0,
installsCurrent: 0,
installsAllTime: 0,
versions: 1,
comments: 0,
},
createdAt: 1,
updatedAt: 1,
};
const { ctx, patch } = makeCtx(skill._id, skill);
await setSkillCatalogMetadataHandler(ctx, {
skillId: skill._id,
categories: [],
topics: [],
});
expect(patch).toHaveBeenCalledWith(
skill._id,
expect.objectContaining({
categories: ["other"],
topics: undefined,
inferredTopics: undefined,
inferredTopicConfidence: undefined,
}),
);
expect(upsertSkillSearchDigestMock).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
categories: ["other"],
}),
);
});
it("uses Other when stored plugin categories are cleared", async () => {
const release = {
_id: "packageReleases:demo",
extractedPluginManifest: { contracts: { tools: ["demo"] } },
};
const pkg = {
_id: "packages:demo",
name: "demo-plugin",
normalizedName: "demo-plugin",
displayName: "Demo Plugin",
family: "code-plugin",
channel: "community",
isOfficial: false,
categories: ["models"],
latestReleaseId: release._id,
ownerUserId: user._id,
stats: { downloads: 0, installs: 0, stars: 0, versions: 1 },
scanStatus: "clean",
createdAt: 1,
updatedAt: 1,
};
const { ctx, patch } = makeCtx(pkg._id, pkg, { [release._id]: release });
await setPackageCatalogMetadataHandler(ctx, {
packageId: pkg._id,
topics: ["Local models"],
});
expect(patch).toHaveBeenCalledWith(
pkg._id,
expect.objectContaining({
categories: ["other"],
topics: ["Local models"],
}),
);
expect(upsertPackageSearchDigestMock).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
categories: ["other"],
pluginCategoryTags: ["other"],
}),
);
});
it("persists Other when plugin categories are explicitly cleared", async () => {
const pkg = {
_id: "packages:legacy-bundle",
name: "legacy-bundle",
normalizedName: "legacy-bundle",
displayName: "Legacy Bundle",
family: "bundle-plugin",
channel: "community",
isOfficial: false,
categories: ["tools"],
inferredTopics: ["Old inference"],
inferredTopicConfidence: "medium",
ownerUserId: user._id,
stats: { downloads: 0, installs: 0, stars: 0, versions: 1 },
scanStatus: "clean",
createdAt: 1,
updatedAt: 1,
};
const { ctx, patch } = makeCtx(pkg._id, pkg);
await setPackageCatalogMetadataHandler(ctx, {
packageId: pkg._id,
categories: [],
topics: [],
});
expect(patch).toHaveBeenCalledWith(
pkg._id,
expect.objectContaining({
categories: ["other"],
topics: undefined,
inferredTopics: undefined,
inferredTopicConfidence: undefined,
}),
);
expect(upsertPackageSearchDigestMock).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
categories: ["other"],
pluginCategoryTags: ["other"],
}),
);
});
it("uses Other when legacy plugin categories are cleared", async () => {
const pkg = {
_id: "packages:legacy-bundle",
name: "legacy-bundle",
normalizedName: "legacy-bundle",
displayName: "Legacy Bundle",
family: "bundle-plugin",
channel: "community",
isOfficial: false,
categories: ["tools"],
ownerUserId: user._id,
stats: { downloads: 0, installs: 0, stars: 0, versions: 1 },
scanStatus: "clean",
createdAt: 1,
updatedAt: 1,
};
const { ctx, patch } = makeCtx(pkg._id, pkg);
await setPackageCatalogMetadataHandler(ctx, {
packageId: pkg._id,
topics: [],
});
expect(patch).toHaveBeenCalledWith(
pkg._id,
expect.objectContaining({
categories: ["other"],
}),
);
});
});
+210
View File
@@ -0,0 +1,210 @@
/* @vitest-environment node */
import { describe, expect, it, vi } from "vitest";
import { listTopByCategory, rankTopCatalogTopics } from "./catalogTopics";
type WrappedHandler<TArgs, TResult> = {
_handler: (ctx: unknown, args: TArgs) => Promise<TResult>;
};
const listTopByCategoryHandler = (
listTopByCategory as unknown as WrappedHandler<
{ kind: "skill" | "plugin"; category: string },
string[]
>
)._handler;
function makeQueryCtx(rowsByTable: Record<string, Array<Record<string, unknown>>>) {
const indexNames: string[] = [];
const filters: Array<{ field: string; value: unknown }> = [];
return {
indexNames,
filters,
db: {
query: vi.fn((table: string) => ({
withIndex: vi.fn(
(
indexName: string,
build: (query: { eq: (field: string, value: unknown) => unknown }) => unknown,
) => {
indexNames.push(indexName);
const query = {
eq: (field: string, value: unknown) => {
filters.push({ field, value });
return query;
},
};
build(query);
return {
order: vi.fn(() => ({
take: vi.fn(async (limit: number) => (rowsByTable[table] ?? []).slice(0, limit)),
})),
};
},
),
})),
},
};
}
function makeSkillDigest(overrides: Record<string, unknown> = {}) {
return {
skillId: "skills:demo",
slug: "demo",
displayName: "Demo",
ownerUserId: "users:owner",
forkOf: undefined,
tags: {},
badges: {},
stats: { downloads: 0, stars: 0, versions: 1, comments: 0 },
moderationStatus: "active",
createdAt: 1,
updatedAt: 2,
...overrides,
};
}
function makePluginCategoryDigest(overrides: Record<string, unknown> = {}) {
return {
packageId: "packages:demo",
name: "demo",
normalizedName: "demo",
displayName: "Demo",
family: "code-plugin",
channel: "community",
isOfficial: false,
ownerUserId: "users:owner",
pluginCategory: "runtime",
scanStatus: "clean",
createdAt: 1,
updatedAt: 2,
...overrides,
};
}
describe("catalog topic ranking", () => {
it("returns the most frequent normalized topics and excludes the selected category", () => {
expect(
rankTopCatalogTopics(
[
{ topics: ["TypeScript", "Development", "Docker"] },
{ topics: ["typescript", "GitHub", "Debugging"] },
{ topics: ["docker", "typescript", "Coding"] },
{ topics: ["Automation", "GitHub"] },
],
"development",
),
).toEqual(["typescript", "docker", "github", "debugging", "coding", "automation"]);
});
});
describe("listTopByCategory", () => {
it("returns top public skill topics from the selected category", async () => {
const ctx = makeQueryCtx({
skillSearchDigest: [
makeSkillDigest({
categories: ["development"],
topics: ["TypeScript", "Docker"],
}),
makeSkillDigest({
skillId: "skills:second",
slug: "second",
categories: ["development"],
topics: ["typescript", "GitHub"],
}),
makeSkillDigest({
skillId: "skills:other",
slug: "other",
categories: ["productivity"],
topics: ["notes"],
}),
makeSkillDigest({
skillId: "skills:blocked",
slug: "blocked",
categories: ["development"],
topics: ["malware"],
moderationStatus: "rejected",
}),
],
});
await expect(
listTopByCategoryHandler(ctx, { kind: "skill", category: "development" }),
).resolves.toEqual(["typescript", "docker", "github"]);
expect(ctx.indexNames).toEqual(["by_active_recommended_score"]);
expect(ctx.filters).toContainEqual({ field: "softDeletedAt", value: undefined });
});
it("collects a category sample beyond the first global topic sample", async () => {
const globallyHigherRanked = Array.from({ length: 240 }, (_, index) =>
makeSkillDigest({
skillId: `skills:global-${index}`,
slug: `global-${index}`,
categories: ["productivity"],
topics: ["notes"],
}),
);
const ctx = makeQueryCtx({
skillSearchDigest: [
...globallyHigherRanked,
makeSkillDigest({
skillId: "skills:development",
slug: "development",
categories: ["development"],
topics: ["TypeScript", "Docker"],
}),
],
});
await expect(
listTopByCategoryHandler(ctx, { kind: "skill", category: "development" }),
).resolves.toEqual(["typescript", "docker"]);
});
it("uses the plugin category index and excludes private or blocked plugins", async () => {
const ctx = makeQueryCtx({
packagePluginCategorySearchDigest: [
makePluginCategoryDigest({ topics: ["Docker", "TypeScript"] }),
makePluginCategoryDigest({
packageId: "packages:second",
name: "second",
normalizedName: "second",
topics: ["docker", "GitHub"],
}),
makePluginCategoryDigest({
packageId: "packages:private",
name: "private",
normalizedName: "private",
topics: ["secret"],
channel: "private",
}),
makePluginCategoryDigest({
packageId: "packages:blocked",
name: "blocked",
normalizedName: "blocked",
topics: ["malware"],
scanStatus: "malicious",
}),
],
});
await expect(
listTopByCategoryHandler(ctx, { kind: "plugin", category: "runtime" }),
).resolves.toEqual(["docker", "typescript", "github"]);
expect(ctx.indexNames).toEqual(["by_active_category_installs"]);
expect(ctx.filters).toContainEqual({ field: "pluginCategory", value: "runtime" });
});
it("rejects categories that do not belong to the requested catalog kind", async () => {
const ctx = makeQueryCtx({});
await expect(
listTopByCategoryHandler(ctx, { kind: "skill", category: "runtime" }),
).resolves.toEqual([]);
await expect(
listTopByCategoryHandler(ctx, { kind: "plugin", category: "development" }),
).resolves.toEqual([]);
expect(ctx.db.query).not.toHaveBeenCalled();
});
});
+93
View File
@@ -0,0 +1,93 @@
import {
getCatalogTopicSlugs,
isPluginCategorySlug,
isSkillCategorySlug,
resolveStoredSkillCategories,
} from "clawhub-schema";
import { v } from "convex/values";
import type { Doc } from "./_generated/dataModel";
import type { QueryCtx } from "./_generated/server";
import { query } from "./functions";
import { isPublicPluginDoc, isPublicSkillDoc } from "./lib/globalStats";
const TOP_CATEGORY_TOPIC_LIMIT = 8;
const TOP_CATEGORY_TOPIC_SAMPLE_LIMIT = 240;
const TOP_SKILL_CATEGORY_TOPIC_SCAN_LIMIT = TOP_CATEGORY_TOPIC_SAMPLE_LIMIT * 10;
type CatalogTopicSource = {
topics?: readonly string[] | null;
};
export function rankTopCatalogTopics(
sources: readonly CatalogTopicSource[],
selectedCategory: string,
limit = TOP_CATEGORY_TOPIC_LIMIT,
) {
const counts = new Map<string, { count: number; firstSeen: number }>();
let firstSeen = 0;
for (const source of sources) {
for (const topic of getCatalogTopicSlugs(source.topics)) {
if (topic === selectedCategory) continue;
const existing = counts.get(topic);
if (existing) {
existing.count += 1;
} else {
counts.set(topic, { count: 1, firstSeen });
firstSeen += 1;
}
}
}
return [...counts.entries()]
.sort(
([leftTopic, left], [rightTopic, right]) =>
right.count - left.count ||
left.firstSeen - right.firstSeen ||
leftTopic.localeCompare(rightTopic),
)
.slice(0, Math.max(0, limit))
.map(([topic]) => topic);
}
async function listTopSkillTopics(ctx: QueryCtx, category: string) {
if (!isSkillCategorySlug(category)) return [];
const digests = await ctx.db
.query("skillSearchDigest")
.withIndex("by_active_recommended_score", (q) => q.eq("softDeletedAt", undefined))
.order("desc")
.take(TOP_SKILL_CATEGORY_TOPIC_SCAN_LIMIT);
const matching: Doc<"skillSearchDigest">[] = [];
// Skill categories are multi-valued, so collect a bounded category sample from the ranked digest.
for (const digest of digests) {
if (!isPublicSkillDoc(digest)) continue;
if (!resolveStoredSkillCategories(digest).includes(category)) continue;
matching.push(digest);
if (matching.length >= TOP_CATEGORY_TOPIC_SAMPLE_LIMIT) break;
}
return rankTopCatalogTopics(matching, category);
}
async function listTopPluginTopics(ctx: QueryCtx, category: string) {
if (!isPluginCategorySlug(category)) return [];
const digests: Doc<"packagePluginCategorySearchDigest">[] = await ctx.db
.query("packagePluginCategorySearchDigest")
.withIndex("by_active_category_installs", (q) =>
q.eq("softDeletedAt", undefined).eq("pluginCategory", category),
)
.order("desc")
.take(TOP_CATEGORY_TOPIC_SAMPLE_LIMIT);
return rankTopCatalogTopics(digests.filter(isPublicPluginDoc), category);
}
export const listTopByCategory = query({
args: {
kind: v.union(v.literal("skill"), v.literal("plugin")),
category: v.string(),
},
handler: async (ctx, args) => {
return args.kind === "skill"
? await listTopSkillTopics(ctx, args.category)
: await listTopPluginTopics(ctx, args.category);
},
});
-290
View File
@@ -1,290 +0,0 @@
/* @vitest-environment node */
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
vi.mock("./_generated/api", () => ({
internal: {
commentModeration: {
getCommentScamBackfillPageInternal: Symbol(
"commentModeration.getCommentScamBackfillPageInternal",
),
applyCommentScamResultInternal: Symbol("commentModeration.applyCommentScamResultInternal"),
backfillCommentScamModerationInternal: Symbol(
"commentModeration.backfillCommentScamModerationInternal",
),
continueCommentScamModerationJobInternal: Symbol(
"commentModeration.continueCommentScamModerationJobInternal",
),
},
llmEval: {
evaluateCommentForScam: Symbol("llmEval.evaluateCommentForScam"),
},
users: {
banUserInternal: Symbol("users.banUserInternal"),
},
},
}));
const { applyCommentScamResultInternalHandler, backfillCommentScamModerationInternalHandler } =
await import("./commentModeration");
const { internal } = await import("./_generated/api");
const previousOpenAiApiKey = process.env.OPENAI_API_KEY;
beforeEach(() => {
process.env.OPENAI_API_KEY = "test-key";
});
afterEach(() => {
if (previousOpenAiApiKey === undefined) {
delete process.env.OPENAI_API_KEY;
return;
}
process.env.OPENAI_API_KEY = previousOpenAiApiKey;
});
describe("commentModeration backfill", () => {
it("evaluates comments and bans on certain/high scams", async () => {
const runQuery = vi.fn().mockResolvedValueOnce({
items: [
{
commentId: "comments:1",
skillId: "skills:1",
userId: "users:2",
body: 'echo "mal" | base64 -D | bash',
softDeletedAt: undefined,
scamScanCheckedAt: undefined,
},
],
cursor: null,
isDone: true,
});
const runAction = vi.fn().mockResolvedValue({
ok: true,
model: "gpt-5-mini",
verdict: "certain_scam",
confidence: "high",
explanation: "Obfuscated shell execution payload.",
evidence: ["base64 decode piped to bash"],
});
const runMutation = vi.fn().mockResolvedValue({
ok: true,
shouldBan: true,
banned: true,
alreadyBanned: false,
protectedRole: false,
wouldBan: false,
});
const result = await backfillCommentScamModerationInternalHandler(
{ runQuery, runAction, runMutation } as never,
{
actorUserId: "users:admin",
dryRun: false,
batchSize: 10,
maxBatches: 1,
} as never,
);
expect(result.ok).toBe(true);
expect(result.stats.commentsScanned).toBe(1);
expect(result.stats.commentsEvaluated).toBe(1);
expect(result.stats.certainScams).toBe(1);
expect(result.stats.banCandidates).toBe(1);
expect(result.stats.usersBanned).toBe(1);
expect(runAction).toHaveBeenCalledWith(internal.llmEval.evaluateCommentForScam, {
commentId: "comments:1",
skillId: "skills:1",
userId: "users:2",
body: 'echo "mal" | base64 -D | bash',
});
expect(runMutation).toHaveBeenCalledWith(
internal.commentModeration.applyCommentScamResultInternal,
{
actorUserId: "users:admin",
commentId: "comments:1",
verdict: "certain_scam",
confidence: "high",
explanation: "Obfuscated shell execution payload.",
evidence: ["base64 decode piped to bash"],
model: "gpt-5-mini",
checkedAt: expect.any(Number),
dryRun: false,
},
);
});
it("skips previously scanned comments unless rescan=true", async () => {
const runQuery = vi.fn().mockResolvedValue({
items: [
{
commentId: "comments:1",
skillId: "skills:1",
userId: "users:2",
body: "something",
softDeletedAt: undefined,
scamScanCheckedAt: 123,
},
],
cursor: null,
isDone: true,
});
const runAction = vi.fn();
const runMutation = vi.fn();
const result = await backfillCommentScamModerationInternalHandler(
{ runQuery, runAction, runMutation } as never,
{
actorUserId: "users:admin",
batchSize: 10,
maxBatches: 1,
} as never,
);
expect(result.stats.commentsScanned).toBe(1);
expect(result.stats.skippedAlreadyScanned).toBe(1);
expect(runAction).not.toHaveBeenCalled();
expect(runMutation).not.toHaveBeenCalled();
});
it("tracks dry-run ban candidates without banning", async () => {
const runQuery = vi.fn().mockResolvedValue({
items: [
{
commentId: "comments:9",
skillId: "skills:7",
userId: "users:5",
body: "run this update installer from random domain",
softDeletedAt: undefined,
scamScanCheckedAt: undefined,
},
],
cursor: null,
isDone: true,
});
const runAction = vi.fn().mockResolvedValue({
ok: true,
model: "gpt-5-mini",
verdict: "certain_scam",
confidence: "high",
explanation: "Social-engineering install command.",
evidence: ["unknown update domain"],
});
const runMutation = vi.fn().mockResolvedValue({
ok: true,
shouldBan: true,
banned: false,
alreadyBanned: false,
protectedRole: false,
wouldBan: true,
});
const result = await backfillCommentScamModerationInternalHandler(
{ runQuery, runAction, runMutation } as never,
{
actorUserId: "users:admin",
dryRun: true,
batchSize: 10,
maxBatches: 1,
} as never,
);
expect(result.stats.usersBanned).toBe(0);
expect(result.stats.usersWouldBeBanned).toBe(1);
});
});
describe("applyCommentScamResultInternalHandler", () => {
it("persists scan metadata and triggers ban with bounded reason", async () => {
const get = vi
.fn()
.mockResolvedValueOnce({
_id: "comments:1",
skillId: "skills:1",
userId: "users:2",
})
.mockResolvedValueOnce({
_id: "users:2",
role: "user",
});
const patch = vi.fn();
const insert = vi.fn();
const runMutation = vi
.fn()
.mockResolvedValue({ ok: true, alreadyBanned: false, deletedSkills: 0 });
const result = await applyCommentScamResultInternalHandler(
{ db: { get, patch, insert }, runMutation } as never,
{
actorUserId: "users:admin",
commentId: "comments:1",
verdict: "certain_scam",
confidence: "high",
explanation: "X".repeat(700),
evidence: ["Y".repeat(280), "Z".repeat(280)],
model: "gpt-5-mini",
checkedAt: 123,
} as never,
);
expect(result.banned).toBe(true);
expect(insert).toHaveBeenCalledWith("auditLogs", {
actorUserId: "users:admin",
action: "comment.scam_scan",
targetType: "comment",
targetId: "comments:1",
metadata: {
skillId: "skills:1",
commentAuthorId: "users:2",
verdict: "certain_scam",
confidence: "high",
shouldBan: true,
model: "gpt-5-mini",
},
createdAt: 123,
});
const banCall = runMutation.mock.calls.find(
(call) => call[0] === internal.users.banUserInternal,
);
expect(banCall).toBeTruthy();
if (!banCall) throw new Error("Expected ban mutation to be called");
expect((banCall[1] as { reason: string }).reason.length).toBeLessThanOrEqual(500);
expect(patch).toHaveBeenCalledWith("comments:1", {
scamBanTriggeredAt: 123,
});
});
it("skips banning moderator/admin accounts", async () => {
const get = vi
.fn()
.mockResolvedValueOnce({
_id: "comments:2",
skillId: "skills:2",
userId: "users:staff",
})
.mockResolvedValueOnce({
_id: "users:staff",
role: "moderator",
});
const patch = vi.fn();
const insert = vi.fn();
const runMutation = vi.fn();
const result = await applyCommentScamResultInternalHandler(
{ db: { get, patch, insert }, runMutation } as never,
{
actorUserId: "users:admin",
commentId: "comments:2",
verdict: "certain_scam",
confidence: "high",
explanation: "Malicious command spam.",
evidence: ["base64|bash"],
model: "gpt-5-mini",
checkedAt: 300,
} as never,
);
expect(result.protectedRole).toBe(true);
expect(runMutation).not.toHaveBeenCalled();
});
});
-479
View File
@@ -1,479 +0,0 @@
import { ConvexError, v } from "convex/values";
import { internal } from "./_generated/api";
import type { Id } from "./_generated/dataModel";
import type { ActionCtx, MutationCtx } from "./_generated/server";
import { action, internalAction, internalMutation, internalQuery } from "./functions";
import { assertRole, requireUserFromAction } from "./lib/access";
import {
buildCommentScamBanReason,
isCertainScam,
type CommentScamConfidence,
type CommentScamVerdict,
} from "./lib/commentScamPrompt";
const DEFAULT_BATCH_SIZE = 25;
const MAX_BATCH_SIZE = 100;
const DEFAULT_MAX_BATCHES = 10;
const MAX_MAX_BATCHES = 200;
type CommentBackfillPageItem = {
commentId: Id<"comments">;
skillId: Id<"skills">;
userId: Id<"users">;
body: string;
softDeletedAt?: number;
scamScanCheckedAt?: number;
};
type CommentBackfillPageResult = {
items: CommentBackfillPageItem[];
cursor: string | null;
isDone: boolean;
};
type ApplyCommentScamResult = {
ok: true;
shouldBan: boolean;
banned: boolean;
alreadyBanned: boolean;
protectedRole: boolean;
wouldBan: boolean;
};
export type CommentScamBackfillStats = {
commentsScanned: number;
commentsEvaluated: number;
certainScams: number;
banCandidates: number;
usersBanned: number;
usersAlreadyBanned: number;
usersWouldBeBanned: number;
protectedRoleSkips: number;
skippedSoftDeleted: number;
skippedAlreadyScanned: number;
skippedEmptyBody: number;
evalErrors: number;
};
export type CommentScamBackfillActionArgs = {
actorUserId: Id<"users">;
dryRun?: boolean;
batchSize?: number;
maxBatches?: number;
cursor?: string;
rescan?: boolean;
includeSoftDeleted?: boolean;
};
export type CommentScamBackfillActionResult = {
ok: true;
stats: CommentScamBackfillStats;
isDone: boolean;
cursor: string | null;
};
export const getCommentScamBackfillPageInternal = internalQuery({
args: {
cursor: v.optional(v.string()),
batchSize: v.optional(v.number()),
},
handler: async (ctx, args): Promise<CommentBackfillPageResult> => {
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE);
const { page, isDone, continueCursor } = await ctx.db
.query("comments")
.order("asc")
.paginate({ cursor: args.cursor ?? null, numItems: batchSize });
return {
items: page.map((comment) => ({
commentId: comment._id,
skillId: comment.skillId,
userId: comment.userId,
body: comment.body,
softDeletedAt: comment.softDeletedAt,
scamScanCheckedAt: comment.scamScanCheckedAt,
})),
cursor: continueCursor,
isDone,
};
},
});
export async function applyCommentScamResultInternalHandler(
ctx: MutationCtx,
args: {
actorUserId: Id<"users">;
commentId: Id<"comments">;
verdict: CommentScamVerdict;
confidence: CommentScamConfidence;
explanation: string;
evidence: string[];
model: string;
checkedAt: number;
dryRun?: boolean;
},
): Promise<ApplyCommentScamResult> {
const comment = await ctx.db.get(args.commentId);
if (!comment) {
throw new ConvexError("Comment not found");
}
const user = await ctx.db.get(comment.userId);
if (!user) {
throw new ConvexError("Comment author not found");
}
const dryRun = Boolean(args.dryRun);
const shouldBan = isCertainScam({
verdict: args.verdict,
confidence: args.confidence,
});
const explanation = args.explanation.trim().slice(0, 1200);
const evidence = args.evidence
.map((item) => item.trim())
.filter(Boolean)
.slice(0, 5);
if (!dryRun) {
await ctx.db.patch(comment._id, {
scamScanVerdict: args.verdict,
scamScanConfidence: args.confidence,
scamScanExplanation: explanation,
scamScanEvidence: evidence,
scamScanModel: args.model,
scamScanCheckedAt: args.checkedAt,
});
await ctx.db.insert("auditLogs", {
actorUserId: args.actorUserId,
action: "comment.scam_scan",
targetType: "comment",
targetId: comment._id,
metadata: {
skillId: comment.skillId,
commentAuthorId: comment.userId,
verdict: args.verdict,
confidence: args.confidence,
shouldBan,
model: args.model,
},
createdAt: args.checkedAt,
});
}
if (!shouldBan) {
return {
ok: true,
shouldBan,
banned: false,
alreadyBanned: false,
protectedRole: false,
wouldBan: false,
};
}
if (user.role === "admin" || user.role === "moderator") {
return {
ok: true,
shouldBan,
banned: false,
alreadyBanned: false,
protectedRole: true,
wouldBan: false,
};
}
if (user.deletedAt || user.deactivatedAt) {
return {
ok: true,
shouldBan,
banned: false,
alreadyBanned: true,
protectedRole: false,
wouldBan: false,
};
}
if (dryRun) {
return {
ok: true,
shouldBan,
banned: false,
alreadyBanned: false,
protectedRole: false,
wouldBan: true,
};
}
const reason = buildCommentScamBanReason({
commentId: String(comment._id),
skillId: String(comment.skillId),
explanation,
evidence,
});
const banResult = await ctx.runMutation(internal.users.banUserInternal, {
actorUserId: args.actorUserId,
targetUserId: comment.userId,
reason,
});
if (!banResult.alreadyBanned) {
await ctx.db.patch(comment._id, {
scamBanTriggeredAt: args.checkedAt,
});
}
return {
ok: true,
shouldBan,
banned: !banResult.alreadyBanned,
alreadyBanned: banResult.alreadyBanned,
protectedRole: false,
wouldBan: false,
};
}
export const applyCommentScamResultInternal = internalMutation({
args: {
actorUserId: v.id("users"),
commentId: v.id("comments"),
verdict: v.union(v.literal("not_scam"), v.literal("likely_scam"), v.literal("certain_scam")),
confidence: v.union(v.literal("low"), v.literal("medium"), v.literal("high")),
explanation: v.string(),
evidence: v.array(v.string()),
model: v.string(),
checkedAt: v.number(),
dryRun: v.optional(v.boolean()),
},
handler: applyCommentScamResultInternalHandler,
});
export async function backfillCommentScamModerationInternalHandler(
ctx: ActionCtx,
args: CommentScamBackfillActionArgs,
): Promise<CommentScamBackfillActionResult> {
if (!process.env.OPENAI_API_KEY) {
throw new ConvexError("OPENAI_API_KEY not configured");
}
const dryRun = Boolean(args.dryRun);
const rescan = Boolean(args.rescan);
const includeSoftDeleted = Boolean(args.includeSoftDeleted);
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE);
const maxBatches = clampInt(args.maxBatches ?? DEFAULT_MAX_BATCHES, 1, MAX_MAX_BATCHES);
let cursor: string | null = args.cursor ?? null;
let isDone = false;
const stats: CommentScamBackfillStats = {
commentsScanned: 0,
commentsEvaluated: 0,
certainScams: 0,
banCandidates: 0,
usersBanned: 0,
usersAlreadyBanned: 0,
usersWouldBeBanned: 0,
protectedRoleSkips: 0,
skippedSoftDeleted: 0,
skippedAlreadyScanned: 0,
skippedEmptyBody: 0,
evalErrors: 0,
};
for (let i = 0; i < maxBatches; i++) {
const page = (await ctx.runQuery(
internal.commentModeration.getCommentScamBackfillPageInternal,
{
cursor: cursor ?? undefined,
batchSize,
},
)) as CommentBackfillPageResult;
cursor = page.cursor;
isDone = page.isDone;
for (const comment of page.items) {
stats.commentsScanned++;
if (!includeSoftDeleted && comment.softDeletedAt) {
stats.skippedSoftDeleted++;
continue;
}
if (!rescan && comment.scamScanCheckedAt) {
stats.skippedAlreadyScanned++;
continue;
}
const body = comment.body.trim();
if (!body) {
stats.skippedEmptyBody++;
continue;
}
const evalResult = (await ctx.runAction(internal.llmEval.evaluateCommentForScam, {
commentId: comment.commentId,
skillId: comment.skillId,
userId: comment.userId,
body,
})) as
| {
ok: true;
model: string;
verdict: CommentScamVerdict;
confidence: CommentScamConfidence;
explanation: string;
evidence: string[];
}
| { ok: false; error: string };
if (!evalResult.ok) {
stats.evalErrors++;
continue;
}
stats.commentsEvaluated++;
const shouldBan = isCertainScam(evalResult);
if (evalResult.verdict === "certain_scam") {
stats.certainScams++;
}
if (shouldBan) {
stats.banCandidates++;
}
const applyResult = (await ctx.runMutation(
internal.commentModeration.applyCommentScamResultInternal,
{
actorUserId: args.actorUserId,
commentId: comment.commentId,
verdict: evalResult.verdict,
confidence: evalResult.confidence,
explanation: evalResult.explanation,
evidence: evalResult.evidence,
model: evalResult.model,
checkedAt: Date.now(),
dryRun,
},
)) as ApplyCommentScamResult;
if (applyResult.banned) stats.usersBanned++;
if (applyResult.alreadyBanned) stats.usersAlreadyBanned++;
if (applyResult.wouldBan) stats.usersWouldBeBanned++;
if (applyResult.protectedRole) stats.protectedRoleSkips++;
}
if (isDone) break;
}
return {
ok: true,
stats,
isDone,
cursor,
};
}
export const backfillCommentScamModerationInternal = internalAction({
args: {
actorUserId: v.id("users"),
dryRun: v.optional(v.boolean()),
batchSize: v.optional(v.number()),
maxBatches: v.optional(v.number()),
cursor: v.optional(v.string()),
rescan: v.optional(v.boolean()),
includeSoftDeleted: v.optional(v.boolean()),
},
handler: backfillCommentScamModerationInternalHandler,
});
export const backfillCommentScamModeration: ReturnType<typeof action> = action({
args: {
dryRun: v.optional(v.boolean()),
batchSize: v.optional(v.number()),
maxBatches: v.optional(v.number()),
cursor: v.optional(v.string()),
rescan: v.optional(v.boolean()),
includeSoftDeleted: v.optional(v.boolean()),
},
handler: async (ctx, args): Promise<CommentScamBackfillActionResult> => {
const { user } = await requireUserFromAction(ctx);
assertRole(user, ["admin", "moderator"]);
return ctx.runAction(internal.commentModeration.backfillCommentScamModerationInternal, {
actorUserId: user._id,
...args,
}) as Promise<CommentScamBackfillActionResult>;
},
});
export const continueCommentScamModerationJobInternal = internalAction({
args: {
actorUserId: v.id("users"),
dryRun: v.optional(v.boolean()),
batchSize: v.optional(v.number()),
cursor: v.optional(v.string()),
rescan: v.optional(v.boolean()),
includeSoftDeleted: v.optional(v.boolean()),
},
handler: async (ctx, args) => {
const result = await backfillCommentScamModerationInternalHandler(ctx, {
actorUserId: args.actorUserId,
dryRun: args.dryRun,
batchSize: args.batchSize,
cursor: args.cursor,
maxBatches: 1,
rescan: args.rescan,
includeSoftDeleted: args.includeSoftDeleted,
});
if (!result.isDone && result.cursor) {
await ctx.scheduler.runAfter(
2_000,
internal.commentModeration.continueCommentScamModerationJobInternal,
{
actorUserId: args.actorUserId,
dryRun: Boolean(args.dryRun),
batchSize: args.batchSize ?? DEFAULT_BATCH_SIZE,
cursor: result.cursor,
rescan: Boolean(args.rescan),
includeSoftDeleted: Boolean(args.includeSoftDeleted),
},
);
}
return result;
},
});
export const scheduleCommentScamModeration: ReturnType<typeof action> = action({
args: {
dryRun: v.optional(v.boolean()),
batchSize: v.optional(v.number()),
rescan: v.optional(v.boolean()),
includeSoftDeleted: v.optional(v.boolean()),
},
handler: async (ctx, args): Promise<{ ok: true }> => {
const { user } = await requireUserFromAction(ctx);
assertRole(user, ["admin", "moderator"]);
await ctx.scheduler.runAfter(
0,
internal.commentModeration.continueCommentScamModerationJobInternal,
{
actorUserId: user._id,
dryRun: Boolean(args.dryRun),
batchSize: clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE),
cursor: undefined,
rescan: Boolean(args.rescan),
includeSoftDeleted: Boolean(args.includeSoftDeleted),
},
);
return { ok: true as const };
},
});
function clampInt(value: number, min: number, max: number) {
return Math.min(Math.max(Math.trunc(value), min), max);
}
-151
View File
@@ -1,151 +0,0 @@
import type { Id } from "./_generated/dataModel";
import type { MutationCtx } from "./_generated/server";
import { assertModerator, requireUser } from "./lib/access";
import { requireGitHubAccountAge } from "./lib/githubAccount";
import {
AUTO_HIDE_REPORT_THRESHOLD,
MAX_ACTIVE_REPORTS_PER_USER,
MAX_REPORT_REASON_LENGTH,
} from "./lib/reporting";
import { insertStatEvent } from "./skillStatEvents";
export async function addHandler(ctx: MutationCtx, args: { skillId: Id<"skills">; body: string }) {
const { userId } = await requireUser(ctx);
await requireGitHubAccountAge(ctx, userId);
const body = args.body.trim();
if (!body) throw new Error("Comment body required");
const skill = await ctx.db.get(args.skillId);
if (!skill) throw new Error("Skill not found");
await ctx.db.insert("comments", {
skillId: args.skillId,
userId,
body,
createdAt: Date.now(),
softDeletedAt: undefined,
deletedBy: undefined,
});
await insertStatEvent(ctx, { skillId: skill._id, kind: "comment" });
}
export async function removeHandler(ctx: MutationCtx, args: { commentId: Id<"comments"> }) {
const { user } = await requireUser(ctx);
const comment = await ctx.db.get(args.commentId);
if (!comment) throw new Error("Comment not found");
if (comment.softDeletedAt) return;
const isOwner = comment.userId === user._id;
if (!isOwner) {
assertModerator(user);
}
await ctx.db.patch(comment._id, {
softDeletedAt: Date.now(),
deletedBy: user._id,
});
await insertStatEvent(ctx, { skillId: comment.skillId, kind: "uncomment" });
await ctx.db.insert("auditLogs", {
actorUserId: user._id,
action: "comment.delete",
targetType: "comment",
targetId: comment._id,
metadata: { skillId: comment.skillId },
createdAt: Date.now(),
});
}
async function countActiveReportsForUser(ctx: MutationCtx, userId: Id<"users">) {
const reports = await ctx.db
.query("commentReports")
.withIndex("by_user", (q) => q.eq("userId", userId))
.collect();
let count = 0;
for (const report of reports) {
const comment = await ctx.db.get(report.commentId);
if (!comment || comment.softDeletedAt) continue;
const skill = await ctx.db.get(comment.skillId);
if (!skill || skill.softDeletedAt || skill.moderationStatus === "removed") continue;
const owner = await ctx.db.get(comment.userId);
if (!owner || owner.deletedAt || owner.deactivatedAt) continue;
count += 1;
if (count >= MAX_ACTIVE_REPORTS_PER_USER) break;
}
return count;
}
export async function reportHandler(
ctx: MutationCtx,
args: { commentId: Id<"comments">; reason: string },
) {
const { userId } = await requireUser(ctx);
const comment = await ctx.db.get(args.commentId);
if (!comment || comment.softDeletedAt) {
throw new Error("Comment not found");
}
const skill = await ctx.db.get(comment.skillId);
if (!skill || skill.softDeletedAt || skill.moderationStatus === "removed") {
throw new Error("Comment not found");
}
const reason = args.reason.trim();
if (!reason) {
throw new Error("Report reason required.");
}
const existing = await ctx.db
.query("commentReports")
.withIndex("by_comment_user", (q) => q.eq("commentId", args.commentId).eq("userId", userId))
.unique();
if (existing) return { ok: true as const, reported: false, alreadyReported: true };
const activeReports = await countActiveReportsForUser(ctx, userId);
if (activeReports >= MAX_ACTIVE_REPORTS_PER_USER) {
throw new Error("Report limit reached. Please wait for moderation before reporting more.");
}
const now = Date.now();
await ctx.db.insert("commentReports", {
commentId: args.commentId,
skillId: comment.skillId,
userId,
reason: reason.slice(0, MAX_REPORT_REASON_LENGTH),
createdAt: now,
});
const nextReportCount = (comment.reportCount ?? 0) + 1;
const shouldAutoHide = nextReportCount > AUTO_HIDE_REPORT_THRESHOLD && !comment.softDeletedAt;
const updates: {
reportCount: number;
lastReportedAt: number;
softDeletedAt?: number;
} = {
reportCount: nextReportCount,
lastReportedAt: now,
};
if (shouldAutoHide) {
updates.softDeletedAt = now;
}
await ctx.db.patch(comment._id, updates);
if (shouldAutoHide) {
await insertStatEvent(ctx, { skillId: comment.skillId, kind: "uncomment" });
await ctx.db.insert("auditLogs", {
actorUserId: userId,
action: "comment.auto_hide",
targetType: "comment",
targetId: comment._id,
metadata: { skillId: comment.skillId, reportCount: nextReportCount },
createdAt: now,
});
}
return { ok: true as const, reported: true, alreadyReported: false };
}
-127
View File
@@ -1,127 +0,0 @@
/* @vitest-environment node */
import { describe, expect, it } from "vitest";
import { listBySkillHandler } from "./comments";
function makeCtx(args: {
comments: Array<Record<string, unknown>>;
usersById: Record<string, Record<string, unknown> | null>;
}) {
const get = async (id: string) => args.usersById[id] ?? null;
const take = async () => args.comments;
const order = () => ({ take });
const withIndex = () => ({ order });
const query = () => ({ withIndex });
return { db: { get, query } } as never;
}
describe("comments.listBySkill", () => {
it("skips soft-deleted comments", async () => {
const ctx = makeCtx({
comments: [
{
_id: "comments:live",
skillId: "skills:1",
userId: "users:live",
body: "hello",
},
{
_id: "comments:deleted",
skillId: "skills:1",
userId: "users:live",
body: "bye",
softDeletedAt: 123,
},
],
usersById: {
"users:live": {
_id: "users:live",
_creationTime: 1,
handle: "live",
name: "live",
displayName: "Live",
image: null,
bio: null,
},
},
});
const result = await listBySkillHandler(ctx, {
skillId: "skills:1",
limit: 50,
} as never);
expect(result).toHaveLength(1);
expect(result[0]?.comment._id).toBe("comments:live");
});
it("skips comments whose author is deleted/deactivated/missing", async () => {
const ctx = makeCtx({
comments: [
{
_id: "comments:ok",
skillId: "skills:1",
userId: "users:ok",
body: "ok",
},
{
_id: "comments:deleted-user",
skillId: "skills:1",
userId: "users:deleted",
body: "hidden",
},
{
_id: "comments:deactivated-user",
skillId: "skills:1",
userId: "users:deactivated",
body: "hidden",
},
{
_id: "comments:missing-user",
skillId: "skills:1",
userId: "users:missing",
body: "hidden",
},
],
usersById: {
"users:ok": {
_id: "users:ok",
_creationTime: 1,
handle: "ok",
name: "ok",
displayName: "Ok",
image: null,
bio: null,
},
"users:deleted": {
_id: "users:deleted",
_creationTime: 1,
handle: "deleted",
name: "deleted",
displayName: "Deleted",
image: null,
bio: null,
deletedAt: 123,
},
"users:deactivated": {
_id: "users:deactivated",
_creationTime: 1,
handle: "deactivated",
name: "deactivated",
displayName: "Deactivated",
image: null,
bio: null,
deactivatedAt: 456,
},
},
});
const result = await listBySkillHandler(ctx, {
skillId: "skills:1",
limit: 50,
} as never);
expect(result).toHaveLength(1);
expect(result[0]?.comment._id).toBe("comments:ok");
expect(result[0]?.user._id).toBe("users:ok");
});
});
-614
View File
@@ -1,614 +0,0 @@
/* @vitest-environment node */
import { afterEach, describe, expect, it, vi } from "vitest";
vi.mock("./lib/access", () => ({
assertModerator: vi.fn(),
requireUser: vi.fn(),
}));
vi.mock("./skillStatEvents", () => ({
insertStatEvent: vi.fn(),
}));
vi.mock("./lib/githubAccount", () => ({
requireGitHubAccountAge: vi.fn(),
}));
const { requireUser, assertModerator } = await import("./lib/access");
const { insertStatEvent } = await import("./skillStatEvents");
const { requireGitHubAccountAge } = await import("./lib/githubAccount");
const { addHandler, removeHandler, reportHandler } = await import("./comments.handlers");
describe("comments mutations", () => {
afterEach(() => {
vi.mocked(assertModerator).mockReset();
vi.mocked(requireUser).mockReset();
vi.mocked(insertStatEvent).mockReset();
vi.mocked(requireGitHubAccountAge).mockReset();
vi.restoreAllMocks();
});
it("add avoids direct skill patch and records stat event", async () => {
vi.mocked(requireUser).mockResolvedValue({
userId: "users:1",
user: { _id: "users:1", role: "user" },
} as never);
vi.mocked(requireGitHubAccountAge).mockResolvedValue(undefined as never);
const get = vi.fn().mockResolvedValue({
_id: "skills:1",
});
const insert = vi.fn();
const patch = vi.fn();
const ctx = { db: { get, insert, patch } } as never;
await addHandler(ctx, { skillId: "skills:1", body: " hello " } as never);
expect(requireGitHubAccountAge).toHaveBeenCalledWith(ctx, "users:1");
expect(patch).not.toHaveBeenCalled();
expect(insertStatEvent).toHaveBeenCalledWith(ctx, {
skillId: "skills:1",
kind: "comment",
});
});
it("add blocks new comments when github account age gate fails", async () => {
vi.mocked(requireUser).mockResolvedValue({
userId: "users:new",
user: { _id: "users:new", role: "user" },
} as never);
vi.mocked(requireGitHubAccountAge).mockRejectedValue(
new Error(
"GitHub account must be at least 14 days old to upload skills. Try again in 3 days.",
),
);
const get = vi.fn();
const insert = vi.fn();
const patch = vi.fn();
const ctx = { db: { get, insert, patch } } as never;
await expect(addHandler(ctx, { skillId: "skills:1", body: "hello" } as never)).rejects.toThrow(
/at least 14 days old/i,
);
expect(get).not.toHaveBeenCalled();
expect(insert).not.toHaveBeenCalled();
expect(patch).not.toHaveBeenCalled();
expect(insertStatEvent).not.toHaveBeenCalled();
});
it("remove keeps comment soft-delete patch free of updatedAt", async () => {
vi.mocked(requireUser).mockResolvedValue({
userId: "users:2",
user: { _id: "users:2", role: "moderator" },
} as never);
const comment = {
_id: "comments:1",
skillId: "skills:1",
userId: "users:2",
softDeletedAt: undefined,
};
const get = vi.fn(async (id: string) => {
if (id === "comments:1") return comment;
if (id === "skills:1") {
return { _id: "skills:1", softDeletedAt: undefined, moderationStatus: "active" };
}
return null;
});
const insert = vi.fn();
const patch = vi.fn();
const ctx = { db: { get, insert, patch } } as never;
await removeHandler(ctx, { commentId: "comments:1" } as never);
expect(patch).toHaveBeenCalledTimes(1);
const deletePatch = vi.mocked(patch).mock.calls[0]?.[1] as Record<string, unknown>;
expect(deletePatch.updatedAt).toBeUndefined();
expect(insertStatEvent).toHaveBeenCalledWith(ctx, {
skillId: "skills:1",
kind: "uncomment",
});
});
it("remove rejects non-owner without moderator permission", async () => {
vi.mocked(requireUser).mockResolvedValue({
userId: "users:3",
user: { _id: "users:3", role: "user" },
} as never);
vi.mocked(assertModerator).mockImplementation(() => {
throw new Error("Moderator role required");
});
const comment = {
_id: "comments:2",
skillId: "skills:2",
userId: "users:9",
softDeletedAt: undefined,
};
const get = vi.fn().mockResolvedValue(comment);
const insert = vi.fn();
const patch = vi.fn();
const ctx = { db: { get, insert, patch } } as never;
await expect(removeHandler(ctx, { commentId: "comments:2" } as never)).rejects.toThrow(
"Moderator role required",
);
expect(patch).not.toHaveBeenCalled();
expect(insertStatEvent).not.toHaveBeenCalled();
});
it("remove no-ops for soft-deleted comment", async () => {
vi.mocked(requireUser).mockResolvedValue({
userId: "users:4",
user: { _id: "users:4", role: "moderator" },
} as never);
const comment = {
_id: "comments:3",
skillId: "skills:3",
userId: "users:4",
softDeletedAt: 123,
};
const get = vi.fn().mockResolvedValue(comment);
const insert = vi.fn();
const patch = vi.fn();
const ctx = { db: { get, insert, patch } } as never;
await removeHandler(ctx, { commentId: "comments:3" } as never);
expect(patch).not.toHaveBeenCalled();
expect(insert).not.toHaveBeenCalled();
expect(insertStatEvent).not.toHaveBeenCalled();
});
it("report increments count and stores reason", async () => {
vi.spyOn(Date, "now").mockReturnValue(1_700_000_000_000);
vi.mocked(requireUser).mockResolvedValue({
userId: "users:1",
user: { _id: "users:1", role: "user" },
} as never);
const comment = {
_id: "comments:1",
skillId: "skills:1",
userId: "users:2",
softDeletedAt: undefined,
reportCount: 1,
};
const get = vi.fn(async (id: string) => {
if (id === "comments:1") return comment;
if (id === "skills:1") {
return { _id: "skills:1", softDeletedAt: undefined, moderationStatus: "active" };
}
return null;
});
const insert = vi.fn();
const patch = vi.fn();
const query = vi.fn((table: string) => {
if (table === "commentReports") {
return {
withIndex: (index: string) => {
if (index === "by_comment_user") {
return { unique: vi.fn().mockResolvedValue(null) };
}
if (index === "by_user") {
return { collect: vi.fn().mockResolvedValue([]) };
}
throw new Error(`Unexpected index ${index}`);
},
};
}
throw new Error(`Unexpected table ${table}`);
});
const ctx = { db: { get, insert, patch, query } } as never;
const result = await reportHandler(ctx, {
commentId: "comments:1",
reason: " spam ",
} as never);
expect(result).toEqual({ ok: true, reported: true, alreadyReported: false });
expect(insert).toHaveBeenCalledWith("commentReports", {
commentId: "comments:1",
skillId: "skills:1",
userId: "users:1",
reason: "spam",
createdAt: 1_700_000_000_000,
});
expect(patch).toHaveBeenCalledWith("comments:1", {
reportCount: 2,
lastReportedAt: 1_700_000_000_000,
});
expect(insertStatEvent).not.toHaveBeenCalled();
});
it("report returns alreadyReported for duplicate reporter/comment pair", async () => {
vi.mocked(requireUser).mockResolvedValue({
userId: "users:1",
user: { _id: "users:1", role: "user" },
} as never);
const comment = {
_id: "comments:dup",
skillId: "skills:1",
userId: "users:2",
softDeletedAt: undefined,
reportCount: 0,
};
const get = vi.fn(async (id: string) => {
if (id === "comments:dup") return comment;
if (id === "skills:1") {
return { _id: "skills:1", softDeletedAt: undefined, moderationStatus: "active" };
}
return null;
});
const insert = vi.fn();
const patch = vi.fn();
const query = vi.fn((table: string) => {
if (table !== "commentReports") throw new Error(`Unexpected table ${table}`);
return {
withIndex: (index: string) => {
if (index === "by_comment_user") {
return { unique: vi.fn().mockResolvedValue({ _id: "commentReports:existing" }) };
}
throw new Error(`Unexpected index ${index}`);
},
};
});
const ctx = { db: { get, insert, patch, query } } as never;
const result = await reportHandler(ctx, { commentId: "comments:dup", reason: "spam" } as never);
expect(result).toEqual({ ok: true, reported: false, alreadyReported: true });
expect(insert).not.toHaveBeenCalled();
expect(patch).not.toHaveBeenCalled();
});
it("report rejects empty reason", async () => {
vi.mocked(requireUser).mockResolvedValue({
userId: "users:1",
user: { _id: "users:1", role: "user" },
} as never);
const comment = {
_id: "comments:empty",
skillId: "skills:1",
userId: "users:2",
softDeletedAt: undefined,
reportCount: 0,
};
const get = vi.fn(async (id: string) => {
if (id === "comments:empty") return comment;
if (id === "skills:1") {
return { _id: "skills:1", softDeletedAt: undefined, moderationStatus: "active" };
}
return null;
});
const insert = vi.fn();
const patch = vi.fn();
const query = vi.fn();
const ctx = { db: { get, insert, patch, query } } as never;
await expect(
reportHandler(ctx, { commentId: "comments:empty", reason: " " } as never),
).rejects.toThrow("Report reason required.");
expect(query).not.toHaveBeenCalled();
expect(insert).not.toHaveBeenCalled();
expect(patch).not.toHaveBeenCalled();
});
it("report rejects comment when parent skill is hidden/removed", async () => {
vi.mocked(requireUser).mockResolvedValue({
userId: "users:1",
user: { _id: "users:1", role: "user" },
} as never);
const comment = {
_id: "comments:hidden-parent",
skillId: "skills:hidden",
userId: "users:2",
softDeletedAt: undefined,
reportCount: 0,
};
const get = vi.fn(async (id: string) => {
if (id === "comments:hidden-parent") return comment;
if (id === "skills:hidden") {
return { _id: "skills:hidden", softDeletedAt: 123, moderationStatus: "removed" };
}
return null;
});
const insert = vi.fn();
const patch = vi.fn();
const query = vi.fn();
const ctx = { db: { get, insert, patch, query } } as never;
await expect(
reportHandler(ctx, { commentId: "comments:hidden-parent", reason: "abuse" } as never),
).rejects.toThrow("Comment not found");
expect(query).not.toHaveBeenCalled();
expect(insert).not.toHaveBeenCalled();
expect(patch).not.toHaveBeenCalled();
});
it("report truncates long reason to 500 chars", async () => {
vi.spyOn(Date, "now").mockReturnValue(1_700_000_000_050);
vi.mocked(requireUser).mockResolvedValue({
userId: "users:1",
user: { _id: "users:1", role: "user" },
} as never);
const comment = {
_id: "comments:long",
skillId: "skills:1",
userId: "users:2",
softDeletedAt: undefined,
reportCount: 0,
};
const get = vi.fn(async (id: string) => {
if (id === "comments:long") return comment;
if (id === "skills:1") {
return { _id: "skills:1", softDeletedAt: undefined, moderationStatus: "active" };
}
return null;
});
const insert = vi.fn();
const patch = vi.fn();
const query = vi.fn((table: string) => {
if (table !== "commentReports") throw new Error(`Unexpected table ${table}`);
return {
withIndex: (index: string) => {
if (index === "by_comment_user") return { unique: vi.fn().mockResolvedValue(null) };
if (index === "by_user") return { collect: vi.fn().mockResolvedValue([]) };
throw new Error(`Unexpected index ${index}`);
},
};
});
const ctx = { db: { get, insert, patch, query } } as never;
await reportHandler(ctx, { commentId: "comments:long", reason: "x".repeat(700) } as never);
const reportInsert = vi.mocked(insert).mock.calls.find((call) => call[0] === "commentReports");
expect(reportInsert?.[1]).toMatchObject({
commentId: "comments:long",
reason: "x".repeat(500),
});
});
it("report active-count filter ignores stale/non-active report targets", async () => {
vi.mocked(requireUser).mockResolvedValue({
userId: "users:1",
user: { _id: "users:1", role: "user" },
} as never);
const comment = {
_id: "comments:target2",
skillId: "skills:1",
userId: "users:2",
softDeletedAt: undefined,
reportCount: 0,
};
const reports = [
{
_id: "commentReports:1",
commentId: "comments:deleted",
userId: "users:1",
skillId: "skills:1",
},
{
_id: "commentReports:2",
commentId: "comments:removed-skill",
userId: "users:1",
skillId: "skills:removed",
},
{
_id: "commentReports:3",
commentId: "comments:deleted-owner",
userId: "users:1",
skillId: "skills:active",
},
];
const get = vi.fn(async (id: string) => {
if (id === "comments:target2") return comment;
if (id === "skills:1") {
return { _id: "skills:1", softDeletedAt: undefined, moderationStatus: "active" };
}
if (id === "comments:deleted") {
return {
_id: "comments:deleted",
softDeletedAt: 123,
skillId: "skills:1",
userId: "users:2",
};
}
if (id === "comments:removed-skill") {
return {
_id: "comments:removed-skill",
softDeletedAt: undefined,
skillId: "skills:removed",
userId: "users:2",
};
}
if (id === "skills:removed") {
return { _id: "skills:removed", softDeletedAt: undefined, moderationStatus: "removed" };
}
if (id === "comments:deleted-owner") {
return {
_id: "comments:deleted-owner",
softDeletedAt: undefined,
skillId: "skills:active",
userId: "users:deleted-owner",
};
}
if (id === "skills:active") {
return { _id: "skills:active", softDeletedAt: undefined, moderationStatus: "active" };
}
if (id === "users:deleted-owner") {
return { _id: "users:deleted-owner", deletedAt: 1, deactivatedAt: undefined };
}
return null;
});
const insert = vi.fn();
const patch = vi.fn();
const query = vi.fn((table: string) => {
if (table !== "commentReports") throw new Error(`Unexpected table ${table}`);
return {
withIndex: (index: string) => {
if (index === "by_comment_user") return { unique: vi.fn().mockResolvedValue(null) };
if (index === "by_user") return { collect: vi.fn().mockResolvedValue(reports) };
throw new Error(`Unexpected index ${index}`);
},
};
});
const ctx = { db: { get, insert, patch, query } } as never;
const result = await reportHandler(ctx, {
commentId: "comments:target2",
reason: "still allowed",
} as never);
expect(result).toEqual({ ok: true, reported: true, alreadyReported: false });
expect(insert).toHaveBeenCalledWith(
"commentReports",
expect.objectContaining({ commentId: "comments:target2", userId: "users:1" }),
);
});
it("report rejects when active report limit is reached", async () => {
vi.mocked(requireUser).mockResolvedValue({
userId: "users:1",
user: { _id: "users:1", role: "user" },
} as never);
const comment = {
_id: "comments:target",
skillId: "skills:1",
userId: "users:2",
softDeletedAt: undefined,
reportCount: 0,
};
const reportedComment = {
_id: "comments:reported",
skillId: "skills:active",
userId: "users:owner",
softDeletedAt: undefined,
};
const reports = Array.from({ length: 20 }, (_, i) => ({
_id: `commentReports:${i + 1}`,
commentId: `comments:reported-${i + 1}`,
userId: "users:1",
skillId: "skills:active",
createdAt: i + 1,
}));
const get = vi.fn(async (id: string) => {
if (id === "comments:target") return comment;
if (id === "skills:1") {
return { _id: "skills:1", softDeletedAt: undefined, moderationStatus: "active" };
}
if (id.startsWith("comments:reported-")) return reportedComment;
if (id === "skills:active") {
return { _id: "skills:active", softDeletedAt: undefined, moderationStatus: "active" };
}
if (id === "users:owner") {
return { _id: "users:owner", deletedAt: undefined, deactivatedAt: undefined };
}
return null;
});
const insert = vi.fn();
const patch = vi.fn();
const query = vi.fn((table: string) => {
if (table === "commentReports") {
return {
withIndex: (index: string) => {
if (index === "by_comment_user") {
return { unique: vi.fn().mockResolvedValue(null) };
}
if (index === "by_user") {
return { collect: vi.fn().mockResolvedValue(reports) };
}
throw new Error(`Unexpected index ${index}`);
},
};
}
throw new Error(`Unexpected table ${table}`);
});
const ctx = { db: { get, insert, patch, query } } as never;
await expect(
reportHandler(ctx, { commentId: "comments:target", reason: "abuse" } as never),
).rejects.toThrow("Report limit reached. Please wait for moderation before reporting more.");
expect(insert).not.toHaveBeenCalled();
expect(patch).not.toHaveBeenCalled();
});
it("report auto-hides comment after fourth unique report", async () => {
vi.spyOn(Date, "now").mockReturnValue(1_700_000_000_100);
vi.mocked(requireUser).mockResolvedValue({
userId: "users:3",
user: { _id: "users:3", role: "user" },
} as never);
const comment = {
_id: "comments:4",
skillId: "skills:9",
userId: "users:2",
softDeletedAt: undefined,
reportCount: 3,
};
const get = vi.fn(async (id: string) => {
if (id === "comments:4") return comment;
if (id === "skills:9") {
return { _id: "skills:9", softDeletedAt: undefined, moderationStatus: "active" };
}
return null;
});
const insert = vi.fn();
const patch = vi.fn();
const query = vi.fn((table: string) => {
if (table === "commentReports") {
return {
withIndex: (index: string) => {
if (index === "by_comment_user") {
return { unique: vi.fn().mockResolvedValue(null) };
}
if (index === "by_user") {
return { collect: vi.fn().mockResolvedValue([]) };
}
throw new Error(`Unexpected index ${index}`);
},
};
}
throw new Error(`Unexpected table ${table}`);
});
const ctx = { db: { get, insert, patch, query } } as never;
const result = await reportHandler(ctx, {
commentId: "comments:4",
reason: " hate ",
} as never);
expect(result).toEqual({ ok: true, reported: true, alreadyReported: false });
expect(patch).toHaveBeenCalledWith("comments:4", {
reportCount: 4,
lastReportedAt: 1_700_000_000_100,
softDeletedAt: 1_700_000_000_100,
});
expect(insertStatEvent).toHaveBeenCalledWith(ctx, {
skillId: "skills:9",
kind: "uncomment",
});
expect(insert).toHaveBeenCalledWith("auditLogs", {
actorUserId: "users:3",
action: "comment.auto_hide",
targetType: "comment",
targetId: "comments:4",
metadata: { skillId: "skills:9", reportCount: 4 },
createdAt: 1_700_000_000_100,
});
});
});
-49
View File
@@ -1,49 +0,0 @@
import { v } from "convex/values";
import type { Doc } from "./_generated/dataModel";
import { addHandler, removeHandler, reportHandler } from "./comments.handlers";
import { mutation, query } from "./functions";
import { type PublicUser, toPublicUser } from "./lib/public";
export const listBySkill = query({
args: { skillId: v.id("skills"), limit: v.optional(v.number()) },
handler: listBySkillHandler,
});
export async function listBySkillHandler(
ctx: import("./_generated/server").QueryCtx,
args: { skillId: import("./_generated/dataModel").Id<"skills">; limit?: number },
) {
const limit = args.limit ?? 50;
const comments = await ctx.db
.query("comments")
.withIndex("by_skill", (q) => q.eq("skillId", args.skillId))
.order("desc")
.take(limit);
const rows = await Promise.all(
comments.map(
async (comment): Promise<{ comment: Doc<"comments">; user: PublicUser } | null> => {
if (comment.softDeletedAt) return null;
const user = toPublicUser(await ctx.db.get(comment.userId));
if (!user) return null;
return { comment, user };
},
),
);
return rows.filter((row): row is { comment: Doc<"comments">; user: PublicUser } => row !== null);
}
export const add = mutation({
args: { skillId: v.id("skills"), body: v.string() },
handler: addHandler,
});
export const remove = mutation({
args: { commentId: v.id("comments") },
handler: removeHandler,
});
export const report = mutation({
args: { commentId: v.id("comments"), reason: v.string() },
handler: reportHandler,
});
+7
View File
@@ -0,0 +1,7 @@
import migrations from "@convex-dev/migrations/convex.config.js";
import { defineApp } from "convex/server";
const app = defineApp();
app.use(migrations);
export default app;
+134 -6
View File
@@ -1,10 +1,25 @@
/* @vitest-environment node */
import { describe, expect, it, vi } from "vitest";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => {
const interval = vi.fn();
const githubSkillSyncRef = Symbol("github-skill-source-sync");
return { interval, githubSkillSyncRef };
const installTelemetryDedupePruneRef = Symbol("install-telemetry-dedupe-prune");
const rateLimitCountersPruneRef = Symbol("rate-limit-counters-prune");
const skillStatEventPruneRef = Symbol("skill-stat-event-prune");
const packageStatEventPruneRef = Symbol("package-stat-event-prune");
const authSessionsPruneRef = Symbol("auth-sessions-prune");
const authRefreshTokensPruneRef = Symbol("auth-refresh-tokens-prune");
return {
interval,
githubSkillSyncRef,
installTelemetryDedupePruneRef,
rateLimitCountersPruneRef,
skillStatEventPruneRef,
packageStatEventPruneRef,
authSessionsPruneRef,
authRefreshTokensPruneRef,
};
});
vi.mock("convex/server", () => ({
@@ -15,16 +30,20 @@ vi.mock("convex/server", () => ({
vi.mock("./_generated/api", () => ({
internal: {
githubBackupsNode: { syncGitHubBackupsInternal: Symbol("github-backup-sync") },
githubSkillSync: { syncGitHubSkillSourcesInternal: mocks.githubSkillSyncRef },
githubSkillSyncNode: { syncGitHubSkillSourcesInternal: mocks.githubSkillSyncRef },
leaderboards: { rebuildTrendingLeaderboardAction: Symbol("trending-leaderboard") },
statsMaintenance: {
runSkillStatBackfillInternal: Symbol("skill-stats-backfill"),
updateGlobalStatsAction: Symbol("global-stats-update"),
},
skillStatEvents: { processSkillStatEventsAction: Symbol("skill-stat-events") },
skillStatEvents: {
processSkillStatEventsAction: Symbol("skill-stat-events"),
processSkillStatEventsInternal: Symbol("skill-doc-stat-sync"),
pruneProcessedSkillStatEventsInternal: mocks.skillStatEventPruneRef,
},
packages: {
processPackageStatEventsInternal: Symbol("package-stat-events"),
pruneProcessedPackageStatEventsInternal: mocks.packageStatEventPruneRef,
backfillPackageReleaseScansInternal: Symbol("package-scan-backfill"),
},
publisherAbuse: {
@@ -37,14 +56,41 @@ vi.mock("./_generated/api", () => ({
securityScan: {
pruneExpiredSkillScanRequestsInternal: Symbol("skill-scan-request-prune"),
},
downloads: { pruneDownloadDedupesInternal: Symbol("download-dedupe-prune") },
downloadMetrics: {
pruneDownloadMetricDedupesInternal: Symbol("download-metric-dedupe-prune"),
},
telemetry: {
pruneInstallTelemetryDedupesInternal: mocks.installTelemetryDedupePruneRef,
},
rateLimits: {
pruneRateLimitCountersInternal: mocks.rateLimitCountersPruneRef,
},
retention: {
pruneExpiredAuthSessionsInternal: mocks.authSessionsPruneRef,
pruneExpiredAuthRefreshTokensInternal: mocks.authRefreshTokensPruneRef,
},
},
}));
describe("crons", () => {
beforeEach(() => {
vi.resetModules();
mocks.interval.mockReset();
delete process.env.CLAWHUB_DISABLE_CRONS;
});
afterEach(() => {
delete process.env.CLAWHUB_DISABLE_CRONS;
});
it("does not register production cron work when explicitly disabled", async () => {
process.env.CLAWHUB_DISABLE_CRONS = "1";
await import("./crons");
expect(mocks.interval).not.toHaveBeenCalled();
});
it("runs GitHub skill source sync every 15 minutes", async () => {
await import("./crons");
@@ -55,4 +101,86 @@ describe("crons", () => {
{},
);
});
it("prunes expired skill scan requests in bounded continuation batches", async () => {
await import("./crons");
expect(mocks.interval).toHaveBeenCalledWith(
"skill-scan-request-prune",
{ hours: 6 },
expect.anything(),
{ batchSize: 10 },
);
});
it("prunes install telemetry dedupe rows daily", async () => {
await import("./crons");
expect(mocks.interval).toHaveBeenCalledWith(
"install-telemetry-dedupe-prune",
{ hours: 24 },
mocks.installTelemetryDedupePruneRef,
{},
);
});
it("prunes expired rate limit counters frequently", async () => {
await import("./crons");
expect(mocks.interval).toHaveBeenCalledWith(
"rate-limit-counters-prune",
{ minutes: 15 },
mocks.rateLimitCountersPruneRef,
{ batchSize: 500 },
);
});
it("prunes expired auth sessions and refresh tokens with the standard batch size", async () => {
await import("./crons");
expect(mocks.interval).toHaveBeenCalledWith(
"auth-session-retention-prune",
{ hours: 1 },
mocks.authSessionsPruneRef,
{ batchSize: 500 },
);
expect(mocks.interval).toHaveBeenCalledWith(
"auth-refresh-token-retention-prune",
{ hours: 6 },
mocks.authRefreshTokensPruneRef,
{ batchSize: 500 },
);
});
it("prunes processed skill stat events daily with a seven-day retention window", async () => {
await import("./crons");
expect(mocks.interval).toHaveBeenCalledWith(
"skill-stat-events-prune",
{ hours: 24 },
mocks.skillStatEventPruneRef,
{
retentionDays: 7,
batchSize: 1000,
maxBatches: 20,
confirmationToken: "PRUNE_PROCESSED_SKILL_STAT_EVENTS",
},
);
});
it("prunes processed package stat events daily with a seven-day retention window", async () => {
await import("./crons");
expect(mocks.interval).toHaveBeenCalledWith(
"package-stat-events-prune",
{ hours: 24 },
mocks.packageStatEventPruneRef,
{
retentionDays: 7,
batchSize: 1000,
maxBatches: 20,
confirmationToken: "PRUNE_PROCESSED_PACKAGE_STAT_EVENTS",
},
);
});
});
+143 -102
View File
@@ -1,124 +1,165 @@
import { cronJobs } from "convex/server";
import { internal } from "./_generated/api";
import { RETENTION_STANDARD_BATCH_SIZE } from "./lib/retentionPolicy";
const crons = cronJobs();
crons.interval(
"github-backup-sync",
{ minutes: 30 },
internal.githubBackupsNode.syncGitHubBackupsInternal,
{ batchSize: 50, maxBatches: 5 },
);
if (process.env.CLAWHUB_DISABLE_CRONS !== "1") {
crons.interval(
"github-skill-source-sync",
{ minutes: 15 },
internal.githubSkillSyncNode.syncGitHubSkillSourcesInternal,
{},
);
crons.interval(
"github-skill-source-sync",
{ minutes: 15 },
internal.githubSkillSync.syncGitHubSkillSourcesInternal,
{},
);
crons.interval(
"trending-leaderboard",
{ minutes: 60 },
internal.leaderboards.rebuildTrendingLeaderboardAction,
{ limit: 200 },
);
crons.interval(
"trending-leaderboard",
{ minutes: 60 },
internal.leaderboards.rebuildTrendingLeaderboardAction,
{ limit: 200 },
);
crons.interval(
"skill-stats-backfill",
{ hours: 6 },
internal.statsMaintenance.runSkillStatBackfillInternal,
{ batchSize: 200, maxBatches: 5 },
);
crons.interval(
"skill-stats-backfill",
{ hours: 6 },
internal.statsMaintenance.runSkillStatBackfillInternal,
{ batchSize: 200, maxBatches: 5 },
);
// Runs frequently to keep dailyStats/trending accurate,
// but does NOT patch skill documents (only writes to skillDailyStats).
crons.interval(
"skill-stat-events",
{ minutes: 15 },
internal.skillStatEvents.processSkillStatEventsAction,
{},
);
// Runs frequently to keep dailyStats/trending accurate,
// but does NOT patch skill documents (only writes to skillDailyStats).
crons.interval(
"skill-stat-events",
{ minutes: 15 },
internal.skillStatEvents.processSkillStatEventsAction,
{},
);
crons.interval(
"package-stat-events",
{ minutes: 15 },
internal.packages.processPackageStatEventsInternal,
{ batchSize: 100 },
);
crons.interval(
"package-stat-events",
{ minutes: 15 },
internal.packages.processPackageStatEventsInternal,
{ batchSize: 500 },
);
// Syncs accumulated stat deltas to skill documents every 6 hours.
// Runs infrequently to avoid thundering-herd reactive query invalidation.
// Uses processedAt field to track progress (independent of the action cursor).
crons.interval(
"skill-doc-stat-sync",
{ hours: 6 },
internal.skillStatEvents.processSkillStatEventsInternal,
{ batchSize: 100 },
);
// Syncs accumulated stat deltas to skill documents every 6 hours.
// Runs infrequently to avoid thundering-herd reactive query invalidation.
// Uses processedAt field to track progress (independent of the action cursor).
crons.interval(
"skill-doc-stat-sync",
{ hours: 6 },
internal.skillStatEvents.processSkillStatEventsInternal,
{ batchSize: 100 },
);
crons.interval(
"skill-stat-events-prune",
{ hours: 24 },
internal.skillStatEvents.pruneProcessedSkillStatEventsInternal,
{
retentionDays: 7,
batchSize: 1000,
maxBatches: 20,
confirmationToken: "PRUNE_PROCESSED_SKILL_STAT_EVENTS",
},
);
crons.interval(
"global-stats-update",
{ hours: 24 },
internal.statsMaintenance.updateGlobalStatsAction,
{},
);
crons.interval(
"package-stat-events-prune",
{ hours: 24 },
internal.packages.pruneProcessedPackageStatEventsInternal,
{
retentionDays: 7,
batchSize: 1000,
maxBatches: 20,
confirmationToken: "PRUNE_PROCESSED_PACKAGE_STAT_EVENTS",
},
);
crons.interval(
"publisher-abuse-score-refresh",
{ hours: 24 },
internal.publisherAbuse.runPublisherAbuseScoreRunInternal,
{ batchSize: 250, maxPages: 5, trigger: "cron" },
);
crons.interval(
"global-stats-update",
{ hours: 24 },
internal.statsMaintenance.updateGlobalStatsAction,
{},
);
crons.interval(
"publisher-temporal-abuse-scan",
{ hours: 24 },
internal.publisherAbuse.runTemporalPublisherAbuseScanInternal,
{
mode: "current",
dryRun: false,
candidateLimit: 1000,
batchSize: 50,
maxPages: 20,
trigger: "cron",
},
);
crons.interval(
"publisher-abuse-score-refresh",
{ hours: 24 },
internal.publisherAbuse.runPublisherAbuseScoreRunInternal,
{ batchSize: 250, maxPages: 5, trigger: "cron" },
);
crons.interval("vt-pending-scans", { minutes: 5 }, internal.vt.pollPendingScans, {
batchSize: 100,
});
crons.interval(
"publisher-temporal-abuse-scan",
{ hours: 24 },
internal.publisherAbuse.runTemporalPublisherAbuseScanInternal,
{
mode: "current",
dryRun: false,
candidateLimit: 1000,
batchSize: 50,
maxPages: 20,
trigger: "cron",
},
);
crons.interval("vt-cache-backfill", { minutes: 30 }, internal.vt.backfillActiveSkillsVTCache, {
batchSize: 100,
});
crons.interval("vt-pending-scans", { minutes: 5 }, internal.vt.pollPendingScans, {
batchSize: 100,
});
crons.interval(
"package-scan-backfill",
{ minutes: 30 },
internal.packages.backfillPackageReleaseScansInternal,
{ batchSize: 100 },
);
crons.interval("vt-cache-backfill", { minutes: 30 }, internal.vt.backfillActiveSkillsVTCache, {
batchSize: 100,
});
crons.interval(
"skill-scan-request-prune",
{ hours: 6 },
internal.securityScan.pruneExpiredSkillScanRequestsInternal,
{ batchSize: 250 },
);
crons.interval(
"package-scan-backfill",
{ minutes: 30 },
internal.packages.backfillPackageReleaseScansInternal,
{ batchSize: 100 },
);
crons.interval(
"download-dedupe-prune",
{ hours: 24 },
internal.downloads.pruneDownloadDedupesInternal,
{},
);
crons.interval(
"skill-scan-request-prune",
{ hours: 6 },
internal.securityScan.pruneExpiredSkillScanRequestsInternal,
{ batchSize: 10 },
);
crons.interval(
"download-metric-dedupe-prune",
{ hours: 24 },
internal.downloadMetrics.pruneDownloadMetricDedupesInternal,
{},
);
crons.interval(
"download-metric-dedupe-prune",
{ hours: 24 },
internal.downloadMetrics.pruneDownloadMetricDedupesInternal,
{},
);
crons.interval(
"install-telemetry-dedupe-prune",
{ hours: 24 },
internal.telemetry.pruneInstallTelemetryDedupesInternal,
{},
);
crons.interval(
"auth-session-retention-prune",
{ hours: 1 },
internal.retention.pruneExpiredAuthSessionsInternal,
{ batchSize: RETENTION_STANDARD_BATCH_SIZE },
);
crons.interval(
"auth-refresh-token-retention-prune",
{ hours: 6 },
internal.retention.pruneExpiredAuthRefreshTokensInternal,
{ batchSize: RETENTION_STANDARD_BATCH_SIZE },
);
crons.interval(
"rate-limit-counters-prune",
{ minutes: 15 },
internal.rateLimits.pruneRateLimitCountersInternal,
{ batchSize: RETENTION_STANDARD_BATCH_SIZE },
);
}
export default crons;
+684
View File
@@ -1,12 +1,14 @@
import { describe, expect, it } from "vitest";
import type { Id } from "./_generated/dataModel";
import {
backfillExistingPublicCorpusBatchRows,
currentUserSeedPackageName,
currentUserSeedSkillSlug,
seedFeaturedPluginPackagesMutation,
seedGitHubBackedSkillSourceMutation,
seedLocalFixtures,
seedLocalModerationFixturesHandler,
seedPublicCorpusBatch,
seedPublicCorpusBatchMutation,
seedSkillMutation,
} from "./devSeed";
@@ -27,9 +29,15 @@ const seedGitHubBackedSkillSourceHandler = (
const seedLocalFixturesHandler = (
seedLocalFixtures as unknown as WrappedHandler<{ reset?: boolean }>
)._handler;
const seedPublicCorpusBatchActionHandler = (
seedPublicCorpusBatch as unknown as WrappedHandler<Record<string, unknown>>
)._handler;
const seedPublicCorpusBatchHandler = (
seedPublicCorpusBatchMutation as unknown as WrappedHandler<Record<string, unknown>>
)._handler;
const backfillExistingPublicCorpusBatchRowsHandler = (
backfillExistingPublicCorpusBatchRows as unknown as WrappedHandler<Record<string, unknown>>
)._handler;
function chainEq(constraints: Record<string, unknown>) {
return {
@@ -261,6 +269,651 @@ describe("devSeed local fixtures", () => {
}),
);
expect(tables.skillEmbeddings?.[0]).not.toHaveProperty("ownerPublisherId");
expect(
(tables.skillDailyStats ?? []).reduce((sum, row) => sum + Number(row.downloads), 0),
).toBe(tables.skills?.[0]?.statsDownloads);
expect((tables.skillDailyStats ?? []).reduce((sum, row) => sum + Number(row.installs), 0)).toBe(
tables.skills?.[0]?.statsInstallsAllTime,
);
});
it("backfills daily activity for existing public corpus skills", async () => {
const { db, tables } = createDb();
const userId = (await db.insert("users", {
handle: "corpus-owner",
displayName: "Corpus Owner",
role: "user",
createdAt: 1,
updatedAt: 1,
})) as Id<"users">;
const publisherId = (await db.insert("publishers", {
kind: "user",
handle: "corpus-owner",
displayName: "Corpus Owner",
linkedUserId: userId,
createdAt: 1,
updatedAt: 1,
})) as Id<"publishers">;
await db.insert("skills", {
slug: "corpus-demo",
displayName: "Corpus Demo",
ownerUserId: userId,
ownerPublisherId: publisherId,
batch: "public-corpus-v1",
tags: {},
badges: {},
statsDownloads: 143,
statsStars: 7,
statsInstallsCurrent: 18,
statsInstallsAllTime: 23,
stats: {
downloads: 143,
stars: 7,
installsCurrent: 18,
installsAllTime: 23,
versions: 1,
comments: 0,
},
createdAt: 1,
updatedAt: 1,
});
const result = await seedPublicCorpusBatchHandler(
createMutationCtx(db) as never,
{
rows: [
{
kind: "skill",
slug: "corpus-demo",
displayName: "Corpus Demo",
version: "0.1.0",
skillMd: "---\ndescription: Corpus demo\n---\n# Corpus demo",
storageId: "storage:corpus-demo",
embedding: [0, 1, 2],
dummyOwner: {
handle: "corpus-owner",
displayName: "Corpus Owner",
image: "https://example.invalid/avatar.png",
},
},
],
} as never,
);
const rows = tables.skillDailyStats ?? [];
expect(result).toEqual({ ok: true, seeded: [], skipped: ["skill:corpus-demo"] });
expect(rows).toHaveLength(30);
expect(rows.reduce((sum, row) => sum + Number(row.downloads), 0)).toBe(143);
expect(rows.reduce((sum, row) => sum + Number(row.installs), 0)).toBe(23);
});
it("pre-skips existing public corpus rows before storage and embedding prep", async () => {
const { db, tables } = createDb();
const userId = (await db.insert("users", {
handle: "corpus-owner",
displayName: "Corpus Owner",
role: "user",
createdAt: 1,
updatedAt: 1,
})) as Id<"users">;
const publisherId = (await db.insert("publishers", {
kind: "user",
handle: "corpus-owner",
displayName: "Corpus Owner",
linkedUserId: userId,
createdAt: 1,
updatedAt: 1,
})) as Id<"publishers">;
await db.insert("skills", {
slug: "corpus-demo",
displayName: "Corpus Demo",
ownerUserId: userId,
ownerPublisherId: publisherId,
batch: "public-corpus-v1",
tags: {},
badges: {},
statsDownloads: 143,
statsStars: 7,
statsInstallsCurrent: 18,
statsInstallsAllTime: 23,
stats: {
downloads: 143,
stars: 7,
installsCurrent: 18,
installsAllTime: 23,
versions: 1,
comments: 0,
},
createdAt: 1,
updatedAt: 1,
});
await db.insert("packages", {
name: "demo-plugin",
normalizedName: "demo-plugin",
displayName: "Demo Plugin",
ownerUserId: userId,
ownerPublisherId: publisherId,
stats: { downloads: 57, installs: 13, stars: 2, versions: 1 },
createdAt: 1,
updatedAt: 1,
});
const mutationCtx = createMutationCtx(db);
const storageStore = async () => {
throw new Error("existing public corpus rows should not store files");
};
const result = await seedPublicCorpusBatchActionHandler(
{
storage: { store: storageStore },
runMutation: async (_ref: unknown, args: Record<string, unknown>) =>
backfillExistingPublicCorpusBatchRowsHandler(mutationCtx as never, args),
} as never,
{
rows: [
{
kind: "skill",
slug: "corpus-demo",
displayName: "Corpus Demo",
version: "0.1.0",
skillMd: "---\ndescription: Corpus demo\n---\n# Corpus demo",
dummyOwner: {
handle: "corpus-owner",
displayName: "Corpus Owner",
image: "https://example.invalid/avatar.png",
},
},
{
kind: "plugin",
name: "demo-plugin",
displayName: "Demo Plugin",
version: "0.1.0",
readme: "# Demo plugin",
dummyOwner: {
handle: "corpus-owner",
displayName: "Corpus Owner",
image: "https://example.invalid/avatar.png",
},
},
],
} as never,
);
expect(result).toEqual({
ok: true,
seeded: [],
skipped: ["skill:corpus-demo", "plugin:demo-plugin"],
});
expect(tables.skillDailyStats).toHaveLength(30);
expect((tables.packageDailyStats ?? []).length).toBeGreaterThan(0);
expect(
(tables.packageDailyStats ?? []).reduce((sum, row) => sum + Number(row.downloads), 0),
).toBe(57);
expect(
(tables.packageDailyStats ?? []).reduce((sum, row) => sum + Number(row.installs), 0),
).toBe(13);
});
it("seeds daily activity for new public corpus packages", async () => {
const { db, tables } = createDb();
await seedPublicCorpusBatchHandler(
createMutationCtx(db) as never,
{
rows: [
{
kind: "plugin",
name: "demo-plugin",
displayName: "Demo Plugin",
version: "0.1.0",
readme: "# Demo plugin",
storageId: "storage:demo-plugin",
dummyOwner: {
handle: "corpus-owner",
displayName: "Corpus Owner",
image: "https://example.invalid/avatar.png",
},
},
],
} as never,
);
const pkg = tables.packages?.find((candidate) => candidate.name === "demo-plugin");
const stats = pkg?.stats;
const downloads =
stats &&
typeof stats === "object" &&
"downloads" in stats &&
typeof stats.downloads === "number"
? stats.downloads
: null;
const installs =
stats &&
typeof stats === "object" &&
"installs" in stats &&
typeof stats.installs === "number"
? stats.installs
: null;
expect(pkg).toBeTruthy();
expect(downloads).not.toBeNull();
expect(installs).not.toBeNull();
expect((tables.packageDailyStats ?? []).length).toBeGreaterThan(0);
expect(
(tables.packageDailyStats ?? []).reduce((sum, row) => sum + Number(row.downloads), 0),
).toBe(downloads);
expect(
(tables.packageDailyStats ?? []).reduce((sum, row) => sum + Number(row.installs), 0),
).toBe(installs);
});
it("populates public corpus plugin catalog metadata, digests, and validation findings", async () => {
const { db, tables } = createDb();
await seedPublicCorpusBatchHandler(
createMutationCtx(db) as never,
{
rows: [
{
kind: "plugin",
name: "gmail-agent-plugin",
displayName: "Gmail Agent Plugin",
version: "0.1.0",
readme: "# Gmail Agent Plugin\n\nWatches Gmail and notifies an OpenClaw channel.",
storageId: "storage:gmail-agent-plugin",
categories: ["channels", "tools"],
topics: ["Gmail", "Notifications"],
dummyOwner: {
handle: "corpus-owner",
displayName: "Corpus Owner",
image: "https://example.invalid/avatar.png",
},
},
],
} as never,
);
const pkg = tables.packages?.find((candidate) => candidate.name === "gmail-agent-plugin");
const release = tables.packageReleases?.find((candidate) => candidate.packageId === pkg?._id);
expect(pkg).toEqual(
expect.objectContaining({
categories: ["channels", "tools"],
topics: ["Gmail", "Notifications"],
}),
);
expect(tables.packageSearchDigest?.[0]).toEqual(
expect.objectContaining({
packageId: pkg?._id,
categories: ["channels", "tools"],
topics: ["Gmail", "Notifications"],
pluginCategoryTags: ["channels", "tools"],
}),
);
expect(
tables.packagePluginCategorySearchDigest
?.map((row) => String(row.pluginCategory))
.sort((left, right) => left.localeCompare(right)),
).toEqual(["channels", "tools"]);
expect(
tables.packageTopicSearchDigest
?.map((row) => String(row.topic))
.sort((left, right) => left.localeCompare(right)),
).toEqual(["gmail", "notifications"]);
expect(tables.packageInspectorWarnings).toEqual([
expect.objectContaining({
packageId: pkg?._id,
releaseId: release?._id,
packageName: "gmail-agent-plugin",
version: "0.1.0",
findingKind: "warning",
code: "package-min-host-version-drift",
authorRemediation: expect.objectContaining({
docsUrl:
"https://docs.openclaw.ai/clawhub/plugin-validation-fixes#package-min-host-version-drift",
}),
}),
]);
});
it("backfills catalog metadata and validation findings for existing public corpus packages", async () => {
const { db, tables } = createDb();
const userId = (await db.insert("users", {
handle: "corpus-owner",
displayName: "Corpus Owner",
role: "user",
createdAt: 1,
updatedAt: 1,
})) as Id<"users">;
const publisherId = (await db.insert("publishers", {
kind: "user",
handle: "corpus-owner",
displayName: "Corpus Owner",
linkedUserId: userId,
createdAt: 1,
updatedAt: 1,
})) as Id<"publishers">;
const packageId = (await db.insert("packages", {
name: "gmail-agent-plugin",
normalizedName: "gmail-agent-plugin",
displayName: "Gmail Agent Plugin",
summary: "Existing public corpus plugin fixture.",
ownerUserId: userId,
ownerPublisherId: publisherId,
family: "code-plugin",
channel: "community",
isOfficial: false,
runtimeId: "gmail-agent-plugin",
latestReleaseId: undefined,
latestVersionSummary: undefined,
tags: {},
compatibility: { pluginApiRange: ">=0.1.0" },
verification: {
tier: "structural",
scope: "artifact-only",
summary: "Seeded from the public corpus fixture.",
scanStatus: "clean",
},
scanStatus: "clean",
stats: { downloads: 57, installs: 13, stars: 2, versions: 1 },
createdAt: 1,
updatedAt: 1,
})) as Id<"packages">;
const releaseId = (await db.insert("packageReleases", {
packageId,
version: "0.1.0",
changelog: "Existing public corpus fixture.",
distTags: ["latest"],
files: [],
integritySha256: "existing-integrity",
compatibility: { pluginApiRange: ">=0.1.0" },
verification: {
tier: "structural",
scope: "artifact-only",
summary: "Seeded from the public corpus fixture.",
scanStatus: "clean",
},
createdBy: userId,
publishActor: { kind: "user", userId },
createdAt: 1,
})) as Id<"packageReleases">;
await db.patch(packageId, {
latestReleaseId: releaseId,
latestVersionSummary: {
version: "0.1.0",
createdAt: 1,
changelog: "Existing public corpus fixture.",
compatibility: { pluginApiRange: ">=0.1.0" },
verification: {
tier: "structural",
scope: "artifact-only",
summary: "Seeded from the public corpus fixture.",
scanStatus: "clean",
},
},
});
await seedPublicCorpusBatchHandler(
createMutationCtx(db) as never,
{
rows: [
{
kind: "plugin",
name: "gmail-agent-plugin",
displayName: "Gmail Agent Plugin",
version: "0.1.0",
readme: "# Gmail Agent Plugin\n\nWatches Gmail and notifies an OpenClaw channel.",
storageId: "storage:gmail-agent-plugin",
categories: ["channels", "tools"],
topics: ["Gmail", "Notifications"],
dummyOwner: {
handle: "corpus-owner",
displayName: "Corpus Owner",
image: "https://example.invalid/avatar.png",
},
},
],
} as never,
);
expect(tables.packages?.[0]).toEqual(
expect.objectContaining({
categories: ["channels", "tools"],
topics: ["Gmail", "Notifications"],
}),
);
expect(tables.packageSearchDigest?.[0]).toEqual(
expect.objectContaining({
packageId,
pluginCategoryTags: ["channels", "tools"],
}),
);
expect(tables.packageInspectorWarnings).toHaveLength(1);
});
it("does not backfill catalog metadata onto non-corpus package name collisions", async () => {
const { db, tables } = createDb();
const userId = (await db.insert("users", {
handle: "real-owner",
displayName: "Real Owner",
role: "user",
createdAt: 1,
updatedAt: 1,
})) as Id<"users">;
const publisherId = (await db.insert("publishers", {
kind: "user",
handle: "real-owner",
displayName: "Real Owner",
linkedUserId: userId,
createdAt: 1,
updatedAt: 1,
})) as Id<"publishers">;
const packageId = (await db.insert("packages", {
name: "gmail-agent-plugin",
normalizedName: "gmail-agent-plugin",
displayName: "Gmail Agent Plugin",
summary: "A real package that happens to collide with the corpus fixture.",
ownerUserId: userId,
ownerPublisherId: publisherId,
family: "code-plugin",
channel: "community",
isOfficial: false,
runtimeId: "gmail-agent-plugin",
latestReleaseId: undefined,
latestVersionSummary: undefined,
tags: {},
categories: ["models"],
topics: ["Original Topic"],
compatibility: { pluginApiRange: ">=0.1.0" },
verification: {
tier: "structural",
scope: "artifact-only",
summary: "Real package verification.",
scanStatus: "clean",
},
scanStatus: "clean",
stats: { downloads: 57, installs: 13, stars: 2, versions: 1 },
createdAt: 1,
updatedAt: 1,
})) as Id<"packages">;
const releaseId = (await db.insert("packageReleases", {
packageId,
version: "0.1.0",
changelog: "Real package release.",
distTags: ["latest"],
files: [],
integritySha256: "existing-integrity",
compatibility: { pluginApiRange: ">=0.1.0" },
verification: {
tier: "structural",
scope: "artifact-only",
summary: "Real package verification.",
scanStatus: "clean",
},
createdBy: userId,
publishActor: { kind: "user", userId },
createdAt: 1,
})) as Id<"packageReleases">;
await db.patch(packageId, {
latestReleaseId: releaseId,
latestVersionSummary: {
version: "0.1.0",
createdAt: 1,
changelog: "Real package release.",
compatibility: { pluginApiRange: ">=0.1.0" },
verification: {
tier: "structural",
scope: "artifact-only",
summary: "Real package verification.",
scanStatus: "clean",
},
},
});
await backfillExistingPublicCorpusBatchRowsHandler(
createMutationCtx(db) as never,
{
rows: [
{
kind: "plugin",
name: "gmail-agent-plugin",
displayName: "Gmail Agent Plugin",
version: "0.1.0",
readme: "# Gmail Agent Plugin\n\nWatches Gmail and notifies an OpenClaw channel.",
storageId: "storage:gmail-agent-plugin",
categories: ["channels", "tools"],
topics: ["Gmail", "Notifications"],
dummyOwner: {
handle: "corpus-owner",
displayName: "Corpus Owner",
image: "https://example.invalid/avatar.png",
},
},
],
} as never,
);
expect(tables.packages?.[0]).toEqual(
expect.objectContaining({
categories: ["models"],
topics: ["Original Topic"],
}),
);
expect(tables.packageSearchDigest).toBeUndefined();
expect(tables.packageDailyStats).toBeUndefined();
expect(tables.packageInspectorWarnings).toBeUndefined();
});
it("caps inferred public corpus plugin categories at the catalog limit", async () => {
const { db, tables } = createDb();
await seedPublicCorpusBatchHandler(
createMutationCtx(db) as never,
{
rows: [
{
kind: "plugin",
name: "context-security-openclaw-email-guard-plugin",
displayName: "Context Security Email Guard Plugin",
version: "0.1.0",
readme:
"# Context Security Email Guard Plugin\n\nA runtime plugin for Gmail, model providers, memory, context, web search, GitHub tools, gateway operations, and OAuth policy checks.",
storageId: "storage:context-security-openclaw-email-guard-plugin",
dummyOwner: {
handle: "corpus-owner",
displayName: "Corpus Owner",
image: "https://example.invalid/avatar.png",
},
},
],
} as never,
);
const pkg = tables.packages?.find(
(candidate) => candidate.name === "context-security-openclaw-email-guard-plugin",
);
const categories = Array.isArray(pkg?.categories) ? pkg.categories : [];
expect(categories.length).toBeLessThanOrEqual(3);
expect(categories.length).toBeGreaterThan(0);
});
it("removes public corpus daily activity rows during reset", async () => {
const { db, tables } = createDb();
await db.insert("globalStats", {
key: "default",
activeSkillsCount: 0,
activePluginsCount: 0,
updatedAt: 1,
});
const rows = [
{
kind: "skill",
slug: "corpus-demo",
displayName: "Corpus Demo",
version: "0.1.0",
skillMd: "---\ndescription: Corpus demo\n---\n# Corpus demo",
storageId: "storage:corpus-demo",
embedding: [0, 1, 2],
dummyOwner: {
handle: "corpus-owner",
displayName: "Corpus Owner",
image: "https://example.invalid/avatar.png",
},
},
{
kind: "plugin",
name: "demo-plugin",
displayName: "Demo Plugin",
version: "0.1.0",
readme: "# Demo plugin",
storageId: "storage:demo-plugin",
dummyOwner: {
handle: "corpus-owner",
displayName: "Corpus Owner",
image: "https://example.invalid/avatar.png",
},
},
];
await seedPublicCorpusBatchHandler(createMutationCtx(db) as never, { rows } as never);
const firstSkillId = tables.skills?.[0]?._id;
const firstPackageId = tables.packages?.[0]?._id;
const firstSkillDailyRows = tables.skillDailyStats?.length ?? 0;
const firstPackageDailyRows = tables.packageDailyStats?.length ?? 0;
const firstPackageDigestRows = tables.packageSearchDigest?.length ?? 0;
const firstPackageCategoryDigestRows = tables.packagePluginCategorySearchDigest?.length ?? 0;
const firstPackageTopicDigestRows = tables.packageTopicSearchDigest?.length ?? 0;
const firstPackageInspectorWarningRows = tables.packageInspectorWarnings?.length ?? 0;
const firstActivePluginsCount = tables.globalStats?.[0]?.activePluginsCount;
await seedPublicCorpusBatchHandler(
createMutationCtx(db) as never,
{ reset: true, resetOwnerHandles: ["corpus-owner"], rows } as never,
);
expect(firstSkillDailyRows).toBeGreaterThan(0);
expect(firstPackageDailyRows).toBeGreaterThan(0);
expect(tables.skillDailyStats).toHaveLength(firstSkillDailyRows);
expect(tables.packageDailyStats).toHaveLength(firstPackageDailyRows);
expect(tables.skillDailyStats?.some((row) => row.skillId === firstSkillId)).toBe(false);
expect(tables.packageDailyStats?.some((row) => row.packageId === firstPackageId)).toBe(false);
expect(firstPackageDigestRows).toBeGreaterThan(0);
expect(firstPackageCategoryDigestRows).toBeGreaterThan(0);
expect(firstPackageTopicDigestRows).toBeGreaterThan(0);
expect(firstPackageInspectorWarningRows).toBeGreaterThan(0);
expect(tables.packageSearchDigest?.some((row) => row.packageId === firstPackageId)).toBe(false);
expect(
tables.packagePluginCategorySearchDigest?.some((row) => row.packageId === firstPackageId),
).toBe(false);
expect(tables.packageTopicSearchDigest?.some((row) => row.packageId === firstPackageId)).toBe(
false,
);
expect(tables.packageInspectorWarnings?.some((row) => row.packageId === firstPackageId)).toBe(
false,
);
expect(firstActivePluginsCount).toBe(1);
expect(tables.globalStats?.[0]?.activePluginsCount).toBe(1);
});
it("seeds a GitHub-backed source and skills without creating mirrored versions", async () => {
@@ -501,6 +1154,37 @@ describe("devSeed local fixtures", () => {
scannedPluginName,
]);
expect(tables.packages?.every((pkg) => pkg.ownerUserId === userId)).toBe(true);
expect(tables.packages).toEqual(
expect.arrayContaining([
expect.objectContaining({
name: scannedPluginName,
icon: "https://cdn.simpleicons.org/github/111111",
latestVersionSummary: expect.objectContaining({
icon: "https://cdn.simpleicons.org/github/111111",
}),
}),
]),
);
expect(tables.packageReleases).toEqual(
expect.arrayContaining([
expect.objectContaining({
packageId: expect.stringMatching(/^packages:/),
icon: "https://cdn.simpleicons.org/github/111111",
}),
]),
);
const scannedPackageId = tables.packages?.find((pkg) => pkg.name === scannedPluginName)?._id;
const scannedPackageDailyStats = (tables.packageDailyStats ?? []).filter(
(row) => row.packageId === scannedPackageId,
);
expect(tables.packageDailyStats).toEqual(
expect.arrayContaining([
expect.objectContaining({
packageId: scannedPackageId,
}),
]),
);
expect(scannedPackageDailyStats.reduce((sum, row) => sum + Number(row.downloads), 0)).toBe(7);
expect(tables.packageInspectorWarnings).toEqual(
expect.arrayContaining([
expect.objectContaining({
+1030 -121
View File
File diff suppressed because it is too large Load Diff
+28 -4
View File
@@ -227,14 +227,38 @@ describe("download metric helpers", () => {
expect(result).toEqual({ deleted: 2, hasMore: false });
expect(indexCalls[0]?.table).toBe("downloadMetricDedupes");
expect(indexCalls[0]?.indexName).toBe("by_day");
expect(take).toHaveBeenCalledWith(200);
expect(take).toHaveBeenCalledWith(500);
expect(delete_).toHaveBeenCalledWith("downloadMetricDedupes:one");
expect(delete_).toHaveBeenCalledWith("downloadMetricDedupes:two");
});
it("prunes stale package install metric dedupe rows after download rows", async () => {
vi.setSystemTime(30 * 86_400_000);
const { db, delete_, indexCalls } = makeDb(
{},
{
packageInstallMetricDedupes: [
{ _id: "packageInstallMetricDedupes:one" },
{ _id: "packageInstallMetricDedupes:two" },
],
},
);
const result = await pruneDownloadMetricDedupesHandler({ db }, {});
expect(result).toEqual({ deleted: 2, hasMore: false });
expect(indexCalls.map((call) => call.table)).toEqual([
"downloadMetricDedupes",
"packageInstallMetricDedupes",
]);
expect(indexCalls[1]?.indexName).toBe("by_day");
expect(delete_).toHaveBeenCalledWith("packageInstallMetricDedupes:one");
expect(delete_).toHaveBeenCalledWith("packageInstallMetricDedupes:two");
});
it("reschedules stale dedupe pruning when one bounded batch fills", async () => {
vi.setSystemTime(30 * 86_400_000);
const rows = Array.from({ length: 200 }, (_, index) => ({
const rows = Array.from({ length: 500 }, (_, index) => ({
_id: `downloadMetricDedupes:${index}`,
}));
const { db, delete_ } = makeDb({}, { downloadMetricDedupes: rows });
@@ -242,8 +266,8 @@ describe("download metric helpers", () => {
const result = await pruneDownloadMetricDedupesHandler({ db, scheduler: { runAfter } }, {});
expect(result).toEqual({ deleted: 200, hasMore: true });
expect(delete_).toHaveBeenCalledTimes(200);
expect(result).toEqual({ deleted: 500, hasMore: true });
expect(delete_).toHaveBeenCalledTimes(500);
expect(runAfter).toHaveBeenCalledWith(0, expect.anything(), {});
});
});
+18 -5
View File
@@ -3,12 +3,13 @@ import { internal } from "./_generated/api";
import type { Id } from "./_generated/dataModel";
import { internalMutation } from "./functions";
import { getClientIp } from "./lib/httpRateLimit";
import { RETENTION_STANDARD_BATCH_SIZE } from "./lib/retentionPolicy";
import { hashToken } from "./lib/tokens";
import { insertStatEvent } from "./skillStatEvents";
const DAY_MS = 86_400_000;
const DEDUPE_RETENTION_MS = 14 * DAY_MS;
const PRUNE_BATCH_SIZE = 200;
const PRUNE_BATCH_SIZE = RETENTION_STANDARD_BATCH_SIZE;
const identityKindValidator = v.union(v.literal("user"), v.literal("ip"));
@@ -105,16 +106,28 @@ export const pruneDownloadMetricDedupesInternal = internalMutation({
args: {},
handler: async (ctx) => {
const cutoffDayStart = getDayStart(Date.now() - DEDUPE_RETENTION_MS);
const stale = await ctx.db
const staleDownloads = await ctx.db
.query("downloadMetricDedupes")
.withIndex("by_day", (q) => q.lt("dayStart", cutoffDayStart))
.take(PRUNE_BATCH_SIZE);
const remainingBatchSize = PRUNE_BATCH_SIZE - staleDownloads.length;
const staleInstalls =
remainingBatchSize > 0
? await ctx.db
.query("packageInstallMetricDedupes")
.withIndex("by_day", (q) => q.lt("dayStart", cutoffDayStart))
.take(remainingBatchSize)
: [];
for (const entry of stale) {
for (const entry of staleDownloads) {
await ctx.db.delete(entry._id);
}
for (const entry of staleInstalls) {
await ctx.db.delete(entry._id);
}
const hasMore = stale.length === PRUNE_BATCH_SIZE;
const deleted = staleDownloads.length + staleInstalls.length;
const hasMore = deleted === PRUNE_BATCH_SIZE;
if (hasMore) {
await ctx.scheduler.runAfter(
0,
@@ -123,7 +136,7 @@ export const pruneDownloadMetricDedupesInternal = internalMutation({
);
}
return { deleted: stale.length, hasMore };
return { deleted, hasMore };
},
});
+78
View File
@@ -150,6 +150,56 @@ describe("downloads helpers", () => {
});
});
it("threads owner handle through the skill lookup", async () => {
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
if (isRateLimitArgs(args)) return okRate();
if ("slug" in args) {
return {
skill: {
_id: "skills:1",
ownerUserId: "users:1",
slug: "demo",
tags: {},
latestVersionId: "skillVersions:1",
},
moderationInfo: null,
};
}
if ("versionId" in args) {
return {
_id: "skillVersions:1",
version: "1.0.0",
createdAt: 3,
files: [{ path: "SKILL.md", storageId: "_storage:1" }],
softDeletedAt: undefined,
};
}
return null;
});
const runMutation = vi.fn(async (_mutation: unknown, args: Record<string, unknown>) => {
if (isRateLimitArgs(args)) return okRate();
return null;
});
await downloadZipHandler(
{
runQuery,
runMutation,
scheduler: { runAfter: vi.fn() },
storage: { get: vi.fn().mockResolvedValue(new Blob(["hello"])) },
} as unknown as ActionCtx,
new Request("https://example.com/api/v1/download?slug=demo&ownerHandle=clawkit"),
);
const skillLookup = runQuery.mock.calls.find(([, args]) => {
const value = args as Record<string, unknown>;
return value.slug === "demo";
});
expect(skillLookup?.[1]).toEqual(
expect.objectContaining({ slug: "demo", ownerHandle: "clawkit" }),
);
});
it("does not serve a tag that points at another skill's version", async () => {
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
if (isRateLimitArgs(args)) return okRate();
@@ -210,6 +260,34 @@ describe("downloads helpers", () => {
expect(storageGet).not.toHaveBeenCalled();
});
it("returns ownerHandle guidance when a slug-only download is ambiguous", async () => {
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
if (isRateLimitArgs(args)) return okRate();
if ("slug" in args) return { skill: null, ambiguous: true };
return null;
});
const runMutation = vi.fn(async (_mutation: unknown, args: Record<string, unknown>) => {
if (isRateLimitArgs(args)) return okRate();
return null;
});
const response = await downloadZipHandler(
{
runQuery,
runMutation,
scheduler: { runAfter: vi.fn() },
storage: { get: vi.fn() },
} as unknown as ActionCtx,
new Request("https://example.com/api/v1/download?slug=demo"),
);
expect(response.status).toBe(409);
expect(response.headers.get("access-control-allow-origin")).toBe("*");
const body = await response.text();
expect(body).toContain('Ambiguous skill slug "demo"');
expect(body).toContain("/api/v1/download?slug=demo&ownerHandle=<owner>");
});
it("blocks the exact requested skill version when its ClawScan verdict is malicious", async () => {
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
if (isRateLimitArgs(args)) return okRate();
+17 -61
View File
@@ -1,20 +1,16 @@
import { v } from "convex/values";
import { api, internal } from "./_generated/api";
import type { Id } from "./_generated/dataModel";
import { buildDownloadMetricArgs, getDownloadIdentity } from "./downloadMetrics";
import { httpAction, internalMutation } from "./functions";
import { httpAction } from "./functions";
import { ambiguousSkillSlugResponse } from "./httpApiV1/shared";
import { getOptionalActiveAuthUserIdFromAction } from "./lib/access";
import { getOptionalApiTokenUserId } from "./lib/apiTokenAuth";
import { corsHeaders, mergeHeaders } from "./lib/httpHeaders";
import { applyRateLimit, getClientIp } from "./lib/httpRateLimit";
import { getPublicSkillVersionDownloadBlock, isSkillVersionForSkill } from "./lib/skillFileAccess";
import { buildDeterministicZip } from "./lib/skillZip";
import { insertStatEvent } from "./skillStatEvents";
const HOUR_MS = 3_600_000;
const DEDUPE_RETENTION_MS = 7 * 24 * HOUR_MS;
const PRUNE_BATCH_SIZE = 200;
const PRUNE_MAX_BATCHES = 50;
const DOWNLOAD_STAT_JITTER_MS = 60_000;
export async function downloadZipHandler(
@@ -23,6 +19,10 @@ export async function downloadZipHandler(
) {
const url = new URL(request.url);
const slug = url.searchParams.get("slug")?.trim().toLowerCase();
const ownerHandle =
(url.searchParams.get("ownerHandle") ?? url.searchParams.get("owner"))
?.trim()
.replace(/^@+/, "") || undefined;
const versionParam = url.searchParams.get("version")?.trim();
const tagParam = url.searchParams.get("tag")?.trim();
@@ -36,8 +36,18 @@ export async function downloadZipHandler(
const rate = await applyRateLimit(ctx, request, "download");
if (!rate.ok) return rate.response;
const skillResult = await ctx.runQuery(api.skills.getBySlug, { slug });
const skillResult = await ctx.runQuery(api.skills.getBySlug, {
slug,
...(ownerHandle ? { ownerHandle } : {}),
});
if (!skillResult?.skill) {
if (skillResult?.ambiguous) {
return ambiguousSkillSlugResponse(
slug,
`/api/v1/download?slug=${encodeURIComponent(slug)}&ownerHandle=<owner>`,
mergeHeaders(rate.headers, corsHeaders()),
);
}
return new Response("Skill not found", {
status: 404,
headers: mergeHeaders(rate.headers, corsHeaders()),
@@ -137,60 +147,6 @@ export async function downloadZipHandler(
export const downloadZip = httpAction(downloadZipHandler);
export const recordDownloadInternal = internalMutation({
args: {
skillId: v.id("skills"),
identityHash: v.string(),
hourStart: v.number(),
},
handler: async (ctx, args) => {
const existing = await ctx.db
.query("downloadDedupes")
.withIndex("by_skill_identity_hour", (q) =>
q
.eq("skillId", args.skillId)
.eq("identityHash", args.identityHash)
.eq("hourStart", args.hourStart),
)
.first();
if (existing) return;
await ctx.db.insert("downloadDedupes", {
skillId: args.skillId,
identityHash: args.identityHash,
hourStart: args.hourStart,
createdAt: Date.now(),
});
await insertStatEvent(ctx, {
skillId: args.skillId,
kind: "download",
});
},
});
export const pruneDownloadDedupesInternal = internalMutation({
args: {},
handler: async (ctx) => {
const cutoff = Date.now() - DEDUPE_RETENTION_MS;
for (let batches = 0; batches < PRUNE_MAX_BATCHES; batches += 1) {
const stale = await ctx.db
.query("downloadDedupes")
.withIndex("by_hour", (q) => q.lt("hourStart", cutoff))
.take(PRUNE_BATCH_SIZE);
if (stale.length === 0) break;
for (const entry of stale) {
await ctx.db.delete(entry._id);
}
if (stale.length < PRUNE_BATCH_SIZE) break;
}
},
});
export function getHourStart(timestamp: number) {
return Math.floor(timestamp / HOUR_MS) * HOUR_MS;
}
+70
View File
@@ -0,0 +1,70 @@
/* @vitest-environment node */
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const { resendConstructorMock, resendSendMock } = vi.hoisted(() => ({
resendConstructorMock: vi.fn(function ResendMock() {
return { emails: { send: resendSendMock } };
}),
resendSendMock: vi.fn(),
}));
vi.mock("resend", () => ({
Resend: resendConstructorMock,
}));
const { sendBanNotificationInternal } = await import("./emailsNode");
type SendBanNotificationHandler = {
_handler: (
ctx: unknown,
args: {
userId: string;
bannedAt: number;
to: string;
handle?: string;
source: "manual" | "autoban";
reason?: string;
},
) => Promise<unknown>;
};
describe("transactional account emails", () => {
beforeEach(() => {
vi.stubEnv("RESEND_API_KEY", "resend_test");
resendConstructorMock.mockClear();
resendSendMock.mockReset();
resendSendMock.mockResolvedValue({ data: { id: "email_123" }, error: null });
});
afterEach(() => {
vi.unstubAllEnvs();
});
it("sends ban notifications without a Reply-To header", async () => {
const result = await (
sendBanNotificationInternal as unknown as SendBanNotificationHandler
)._handler(
{},
{
userId: "users:target",
bannedAt: 1_700_000_000_000,
to: "target@example.com",
handle: "target",
source: "manual",
reason: "security review",
},
);
expect(result).toEqual({ ok: true, id: "email_123" });
expect(resendSendMock).toHaveBeenCalledTimes(1);
const [payload, options] = resendSendMock.mock.calls[0] ?? [];
expect(payload).toMatchObject({
from: "ClawHub Security <noreply@notifications.openclaw.ai>",
to: "target@example.com",
subject: "Your ClawHub account has been suspended",
});
expect(payload).not.toHaveProperty("replyTo");
expect(options).toEqual({ idempotencyKey: "ban:users:target:1700000000000" });
});
});
-3
View File
@@ -13,7 +13,6 @@ import {
} from "./lib/emails";
const DEFAULT_FROM = "ClawHub Security <noreply@notifications.openclaw.ai>";
const DEFAULT_REPLY_TO = "security@notifications.openclaw.ai";
const notificationArtifactValidator = v.object({
kind: v.union(v.literal("skill"), v.literal("plugin")),
@@ -32,7 +31,6 @@ function getEmailConfig() {
return {
apiKey: process.env.RESEND_API_KEY,
from: process.env.CLAWHUB_SECURITY_EMAIL_FROM || DEFAULT_FROM,
replyTo: process.env.CLAWHUB_SECURITY_EMAIL || DEFAULT_REPLY_TO,
};
}
@@ -60,7 +58,6 @@ async function sendTransactionalEmail(args: SendEmailArgs) {
{
from: config.from,
to: args.to,
replyTo: config.replyTo,
subject: args.subject,
text: args.text,
html: args.html,
+3 -109
View File
@@ -5,9 +5,7 @@ import { internal } from "./_generated/api";
import type { Doc, Id, TableNames } from "./_generated/dataModel";
import {
internalMutation,
isGitHubMirrorEligibleSkillDoc,
repointPackageLatestRelease,
scheduleGitHubBackupDeletionForSkill,
scheduleOwnerPublisherDigestSync,
shouldScheduleOwnerPublisherDigestSyncForPublisherChange,
shouldScheduleOwnerUserPackageDigestSyncForUserChange,
@@ -48,89 +46,6 @@ function isRecord(value: unknown): value is Record<string, unknown> {
}
describe("package digest sync", () => {
it("identifies GitHub mirror eligibility from skill visibility fields", () => {
expect(isGitHubMirrorEligibleSkillDoc({ softDeletedAt: undefined })).toBe(true);
expect(
isGitHubMirrorEligibleSkillDoc({
softDeletedAt: undefined,
moderationStatus: "active",
}),
).toBe(true);
expect(
isGitHubMirrorEligibleSkillDoc({
softDeletedAt: undefined,
moderationStatus: "active",
moderationVerdict: "malicious",
}),
).toBe(false);
expect(
isGitHubMirrorEligibleSkillDoc({
softDeletedAt: undefined,
moderationStatus: "active",
moderationFlags: ["blocked.malware"],
}),
).toBe(false);
expect(
isGitHubMirrorEligibleSkillDoc({
softDeletedAt: undefined,
moderationStatus: "hidden",
}),
).toBe(false);
expect(
isGitHubMirrorEligibleSkillDoc({
softDeletedAt: undefined,
moderationStatus: "removed",
}),
).toBe(false);
expect(isGitHubMirrorEligibleSkillDoc({ softDeletedAt: 123 })).toBe(false);
});
it("schedules GitHub mirror deletion for a skill using the owner handle", async () => {
const ctx = {
db: {
get: vi.fn(async (id: string) => {
if (id === "users:owner") {
return {
_id: "users:owner",
handle: "alice",
deletedAt: undefined,
deactivatedAt: undefined,
};
}
return null;
}),
query: vi.fn(() => ({
withIndex: vi.fn(() => ({
unique: vi.fn().mockResolvedValue(null),
})),
})),
},
scheduler: {
runAfter: vi.fn(),
},
};
await scheduleGitHubBackupDeletionForSkill(
ctx as never,
{
slug: "hidden-skill",
ownerUserId: "users:owner",
ownerPublisherId: undefined,
softDeletedAt: 123,
moderationStatus: "hidden",
} as never,
);
expect(ctx.scheduler.runAfter).toHaveBeenCalledWith(
0,
internal.githubBackupsNode.deleteGitHubBackupForSlugInternal,
{
ownerHandle: "alice",
slug: "hidden-skill",
},
);
});
it("clears latestVersion when the current package release is soft-deleted", async () => {
const pkg = {
_id: "packages:demo",
@@ -142,8 +57,6 @@ describe("package digest sync", () => {
isOfficial: false,
ownerUserId: "users:owner",
summary: "demo",
capabilityTags: ["tools"],
executesCode: true,
runtimeId: null,
softDeletedAt: undefined,
createdAt: 1,
@@ -191,14 +104,6 @@ describe("package digest sync", () => {
packageId: "packages:demo",
latestVersion: undefined,
ownerHandle: "owner",
pluginCategoryTags: ["dev-tools"],
}),
);
expect(ctx.db.insert).toHaveBeenCalledWith(
"packagePluginCategorySearchDigest",
expect.objectContaining({
packageId: "packages:demo",
pluginCategory: "dev-tools",
}),
);
});
@@ -284,10 +189,7 @@ describe("package digest sync", () => {
},
latestReleaseId: "packageReleases:demo-2",
latestVersionSummary: { version: "2.0.0" },
capabilityTags: ["new"],
executesCode: true,
compatibility: { openclaw: "^2.0.0" },
capabilities: { capabilityTags: ["new"], executesCode: true },
verification: { tier: "community" },
runtimeId: null,
softDeletedAt: undefined,
@@ -302,7 +204,6 @@ describe("package digest sync", () => {
changelog: "old stable",
summary: "stable summary",
compatibility: { openclaw: "^1.0.0" },
capabilities: { capabilityTags: ["stable"], executesCode: false },
verification: { tier: "verified" },
distTags: ["stable"],
createdAt: 10,
@@ -316,7 +217,6 @@ describe("package digest sync", () => {
changelog: "legacy hotfix",
summary: "legacy summary",
compatibility: { openclaw: "^0.9.0" },
capabilities: { capabilityTags: ["legacy"], executesCode: false },
verification: { tier: "verified" },
distTags: ["legacy"],
createdAt: 20,
@@ -360,6 +260,7 @@ describe("package digest sync", () => {
}
if (
table === "packageCapabilitySearchDigest" ||
table === "packageTopicSearchDigest" ||
table === "packagePluginCategorySearchDigest"
) {
return {
@@ -393,8 +294,6 @@ describe("package digest sync", () => {
tags: { latest: "packageReleases:demo-1" },
latestVersionSummary: expect.objectContaining({ version: "1.0.0" }),
summary: "stable summary",
capabilityTags: ["stable"],
executesCode: false,
}),
);
expect(ctx.db.insert).toHaveBeenCalledWith(
@@ -423,10 +322,7 @@ describe("package digest sync", () => {
},
latestReleaseId: "packageReleases:bundle-latest",
latestVersionSummary: { version: "latest" },
capabilityTags: ["new"],
executesCode: false,
compatibility: { hosts: ["openclaw"] },
capabilities: { capabilityTags: ["new"], executesCode: false },
verification: { tier: "community" },
runtimeId: "bundle.runtime",
softDeletedAt: undefined,
@@ -441,7 +337,6 @@ describe("package digest sync", () => {
changelog: "older semver",
summary: "older semver summary",
compatibility: { hosts: ["openclaw"] },
capabilities: { capabilityTags: ["semver"], executesCode: false },
verification: { tier: "verified" },
distTags: ["legacy"],
createdAt: 10,
@@ -455,7 +350,6 @@ describe("package digest sync", () => {
changelog: "newest bundle build",
summary: "newest bundle summary",
compatibility: { hosts: ["openclaw"] },
capabilities: { capabilityTags: ["bundle"], executesCode: false },
verification: { tier: "verified" },
distTags: ["release-2024-12"],
createdAt: 20,
@@ -499,6 +393,7 @@ describe("package digest sync", () => {
}
if (
table === "packageCapabilitySearchDigest" ||
table === "packageTopicSearchDigest" ||
table === "packagePluginCategorySearchDigest"
) {
return {
@@ -564,8 +459,6 @@ describe("package digest sync", () => {
tags: {},
latestReleaseId: undefined,
latestVersionSummary: undefined,
capabilityTags: [],
executesCode: false,
runtimeId: null,
softDeletedAt: undefined,
createdAt: 1,
@@ -601,6 +494,7 @@ describe("package digest sync", () => {
}
if (
table === "packageCapabilitySearchDigest" ||
table === "packageTopicSearchDigest" ||
table === "packagePluginCategorySearchDigest"
) {
return {
+10 -61
View File
@@ -14,7 +14,6 @@ import {
httpAction,
} from "./_generated/server";
import type { MutationCtx } from "./_generated/server";
import { isPublicSkillDoc } from "./lib/globalStats";
import {
deletePackageSearchDigests,
extractPackageDigestFields,
@@ -25,7 +24,11 @@ import {
adjustPublisherStatsForPackageChange,
adjustPublisherStatsForSkillChange,
} from "./lib/publisherStats";
import { extractValidatedDigestFields, upsertSkillSearchDigest } from "./lib/skillSearchDigest";
import {
deleteSkillSearchDigests,
extractValidatedDigestFields,
upsertSkillSearchDigest,
} from "./lib/skillSearchDigest";
const triggers = new Triggers<DataModel>();
@@ -38,7 +41,6 @@ function isMissingTableError(error: unknown, table: string) {
type PackageDigestSyncCtx = Pick<MutationCtx, "db">;
type OwnerPublisherDigestScheduleCtx = Pick<Partial<MutationCtx>, "scheduler">;
type GitHubBackupDeletionCtx = Pick<MutationCtx, "db" | "scheduler">;
const OWNER_PUBLISHER_DIGEST_PAGE_SIZE = 100;
type LatestPackageRelease = Pick<
Doc<"packageReleases">,
@@ -47,8 +49,8 @@ type LatestPackageRelease = Pick<
| "version"
| "changelog"
| "summary"
| "icon"
| "compatibility"
| "capabilities"
| "verification"
| "distTags"
> & {
@@ -63,8 +65,8 @@ function toPackageLatestVersionSummary(
version: release.version,
createdAt: release.createdAt,
changelog: release.changelog,
icon: release.icon,
compatibility: release.compatibility,
capabilities: release.capabilities,
verification: release.verification,
};
}
@@ -109,8 +111,8 @@ async function getPreferredFallbackPackageRelease(
version: release.version,
changelog: release.changelog,
summary: release.summary,
icon: release.icon,
compatibility: release.compatibility,
capabilities: release.capabilities,
verification: release.verification,
scanStatus: release.verification?.scanStatus,
distTags: release.distTags,
@@ -229,42 +231,6 @@ async function syncSkillSearchDigestForSkill(
});
}
export function isGitHubMirrorEligibleSkillDoc(
skill:
| Pick<
Doc<"skills">,
"softDeletedAt" | "moderationStatus" | "moderationFlags" | "moderationVerdict"
>
| null
| undefined,
) {
return isPublicSkillDoc(skill);
}
export async function scheduleGitHubBackupDeletionForSkill(
ctx: GitHubBackupDeletionCtx,
skill: Pick<
Doc<"skills">,
| "slug"
| "ownerPublisherId"
| "ownerUserId"
| "softDeletedAt"
| "moderationStatus"
| "moderationFlags"
| "moderationVerdict"
>,
) {
const owner = await getOwnerPublisher(ctx, {
ownerPublisherId: skill.ownerPublisherId,
ownerUserId: skill.ownerUserId,
});
const ownerHandle = owner?.handle ?? String(skill.ownerPublisherId ?? skill.ownerUserId);
await ctx.scheduler.runAfter(0, internal.githubBackupsNode.deleteGitHubBackupForSlugInternal, {
ownerHandle,
slug: skill.slug,
});
}
export async function syncSkillSearchDigestsForOwnerPublisherId(
ctx: PackageDigestSyncCtx & OwnerPublisherDigestScheduleCtx,
ownerPublisherId: Id<"publishers"> | null | undefined,
@@ -437,13 +403,8 @@ export async function repointPackageLatestRelease(
patch.latestReleaseId = nextLatest?._id;
patch.latestVersionSummary = toPackageLatestVersionSummary(nextLatest);
patch.summary = nextLatest?.summary;
patch.capabilityTags = nextLatest?.capabilities?.capabilityTags;
patch.executesCode =
typeof nextLatest?.capabilities?.executesCode === "boolean"
? nextLatest.capabilities.executesCode
: undefined;
patch.icon = nextLatest?.icon;
patch.compatibility = nextLatest?.compatibility;
patch.capabilities = nextLatest?.capabilities;
patch.verification = nextLatest?.verification;
patch.scanStatus = nextLatest?.scanStatus;
}
@@ -458,20 +419,8 @@ triggers.register("skills", async (ctx, change) => {
change.operation === "delete" ? null : change.newDoc,
);
if (change.operation === "delete") {
await scheduleGitHubBackupDeletionForSkill(ctx, change.oldDoc);
const existing = await ctx.db
.query("skillSearchDigest")
.withIndex("by_skill", (q) => q.eq("skillId", change.id))
.unique();
if (existing) await ctx.db.delete(existing._id);
await deleteSkillSearchDigests(ctx, change.id);
} else {
if (
change.operation === "update" &&
isGitHubMirrorEligibleSkillDoc(change.oldDoc) &&
!isGitHubMirrorEligibleSkillDoc(change.newDoc)
) {
await scheduleGitHubBackupDeletionForSkill(ctx, change.oldDoc);
}
await syncSkillSearchDigestForSkill(ctx, change.newDoc);
}
});
-204
View File
@@ -1,204 +0,0 @@
import { describe, expect, it, vi } from "vitest";
import { getGitHubBackupPageInternal } from "./githubBackups";
const handler = (getGitHubBackupPageInternal as unknown as { _handler: Function })._handler;
describe("githubBackups page filtering", () => {
it("skips non-public digests (soft-deleted, hidden, removed)", async () => {
const activeDigest = {
_id: "skillSearchDigest:active",
skillId: "skills:active",
slug: "active-skill",
displayName: "Active Skill",
ownerUserId: "users:active",
ownerHandle: "alice",
latestVersionId: "skillVersions:active",
latestVersionSummary: {
version: "1.0.0",
createdAt: 1_700_000_000_000,
changelog: "init",
},
softDeletedAt: undefined,
moderationStatus: "active",
};
const hiddenDigest = {
_id: "skillSearchDigest:hidden",
skillId: "skills:hidden",
slug: "hidden-skill",
displayName: "Hidden Skill",
ownerUserId: "users:hidden",
ownerHandle: "bob",
latestVersionId: "skillVersions:hidden",
latestVersionSummary: {
version: "1.0.0",
createdAt: 1_700_000_000_000,
changelog: "init",
},
softDeletedAt: undefined,
moderationStatus: "hidden",
};
const removedDigest = {
_id: "skillSearchDigest:removed",
skillId: "skills:removed",
slug: "removed-skill",
displayName: "Removed Skill",
ownerUserId: "users:removed",
ownerHandle: "carol",
latestVersionId: "skillVersions:removed",
latestVersionSummary: {
version: "1.0.0",
createdAt: 1_700_000_000_000,
changelog: "init",
},
softDeletedAt: undefined,
moderationStatus: "removed",
};
const softDeletedDigest = {
_id: "skillSearchDigest:soft",
skillId: "skills:soft",
slug: "soft-skill",
displayName: "Soft Skill",
ownerUserId: "users:soft",
ownerHandle: "dave",
latestVersionId: "skillVersions:soft",
latestVersionSummary: {
version: "1.0.0",
createdAt: 1_700_000_000_000,
changelog: "init",
},
softDeletedAt: 1,
moderationStatus: "active",
};
const paginate = vi.fn().mockResolvedValue({
page: [activeDigest, hiddenDigest, removedDigest, softDeletedDigest],
isDone: true,
continueCursor: null,
});
const order = vi.fn().mockReturnValue({ paginate });
const query = vi.fn().mockReturnValue({ order });
const result = await handler(
{
db: { query },
} as never,
{ batchSize: 50 },
);
expect(query).toHaveBeenCalledWith("skillSearchDigest");
expect(result).toMatchObject({
isDone: true,
cursor: null,
items: [
{
kind: "ok",
slug: "active-skill",
ownerHandle: "alice",
version: "1.0.0",
},
],
});
});
it("keeps legacy digests with undefined moderationStatus eligible", async () => {
const legacyDigest = {
_id: "skillSearchDigest:legacy",
skillId: "skills:legacy",
slug: "legacy-skill",
displayName: "Legacy Skill",
ownerUserId: "users:legacy",
ownerHandle: "",
latestVersionId: "skillVersions:legacy",
latestVersionSummary: {
version: "2.0.0",
createdAt: 1_700_000_000_100,
changelog: "update",
},
softDeletedAt: undefined,
moderationStatus: undefined,
};
const paginate = vi.fn().mockResolvedValue({
page: [legacyDigest],
isDone: true,
continueCursor: null,
});
const order = vi.fn().mockReturnValue({ paginate });
const query = vi.fn().mockReturnValue({ order });
const result = await handler(
{
db: { query },
} as never,
{},
);
expect(result.items).toHaveLength(1);
expect(result.items[0]).toMatchObject({
kind: "ok",
slug: "legacy-skill",
ownerHandle: "users:legacy",
version: "2.0.0",
});
});
it("skips digests without ownerHandle or latestVersionSummary", async () => {
const noOwnerHandle = {
_id: "skillSearchDigest:no-owner",
skillId: "skills:no-owner",
slug: "no-owner",
displayName: "No Owner",
ownerUserId: "users:no-owner",
ownerHandle: undefined,
latestVersionId: "skillVersions:no-owner",
latestVersionSummary: { version: "1.0.0", createdAt: 1, changelog: "init" },
softDeletedAt: undefined,
moderationStatus: "active",
};
const noVersion = {
_id: "skillSearchDigest:no-version",
skillId: "skills:no-version",
slug: "no-version",
displayName: "No Version",
ownerUserId: "users:no-version",
ownerHandle: "frank",
latestVersionId: undefined,
latestVersionSummary: undefined,
softDeletedAt: undefined,
moderationStatus: "active",
};
const paginate = vi.fn().mockResolvedValue({
page: [noOwnerHandle, noVersion],
isDone: true,
continueCursor: null,
});
const order = vi.fn().mockReturnValue({ paginate });
const query = vi.fn().mockReturnValue({ order });
const result = await handler({ db: { query } } as never, {});
expect(result.items).toEqual([
{ kind: "missingOwner", skillId: "skills:no-owner", ownerUserId: "users:no-owner" },
{ kind: "missingLatestVersion", skillId: "skills:no-version" },
]);
});
it("resets stale skills-table cursors after switching to digest pagination", async () => {
const paginate = vi
.fn()
.mockRejectedValueOnce(new Error("cursor is from a different query"))
.mockResolvedValueOnce({ page: [], isDone: true, continueCursor: null });
const order = vi.fn().mockReturnValue({ paginate });
const query = vi.fn().mockReturnValue({ order });
const result = await handler({ db: { query } } as never, { cursor: "stale-cursor" });
expect(result).toMatchObject({ items: [], isDone: true, cursor: null });
expect(paginate).toHaveBeenNthCalledWith(1, { cursor: "stale-cursor", numItems: 50 });
expect(paginate).toHaveBeenNthCalledWith(2, { cursor: null, numItems: 50 });
});
});
-203
View File
@@ -1,203 +0,0 @@
import { v } from "convex/values";
import { internal } from "./_generated/api";
import type { Doc, Id } from "./_generated/dataModel";
import { action, internalMutation, internalQuery } from "./functions";
import { assertRole, requireUserFromAction } from "./lib/access";
import { isPublicSkillDoc } from "./lib/globalStats";
const DEFAULT_BATCH_SIZE = 50;
const MAX_BATCH_SIZE = 200;
const SYNC_STATE_KEY = "default";
type BackupPageItem =
| {
kind: "ok";
skillId: Id<"skills">;
versionId: Id<"skillVersions">;
slug: string;
displayName: string;
version: string;
ownerHandle: string;
publishedAt: number;
}
| { kind: "missingLatestVersion"; skillId: Id<"skills"> }
| { kind: "missingOwner"; skillId: Id<"skills">; ownerUserId: Id<"users"> };
type BackupPageResult = {
items: BackupPageItem[];
cursor: string | null;
isDone: boolean;
};
type BackupSyncState = {
cursor: string | null;
pruneCursor: string | null;
};
export type SyncGitHubBackupsResult = {
stats: {
skillsScanned: number;
skillsSkipped: number;
skillsBackedUp: number;
skillsDeleted: number;
skillsMissingVersion: number;
skillsMissingOwner: number;
errors: number;
};
cursor: string | null;
pruneCursor: string | null;
isDone: boolean;
};
export const getGitHubBackupPageInternal = internalQuery({
args: {
cursor: v.optional(v.string()),
batchSize: v.optional(v.number()),
},
handler: async (ctx, args): Promise<BackupPageResult> => {
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE);
let pageResult;
try {
pageResult = await ctx.db
.query("skillSearchDigest")
.order("asc")
.paginate({ cursor: args.cursor ?? null, numItems: batchSize });
} catch (error) {
if (!args.cursor || !isStaleCursorError(error)) throw error;
pageResult = await ctx.db
.query("skillSearchDigest")
.order("asc")
.paginate({ cursor: null, numItems: batchSize });
}
const items: BackupPageItem[] = [];
for (const digest of pageResult.page) {
if (!isPubliclyAvailableSkill(digest)) continue;
if (!digest.latestVersionId || !digest.latestVersionSummary) {
items.push({ kind: "missingLatestVersion", skillId: digest.skillId });
continue;
}
if (digest.ownerHandle === undefined) {
items.push({
kind: "missingOwner",
skillId: digest.skillId,
ownerUserId: digest.ownerUserId,
});
continue;
}
const ownerHandle =
digest.ownerHandle || String(digest.ownerPublisherId ?? digest.ownerUserId);
items.push({
kind: "ok",
skillId: digest.skillId,
versionId: digest.latestVersionId,
slug: digest.slug,
displayName: digest.displayName,
version: digest.latestVersionSummary.version,
ownerHandle,
publishedAt: digest.latestVersionSummary.createdAt,
});
}
return { items, cursor: pageResult.continueCursor, isDone: pageResult.isDone };
},
});
function isPubliclyAvailableSkill(
skill: Pick<
Doc<"skillSearchDigest">,
"softDeletedAt" | "moderationStatus" | "moderationFlags" | "moderationVerdict"
>,
) {
return isPublicSkillDoc(skill);
}
function isStaleCursorError(error: unknown) {
const message =
typeof error === "string"
? error
: error && typeof error === "object" && "message" in error
? String((error as { message?: unknown }).message)
: "";
return (
message.includes("Failed to parse cursor") ||
message.includes("cursor is from a different query")
);
}
export const getGitHubBackupSyncStateInternal = internalQuery({
args: {},
handler: async (ctx): Promise<BackupSyncState> => {
const state = await ctx.db
.query("githubBackupSyncState")
.withIndex("by_key", (q) => q.eq("key", SYNC_STATE_KEY))
.unique();
return { cursor: state?.cursor ?? null, pruneCursor: state?.pruneCursor ?? null };
},
});
export const setGitHubBackupSyncStateInternal = internalMutation({
args: {
cursor: v.optional(v.string()),
pruneCursor: v.optional(v.string()),
},
handler: async (ctx, args) => {
const now = Date.now();
const state = await ctx.db
.query("githubBackupSyncState")
.withIndex("by_key", (q) => q.eq("key", SYNC_STATE_KEY))
.unique();
if (!state) {
await ctx.db.insert("githubBackupSyncState", {
key: SYNC_STATE_KEY,
cursor: args.cursor,
pruneCursor: args.pruneCursor,
updatedAt: now,
});
return { ok: true as const };
}
await ctx.db.patch(state._id, {
cursor: args.cursor,
pruneCursor: args.pruneCursor,
updatedAt: now,
});
return { ok: true as const };
},
});
export const syncGitHubBackups: ReturnType<typeof action> = action({
args: {
dryRun: v.optional(v.boolean()),
batchSize: v.optional(v.number()),
maxBatches: v.optional(v.number()),
pruneBatchSize: v.optional(v.number()),
resetCursor: v.optional(v.boolean()),
},
handler: async (ctx, args): Promise<SyncGitHubBackupsResult> => {
const { user } = await requireUserFromAction(ctx);
assertRole(user, ["admin"]);
if (args.resetCursor && !args.dryRun) {
await ctx.runMutation(internal.githubBackups.setGitHubBackupSyncStateInternal, {
cursor: undefined,
pruneCursor: undefined,
});
}
return ctx.runAction(internal.githubBackupsNode.syncGitHubBackupsInternal, {
dryRun: args.dryRun,
batchSize: args.batchSize,
maxBatches: args.maxBatches,
pruneBatchSize: args.pruneBatchSize,
}) as Promise<SyncGitHubBackupsResult>;
},
});
function clampInt(value: number, min: number, max: number) {
return Math.max(min, Math.min(max, Math.floor(value)));
}
-325
View File
@@ -1,325 +0,0 @@
"use node";
import { v } from "convex/values";
import { internal } from "./_generated/api";
import type { Doc } from "./_generated/dataModel";
import type { ActionCtx } from "./_generated/server";
import { internalAction } from "./functions";
import {
backupSkillToGitHub,
deleteGitHubSkillBackup,
fetchGitHubSkillMeta,
getGitHubBackupContext,
isGitHubBackupConfigured,
listGitHubSkillBackupEntries,
normalizeOwner,
} from "./lib/githubBackup";
import { isPublicSkillDoc } from "./lib/globalStats";
const DEFAULT_BATCH_SIZE = 50;
const MAX_BATCH_SIZE = 200;
const DEFAULT_MAX_BATCHES = 5;
const MAX_MAX_BATCHES = 200;
const DEFAULT_PRUNE_BATCH_SIZE = 10;
const MAX_PRUNE_BATCH_SIZE = 100;
type BackupPageItem =
| {
kind: "ok";
versionId: Doc<"skillVersions">["_id"];
slug: string;
version: string;
displayName: string;
ownerHandle: string;
publishedAt: number;
}
| { kind: "missingLatestVersion" }
| { kind: "missingOwner" };
export type GitHubBackupSyncStats = {
skillsScanned: number;
skillsSkipped: number;
skillsBackedUp: number;
skillsDeleted: number;
skillsMissingVersion: number;
skillsMissingOwner: number;
errors: number;
};
export type SyncGitHubBackupsInternalArgs = {
dryRun?: boolean;
batchSize?: number;
maxBatches?: number;
pruneBatchSize?: number;
};
export type SyncGitHubBackupsInternalResult = {
stats: GitHubBackupSyncStats;
cursor: string | null;
pruneCursor: string | null;
isDone: boolean;
};
export const backupSkillForPublishInternal = internalAction({
args: {
slug: v.string(),
version: v.string(),
displayName: v.string(),
ownerHandle: v.string(),
files: v.array(
v.object({
path: v.string(),
size: v.number(),
storageId: v.id("_storage"),
sha256: v.string(),
contentType: v.optional(v.string()),
}),
),
publishedAt: v.number(),
},
handler: async (ctx, args) => {
if (!isGitHubBackupConfigured()) {
return { skipped: true as const };
}
await backupSkillToGitHub(ctx, args);
return { skipped: false as const };
},
});
export async function syncGitHubBackupsInternalHandler(
ctx: ActionCtx,
args: SyncGitHubBackupsInternalArgs,
): Promise<SyncGitHubBackupsInternalResult> {
const dryRun = Boolean(args.dryRun);
const stats: GitHubBackupSyncStats = {
skillsScanned: 0,
skillsSkipped: 0,
skillsBackedUp: 0,
skillsDeleted: 0,
skillsMissingVersion: 0,
skillsMissingOwner: 0,
errors: 0,
};
if (!isGitHubBackupConfigured()) {
return { stats, cursor: null, pruneCursor: null, isDone: true };
}
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE);
const maxBatches = clampInt(args.maxBatches ?? DEFAULT_MAX_BATCHES, 1, MAX_MAX_BATCHES);
const pruneBatchSize = clampInt(
args.pruneBatchSize ?? DEFAULT_PRUNE_BATCH_SIZE,
1,
MAX_PRUNE_BATCH_SIZE,
);
const context = await getGitHubBackupContext();
const state = dryRun
? { cursor: null as string | null, pruneCursor: null as string | null }
: ((await ctx.runQuery(internal.githubBackups.getGitHubBackupSyncStateInternal, {})) as {
cursor: string | null;
pruneCursor: string | null;
});
let cursor: string | null = state.cursor;
let pruneCursor: string | null = state.pruneCursor;
let isDone = false;
for (let batch = 0; batch < maxBatches; batch++) {
const page = (await ctx.runQuery(internal.githubBackups.getGitHubBackupPageInternal, {
cursor: cursor ?? undefined,
batchSize,
})) as { items: BackupPageItem[]; cursor: string | null; isDone: boolean };
cursor = page.cursor;
isDone = page.isDone;
for (const item of page.items) {
if (item.kind !== "ok") {
if (item.kind === "missingLatestVersion") {
stats.skillsMissingVersion += 1;
} else if (item.kind === "missingOwner") {
stats.skillsMissingOwner += 1;
}
continue;
}
stats.skillsScanned += 1;
try {
const meta = await fetchGitHubSkillMeta(context, item.ownerHandle, item.slug);
if (meta?.latest?.version === item.version) {
stats.skillsSkipped += 1;
continue;
}
const version = (await ctx.runQuery(internal.skills.getVersionByIdInternal, {
versionId: item.versionId,
})) as Doc<"skillVersions"> | null;
if (!version) {
stats.skillsMissingVersion += 1;
continue;
}
if (!dryRun) {
await backupSkillToGitHub(
ctx,
{
slug: item.slug,
version: item.version,
displayName: item.displayName,
ownerHandle: item.ownerHandle,
files: version.files,
publishedAt: item.publishedAt,
},
context,
);
stats.skillsBackedUp += 1;
}
} catch (error) {
console.error("GitHub backup sync failed", error);
stats.errors += 1;
}
}
if (!dryRun) {
await ctx.runMutation(internal.githubBackups.setGitHubBackupSyncStateInternal, {
cursor: isDone ? undefined : (cursor ?? undefined),
pruneCursor: pruneCursor ?? undefined,
});
}
if (isDone) break;
}
pruneCursor = await pruneDeletedSkillBackups(
ctx,
context,
dryRun,
stats,
pruneCursor,
pruneBatchSize,
);
if (!dryRun) {
await ctx.runMutation(internal.githubBackups.setGitHubBackupSyncStateInternal, {
cursor: isDone ? undefined : (cursor ?? undefined),
pruneCursor: pruneCursor ?? undefined,
});
}
return { stats, cursor, pruneCursor, isDone };
}
async function pruneDeletedSkillBackups(
ctx: ActionCtx,
context: Awaited<ReturnType<typeof getGitHubBackupContext>>,
dryRun: boolean,
stats: GitHubBackupSyncStats,
pruneCursor: string | null,
pruneBatchSize: number,
): Promise<string | null> {
let entries: Awaited<ReturnType<typeof listGitHubSkillBackupEntries>>;
try {
entries = await listGitHubSkillBackupEntries(context);
} catch (error) {
console.error("GitHub backup cleanup list failed", error);
stats.errors += 1;
return pruneCursor;
}
if (!entries.length) return null;
const sortedEntries = [...entries].sort((a, b) => a.rootPath.localeCompare(b.rootPath));
const startIndex =
pruneCursor == null
? 0
: sortedEntries.findIndex((entry) => entry.rootPath.localeCompare(pruneCursor) > 0);
if (startIndex === -1) return null;
const chunk = sortedEntries.slice(startIndex, startIndex + pruneBatchSize);
if (!chunk.length) return null;
let lastProcessed = pruneCursor;
for (const entry of chunk) {
lastProcessed = entry.rootPath;
try {
const skill = (await ctx.runQuery(internal.skills.getSkillBySlugInternal, {
slug: entry.slug,
})) as Doc<"skills"> | null;
if (!isMirrorEligibleSkill(skill)) {
await deleteBackupIfNeeded(context, entry, dryRun, stats);
continue;
}
const owner = (await ctx.runQuery(internal.users.getByIdInternal, {
userId: skill.ownerUserId,
})) as Doc<"users"> | null;
if (!owner || owner.deletedAt || owner.deactivatedAt) {
await deleteBackupIfNeeded(context, entry, dryRun, stats);
continue;
}
const ownerHandle = normalizeOwner(owner.handle ?? owner._id);
if (ownerHandle !== entry.owner) {
await deleteBackupIfNeeded(context, entry, dryRun, stats);
}
} catch (error) {
console.error("GitHub backup cleanup failed", error);
stats.errors += 1;
}
}
const reachedEnd = startIndex + chunk.length >= sortedEntries.length;
return reachedEnd ? null : (lastProcessed ?? null);
}
function isMirrorEligibleSkill(skill: Doc<"skills"> | null): skill is Doc<"skills"> {
return isPublicSkillDoc(skill);
}
async function deleteBackupIfNeeded(
context: Awaited<ReturnType<typeof getGitHubBackupContext>>,
entry: Awaited<ReturnType<typeof listGitHubSkillBackupEntries>>[number],
dryRun: boolean,
stats: GitHubBackupSyncStats,
) {
const result = dryRun
? { deleted: true as const }
: await deleteGitHubSkillBackup(context, entry.owner, entry.slug);
if (result.deleted) {
stats.skillsDeleted += 1;
}
}
export const syncGitHubBackupsInternal = internalAction({
args: {
dryRun: v.optional(v.boolean()),
batchSize: v.optional(v.number()),
maxBatches: v.optional(v.number()),
pruneBatchSize: v.optional(v.number()),
},
handler: syncGitHubBackupsInternalHandler,
});
export const deleteGitHubBackupForSlugInternal = internalAction({
args: {
ownerHandle: v.string(),
slug: v.string(),
dryRun: v.optional(v.boolean()),
},
handler: async (_ctx, args) => {
if (!isGitHubBackupConfigured()) {
return { skipped: true as const, deleted: false as const };
}
if (args.dryRun) {
return { skipped: false as const, deleted: true as const, dryRun: true as const };
}
const context = await getGitHubBackupContext();
const result = await deleteGitHubSkillBackup(context, args.ownerHandle, args.slug);
return { skipped: false as const, ...result };
},
});
function clampInt(value: number, min: number, max: number) {
return Math.max(min, Math.min(max, Math.floor(value)));
}
+557 -1
View File
@@ -1,9 +1,35 @@
/* @vitest-environment node */
import { describe, expect, it } from "vitest";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { internal } from "./_generated/api";
import { __test } from "./githubImport";
import { buildGitHubZipForTests } from "./lib/githubImport";
vi.mock("./_generated/api", () => ({
internal: {
githubIdentity: {
getGitHubProviderAccountIdInternal: Symbol("getGitHubProviderAccountIdInternal"),
},
skills: {
getSkillBySlugInternal: Symbol("getSkillBySlugInternal"),
},
},
}));
const originalGitHubToken = process.env.GITHUB_TOKEN;
describe("githubImport", () => {
beforeEach(() => {
delete process.env.GITHUB_TOKEN;
});
afterEach(() => {
if (originalGitHubToken) {
process.env.GITHUB_TOKEN = originalGitHubToken;
} else {
delete process.env.GITHUB_TOKEN;
}
});
it("formats storage failure message with file context", () => {
const message = __test.buildStoreFailureMessage("skill/SKILL.md", 123, new Error("disk full"));
expect(message).toBe('Failed to store file "skill/SKILL.md" (123 bytes). disk full');
@@ -13,6 +39,15 @@ describe("githubImport", () => {
expect(__test.buildPublishFailureMessage(new Error("slug exists"))).toBe(
"Import failed during publish: slug exists. Check skill format, slug availability, and try again.",
);
expect(
__test.buildPublishFailureMessage(
new Error(
'Uncaught ConvexError: Publisher handle "@local-owner" is already claimed at ensurePersonalPublisherForUser (../../convex/lib/publishers.ts:235:4)',
),
),
).toBe(
'Import failed during publish: Publisher handle "@local-owner" is already claimed. Check skill format, slug availability, and try again.',
);
expect(__test.buildPublishFailureMessage("unexpected")).toBe(
"Import failed during publish: unexpected. Check skill format, slug availability, and try again.",
);
@@ -33,4 +68,525 @@ describe("githubImport", () => {
"demo-repo/skill/notes.md",
]);
});
it("uses publish-supported text extensions for tree path imports", () => {
expect(__test.isPreviewFetchableTextPath("skill/SKILL.md")).toBe(true);
expect(__test.isPreviewFetchableTextPath("skill/icon.svg")).toBe(true);
expect(__test.isPreviewFetchableTextPath("skill/styles.scss")).toBe(true);
expect(__test.isPreviewFetchableTextPath("skill/install.ps1")).toBe(true);
expect(__test.isPreviewFetchableTextPath("skill/config.conf")).toBe(true);
expect(__test.isPreviewFetchableTextPath("skill/binary.exe")).toBe(false);
});
it("rejects a public repo owned by another GitHub account before repo lookup", async () => {
const ctx = {
runQuery: vi.fn().mockResolvedValue("123"),
};
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({
id: 123,
login: "vyctorbrzezowski",
avatar_url: "https://avatars.githubusercontent.com/u/123?v=4",
}),
});
await expect(
__test.requireOwnedPublicGitHubRepoForImport(
ctx as never,
"users:1" as never,
"someone-else",
"public-skill",
fetchMock as never,
),
).rejects.toThrow(/owned by your GitHub account/i);
expect(ctx.runQuery).toHaveBeenCalledWith(
internal.githubIdentity.getGitHubProviderAccountIdInternal,
{ userId: "users:1" },
);
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(fetchMock).toHaveBeenCalledWith(
"https://api.github.com/user/123",
expect.objectContaining({
headers: expect.objectContaining({ "User-Agent": "clawhub/github-import" }),
}),
);
});
it("rejects a public repo when GitHub metadata owner id does not match the signed-in user", async () => {
const ctx = {
runQuery: vi.fn().mockResolvedValue("123"),
};
const fetchMock = vi
.fn()
.mockResolvedValueOnce({
ok: true,
json: async () => ({
id: 123,
login: "vyctorbrzezowski",
avatar_url: "https://avatars.githubusercontent.com/u/123?v=4",
}),
})
.mockResolvedValueOnce({
ok: true,
json: async () => ({
name: "public-skill",
full_name: "vyctorbrzezowski/public-skill",
private: false,
visibility: "public",
owner: { id: 456, login: "vyctorbrzezowski" },
}),
});
await expect(
__test.requireOwnedPublicGitHubRepoForImport(
ctx as never,
"users:1" as never,
"vyctorbrzezowski",
"public-skill",
fetchMock as never,
),
).rejects.toThrow(/owned by your GitHub account/i);
expect(fetchMock).toHaveBeenCalledWith(
"https://api.github.com/repos/vyctorbrzezowski/public-skill",
expect.objectContaining({ headers: expect.any(Object) }),
);
});
it("rejects direct URL preview from another public GitHub owner before repo lookup", async () => {
const ctx = {
runQuery: vi.fn().mockResolvedValue("123"),
};
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({
id: 123,
login: "vyctorbrzezowski",
avatar_url: "https://avatars.githubusercontent.com/u/123?v=4",
}),
});
await expect(
__test.previewGitHubImportForUser(
ctx as never,
"users:1" as never,
{ url: "https://github.com/someone-else/public-skill" },
fetchMock as never,
),
).rejects.toThrow(/owned by your GitHub account/i);
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(fetchMock).toHaveBeenCalledWith(
"https://api.github.com/user/123",
expect.objectContaining({ headers: expect.any(Object) }),
);
});
it("rejects direct URL candidate preview from another public GitHub owner before repo lookup", async () => {
const ctx = {
runQuery: vi.fn().mockResolvedValue("123"),
};
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({
id: 123,
login: "vyctorbrzezowski",
avatar_url: "https://avatars.githubusercontent.com/u/123?v=4",
}),
});
await expect(
__test.previewGitHubImportCandidateForUser(
ctx as never,
"users:1" as never,
{
url: "https://github.com/someone-else/public-skill",
candidatePath: "",
},
fetchMock as never,
),
).rejects.toThrow(/owned by your GitHub account/i);
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(fetchMock).toHaveBeenCalledWith(
"https://api.github.com/user/123",
expect.objectContaining({ headers: expect.any(Object) }),
);
});
it("rejects direct URL publish from another public GitHub owner before repo lookup", async () => {
const ctx = {
runQuery: vi.fn().mockResolvedValue("123"),
};
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({
id: 123,
login: "vyctorbrzezowski",
avatar_url: "https://avatars.githubusercontent.com/u/123?v=4",
}),
});
await expect(
__test.importGitHubSkillForUser(
ctx as never,
"users:1" as never,
{
url: "https://github.com/someone-else/public-skill",
commit: "a".repeat(40),
candidatePath: "",
selectedPaths: ["SKILL.md"],
slug: "public-skill",
displayName: "Public Skill",
version: "1.0.0",
tags: ["latest"],
acceptLicenseTerms: true,
},
fetchMock as never,
),
).rejects.toThrow(/owned by your GitHub account/i);
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(fetchMock).toHaveBeenCalledWith(
"https://api.github.com/user/123",
expect.objectContaining({ headers: expect.any(Object) }),
);
});
it("lists only owned public skill file candidates", async () => {
const ctx = {
runQuery: vi.fn().mockResolvedValue("123"),
};
const fetchMock = vi
.fn()
.mockResolvedValueOnce({
ok: true,
json: async () => ({
id: 123,
login: "vyctorbrzezowski",
avatar_url: "https://avatars.githubusercontent.com/u/123?v=4",
}),
})
.mockResolvedValueOnce({
ok: true,
json: async () => [
{
name: "clawhub",
full_name: "vyctorbrzezowski/clawhub",
html_url: "https://github.com/vyctorbrzezowski/clawhub",
default_branch: "main",
pushed_at: "2026-05-27T00:00:00Z",
updated_at: "2026-05-27T00:00:00Z",
language: "TypeScript",
fork: false,
archived: false,
disabled: false,
private: false,
visibility: "public",
owner: { id: 123, login: "vyctorbrzezowski" },
},
{
name: "docs",
full_name: "vyctorbrzezowski/docs",
html_url: "https://github.com/vyctorbrzezowski/docs",
default_branch: "main",
pushed_at: "2026-05-26T00:00:00Z",
updated_at: "2026-05-26T00:00:00Z",
fork: false,
archived: false,
disabled: false,
private: false,
visibility: "public",
owner: { id: 123, login: "vyctorbrzezowski" },
},
{
name: "forked-skill",
full_name: "vyctorbrzezowski/forked-skill",
default_branch: "main",
fork: true,
archived: false,
disabled: false,
private: false,
visibility: "public",
owner: { id: 123, login: "vyctorbrzezowski" },
},
{
name: "archived-skill",
full_name: "vyctorbrzezowski/archived-skill",
default_branch: "main",
fork: false,
archived: true,
disabled: false,
private: false,
visibility: "public",
owner: { id: 123, login: "vyctorbrzezowski" },
},
{
name: "private-skill",
private: true,
visibility: "private",
owner: { id: 123, login: "vyctorbrzezowski" },
},
{
name: "org-skill",
private: false,
visibility: "public",
owner: { id: 456, login: "openclaw" },
},
],
})
.mockResolvedValueOnce({
ok: true,
json: async () => ({
truncated: false,
tree: [
{ path: "SKILL.md", type: "blob" },
{ path: "skills/copilot/SKILL.md", type: "blob" },
{ path: "legacy/skills.md", type: "blob" },
{ path: ".agents/skills/internal/SKILL.md", type: "blob" },
{ path: "README.md", type: "blob" },
{ path: "skill.md", type: "tree" },
],
}),
})
.mockResolvedValueOnce({
ok: true,
json: async () => ({
truncated: false,
tree: [
{ path: "README.md", type: "blob" },
{ path: "guides/usage.md", type: "blob" },
],
}),
});
const result = await __test.listOwnedPublicGitHubReposForUser(
ctx as never,
"users:1" as never,
{ page: 1, perPage: 30 },
fetchMock as never,
);
expect(result.account.login).toBe("vyctorbrzezowski");
expect(result.account.avatarUrl).toBe("https://avatars.githubusercontent.com/u/123?v=4");
expect(result.repos).toEqual([
expect.objectContaining({
owner: "vyctorbrzezowski",
name: "clawhub",
repoName: "clawhub",
repoFullName: "vyctorbrzezowski/clawhub",
fullName: "vyctorbrzezowski/clawhub",
htmlUrl: "https://github.com/vyctorbrzezowski/clawhub",
candidatePath: "",
skillPath: "SKILL.md",
importable: true,
}),
expect.objectContaining({
owner: "vyctorbrzezowski",
name: "copilot",
repoName: "clawhub",
repoFullName: "vyctorbrzezowski/clawhub",
fullName: "vyctorbrzezowski/clawhub/skills/copilot",
htmlUrl: "https://github.com/vyctorbrzezowski/clawhub/tree/main/skills/copilot",
candidatePath: "skills/copilot",
skillPath: "skills/copilot/SKILL.md",
importable: true,
}),
expect.objectContaining({
owner: "vyctorbrzezowski",
name: "legacy",
repoName: "clawhub",
repoFullName: "vyctorbrzezowski/clawhub",
fullName: "vyctorbrzezowski/clawhub/legacy",
htmlUrl: "https://github.com/vyctorbrzezowski/clawhub/tree/main/legacy",
candidatePath: "legacy",
skillPath: "legacy/skills.md",
importable: true,
}),
]);
expect(fetchMock).toHaveBeenNthCalledWith(
2,
"https://api.github.com/users/vyctorbrzezowski/repos?type=owner&sort=pushed&direction=desc&per_page=30&page=1",
expect.objectContaining({ headers: expect.any(Object) }),
);
expect(fetchMock).toHaveBeenNthCalledWith(
3,
"https://api.github.com/repos/vyctorbrzezowski/clawhub/git/trees/main?recursive=1",
expect.objectContaining({ headers: expect.any(Object) }),
);
expect(fetchMock).toHaveBeenNthCalledWith(
4,
"https://api.github.com/repos/vyctorbrzezowski/docs/git/trees/main?recursive=1",
expect.objectContaining({ headers: expect.any(Object) }),
);
});
it("uses GitHub code search for owned skill file discovery when a token is configured", async () => {
process.env.GITHUB_TOKEN = "github-token";
const ctx = {
runQuery: vi.fn().mockResolvedValue("123"),
};
const ownedRepo = {
name: "skills",
full_name: "vyctorbrzezowski/skills",
html_url: "https://github.com/vyctorbrzezowski/skills",
default_branch: "main",
pushed_at: "2026-05-27T00:00:00Z",
updated_at: "2026-05-27T00:00:00Z",
fork: false,
archived: false,
disabled: false,
private: false,
visibility: "public",
owner: { id: 123, login: "vyctorbrzezowski" },
};
const fetchMock = vi
.fn()
.mockResolvedValueOnce({
ok: true,
json: async () => ({
id: 123,
login: "vyctorbrzezowski",
avatar_url: "https://avatars.githubusercontent.com/u/123?v=4",
}),
})
.mockResolvedValueOnce({
ok: true,
json: async () => ({
items: [
{ path: "SKILL.md", repository: ownedRepo },
{ path: "tools/review/SKILL.md", repository: ownedRepo },
{ path: ".agents/skills/internal/SKILL.md", repository: ownedRepo },
{
path: "SKILL.md",
repository: {
...ownedRepo,
name: "forked",
full_name: "vyctorbrzezowski/forked",
fork: true,
},
},
{ path: "README.md", repository: ownedRepo },
],
}),
})
.mockResolvedValueOnce({
ok: true,
json: async () => ({
items: [{ path: "legacy/skills.md", repository: ownedRepo }],
}),
});
const result = await __test.listOwnedPublicGitHubReposForUser(
ctx as never,
"users:1" as never,
{ page: 1, perPage: 30 },
fetchMock as never,
);
expect(result.repos).toEqual([
expect.objectContaining({
name: "skills",
repoName: "skills",
candidatePath: "",
skillPath: "SKILL.md",
}),
expect.objectContaining({
name: "review",
repoName: "skills",
candidatePath: "tools/review",
skillPath: "tools/review/SKILL.md",
}),
expect.objectContaining({
name: "legacy",
repoName: "skills",
candidatePath: "legacy",
skillPath: "legacy/skills.md",
}),
]);
expect(fetchMock).toHaveBeenCalledTimes(3);
const searchUrl = new URL(fetchMock.mock.calls[1]?.[0] as string);
expect(searchUrl.pathname).toBe("/search/code");
expect(searchUrl.searchParams.get("q")).toBe("filename:SKILL.md user:vyctorbrzezowski");
const legacySearchUrl = new URL(fetchMock.mock.calls[2]?.[0] as string);
expect(legacySearchUrl.pathname).toBe("/search/code");
expect(legacySearchUrl.searchParams.get("q")).toBe("filename:skills.md user:vyctorbrzezowski");
expect(fetchMock.mock.calls[1]?.[1]).toEqual(
expect.objectContaining({
headers: expect.objectContaining({ Authorization: "Bearer github-token" }),
}),
);
});
it("falls back to the repo archive when GitHub truncates the discovery tree", async () => {
const ctx = {
runQuery: vi.fn().mockResolvedValue("123"),
};
const zip = buildGitHubZipForTests({
"large-repo/tools/review/SKILL.md": "# Review",
"large-repo/tools/review/notes.md": "notes",
});
const fetchMock = vi
.fn()
.mockResolvedValueOnce({
ok: true,
json: async () => ({
id: 123,
login: "vyctorbrzezowski",
avatar_url: "https://avatars.githubusercontent.com/u/123?v=4",
}),
})
.mockResolvedValueOnce({
ok: true,
json: async () => [
{
name: "large-repo",
full_name: "vyctorbrzezowski/large-repo",
html_url: "https://github.com/vyctorbrzezowski/large-repo",
default_branch: "main",
fork: false,
archived: false,
disabled: false,
private: false,
visibility: "public",
owner: { id: 123, login: "vyctorbrzezowski" },
},
],
})
.mockResolvedValueOnce({
ok: true,
json: async () => ({
truncated: true,
tree: [{ path: "README.md", type: "blob" }],
}),
})
.mockResolvedValueOnce({
ok: true,
headers: { get: () => null },
arrayBuffer: async () => zip.buffer.slice(zip.byteOffset, zip.byteOffset + zip.byteLength),
});
const result = await __test.listOwnedPublicGitHubReposForUser(
ctx as never,
"users:1" as never,
{ page: 1, perPage: 30 },
fetchMock as never,
);
expect(result.repos).toEqual([
expect.objectContaining({
name: "review",
repoName: "large-repo",
candidatePath: "tools/review",
skillPath: "tools/review/SKILL.md",
}),
]);
expect(fetchMock).toHaveBeenNthCalledWith(
4,
"https://codeload.github.com/vyctorbrzezowski/large-repo/zip/main",
expect.objectContaining({ headers: expect.any(Object) }),
);
});
});
+838 -199
View File
File diff suppressed because it is too large Load Diff
-225
View File
@@ -1,225 +0,0 @@
"use node";
import { v } from "convex/values";
import { internal } from "./_generated/api";
import type { Doc, Id } from "./_generated/dataModel";
import { internalAction } from "./functions";
import { assertAdmin } from "./lib/access";
import { guessContentTypeForPath } from "./lib/contentTypes";
import {
fetchGitHubSkillMeta,
getGitHubBackupContext,
isGitHubBackupConfigured,
} from "./lib/githubBackup";
import { listGitHubBackupFiles, readGitHubBackupFile } from "./lib/githubRestoreHelpers";
import { publishVersionForUser } from "./lib/skillPublish";
type RestoreResult = {
slug: string;
status: "restored" | "slug_conflict" | "already_exists" | "no_backup" | "error";
detail?: string;
};
type BulkRestoreResult = {
results: RestoreResult[];
totalRestored: number;
totalConflicts: number;
totalSkipped: number;
totalErrors: number;
};
/**
* Admin-only: restore a single skill from GitHub backup.
* Reads the backup files from the GitHub repo and re-creates the skill in the database.
*/
export const restoreSkillFromBackup = internalAction({
args: {
actorUserId: v.id("users"),
ownerHandle: v.string(),
ownerUserId: v.id("users"),
slug: v.string(),
forceOverwriteSquatter: v.optional(v.boolean()),
},
handler: async (ctx, args): Promise<RestoreResult> => {
try {
const actor = await ctx.runQuery(internal.users.getByIdInternal, {
userId: args.actorUserId,
});
if (!actor || actor.deletedAt || actor.deactivatedAt) {
return { slug: args.slug, status: "error", detail: "Actor not found" };
}
assertAdmin(actor as Doc<"users">);
if (!isGitHubBackupConfigured()) {
return { slug: args.slug, status: "error", detail: "GitHub backup not configured" };
}
const ghContext = await getGitHubBackupContext();
// Check if skill already exists in the DB
const existingSkill = (await ctx.runQuery(internal.skills.getSkillBySlugInternal, {
slug: args.slug,
})) as Doc<"skills"> | null;
if (existingSkill) {
if (existingSkill.ownerUserId === args.ownerUserId) {
return {
slug: args.slug,
status: "already_exists",
detail: "Skill already owned by user",
};
}
if (!args.forceOverwriteSquatter) {
return {
slug: args.slug,
status: "slug_conflict",
detail: `Slug occupied by another user. Set forceOverwriteSquatter=true to reclaim.`,
};
}
// Free the slug in-transaction by renaming the squatter, then enqueue cleanup.
await ctx.runMutation(
internal.githubRestoreMutations.evictSquatterSkillForRestoreInternal,
{
actorUserId: args.actorUserId,
slug: args.slug,
rightfulOwnerUserId: args.ownerUserId,
},
);
}
// Fetch metadata from GitHub backup
const meta = await fetchGitHubSkillMeta(ghContext, args.ownerHandle, args.slug);
if (!meta) {
return { slug: args.slug, status: "no_backup", detail: "No backup found in GitHub repo" };
}
// Read the actual files from the backup
const backupFiles = await listGitHubBackupFiles(ghContext, args.ownerHandle, args.slug);
if (backupFiles.length === 0) {
return { slug: args.slug, status: "no_backup", detail: "Backup has no files" };
}
// Download and store each file in Convex storage
const storedFiles: Array<{
path: string;
size: number;
storageId: Id<"_storage">;
sha256: string;
contentType: string;
}> = [];
for (const filePath of backupFiles) {
const fileContent = await readGitHubBackupFile(
ghContext,
args.ownerHandle,
args.slug,
filePath,
);
if (!fileContent) continue;
const sha256 = await sha256Hex(fileContent);
const contentType = guessContentTypeForPath(filePath);
const blob = new Blob([Buffer.from(fileContent)], { type: contentType });
const storageId = await ctx.storage.store(blob);
storedFiles.push({
path: filePath,
size: fileContent.byteLength,
storageId,
sha256,
contentType,
});
}
if (storedFiles.length === 0) {
return { slug: args.slug, status: "error", detail: "Could not download any backup files" };
}
await publishVersionForUser(
ctx,
args.ownerUserId,
{
slug: args.slug,
displayName: meta.displayName,
version: meta.latest.version,
changelog: "Restored from GitHub backup",
files: storedFiles,
},
{
bypassGitHubAccountAge: true,
bypassNewSkillRateLimit: true,
bypassQualityGate: true,
skipBackup: true,
skipWebhook: true,
},
);
return { slug: args.slug, status: "restored" };
} catch (error) {
const message = error instanceof Error ? error.message : "Unknown error";
console.error(`[restore] Failed to restore ${args.slug}:`, message);
return { slug: args.slug, status: "error", detail: message };
}
},
});
/**
* Admin-only: bulk restore all skills for a user from GitHub backup.
*/
export const restoreUserSkillsFromBackup = internalAction({
args: {
actorUserId: v.id("users"),
ownerHandle: v.string(),
ownerUserId: v.id("users"),
slugs: v.array(v.string()),
forceOverwriteSquatter: v.optional(v.boolean()),
},
handler: async (ctx, args): Promise<BulkRestoreResult> => {
const results: RestoreResult[] = [];
let totalRestored = 0;
let totalConflicts = 0;
let totalSkipped = 0;
let totalErrors = 0;
for (const slug of args.slugs) {
const result = (await ctx.runAction(internal.githubRestore.restoreSkillFromBackup, {
actorUserId: args.actorUserId,
ownerHandle: args.ownerHandle,
ownerUserId: args.ownerUserId,
slug,
forceOverwriteSquatter: args.forceOverwriteSquatter,
})) as RestoreResult;
results.push(result);
switch (result.status) {
case "restored":
totalRestored += 1;
break;
case "slug_conflict":
totalConflicts += 1;
break;
case "already_exists":
case "no_backup":
totalSkipped += 1;
break;
case "error":
totalErrors += 1;
break;
}
}
return { results, totalRestored, totalConflicts, totalSkipped, totalErrors };
},
});
async function sha256Hex(bytes: Uint8Array) {
const { createHash } = await import("node:crypto");
const hash = createHash("sha256");
hash.update(bytes);
return hash.digest("hex");
}
// guessContentTypeForPath in lib/contentTypes.ts
-83
View File
@@ -1,83 +0,0 @@
import { v } from "convex/values";
import { internal } from "./_generated/api";
import { internalMutation } from "./functions";
import { assertAdmin } from "./lib/access";
export const evictSquatterSkillForRestoreInternal = internalMutation({
args: {
actorUserId: v.id("users"),
slug: v.string(),
rightfulOwnerUserId: v.id("users"),
},
handler: async (ctx, args) => {
const actor = await ctx.db.get(args.actorUserId);
if (!actor || actor.deletedAt || actor.deactivatedAt) throw new Error("Actor not found");
assertAdmin(actor);
const slug = args.slug.trim().toLowerCase();
if (!slug) throw new Error("Slug required");
const now = Date.now();
const existingSkill = await ctx.db
.query("skills")
.withIndex("by_slug", (q) => q.eq("slug", slug))
.unique();
if (!existingSkill) return { ok: true as const, action: "noop" as const };
if (existingSkill.ownerUserId === args.rightfulOwnerUserId) {
return { ok: true as const, action: "already_owned" as const };
}
const evictedSlug = buildEvictedSlug(slug, now);
// Free the slug immediately (same transaction) by renaming the squatter's skill.
await ctx.db.patch(existingSkill._id, {
slug: evictedSlug,
softDeletedAt: now,
hiddenAt: existingSkill.hiddenAt ?? now,
hiddenBy: existingSkill.hiddenBy ?? actor._id,
updatedAt: now,
});
// Remove from vector search ASAP.
const embeddings = await ctx.db
.query("skillEmbeddings")
.withIndex("by_skill", (q) => q.eq("skillId", existingSkill._id))
.collect();
for (const embedding of embeddings) {
await ctx.db.patch(embedding._id, {
visibility: "deleted",
updatedAt: now,
});
}
// Cleanup the rest asynchronously (versions, fingerprints, installs, etc.)
await ctx.scheduler.runAfter(0, internal.skills.hardDeleteInternal, {
skillId: existingSkill._id,
actorUserId: actor._id,
phase: "versions",
});
await ctx.db.insert("auditLogs", {
actorUserId: actor._id,
action: "slug.reclaim.sync",
targetType: "skill",
targetId: existingSkill._id,
metadata: {
slug,
evictedSlug,
squatterUserId: existingSkill.ownerUserId,
rightfulOwnerUserId: args.rightfulOwnerUserId,
reason: "Synchronous eviction during GitHub restore",
},
createdAt: now,
});
return { ok: true as const, action: "evicted" as const, evictedSlug };
},
});
function buildEvictedSlug(slug: string, now: number) {
const suffix = now.toString(36);
return `${slug}-evicted-${suffix}`;
}
+214 -2
View File
@@ -15,10 +15,24 @@ vi.mock("./lib/publishers", async () => {
const { requireUser } = await import("./lib/access");
const { requirePublisherRole } = await import("./lib/publishers");
const { deleteForPublisherHandler } = await import("./githubSkillSources");
const {
cleanupDeletedSourceScansHandler,
deleteForPublisherHandler,
listForManageableOfficialPublishers,
} = await import("./githubSkillSources");
const { buildSkillInstallResolution } = await import("./lib/installResolver");
type Row = Record<string, unknown> & { _id: string };
type WrappedHandler<TArgs, TResult = unknown> = {
_handler: (ctx: unknown, args: TArgs) => Promise<TResult>;
};
const listForManageableOfficialPublishersHandler = (
listForManageableOfficialPublishers as unknown as WrappedHandler<
Record<string, never>,
Array<{ _id: string; repo: string; ownerPublisher: { handle: string } | null }>
>
)._handler;
function chainEq(constraints: Record<string, unknown>) {
return {
@@ -74,6 +88,7 @@ function createDb(initial: Record<string, Row[]> = {}) {
const matched = () => list(table).filter((row) => matches(row, constraints));
return {
collect: async () => matched(),
take: async (limit: number) => matched().slice(0, limit),
unique: async () => matched()[0] ?? null,
};
},
@@ -107,6 +122,20 @@ describe("githubSkillSources.deleteForPublisherHandler", () => {
githubSourceId: "githubSkillSources:matt",
},
],
githubSkillScans: [
{
_id: "githubSkillScans:matt",
skillId: "skills:github",
githubSourceId: "githubSkillSources:matt",
contentHash: "hash-source-backed",
},
{
_id: "githubSkillScans:other",
skillId: "skills:other-source",
githubSourceId: "githubSkillSources:other",
contentHash: "hash-other-source",
},
],
skills: [
{
_id: "skills:github",
@@ -163,9 +192,10 @@ describe("githubSkillSources.deleteForPublisherHandler", () => {
},
],
});
const scheduler = { runAfter: vi.fn(async () => undefined) };
await expect(
deleteForPublisherHandler({ db } as never, {
deleteForPublisherHandler({ db, scheduler } as never, {
ownerPublisherId: "publishers:openclaw" as never,
sourceId: "githubSkillSources:matt" as never,
now: 123,
@@ -182,6 +212,10 @@ describe("githubSkillSources.deleteForPublisherHandler", () => {
);
expect(tables.githubSkillSources).toHaveLength(0);
expect(tables.githubSkillContents).toHaveLength(0);
expect(tables.githubSkillScans).toHaveLength(2);
expect(scheduler.runAfter).toHaveBeenCalledWith(0, expect.anything(), {
sourceId: "githubSkillSources:matt",
});
const deletedSkill = tables.skills.find((skill) => skill._id === "skills:github");
expect(deletedSkill).toMatchObject({
softDeletedAt: 123,
@@ -217,6 +251,61 @@ describe("githubSkillSources.deleteForPublisherHandler", () => {
});
});
it("cleans deleted-source scan history in bounded batches", async () => {
const { db, tables } = createDb({
githubSkillScans: [
{
_id: "githubSkillScans:matt",
githubSourceId: "githubSkillSources:matt",
skillScanRequestId: "skillScanRequests:matt",
},
{
_id: "githubSkillScans:other",
githubSourceId: "githubSkillSources:other",
},
],
securityScanJobs: [
{
_id: "securityScanJobs:matt",
targetKind: "skillScanRequest",
status: "queued",
},
],
skillScanRequests: [
{
_id: "skillScanRequests:matt",
sourceKind: "github",
status: "queued",
securityScanJobId: "securityScanJobs:matt",
githubSkillScanId: "githubSkillScans:matt",
expiresAt: Number.MAX_SAFE_INTEGER,
},
],
});
const scheduler = { runAfter: vi.fn(async () => undefined) };
await expect(
cleanupDeletedSourceScansHandler({ db, scheduler } as never, {
sourceId: "githubSkillSources:matt" as never,
}),
).resolves.toEqual({ ok: true, deleted: 1, done: true });
expect(tables.githubSkillScans).toEqual([
expect.objectContaining({ _id: "githubSkillScans:other" }),
]);
expect(tables.securityScanJobs).toEqual([]);
expect(tables.skillScanRequests).toEqual([
expect.objectContaining({
_id: "skillScanRequests:matt",
status: "failed",
}),
]);
expect(tables.skillScanRequests?.[0]).not.toHaveProperty("githubSkillScanId");
expect(tables.skillScanRequests?.[0]).not.toHaveProperty("securityScanJobId");
expect(tables.skillScanRequests?.[0]?.expiresAt).toBeLessThan(Number.MAX_SAFE_INTEGER);
expect(scheduler.runAfter).toHaveBeenCalledWith(0, expect.anything(), { batchSize: 10 });
});
it("rejects deleting a source from another publisher", async () => {
const { db } = createDb({
githubSkillSources: [
@@ -239,3 +328,126 @@ describe("githubSkillSources.deleteForPublisherHandler", () => {
).rejects.toBeInstanceOf(ConvexError);
});
});
describe("githubSkillSources.listForManageableOfficialPublishers", () => {
beforeEach(() => {
vi.mocked(requireUser).mockResolvedValue({
userId: "users:steipete",
user: {
_id: "users:steipete",
handle: "steipete",
displayName: "Peter Steinberger",
personalPublisherId: "publishers:steipete",
createdAt: 1,
updatedAt: 2,
},
} as never);
});
it("includes official personal publishers the user can administer", async () => {
const { db } = createDb({
publisherMembers: [
{
_id: "publisherMembers:steipete-owner",
publisherId: "publishers:steipete",
userId: "users:steipete",
role: "owner",
},
],
publishers: [
{
_id: "publishers:steipete",
kind: "user",
handle: "steipete",
displayName: "Peter Steinberger",
linkedUserId: "users:steipete",
createdAt: 1,
updatedAt: 2,
},
],
officialPublishers: [
{
_id: "officialPublishers:steipete",
publisherId: "publishers:steipete",
reason: "Verified individual publisher",
createdAt: 3,
updatedAt: 3,
},
],
githubSkillSources: [
{
_id: "githubSkillSources:steipete",
ownerPublisherId: "publishers:steipete",
repo: "steipete/agent-rules",
defaultBranch: "main",
lastSyncStatus: "ok",
createdAt: 4,
updatedAt: 5,
},
],
skills: [],
});
await expect(
listForManageableOfficialPublishersHandler({ db } as never, {}),
).resolves.toMatchObject([
{
_id: "githubSkillSources:steipete",
repo: "steipete/agent-rules",
ownerPublisher: {
handle: "steipete",
},
},
]);
});
it("includes linked official personal publishers without a membership row", async () => {
const { db } = createDb({
publisherMembers: [],
publishers: [
{
_id: "publishers:steipete",
kind: "user",
handle: "steipete",
displayName: "Peter Steinberger",
linkedUserId: "users:steipete",
createdAt: 1,
updatedAt: 2,
},
],
officialPublishers: [
{
_id: "officialPublishers:steipete",
publisherId: "publishers:steipete",
reason: "Verified individual publisher",
createdAt: 3,
updatedAt: 3,
},
],
githubSkillSources: [
{
_id: "githubSkillSources:steipete",
ownerPublisherId: "publishers:steipete",
repo: "steipete/agent-rules",
defaultBranch: "main",
lastSyncStatus: "ok",
createdAt: 4,
updatedAt: 5,
},
],
skills: [],
});
await expect(
listForManageableOfficialPublishersHandler({ db } as never, {}),
).resolves.toMatchObject([
{
_id: "githubSkillSources:steipete",
repo: "steipete/agent-rules",
ownerPublisher: {
handle: "steipete",
},
},
]);
});
});
+56 -7
View File
@@ -1,13 +1,22 @@
import { ConvexError, v } from "convex/values";
import { internal } from "./_generated/api";
import type { Doc, Id } from "./_generated/dataModel";
import type { MutationCtx, QueryCtx } from "./_generated/server";
import { internalQuery, mutation, query } from "./functions";
import { internalMutation, internalQuery, mutation, query } from "./functions";
import { requireUser } from "./lib/access";
import { deleteGitHubSkillScansForSource } from "./lib/githubSkillScans";
import { adjustGlobalPublicSkillsCount, getPublicSkillVisibilityDelta } from "./lib/globalStats";
import { isOfficialPublisher } from "./lib/officialPublishers";
import { isPublisherActive, isPublisherRoleAllowed, requirePublisherRole } from "./lib/publishers";
import {
getPersonalPublisherForUserOrFallback,
isPublisherActive,
isPublisherRoleAllowed,
requirePublisherRole,
} from "./lib/publishers";
import { syncSkillSearchDigestForSkill } from "./lib/skillSearchDigest";
const GITHUB_SKILL_SCAN_CLEANUP_BATCH_SIZE = 25;
type PublicGitHubSkillSource = Pick<
Doc<"githubSkillSources">,
| "_id"
@@ -101,27 +110,34 @@ export const listForPublisher = query({
export const listForManageableOfficialPublishers = query({
args: {},
handler: async (ctx): Promise<PublicGitHubSkillSource[]> => {
const { userId } = await requireUser(ctx);
const { userId, user } = await requireUser(ctx);
const memberships = await ctx.db
.query("publisherMembers")
.withIndex("by_user", (q) => q.eq("userId", userId))
.collect();
const ownerPublisherIds: Id<"publishers">[] = [];
const ownerPublisherIds = new Set<Id<"publishers">>();
for (const membership of memberships) {
if (!isPublisherRoleAllowed(membership.role, ["admin"])) continue;
const publisher = await ctx.db.get(membership.publisherId);
if (
!publisher ||
publisher.kind !== "org" ||
!isPublisherActive(publisher) ||
!(await isOfficialPublisher(ctx, publisher))
) {
continue;
}
ownerPublisherIds.push(publisher._id);
ownerPublisherIds.add(publisher._id);
}
const personalPublisher = await getPersonalPublisherForUserOrFallback(ctx, user);
if (
personalPublisher &&
isPublisherActive(personalPublisher) &&
(await isOfficialPublisher(ctx, personalPublisher))
) {
ownerPublisherIds.add(personalPublisher._id);
}
const sourceGroups = await Promise.all(
ownerPublisherIds.map((ownerPublisherId) =>
[...ownerPublisherIds].map((ownerPublisherId) =>
ctx.db
.query("githubSkillSources")
.withIndex("by_owner_publisher", (q) => q.eq("ownerPublisherId", ownerPublisherId))
@@ -161,6 +177,9 @@ export async function deleteForPublisherHandler(
for (const content of contents) {
await ctx.db.delete(content._id);
}
await ctx.scheduler.runAfter(0, internal.githubSkillSources.cleanupDeletedSourceScansInternal, {
sourceId: args.sourceId,
});
const skills = await ctx.db
.query("skills")
@@ -197,6 +216,36 @@ export async function deleteForPublisherHandler(
return { ok: true as const, deletedSkills };
}
export async function cleanupDeletedSourceScansHandler(
ctx: MutationCtx,
args: { sourceId: Id<"githubSkillSources"> },
) {
const deleted = await deleteGitHubSkillScansForSource(
ctx,
args.sourceId,
GITHUB_SKILL_SCAN_CLEANUP_BATCH_SIZE,
);
const done = deleted < GITHUB_SKILL_SCAN_CLEANUP_BATCH_SIZE;
if (deleted > 0) {
await ctx.scheduler.runAfter(0, internal.securityScan.pruneExpiredSkillScanRequestsInternal, {
batchSize: 10,
});
}
if (!done) {
await ctx.scheduler.runAfter(
0,
internal.githubSkillSources.cleanupDeletedSourceScansInternal,
args,
);
}
return { ok: true as const, deleted, done };
}
export const cleanupDeletedSourceScansInternal = internalMutation({
args: { sourceId: v.id("githubSkillSources") },
handler: cleanupDeletedSourceScansHandler,
});
export const deleteForPublisher: ReturnType<typeof mutation> = mutation({
args: {
ownerPublisherId: v.id("publishers"),
+62 -2
View File
@@ -8,6 +8,11 @@ import {
verifyGitHubSkillHandler,
} from "./githubSkillSync";
import { buildSkillInstallResolution } from "./lib/installResolver";
import {
appendGitHubSkillScanRequestFilesInternal,
finalizeGitHubSkillScanRequestInternal,
prepareGitHubSkillScanRequestInternal,
} from "./securityScan";
type Row = Record<string, unknown> & { _id: string };
@@ -35,10 +40,12 @@ function createDb(initial: Record<string, Row[]> = {}) {
};
const db = {
get: async (id: string) => {
get: async (idOrTable: string, maybeId?: string) => {
const id = maybeId ?? idOrTable;
const table = id.split(":")[0] ?? "";
return list(table).find((row) => row._id === id) ?? null;
},
normalizeId: (table: string, id: string) => (id.startsWith(`${table}:`) ? id : null),
insert: async (table: string, doc: Record<string, unknown>) => {
counters[table] = (counters[table] ?? 0) + 1;
const inserted = {
@@ -58,6 +65,18 @@ function createDb(initial: Record<string, Row[]> = {}) {
else row[key] = value;
}
},
replace: async (id: string, doc: Record<string, unknown>) => {
const table = id.split(":")[0] ?? "";
const rows = list(table);
const index = rows.findIndex((candidate) => candidate._id === id);
if (index >= 0) rows[index] = { _id: id, ...doc };
},
delete: async (id: string) => {
const table = id.split(":")[0] ?? "";
const rows = list(table);
const index = rows.findIndex((candidate) => candidate._id === id);
if (index >= 0) rows.splice(index, 1);
},
query: (table: string) => ({
withIndex: (_indexName: string, build?: (q: ReturnType<typeof chainEq>) => unknown) => {
const constraints: Record<string, unknown> = {};
@@ -65,6 +84,7 @@ function createDb(initial: Record<string, Row[]> = {}) {
const matched = () => list(table).filter((row) => matches(row, constraints));
return {
collect: async () => matched(),
take: async (limit: number) => matched().slice(0, limit),
unique: async () => matched()[0] ?? null,
};
},
@@ -114,6 +134,7 @@ describe("GitHub-backed skills live canary", () => {
],
});
const scheduler = { runAfter: async () => undefined };
let storedFile = 0;
let now = Date.now();
const actionCtx = {
runQuery: async (_query: unknown, args: Record<string, unknown>) => {
@@ -165,6 +186,27 @@ describe("GitHub-backed skills live canary", () => {
} as never,
);
}
if ("requestId" in args && "files" in args) {
return await (
appendGitHubSkillScanRequestFilesInternal as unknown as {
_handler: (ctx: never, args: never) => unknown;
}
)._handler({ db } as never, args as never);
}
if ("requestId" in args) {
return await (
finalizeGitHubSkillScanRequestInternal as unknown as {
_handler: (ctx: never, args: never) => unknown;
}
)._handler({ db } as never, args as never);
}
if ("staticScan" in args && "commit" in args && "contentHash" in args) {
return await (
prepareGitHubSkillScanRequestInternal as unknown as {
_handler: (ctx: never, args: never) => unknown;
}
)._handler({ db } as never, args as never);
}
if ("scanStatus" in args && "contentHash" in args) {
return await applyGitHubSkillVerificationResultHandler(
{ db } as never,
@@ -185,6 +227,13 @@ describe("GitHub-backed skills live canary", () => {
}
throw new Error(`unexpected live canary mutation args: ${JSON.stringify(args)}`);
},
storage: {
store: async () => {
storedFile += 1;
return `storage:live-${storedFile}`;
},
delete: async () => undefined,
},
auth: { getUserIdentity: async () => null },
};
@@ -227,7 +276,18 @@ describe("GitHub-backed skills live canary", () => {
fetch,
);
expect(verified).toMatchObject({ ok: true, scanStatus: "clean" });
expect(verified).toMatchObject({ ok: true, queued: true });
expect(storedFile).toBeGreaterThan(0);
expect(resolveInstallFromTables(tables, skillSlug)).toMatchObject({
ok: false,
reason: "github_verification_pending",
});
await applyGitHubSkillVerificationResultHandler({ db } as never, {
skillId: skill._id as never,
contentHash: skill.githubCurrentContentHash as string,
scanStatus: "clean",
now,
});
skill = getSkill(tables, skillSlug);
expect(skill).toMatchObject({
githubCurrentCommit: configured.commit,
+611 -56
View File
@@ -1,3 +1,4 @@
import { getFunctionName } from "convex/server";
import { zipSync } from "fflate";
import { describe, expect, it, vi } from "vitest";
import {
@@ -507,6 +508,30 @@ describe("syncGitHubSkillSourcesHandler", () => {
}
});
it("continues paginated scheduled syncs in the Node runtime", async () => {
const scheduler = {
runAfter: vi.fn(
async (_delayMs: number, _functionRef: unknown, _args: Record<string, unknown>) =>
undefined,
),
};
const runQuery = vi
.fn()
.mockResolvedValueOnce({ sources: [], continueCursor: "next-page", isDone: false });
const result = await syncGitHubSkillSourcesHandler(
{ runQuery, runMutation: vi.fn(), scheduler } as never,
{},
vi.fn() as never,
);
expect(result).toMatchObject({ scheduledNext: true, cursor: "next-page", isDone: false });
const scheduledFunction = scheduler.runAfter.mock.calls[0]?.[1];
expect(getFunctionName(scheduledFunction as Parameters<typeof getFunctionName>[0])).toBe(
"githubSkillSyncNode:syncGitHubSkillSourcesInternal",
);
});
it("rechecks repo visibility before scheduled syncs", async () => {
const runQuery = vi
.fn()
@@ -665,6 +690,7 @@ description: Install from a GitHub-backed source.
],
});
const scheduler = { runAfter: vi.fn(async () => undefined) };
let storedFile = 0;
let now = 100;
const consoleLog = vi.spyOn(console, "log").mockImplementation(() => {});
const actionCtx = {
@@ -715,7 +741,8 @@ description: Install from a GitHub-backed source.
}
throw new Error(`unexpected lifecycle query args: ${JSON.stringify(args)}`);
}),
runMutation: vi.fn(async (_mutation: unknown, args: Record<string, unknown>) => {
runMutation: vi.fn(async (mutation: unknown, args: Record<string, unknown>) => {
const mutationName = getFunctionName(mutation as Parameters<typeof getFunctionName>[0]);
if ("snapshot" in args) {
return await applyGitHubSkillSourceSyncHandler(
{ db, scheduler } as never,
@@ -725,6 +752,26 @@ description: Install from a GitHub-backed source.
} as never,
);
}
if (mutationName === "securityScan:prepareGitHubSkillScanRequestInternal") {
return {
ok: true,
prepared: true,
scanId: "githubSkillScans:1",
requestId: "skillScanRequests:1",
};
}
if (mutationName === "securityScan:appendGitHubSkillScanRequestFilesInternal") {
return { ok: true, appended: true };
}
if (mutationName === "securityScan:finalizeGitHubSkillScanRequestInternal") {
return {
ok: true,
queued: true,
scanId: "githubSkillScans:1",
requestId: "skillScanRequests:1",
jobId: "securityScanJobs:1",
};
}
if ("scanStatus" in args && "contentHash" in args) {
return await applyGitHubSkillVerificationResultHandler(
{ db } as never,
@@ -754,6 +801,13 @@ description: Install from a GitHub-backed source.
}
throw new Error(`unexpected lifecycle mutation args: ${JSON.stringify(args)}`);
}),
storage: {
store: vi.fn(async () => {
storedFile += 1;
return `storage:${storedFile}`;
}),
delete: vi.fn(),
},
auth: { getUserIdentity: vi.fn() },
};
@@ -824,14 +878,27 @@ description: Install from a GitHub-backed source.
});
now = 110;
await verifyGitHubSkillHandler(
actionCtx as never,
{
skillId: skill._id as never,
contentHash: skill.githubCurrentContentHash as string,
},
fakeGitHub.fetcher as never,
);
await expect(
verifyGitHubSkillHandler(
actionCtx as never,
{
skillId: skill._id as never,
contentHash: skill.githubCurrentContentHash as string,
},
fakeGitHub.fetcher as never,
),
).resolves.toMatchObject({ ok: true, queued: true });
expect(resolveInstallFromTables(tables, "demo-source")).toMatchObject({
ok: false,
reason: "github_verification_pending",
status: 423,
});
await applyGitHubSkillVerificationResultHandler({ db } as never, {
skillId: skill._id as never,
contentHash: skill.githubCurrentContentHash as string,
scanStatus: "clean",
now,
});
expect(resolveInstallFromTables(tables, "demo-source")).toMatchObject({
ok: true,
installKind: "github",
@@ -886,14 +953,27 @@ description: Install from a GitHub-backed source.
});
now = 210;
await verifyGitHubSkillHandler(
actionCtx as never,
{
skillId: skill._id as never,
contentHash: skill.githubCurrentContentHash as string,
},
fakeGitHub.fetcher as never,
);
await expect(
verifyGitHubSkillHandler(
actionCtx as never,
{
skillId: skill._id as never,
contentHash: skill.githubCurrentContentHash as string,
},
fakeGitHub.fetcher as never,
),
).resolves.toMatchObject({ ok: true, queued: true });
expect(resolveInstallFromTables(tables, "demo-source")).toMatchObject({
ok: false,
reason: "github_verification_pending",
status: 423,
});
await applyGitHubSkillVerificationResultHandler({ db } as never, {
skillId: skill._id as never,
contentHash: skill.githubCurrentContentHash as string,
scanStatus: "clean",
now,
});
expect(resolveInstallFromTables(tables, "demo-source")).toMatchObject({
ok: true,
installKind: "github",
@@ -987,7 +1067,73 @@ describe("resolveOwnerUserIdForPublisherHandler", () => {
});
describe("applyGitHubSkillSourceSyncHandler", () => {
it("applies a trusted fetched snapshot without overwriting unrelated slug owners", async () => {
it("queues a full scan and blocks legacy clean GitHub skills without a durable result", async () => {
const snapshot = await buildGitHubSkillSourceSnapshot({
repo: "NVIDIA/skills",
defaultBranch: "main",
commit: "2".repeat(40),
entries: {
"skills/aiq-deploy/SKILL.md": new TextEncoder().encode("# AIQ Deploy\n"),
},
});
const contentHash = snapshot.skills[0]?.contentHash;
const { db, tables } = createDb({
githubSkillSources: [
{
_id: "githubSkillSources:nvidia",
repo: "NVIDIA/skills",
ownerPublisherId: "publishers:nvidia",
createdAt: 1,
updatedAt: 1,
},
],
skills: [
{
_id: "skills:aiq-deploy",
slug: "aiq-deploy",
displayName: "AIQ Deploy",
ownerUserId: "users:nvidia",
ownerPublisherId: "publishers:nvidia",
installKind: "github",
githubSourceId: "githubSkillSources:nvidia",
githubPath: "skills/aiq-deploy",
githubCurrentCommit: "1".repeat(40),
githubCurrentContentHash: contentHash,
githubCurrentStatus: "present",
githubScanStatus: "clean",
moderationStatus: "active",
moderationVerdict: "clean",
tags: {},
stats: { downloads: 0, stars: 0, installsCurrent: 0, installsAllTime: 0, versions: 0 },
createdAt: 1,
updatedAt: 1,
},
],
});
const scheduler = { runAfter: vi.fn(async () => undefined) };
await applyGitHubSkillSourceSyncHandler({ db, scheduler } as never, {
sourceId: "githubSkillSources:nvidia" as never,
repo: "NVIDIA/skills",
ownerUserId: "users:nvidia" as never,
ownerPublisherId: "publishers:nvidia" as never,
snapshot,
now: 123,
});
expect(tables.skills[0]).toMatchObject({
githubScanStatus: "pending",
moderationStatus: "active",
moderationReason: "pending.scan",
});
expect(tables.skills[0]).not.toHaveProperty("moderationVerdict");
expect(scheduler.runAfter).toHaveBeenCalledWith(0, expect.anything(), {
skillId: "skills:aiq-deploy",
contentHash,
});
});
it("applies a trusted fetched snapshot without blocking unrelated slug owners", async () => {
const snapshot = await buildGitHubSkillSourceSnapshot({
repo: "NVIDIA/skills",
defaultBranch: "main",
@@ -1072,55 +1218,71 @@ describe("applyGitHubSkillSourceSyncHandler", () => {
expect(result.stats).toMatchObject({
discovered: 2,
changed: 1,
inserted: 0,
conflicts: 1,
inserted: 1,
conflicts: 0,
});
expect(tables.githubSkillSources[0]).toMatchObject({
ownerPublisherId: "publishers:nvidia",
displayManifestStatus: "ok",
displayManifestCommit: "2".repeat(40),
lastSyncIssues: [
{
slug: "vision-helper",
path: "skills/vision-helper",
displayName: "Vision Helper",
kind: "slug_conflict",
severity: "error",
message: "Slug already exists on ClawHub under @jonathanjing.",
existingOwnerHandle: "jonathanjing",
},
],
lastSyncIssues: [],
});
expect(tables.skills.find((skill) => skill._id === "skills:aiq-deploy")).toMatchObject({
githubCurrentCommit: "2".repeat(40),
githubScanStatus: "pending",
moderationStatus: "active",
});
expect(tables.githubSkillContents).toEqual([
expect.objectContaining({
skillId: "skills:aiq-deploy",
githubSourceId: "githubSkillSources:nvidia",
githubPath: "skills/aiq-deploy",
skillMarkdownPath: "skills/aiq-deploy/SKILL.md",
skillMarkdown: "# AIQ Deploy v2\n",
skillCardMarkdownPath: "skills/aiq-deploy/skill-card.md",
skillCardMarkdown: "# AIQ Card v2\n",
githubCommit: "2".repeat(40),
githubContentHash: snapshot.skills.find((skill) => skill.slug === "aiq-deploy")
?.contentHash,
fetchedAt: 123,
}),
]);
expect(tables.githubSkillContents).toEqual(
expect.arrayContaining([
expect.objectContaining({
skillId: "skills:aiq-deploy",
githubSourceId: "githubSkillSources:nvidia",
githubPath: "skills/aiq-deploy",
skillMarkdownPath: "skills/aiq-deploy/SKILL.md",
skillMarkdown: "# AIQ Deploy v2\n",
skillCardMarkdownPath: "skills/aiq-deploy/skill-card.md",
skillCardMarkdown: "# AIQ Card v2\n",
githubCommit: "2".repeat(40),
githubContentHash: snapshot.skills.find((skill) => skill.slug === "aiq-deploy")
?.contentHash,
fetchedAt: 123,
}),
expect.objectContaining({
skillId: "skills:new-1",
githubSourceId: "githubSkillSources:nvidia",
githubPath: "skills/vision-helper",
skillMarkdownPath: "skills/vision-helper/SKILL.md",
skillMarkdown: "# Vision Helper\n",
githubCommit: "2".repeat(40),
githubContentHash: snapshot.skills.find((skill) => skill.slug === "vision-helper")
?.contentHash,
fetchedAt: 123,
}),
]),
);
expect(tables.globalStats[0]).toMatchObject({
activeSkillsCount: 10,
updatedAt: 1,
activeSkillsCount: 11,
updatedAt: 123,
});
const conflict = tables.skills.find((skill) => skill._id === "skills:vision-helper-conflict");
expect(conflict).toMatchObject({
displayName: "Existing Direct Skill",
});
expect(conflict).not.toHaveProperty("installKind");
expect(tables.skills).toHaveLength(2);
expect(tables.skills).toHaveLength(3);
expect(tables.skills.find((skill) => skill._id === "skills:new-1")).toMatchObject({
slug: "vision-helper",
displayName: "Vision Helper",
ownerUserId: "users:nvidia",
ownerPublisherId: "publishers:nvidia",
installKind: "github",
githubSourceId: "githubSkillSources:nvidia",
githubPath: "skills/vision-helper",
githubCurrentCommit: "2".repeat(40),
githubCurrentStatus: "present",
githubScanStatus: "pending",
moderationStatus: "active",
});
});
it("preserves an existing soft delete timestamp when upstream remains missing", async () => {
@@ -1385,7 +1547,7 @@ describe("applyGitHubSkillSourceSyncHandler", () => {
"skills/aiq-deploy/SKILL.md": new TextEncoder().encode("# AIQ Deploy\n"),
},
});
const { db } = createDb({
const { db, tables } = createDb({
githubSkillSources: [
{
_id: "githubSkillSources:nvidia",
@@ -1396,7 +1558,12 @@ describe("applyGitHubSkillSourceSyncHandler", () => {
},
],
});
const scheduler = { runAfter: vi.fn(async () => undefined) };
const scheduler = {
runAfter: vi.fn(
async (_delayMs: number, _functionRef: unknown, _args: Record<string, unknown>) =>
undefined,
),
};
await applyGitHubSkillSourceSyncHandler({ db, scheduler } as never, {
sourceId: "githubSkillSources:nvidia" as never,
@@ -1411,6 +1578,165 @@ describe("applyGitHubSkillSourceSyncHandler", () => {
skillId: "skills:new-1",
contentHash: snapshot.skills[0]?.contentHash,
});
expect(Object.values(tables.githubSkillScans?.[0] ?? {})).not.toContain(undefined);
const scheduledFunction = scheduler.runAfter.mock.calls[0]?.[1];
expect(getFunctionName(scheduledFunction as Parameters<typeof getFunctionName>[0])).toBe(
"githubSkillSyncNode:verifyGitHubSkillInternal",
);
});
it("does not requeue heavy verification while the current content scan job is active", async () => {
const snapshot = await buildGitHubSkillSourceSnapshot({
repo: "NVIDIA/skills",
defaultBranch: "main",
commit: "2".repeat(40),
entries: {
"skills/aiq-deploy/SKILL.md": new TextEncoder().encode("# AIQ Deploy\n"),
},
});
const contentHash = snapshot.skills[0]?.contentHash;
const { db } = createDb({
githubSkillSources: [
{
_id: "githubSkillSources:nvidia",
repo: "NVIDIA/skills",
ownerPublisherId: "publishers:nvidia",
createdAt: 1,
updatedAt: 1,
},
],
skills: [
{
_id: "skills:aiq-deploy",
slug: "aiq-deploy",
displayName: "AIQ Deploy",
ownerUserId: "users:nvidia",
ownerPublisherId: "publishers:nvidia",
installKind: "github",
githubSourceId: "githubSkillSources:nvidia",
githubPath: "skills/aiq-deploy",
githubCurrentCommit: "1".repeat(40),
githubCurrentContentHash: contentHash,
githubCurrentStatus: "present",
githubScanStatus: "pending",
tags: {},
stats: { downloads: 0, stars: 0, installsCurrent: 0, installsAllTime: 0, versions: 0 },
createdAt: 1,
updatedAt: 1,
},
],
githubSkillScans: [
{
_id: "githubSkillScans:aiq-deploy",
skillId: "skills:aiq-deploy",
githubSourceId: "githubSkillSources:nvidia",
contentHash,
status: "pending",
skillScanRequestId: "skillScanRequests:aiq-deploy",
},
],
skillScanRequests: [
{
_id: "skillScanRequests:aiq-deploy",
securityScanJobId: "securityScanJobs:aiq-deploy",
},
],
securityScanJobs: [
{
_id: "securityScanJobs:aiq-deploy",
status: "queued",
},
],
});
const scheduler = { runAfter: vi.fn(async () => undefined) };
await applyGitHubSkillSourceSyncHandler({ db, scheduler } as never, {
sourceId: "githubSkillSources:nvidia" as never,
repo: "NVIDIA/skills",
ownerUserId: "users:nvidia" as never,
ownerPublisherId: "publishers:nvidia" as never,
snapshot,
now: 123,
});
expect(scheduler.runAfter).toHaveBeenCalledTimes(0);
});
it("does not requeue heavy verification while a recent verification action is pending", async () => {
const snapshot = await buildGitHubSkillSourceSnapshot({
repo: "NVIDIA/skills",
defaultBranch: "main",
commit: "2".repeat(40),
entries: {
"skills/aiq-deploy/SKILL.md": new TextEncoder().encode("# AIQ Deploy\n"),
},
});
const contentHash = snapshot.skills[0]?.contentHash;
const { db } = createDb({
githubSkillSources: [
{
_id: "githubSkillSources:nvidia",
repo: "NVIDIA/skills",
ownerPublisherId: "publishers:nvidia",
createdAt: 1,
updatedAt: 1,
},
],
skills: [
{
_id: "skills:aiq-deploy",
slug: "aiq-deploy",
displayName: "AIQ Deploy",
ownerUserId: "users:nvidia",
ownerPublisherId: "publishers:nvidia",
installKind: "github",
githubSourceId: "githubSkillSources:nvidia",
githubPath: "skills/aiq-deploy",
githubCurrentCommit: "1".repeat(40),
githubCurrentContentHash: contentHash,
githubCurrentStatus: "present",
githubScanStatus: "pending",
tags: {},
stats: { downloads: 0, stars: 0, installsCurrent: 0, installsAllTime: 0, versions: 0 },
createdAt: 1,
updatedAt: 1,
},
],
githubSkillScans: [
{
_id: "githubSkillScans:aiq-deploy",
skillId: "skills:aiq-deploy",
githubSourceId: "githubSkillSources:nvidia",
contentHash,
commit: "1".repeat(40),
path: "skills/aiq-deploy",
status: "pending",
skillScanRequestId: "skillScanRequests:aiq-deploy",
createdAt: 1,
updatedAt: 123,
},
],
skillScanRequests: [
{
_id: "skillScanRequests:aiq-deploy",
sourceKind: "github",
createdAt: 123,
updatedAt: 123,
},
],
});
const scheduler = { runAfter: vi.fn(async () => undefined) };
await applyGitHubSkillSourceSyncHandler({ db, scheduler } as never, {
sourceId: "githubSkillSources:nvidia" as never,
repo: "NVIDIA/skills",
ownerUserId: "users:nvidia" as never,
ownerPublisherId: "publishers:nvidia" as never,
snapshot,
now: 123,
});
expect(scheduler.runAfter).toHaveBeenCalledTimes(0);
});
it("refreshes cached GitHub content metadata when bytes are unchanged at a new commit", async () => {
@@ -1643,6 +1969,9 @@ describe("verifyGitHubSkillHandler", () => {
const commit = "3".repeat(40);
const zip = zipSync({
"skills-main/skills/aiq-deploy/SKILL.md": new TextEncoder().encode("# AIQ Deploy\n"),
"skills-main/skills/aiq-deploy/scripts/deploy.sh": new TextEncoder().encode(
"#!/bin/sh\necho deploy\n",
),
});
const snapshot = await buildGitHubSkillSourceSnapshot({
repo: "NVIDIA/skills",
@@ -1653,7 +1982,40 @@ describe("verifyGitHubSkillHandler", () => {
const contentHash = snapshot.skills[0]?.contentHash;
if (!contentHash) throw new Error("missing fixture hash");
const runMutation = vi.fn(async () => ({ ok: true, promoted: false }));
const events: string[] = [];
let storedFile = 0;
const store = vi.fn(async (_blob: Blob) => {
events.push("store");
storedFile += 1;
return `storage:${storedFile}`;
});
const runMutation = vi.fn(async (mutation: unknown, _args: Record<string, unknown>) => {
const name = getFunctionName(mutation as Parameters<typeof getFunctionName>[0]);
if (name === "securityScan:prepareGitHubSkillScanRequestInternal") {
events.push("prepare");
return {
ok: true,
prepared: true,
scanId: "githubSkillScans:1",
requestId: "skillScanRequests:1",
};
}
if (name === "securityScan:appendGitHubSkillScanRequestFilesInternal") {
events.push("append");
return { ok: true, appended: true };
}
if (name === "securityScan:finalizeGitHubSkillScanRequestInternal") {
events.push("finalize");
return {
ok: true,
queued: true,
scanId: "githubSkillScans:1",
requestId: "skillScanRequests:1",
jobId: "securityScanJobs:1",
};
}
throw new Error(`unexpected mutation: ${name}`);
});
const ctx = {
runQuery: vi.fn(async () => ({
skill: {
@@ -1673,6 +2035,7 @@ describe("verifyGitHubSkillHandler", () => {
},
})),
runMutation,
storage: { store, delete: vi.fn() },
};
const fetcher = vi.fn(async (input: RequestInfo | URL) => {
const url =
@@ -1694,15 +2057,207 @@ describe("verifyGitHubSkillHandler", () => {
fetcher as unknown as typeof fetch,
);
expect(result).toMatchObject({ ok: true, scanStatus: "clean" });
expect(runMutation).toHaveBeenCalledWith(
expect.anything(),
expect(result).toMatchObject({ ok: true, queued: true });
expect(store).toHaveBeenCalledTimes(2);
expect((store.mock.calls[0]?.[0] as Blob | undefined)?.type).toBe("application/octet-stream");
expect(runMutation).toHaveBeenCalledTimes(3);
expect(events).toEqual(["prepare", "store", "store", "append", "finalize"]);
const [prepareMutation, prepareArgs] = runMutation.mock.calls[0] ?? [];
expect(getFunctionName(prepareMutation as Parameters<typeof getFunctionName>[0])).toBe(
"securityScan:prepareGitHubSkillScanRequestInternal",
);
expect(prepareArgs).toEqual(
expect.objectContaining({
skillId: "skills:aiq-deploy",
contentHash,
scanStatus: "clean",
commit,
staticScan: expect.objectContaining({ status: "clean" }),
}),
);
expect(prepareArgs).not.toHaveProperty("files");
expect(Object.values(prepareArgs ?? {})).not.toContain(undefined);
const [appendMutation, appendArgs] = runMutation.mock.calls[1] ?? [];
expect(getFunctionName(appendMutation as Parameters<typeof getFunctionName>[0])).toBe(
"securityScan:appendGitHubSkillScanRequestFilesInternal",
);
expect(appendArgs).toEqual(
expect.objectContaining({
requestId: "skillScanRequests:1",
chunkIndex: 0,
files: expect.arrayContaining([
expect.objectContaining({ path: "SKILL.md" }),
expect.objectContaining({ path: "scripts/deploy.sh" }),
]),
}),
);
const [finalizeMutation, finalizeArgs] = runMutation.mock.calls[2] ?? [];
expect(getFunctionName(finalizeMutation as Parameters<typeof getFunctionName>[0])).toBe(
"securityScan:finalizeGitHubSkillScanRequestInternal",
);
expect(finalizeArgs).toEqual({ requestId: "skillScanRequests:1" });
});
it("does not store GitHub skill files when the durable content-hash scan can be reused", async () => {
const commit = "4".repeat(40);
const zip = zipSync({
"skills-main/skills/aiq-deploy/SKILL.md": new TextEncoder().encode("# AIQ Deploy\n"),
});
const snapshot = await buildGitHubSkillSourceSnapshot({
repo: "NVIDIA/skills",
defaultBranch: "main",
commit,
entries: stripGitHubZipRoot(__test.unzipToEntries(zip)),
});
const contentHash = snapshot.skills[0]?.contentHash;
if (!contentHash) throw new Error("missing fixture hash");
const store = vi.fn();
const runMutation = vi.fn(async (mutation: unknown) => {
const name = getFunctionName(mutation as Parameters<typeof getFunctionName>[0]);
if (name === "securityScan:prepareGitHubSkillScanRequestInternal") {
return {
ok: true,
reused: true,
scanId: "githubSkillScans:1",
scanStatus: "clean",
};
}
if (name === "githubSkillSync:applyGitHubSkillVerificationResultInternal") {
return { ok: true, promoted: true };
}
throw new Error(`unexpected mutation: ${name}`);
});
const ctx = {
runQuery: vi.fn(async () => ({
skill: {
_id: "skills:aiq-deploy",
slug: "aiq-deploy",
displayName: "AIQ Deploy",
githubPath: "skills/aiq-deploy",
githubCurrentCommit: commit,
githubCurrentContentHash: contentHash,
githubCurrentStatus: "present",
},
source: {
_id: "githubSkillSources:nvidia",
repo: "NVIDIA/skills",
defaultBranch: "main",
},
})),
runMutation,
storage: { store, delete: vi.fn() },
};
const fetcher = vi.fn(async (input: RequestInfo | URL) => {
const url =
typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
if (url.startsWith("https://api.github.com/")) {
return new Response(JSON.stringify({ sha: commit }), {
headers: { "content-type": "application/json" },
});
}
if (url.startsWith("https://codeload.github.com/")) {
return new Response(zip, { headers: { "content-length": String(zip.byteLength) } });
}
return new Response("not found", { status: 404 });
});
await expect(
verifyGitHubSkillHandler(
ctx as never,
{ skillId: "skills:aiq-deploy" as never, contentHash },
fetcher as unknown as typeof fetch,
),
).resolves.toMatchObject({ ok: true, reused: true, scanStatus: "clean" });
expect(store).not.toHaveBeenCalled();
});
it("deletes the newly stored boundary file when appending the previous chunk fails", async () => {
const commit = "5".repeat(40);
const zipEntries: Record<string, Uint8Array> = {
"skills-main/skills/aiq-deploy/SKILL.md": new TextEncoder().encode("# AIQ Deploy\n"),
};
for (let index = 0; index < 100; index += 1) {
zipEntries[
`skills-main/skills/aiq-deploy/scripts/file-${String(index).padStart(3, "0")}.txt`
] = new TextEncoder().encode(`file ${index}\n`);
}
const zip = zipSync(zipEntries);
const snapshot = await buildGitHubSkillSourceSnapshot({
repo: "NVIDIA/skills",
defaultBranch: "main",
commit,
entries: stripGitHubZipRoot(__test.unzipToEntries(zip)),
});
const contentHash = snapshot.skills[0]?.contentHash;
if (!contentHash) throw new Error("missing fixture hash");
let storedFile = 0;
const store = vi.fn(async () => {
storedFile += 1;
return `storage:${storedFile}`;
});
const deleteFile = vi.fn(async () => undefined);
const runMutation = vi.fn(async (mutation: unknown) => {
const name = getFunctionName(mutation as Parameters<typeof getFunctionName>[0]);
if (name === "securityScan:prepareGitHubSkillScanRequestInternal") {
return {
ok: true,
prepared: true,
scanId: "githubSkillScans:1",
requestId: "skillScanRequests:1",
};
}
if (name === "securityScan:appendGitHubSkillScanRequestFilesInternal") {
throw new Error("append failed");
}
throw new Error(`unexpected mutation: ${name}`);
});
const ctx = {
runQuery: vi.fn(async () => ({
skill: {
_id: "skills:aiq-deploy",
slug: "aiq-deploy",
displayName: "AIQ Deploy",
githubPath: "skills/aiq-deploy",
githubCurrentCommit: commit,
githubCurrentContentHash: contentHash,
githubCurrentStatus: "present",
},
source: {
_id: "githubSkillSources:nvidia",
repo: "NVIDIA/skills",
defaultBranch: "main",
},
})),
runMutation,
storage: { store, delete: deleteFile },
};
const fetcher = vi.fn(async (input: RequestInfo | URL) => {
const url =
typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
if (url.startsWith("https://api.github.com/")) {
return new Response(JSON.stringify({ sha: commit }), {
headers: { "content-type": "application/json" },
});
}
if (url.startsWith("https://codeload.github.com/")) {
return new Response(zip, { headers: { "content-length": String(zip.byteLength) } });
}
return new Response("not found", { status: 404 });
});
await expect(
verifyGitHubSkillHandler(
ctx as never,
{ skillId: "skills:aiq-deploy" as never, contentHash },
fetcher as unknown as typeof fetch,
),
).rejects.toThrow("append failed");
expect(store).toHaveBeenCalledTimes(101);
expect(deleteFile).toHaveBeenCalledTimes(101);
expect(deleteFile).toHaveBeenCalledWith("storage:101");
});
});
+263 -142
View File
@@ -3,7 +3,7 @@ import { unzipSync, type UnzipFileInfo } from "fflate";
import { internal } from "./_generated/api";
import type { Doc, Id } from "./_generated/dataModel";
import type { ActionCtx, MutationCtx, QueryCtx } from "./_generated/server";
import { action, internalAction, internalMutation, internalQuery } from "./functions";
import { action, internalMutation, internalQuery } from "./functions";
import { assertAdmin, requireUserFromAction } from "./lib/access";
import { buildGitHubApiHeaders } from "./lib/githubAuth";
import {
@@ -28,10 +28,12 @@ import { Events, logErrorEvent, logEvent } from "./lib/observabilityEvents";
import { isOfficialPublisher } from "./lib/officialPublishers";
import { requirePublisherRole } from "./lib/publishers";
import { isMacJunkPath, isTextFile, parseFrontmatter } from "./lib/skills";
import { chunkSkillScanRequestFiles } from "./lib/skillScanRequestFiles";
import { syncSkillSearchDigestForSkill } from "./lib/skillSearchDigest";
import { assertValidSkillSlug } from "./lib/skillSlugValidator";
const DEFAULT_BRANCH = "main";
const GITHUB_SKILL_SCAN_ACTION_LEASE_MS = 15 * 60 * 1000;
const PUBLIC_REPO_ONLY_ERROR = "Enter a public GitHub repo.";
const MAX_UNZIPPED_BYTES = 80 * 1024 * 1024;
const MAX_FILE_COUNT = 7_500;
@@ -129,6 +131,20 @@ type GitHubSkillVerificationTarget = {
source: Pick<Doc<"githubSkillSources">, "_id" | "repo" | "defaultBranch">;
};
type GitHubSkillVerificationResult = {
ok: true;
prepared?: true;
queued?: true;
reused?: true;
alreadyQueued?: true;
skipped?: string;
scanStatus?: GitHubSkillScanStatus;
scanId?: Id<"githubSkillScans">;
requestId?: Id<"skillScanRequests">;
jobId?: Id<"securityScanJobs">;
currentContentHash?: string;
};
type GitHubSkillContentTarget = {
skillId: Id<"skills">;
githubPath: string;
@@ -485,6 +501,7 @@ export async function applyGitHubSkillSourceSyncHandler(
skillId: skillPatch.skillId as Id<"skills">,
contentHash: discovered.contentHash,
scanStatus: skillPatch.patch.githubScanStatus,
now,
});
}
}
@@ -528,66 +545,57 @@ export async function applyGitHubSkillSourceSyncHandler(
continue;
}
const existingBySlug = await ctx.db
.query("skills")
.withIndex("by_slug", (q) => q.eq("slug", skillInsert.slug))
.unique();
if (existingBySlug) {
if (canReviveGitHubSkillSlugConflict(existingBySlug, sourceOwnerPublisherId)) {
const previousSkillSnapshot = { ...existingBySlug } as Doc<"skills">;
const doc = stripUndefined(skillInsert.doc) as Partial<Doc<"skills">>;
const patch = {
...doc,
createdAt: existingBySlug.createdAt,
tags: existingBySlug.tags ?? {},
statsDownloads: existingBySlug.statsDownloads ?? doc.statsDownloads,
statsStars: existingBySlug.statsStars ?? doc.statsStars,
statsInstallsCurrent: existingBySlug.statsInstallsCurrent ?? doc.statsInstallsCurrent,
statsInstallsAllTime: existingBySlug.statsInstallsAllTime ?? doc.statsInstallsAllTime,
stats: existingBySlug.stats ?? doc.stats,
badges: existingBySlug.badges,
latestVersionId: undefined,
githubRemovedAt: undefined,
softDeletedAt: undefined,
updatedAt: now,
};
await ctx.db.patch(existingBySlug._id, patch);
const nextSkillSnapshot = { ...previousSkillSnapshot, ...patch } as Doc<"skills">;
const discovered = discoveredBySlug.get(skillInsert.slug);
if (discovered) {
if (hasGitHubSkillContent(discovered)) {
await upsertGitHubSkillContent(ctx, {
skillId: existingBySlug._id,
sourceId,
discovered,
commit: args.snapshot.commit,
now,
});
}
await scheduleGitHubSkillVerification(ctx, {
skillId: existingBySlug._id,
contentHash: discovered.contentHash,
scanStatus: doc.githubScanStatus,
const reviveCandidate = await findGitHubSkillRevivalCandidate(ctx, {
ownerUserId: args.ownerUserId,
ownerPublisherId: sourceOwnerPublisherId,
slug: skillInsert.slug,
});
if (reviveCandidate && canReviveGitHubSkillForSource(reviveCandidate)) {
const previousSkillSnapshot = { ...reviveCandidate } as Doc<"skills">;
const doc = stripUndefined(skillInsert.doc) as Partial<Doc<"skills">>;
const patch = {
...doc,
createdAt: reviveCandidate.createdAt,
tags: reviveCandidate.tags ?? {},
statsDownloads: reviveCandidate.statsDownloads ?? doc.statsDownloads,
statsStars: reviveCandidate.statsStars ?? doc.statsStars,
statsInstallsCurrent: reviveCandidate.statsInstallsCurrent ?? doc.statsInstallsCurrent,
statsInstallsAllTime: reviveCandidate.statsInstallsAllTime ?? doc.statsInstallsAllTime,
stats: reviveCandidate.stats ?? doc.stats,
badges: reviveCandidate.badges,
latestVersionId: undefined,
githubRemovedAt: undefined,
softDeletedAt: undefined,
updatedAt: now,
};
await ctx.db.patch(reviveCandidate._id, patch);
const nextSkillSnapshot = { ...previousSkillSnapshot, ...patch } as Doc<"skills">;
const discovered = discoveredBySlug.get(skillInsert.slug);
if (discovered) {
if (hasGitHubSkillContent(discovered)) {
await upsertGitHubSkillContent(ctx, {
skillId: reviveCandidate._id,
sourceId,
discovered,
commit: args.snapshot.commit,
now,
});
}
await adjustGlobalPublicCountForSkillChange(
ctx,
previousSkillSnapshot,
nextSkillSnapshot,
await scheduleGitHubSkillVerification(ctx, {
skillId: reviveCandidate._id,
contentHash: discovered.contentHash,
scanStatus: doc.githubScanStatus,
now,
);
await syncSkillSearchDigestForSkill(ctx, nextSkillSnapshot);
revived += 1;
continue;
});
}
conflicts += 1;
issues.push(
await buildSlugConflictIssue(ctx, {
skillInsert,
existingSkill: existingBySlug,
discovered: discoveredBySlug.get(skillInsert.slug),
}),
await adjustGlobalPublicCountForSkillChange(
ctx,
previousSkillSnapshot,
nextSkillSnapshot,
now,
);
await syncSkillSearchDigestForSkill(ctx, nextSkillSnapshot);
revived += 1;
continue;
}
@@ -610,6 +618,7 @@ export async function applyGitHubSkillSourceSyncHandler(
skillId,
contentHash: discovered.contentHash,
scanStatus: doc.githubScanStatus,
now,
});
}
await adjustGlobalPublicCountForSkillChange(ctx, null, insertedSkill, now);
@@ -639,58 +648,30 @@ export async function applyGitHubSkillSourceSyncHandler(
};
}
async function buildSlugConflictIssue(
async function findGitHubSkillRevivalCandidate(
ctx: MutationCtx,
args: {
skillInsert: ReturnType<typeof buildGitHubSkillSyncPlan>["skillInserts"][number];
existingSkill: Doc<"skills">;
discovered: GitHubSkillSourceMetadataSnapshot["skills"][number] | undefined;
ownerUserId: Id<"users">;
ownerPublisherId: Id<"publishers"> | undefined;
slug: string;
},
): Promise<GitHubSkillSourceSyncIssue> {
const displayName =
typeof args.skillInsert.doc.displayName === "string"
? args.skillInsert.doc.displayName
: args.skillInsert.slug;
const path =
args.discovered?.path ??
(typeof args.skillInsert.doc.githubPath === "string"
? args.skillInsert.doc.githubPath
: args.skillInsert.slug);
const existingOwnerHandle = await getSkillOwnerHandle(ctx, args.existingSkill);
const ownerPhrase = existingOwnerHandle ? ` under @${existingOwnerHandle}` : "";
return {
slug: args.skillInsert.slug,
path,
displayName,
kind: "slug_conflict",
severity: "error",
message: `Slug already exists on ClawHub${ownerPhrase}.`,
...(existingOwnerHandle ? { existingOwnerHandle } : {}),
};
}
async function getSkillOwnerHandle(ctx: MutationCtx, skill: Doc<"skills">) {
if (skill.ownerPublisherId) {
const publisher = await ctx.db.get(skill.ownerPublisherId);
if (publisher?.handle) return publisher.handle;
}
if (skill.ownerUserId) {
const user = await ctx.db.get(skill.ownerUserId);
if (user?.handle) return user.handle;
}
return null;
}
function canReviveGitHubSkillSlugConflict(
skill: Doc<"skills">,
ownerPublisherId: Id<"publishers"> | undefined,
) {
return (
skill.installKind === "github" &&
typeof skill.softDeletedAt === "number" &&
Boolean(ownerPublisherId) &&
skill.ownerPublisherId === ownerPublisherId
);
if (args.ownerPublisherId) {
return await ctx.db
.query("skills")
.withIndex("by_owner_publisher_slug", (q) =>
q.eq("ownerPublisherId", args.ownerPublisherId).eq("slug", args.slug),
)
.unique();
}
return await ctx.db
.query("skills")
.withIndex("by_owner_slug", (q) => q.eq("ownerUserId", args.ownerUserId).eq("slug", args.slug))
.unique();
}
function canReviveGitHubSkillForSource(skill: Doc<"skills">) {
return skill.installKind === "github" && typeof skill.softDeletedAt === "number";
}
function hasGitHubSkillContent(
@@ -783,10 +764,71 @@ async function scheduleGitHubSkillVerification(
skillId: Id<"skills">;
contentHash: string;
scanStatus: unknown;
now: number;
},
) {
if (args.scanStatus !== "pending") return;
await ctx.scheduler?.runAfter(0, internal.githubSkillSync.verifyGitHubSkillInternal, {
const scan = await ctx.db
.query("githubSkillScans")
.withIndex("by_skill_and_content_hash", (q) =>
q.eq("skillId", args.skillId).eq("contentHash", args.contentHash),
)
.unique();
if (args.scanStatus !== "pending") {
if (scan?.status !== "pending") {
if (scan) return;
await applyGitHubSkillVerificationResultHandler(ctx, {
skillId: args.skillId,
contentHash: args.contentHash,
scanStatus: "pending",
});
}
}
if (scan?.status === "pending" && scan.skillScanRequestId) {
const request = await ctx.db.get(scan.skillScanRequestId);
const job = request?.securityScanJobId ? await ctx.db.get(request.securityScanJobId) : null;
if (job?.status === "queued" || job?.status === "running") return;
if (request && request.updatedAt > args.now - GITHUB_SKILL_SCAN_ACTION_LEASE_MS) return;
}
if (
scan?.status === "pending" &&
!scan.skillScanRequestId &&
scan.updatedAt > args.now - GITHUB_SKILL_SCAN_ACTION_LEASE_MS
) {
return;
}
const skill = await ctx.db.get(args.skillId);
if (
!skill ||
skill.installKind !== "github" ||
!skill.githubSourceId ||
!skill.githubPath ||
skill.githubCurrentStatus !== "present" ||
!skill.githubCurrentCommit ||
skill.githubCurrentContentHash !== args.contentHash
) {
return;
}
const pendingScanInsert = {
githubSourceId: skill.githubSourceId,
commit: skill.githubCurrentCommit,
path: skill.githubPath,
status: "pending" as const,
updatedAt: args.now,
};
if (scan) {
await ctx.db.patch(scan._id, {
...pendingScanInsert,
skillScanRequestId: undefined,
});
} else {
await ctx.db.insert("githubSkillScans", {
skillId: skill._id,
contentHash: args.contentHash,
...pendingScanInsert,
createdAt: args.now,
});
}
await ctx.scheduler?.runAfter(0, internal.githubSkillSyncNode.verifyGitHubSkillInternal, {
skillId: args.skillId,
contentHash: args.contentHash,
});
@@ -860,12 +902,12 @@ export const applyGitHubSkillVerificationResultInternal = internalMutation({
export async function verifyGitHubSkillHandler(
ctx: ActionCtx,
args: { skillId: Id<"skills">; contentHash: string },
args: { skillId: Id<"skills">; contentHash: string; force?: boolean },
fetcher: typeof fetch = fetch,
) {
): Promise<GitHubSkillVerificationResult> {
const target = (await ctx.runQuery(
internal.githubSkillSync.getGitHubSkillVerificationTargetInternal,
args,
{ skillId: args.skillId, contentHash: args.contentHash },
)) as GitHubSkillVerificationTarget | null;
if (!target) return { ok: true as const, skipped: "stale-or-missing" as const };
@@ -895,26 +937,44 @@ export async function verifyGitHubSkillHandler(
fileContents: listGitHubSkillTextContents(entries, discovered.path),
});
await ctx.runMutation(internal.githubSkillSync.applyGitHubSkillVerificationResultInternal, {
skillId: target.skill._id,
contentHash: args.contentHash,
scanStatus: staticScan.status,
const prepared = (await ctx.runMutation(
internal.securityScan.prepareGitHubSkillScanRequestInternal,
{
skillId: target.skill._id,
contentHash: args.contentHash,
commit: target.skill.githubCurrentCommit,
...(args.force ? { force: true } : {}),
parsed: { frontmatter: parseFrontmatter(discovered.skillMarkdown) },
staticScan,
},
)) as GitHubSkillVerificationResult | undefined;
if (!prepared?.prepared || !prepared.requestId) {
if (prepared?.reused && prepared.scanStatus) {
await ctx.runMutation(internal.githubSkillSync.applyGitHubSkillVerificationResultInternal, {
skillId: target.skill._id,
contentHash: args.contentHash,
scanStatus: prepared.scanStatus,
});
}
return prepared ?? { ok: true as const, skipped: "scan-request-not-created" as const };
}
let chunkIndex = 0;
await storeGitHubSkillScanFileChunks(ctx, entries, discovered.path, async (chunk) => {
await ctx.runMutation(internal.securityScan.appendGitHubSkillScanRequestFilesInternal, {
requestId: prepared.requestId as Id<"skillScanRequests">,
chunkIndex,
files: chunk,
});
chunkIndex += 1;
});
return {
ok: true as const,
scanStatus: staticScan.status,
};
return (await ctx.runMutation(internal.securityScan.finalizeGitHubSkillScanRequestInternal, {
requestId: prepared.requestId,
...(args.force ? { force: true } : {}),
})) as typeof prepared;
}
export const verifyGitHubSkillInternal = internalAction({
args: {
skillId: v.id("skills"),
contentHash: v.string(),
},
handler: verifyGitHubSkillHandler,
});
export async function configurePublicGitHubSkillSourceHandler(
ctx: ActionCtx,
args: { ownerPublisherId: Id<"publishers">; repo: string },
@@ -1160,7 +1220,7 @@ export async function syncGitHubSkillSourcesHandler(
let scheduledNext = false;
if (!page.isDone && page.continueCursor && ctx.scheduler) {
await ctx.scheduler.runAfter(0, internal.githubSkillSync.syncGitHubSkillSourcesInternal, {
await ctx.scheduler.runAfter(0, internal.githubSkillSyncNode.syncGitHubSkillSourcesInternal, {
cursor: page.continueCursor,
batchSize,
});
@@ -1179,21 +1239,6 @@ export async function syncGitHubSkillSourcesHandler(
};
}
export const syncGitHubSkillSourcesInternal = internalAction({
args: {
cursor: v.optional(v.union(v.string(), v.null())),
batchSize: v.optional(v.number()),
},
handler: async (ctx, args) => {
try {
return await syncGitHubSkillSourcesHandler(ctx, args);
} catch (error) {
logErrorEvent(Events.GitHubSkillSourceSyncFailed, { error: getErrorMessage(error) });
throw error;
}
},
});
async function fetchGitHubSkillSourceSnapshot(
{
repo,
@@ -1277,6 +1322,82 @@ function listGitHubSkillFolderEntries(entries: Record<string, Uint8Array>, folde
.sort(([a], [b]) => a.localeCompare(b));
}
async function storeGitHubSkillScanFileChunks(
ctx: Pick<ActionCtx, "storage">,
entries: Record<string, Uint8Array>,
folderPath: string,
appendChunk: (
files: Array<{
path: string;
size: number;
storageId: Id<"_storage">;
sha256: string;
}>,
) => Promise<void>,
) {
let pendingChunk: Array<{
path: string;
size: number;
storageId: Id<"_storage">;
sha256: string;
}> = [];
try {
for (const [path, bytes] of listGitHubSkillFolderEntries(entries, folderPath)) {
const safeBytes = new Uint8Array(bytes);
const sha256 = await sha256Hex(safeBytes);
const storageId = await ctx.storage.store(
new Blob([safeBytes], { type: "application/octet-stream" }),
);
const file = {
path,
size: safeBytes.byteLength,
storageId,
sha256,
};
const nextPendingChunk = [...pendingChunk, file];
let candidateChunks;
try {
candidateChunks = chunkSkillScanRequestFiles(nextPendingChunk);
} catch (error) {
pendingChunk = nextPendingChunk;
throw error;
}
if (candidateChunks.length > 1) {
try {
await appendChunk(pendingChunk);
} catch (error) {
pendingChunk = nextPendingChunk;
throw error;
}
pendingChunk = [file];
} else {
pendingChunk = candidateChunks[0] ?? [];
}
}
if (pendingChunk.length > 0) {
await appendChunk(pendingChunk);
}
} catch (error) {
// Prior chunks are owned by the durable request; only this bounded chunk can be orphaned.
await deleteStoredGitHubSkillScanFiles(ctx, pendingChunk);
throw error;
}
}
async function deleteStoredGitHubSkillScanFiles(
ctx: Pick<ActionCtx, "storage">,
files: Array<{ storageId: Id<"_storage"> }>,
) {
await Promise.allSettled(files.map((file) => ctx.storage.delete(file.storageId)));
}
async function sha256Hex(bytes: Uint8Array) {
const digest = await crypto.subtle.digest("SHA-256", new Uint8Array(bytes));
return Array.from(new Uint8Array(digest))
.map((byte) => byte.toString(16).padStart(2, "0"))
.join("");
}
function buildGitHubSourceImport(repo: string, defaultBranch: string): GitHubImportUrl {
const normalizedRepo = normalizeRepo(repo);
const [owner, repoName] = normalizedRepo.split("/") as [string, string];
+40
View File
@@ -0,0 +1,40 @@
"use node";
import { v } from "convex/values";
import { internalAction } from "./functions";
import { syncGitHubSkillSourcesHandler, verifyGitHubSkillHandler } from "./githubSkillSync";
import { Events, logErrorEvent } from "./lib/observabilityEvents";
function getErrorMessage(error: unknown) {
if (error instanceof Error) return error.message;
if (typeof error === "string") return error;
if (error && typeof error === "object" && "message" in error) {
const message = (error as { message?: unknown }).message;
if (typeof message === "string") return message;
}
return String(error);
}
export const syncGitHubSkillSourcesInternal = internalAction({
args: {
cursor: v.optional(v.union(v.string(), v.null())),
batchSize: v.optional(v.number()),
},
handler: async (ctx, args) => {
try {
return await syncGitHubSkillSourcesHandler(ctx, args);
} catch (error) {
logErrorEvent(Events.GitHubSkillSourceSyncFailed, { error: getErrorMessage(error) });
throw error;
}
},
});
export const verifyGitHubSkillInternal = internalAction({
args: {
skillId: v.id("skills"),
contentHash: v.string(),
force: v.optional(v.boolean()),
},
handler: verifyGitHubSkillHandler,
});

Some files were not shown because too many files have changed in this diff Show More