mirror of
https://github.com/openclaw/clawhub.git
synced 2026-08-15 17:32:22 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4a4ad1634c | ||
|
|
cb53d4ab15 | ||
|
|
52970a3e3d | ||
|
|
14beb939ba |
@@ -1,95 +1,68 @@
|
||||
---
|
||||
name: clawhub-moderation
|
||||
description: "Use for ClawHub staff moderation actions with the repo-local clawhub-mod/admin tool: skills, users, org publishers, plugin packages, trusted publishers, official publishers, and guarded staff email."
|
||||
description: "Use for ClawHub staff moderation actions with the repo-local clawhub-mod tool: unhide skills, ban or unban users, change roles, and verify moderation state."
|
||||
---
|
||||
|
||||
# ClawHub Moderation
|
||||
|
||||
Use the repo-local admin tool from a checked-out ClawHub repo. It wraps
|
||||
Use the repo-local `clawhub-mod` tool from a checked-out ClawHub repo. It wraps
|
||||
the existing ClawHub CLI auth/config and HTTP API surfaces. Do not call Convex
|
||||
internal mutations directly for staff actions.
|
||||
|
||||
## Safety Rules
|
||||
|
||||
- Require an explicit target from the user: skill slug, user handle, or user id.
|
||||
- Require a reason for destructive, restorative, ownership, or moderation writes.
|
||||
- Require a reason for `skills unhide`, `users ban`, and `users unban`.
|
||||
- Before any write, show the exact command and ask for confirmation unless the
|
||||
user already said to proceed or supplied `--yes`.
|
||||
- For `email send`, the user must explicitly ask for the email and sign off on
|
||||
the final recipient, subject, and body. Dry-run is fine for drafting. Never
|
||||
send until both are true, and only use `--send --confirm-user-request
|
||||
--confirm-user-signoff` after that approval.
|
||||
- Prefer handles for humans. Use `--id` only when the user provides a user id.
|
||||
- Never bypass API-token auth, server role checks, or audit logging.
|
||||
- After the write, verify state with the CLI/API and report the result.
|
||||
|
||||
## Command Map
|
||||
## Commands
|
||||
|
||||
Run from the ClawHub repo root:
|
||||
|
||||
```sh
|
||||
bun run admin -- --help
|
||||
bun run mod -- --help
|
||||
```
|
||||
|
||||
Authenticate or validate the current token:
|
||||
|
||||
```sh
|
||||
bun run admin -- login
|
||||
bun run admin -- whoami
|
||||
bun run mod -- login
|
||||
bun run mod -- whoami
|
||||
```
|
||||
|
||||
Current top-level command groups:
|
||||
|
||||
```text
|
||||
auth
|
||||
users
|
||||
plugins|plugin
|
||||
packages|package
|
||||
org
|
||||
email
|
||||
skills|skill
|
||||
```
|
||||
|
||||
### Skills
|
||||
|
||||
`bun run admin -- skills --help` exposes:
|
||||
|
||||
```text
|
||||
unhide <slug>
|
||||
rescan <slug>
|
||||
reports
|
||||
triage-report <report-id>
|
||||
```
|
||||
|
||||
Examples:
|
||||
Unhide a skill after moderator review:
|
||||
|
||||
```sh
|
||||
bun run admin -- skills unhide <slug> --reason "<reason>" --yes
|
||||
bun run admin -- skills rescan <slug> --reason "<reason>" --yes
|
||||
bun run admin -- skills reports --status open
|
||||
bun run admin -- skills triage-report <report-id> --status confirmed --action hide --note "<note>" --yes
|
||||
bun run mod -- skills unhide <slug> --reason "<reason>" --yes
|
||||
```
|
||||
|
||||
### Users
|
||||
|
||||
`bun run admin -- users --help` exposes:
|
||||
|
||||
```text
|
||||
ban <handleOrId>
|
||||
unban <handleOrId>
|
||||
set-role <handleOrId> <role>
|
||||
reclassify-ban <handleOrId>
|
||||
remediate-autobans
|
||||
```
|
||||
|
||||
Examples:
|
||||
List and triage skill reports:
|
||||
|
||||
```sh
|
||||
bun run admin -- users ban <handleOrId> --reason "<reason>" --yes
|
||||
bun run admin -- users unban <handleOrId> --reason "<reason>" --yes
|
||||
bun run admin -- users set-role <handleOrId> <user|moderator|admin> --yes
|
||||
bun run admin -- users reclassify-ban <handleOrId> --reason "<reason>" --apply --yes
|
||||
bun run admin -- users remediate-autobans --apply --reason "<reason>"
|
||||
bun run mod -- skills reports --status open
|
||||
bun run mod -- skills triage-report <report-id> --status confirmed --action hide --note "<note>" --yes
|
||||
```
|
||||
|
||||
Ban a user:
|
||||
|
||||
```sh
|
||||
bun run mod -- users ban <handleOrId> --reason "<reason>" --yes
|
||||
```
|
||||
|
||||
Unban a user:
|
||||
|
||||
```sh
|
||||
bun run mod -- users unban <handleOrId> --reason "<reason>" --yes
|
||||
```
|
||||
|
||||
Change a user role:
|
||||
|
||||
```sh
|
||||
bun run mod -- users set-role <handleOrId> <user|moderator|admin> --yes
|
||||
```
|
||||
|
||||
Use `--id` when `<handleOrId>` is a user id. Use `--fuzzy` only when the user
|
||||
@@ -98,108 +71,15 @@ has asked for fuzzy handle resolution or the exact handle is ambiguous.
|
||||
The old top-level aliases still exist for user commands:
|
||||
|
||||
```sh
|
||||
bun run admin -- ban-user <handleOrId> --reason "<reason>" --yes
|
||||
bun run admin -- unban-user <handleOrId> --reason "<reason>" --yes
|
||||
bun run mod -- ban-user <handleOrId> --reason "<reason>" --yes
|
||||
bun run mod -- unban-user <handleOrId> --reason "<reason>" --yes
|
||||
```
|
||||
|
||||
### Org Publishers
|
||||
|
||||
`bun run admin -- org --help` exposes:
|
||||
|
||||
```text
|
||||
official
|
||||
create <handle>
|
||||
remove-member <handle> <member>
|
||||
delete <handle>
|
||||
repair-scoped-packages <csv>
|
||||
```
|
||||
|
||||
Examples:
|
||||
|
||||
```sh
|
||||
bun run admin -- org official list
|
||||
bun run admin -- org official add <handle> --reason "<reason>" --yes
|
||||
bun run admin -- org official remove <handle> --reason "<reason>" --yes
|
||||
bun run admin -- org create <handle> --display-name "<name>" --member <user-handle> --role owner
|
||||
bun run admin -- org remove-member <handle> <member-handle>
|
||||
bun run admin -- org delete <handle> --reason "<reason>" # dry-run
|
||||
bun run admin -- org delete <handle> --reason "<reason>" --apply
|
||||
bun run admin -- org repair-scoped-packages <csv> # dry-run
|
||||
bun run admin -- org repair-scoped-packages <csv> --apply
|
||||
```
|
||||
|
||||
`org create` requires `--member`; it must not add the moderator running the
|
||||
command as an implicit owner. `org delete` only works for empty org publishers
|
||||
and defaults to dry-run.
|
||||
|
||||
### Plugin Packages
|
||||
|
||||
`bun run admin -- packages --help` exposes:
|
||||
|
||||
```text
|
||||
moderate <name>
|
||||
status|moderation-status <name>
|
||||
queue|moderation-queue
|
||||
reports
|
||||
triage-report <report-id>
|
||||
transfer <name>
|
||||
repair-name <name>
|
||||
migrations
|
||||
set-migration <bundled-plugin-id>
|
||||
trusted-publisher
|
||||
```
|
||||
|
||||
Examples:
|
||||
|
||||
```sh
|
||||
bun run admin -- packages status <name>
|
||||
bun run admin -- packages transfer <name> --to <owner> --reason "<reason>" # dry-run
|
||||
bun run admin -- packages transfer <name> --to <owner> --reason "<reason>" --apply
|
||||
bun run admin -- packages repair-name <name> --next-name <name> --reason "<reason>"
|
||||
bun run admin -- packages trusted-publisher get <name>
|
||||
bun run admin -- packages trusted-publisher set <name> --repository <owner/repo> --workflow-filename <file>
|
||||
```
|
||||
|
||||
### Staff Email
|
||||
|
||||
`bun run admin -- email send --help` exposes:
|
||||
|
||||
```text
|
||||
--to <email>
|
||||
--user <handle>
|
||||
--subject <subject>
|
||||
--body-file <path>
|
||||
--body <text>
|
||||
--send
|
||||
--confirm-user-request
|
||||
--confirm-user-signoff
|
||||
--json
|
||||
```
|
||||
|
||||
Draft only:
|
||||
|
||||
```sh
|
||||
bun run admin -- email send --user <handle> --subject "<subject>" --body-file <path>
|
||||
```
|
||||
|
||||
Send only after explicit request and sign-off:
|
||||
|
||||
```sh
|
||||
bun run admin -- email send --user <handle> --subject "<subject>" --body-file <path> --send --confirm-user-request --confirm-user-signoff
|
||||
```
|
||||
|
||||
The server sends through the production noreply provider and writes an audit log
|
||||
only after admin auth succeeds.
|
||||
|
||||
## Verification
|
||||
|
||||
- For skills, inspect the page/API status after `skills unhide`.
|
||||
- For users, prefer user search/admin surfaces for target accounts where
|
||||
available.
|
||||
- For orgs and packages, use the public publisher/plugin pages and the relevant
|
||||
CLI status command after a write.
|
||||
- For email, verify the CLI response and audit expectation; do not send a second
|
||||
email just to test delivery.
|
||||
- For users, prefer `bun run mod -- whoami` for the current token and user
|
||||
search/admin surfaces for target accounts where available.
|
||||
- If verification is blocked by auth or missing admin access, report the command
|
||||
result and the verification blocker plainly.
|
||||
|
||||
@@ -214,7 +94,3 @@ only after admin auth succeeds.
|
||||
hides owned skills, soft-deletes comments, and writes audit logs.
|
||||
- `unban-user` is admin-only. It clears ban state and restores skills that were
|
||||
hidden by the matching ban flow; revoked API tokens stay revoked.
|
||||
- `packages transfer` preserves the package row, stats, releases, and history;
|
||||
it changes the owner publisher.
|
||||
- `org delete` soft-deletes an empty org publisher and retains member rows for
|
||||
history; it refuses orgs with active skills or packages.
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
---
|
||||
name: clawhub-ui-proof
|
||||
description: Use when ClawHub UI changes need visual proof, before/after comparison, new-feature screenshots, temporary Playwright scenarios, or Crabbox desktop recordings.
|
||||
---
|
||||
|
||||
# ClawHub UI Proof
|
||||
|
||||
Use `proof:ui` for human-readable UI evidence. The agent should write a
|
||||
temporary scenario for the feature instead of manually clicking through the UI.
|
||||
|
||||
## Pick A Mode
|
||||
|
||||
- Use `--mode before-after` for bug fixes, regressions, changed copy, changed
|
||||
layout, or anything where main-vs-candidate comparison helps. This is the
|
||||
default and runs baseline `origin/main` plus the candidate worktree.
|
||||
- Use `--mode feature` for new pages, new workflows, or new UI states that do
|
||||
not exist on main. This runs only the candidate lane.
|
||||
- Every proof lane runs full-stack by default: the lane's Git checkout starts
|
||||
its own local Convex backend, pushes that lane's functions/schema, and builds
|
||||
the frontend against that lane-local Convex URL. Add
|
||||
`--seed-command '<command>'` when the scenario needs fixtures.
|
||||
- Dev auth is opt-in. Use `--dev-auth` or explicit `--env KEY=VALUE` entries
|
||||
only for scenarios that need development auth controls.
|
||||
- Do not use `proof:ui` to inspect contributor-provided screenshots, videos, or
|
||||
logs. Review those artifacts directly and cite what they prove or fail to
|
||||
prove.
|
||||
|
||||
## Scenario Shape
|
||||
|
||||
Create a temporary scenario under `.artifacts/proof-scenarios/`:
|
||||
|
||||
```js
|
||||
export default async function scenario({ baseURL, expect, page, proof }) {
|
||||
await proof.step("01 skills list", async () => {
|
||||
await page.goto(`${baseURL}/skills`);
|
||||
await expect(page.getByText("Skills")).toBeVisible();
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
Each `proof.step()` captures a screenshot after the step. The runner compares
|
||||
`origin/main` to the current worktree by default in `before-after` mode.
|
||||
|
||||
## Commands
|
||||
|
||||
Dry-run the plan first. Before/after mode is the default:
|
||||
|
||||
```sh
|
||||
bun run proof:ui -- --mode before-after --scenario .artifacts/proof-scenarios/my-fix.pw.ts --dry-run
|
||||
```
|
||||
|
||||
For new feature proof, run candidate-only:
|
||||
|
||||
```sh
|
||||
bun run proof:ui -- --mode feature --scenario .artifacts/proof-scenarios/my-feature.pw.ts --dry-run
|
||||
```
|
||||
|
||||
Run real desktop proof on a Crabbox-owned provider:
|
||||
|
||||
```sh
|
||||
bun run proof:ui -- --mode before-after --scenario .artifacts/proof-scenarios/my-fix.pw.ts --provider hetzner
|
||||
```
|
||||
|
||||
Run proof with seeded lane-local Convex fixtures:
|
||||
|
||||
```sh
|
||||
bun run proof:ui -- --mode before-after --seed-command 'bunx convex run --no-push devSeed:seedNixSkills' --scenario .artifacts/proof-scenarios/my-fix.pw.ts --provider hetzner
|
||||
```
|
||||
|
||||
Artifacts are written under `.artifacts/clawhub-ui-proof/<timestamp>/` with
|
||||
screenshots, videos when available, `summary.json`, and `report.md`. Feature
|
||||
mode has only candidate artifacts. Promote only broadly useful scenarios into
|
||||
committed `e2e/proofs/`.
|
||||
|
||||
## Publish To A PR
|
||||
|
||||
When UI proof should appear on a GitHub PR, publish the completed proof run
|
||||
instead of posting local paths:
|
||||
|
||||
```sh
|
||||
bun run proof:publish -- --proof-dir .artifacts/clawhub-ui-proof/<timestamp> --target-pr <number>
|
||||
```
|
||||
|
||||
`proof:publish` copies the selected screenshots, video preview GIFs when
|
||||
present, MP4s, `summary.json`, and `report.md` to the `qa-artifacts` branch,
|
||||
then upserts a marker-backed PR comment with inline screenshots/previews and
|
||||
linked MP4s. Use `--dry-run` first when drafting or checking the comment body.
|
||||
|
||||
## Share In GitHub Issues
|
||||
|
||||
When proof images or screenshots should appear in GitHub issues, share
|
||||
`here.now` links instead of uploading image attachments directly to GitHub.
|
||||
Include a short note about what the linked image proves.
|
||||
@@ -16,16 +16,5 @@ AUTH_GITHUB_SECRET=
|
||||
JWT_PRIVATE_KEY=
|
||||
JWKS=
|
||||
|
||||
# Local dev personas
|
||||
DEV_AUTH_ENABLED=
|
||||
DEV_AUTH_CONVEX_DEPLOYMENT=
|
||||
DEV_AUTH_SITE_URL=
|
||||
DEV_AUTH_SECRET=
|
||||
|
||||
# Embeddings
|
||||
OPENAI_API_KEY=
|
||||
|
||||
# Transactional email
|
||||
RESEND_API_KEY=
|
||||
CLAWHUB_SECURITY_EMAIL=security@notifications.openclaw.ai
|
||||
CLAWHUB_SECURITY_EMAIL_FROM=ClawHub Security <noreply@notifications.openclaw.ai>
|
||||
|
||||
@@ -1,19 +1,5 @@
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: "bun"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
day: "monday"
|
||||
time: "09:00"
|
||||
timezone: "America/Los_Angeles"
|
||||
open-pull-requests-limit: 3
|
||||
labels:
|
||||
- dependencies
|
||||
- needs-cli-release
|
||||
allow:
|
||||
- dependency-name: "@openclaw/plugin-inspector"
|
||||
|
||||
- package-ecosystem: "bun"
|
||||
directory: "/"
|
||||
schedule:
|
||||
@@ -23,7 +9,6 @@ updates:
|
||||
timezone: "America/Los_Angeles"
|
||||
open-pull-requests-limit: 10
|
||||
ignore:
|
||||
- dependency-name: "@openclaw/plugin-inspector"
|
||||
- dependency-name: "@auth/core"
|
||||
update-types:
|
||||
- "version-update:semver-minor"
|
||||
|
||||
@@ -22,7 +22,7 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Begin Testbox
|
||||
uses: useblacksmith/begin-testbox@233448af4bfdc6fca509a7f0974411ac6d8a8043
|
||||
uses: useblacksmith/begin-testbox@d0e04585c26905fdd92c94a09c159544c7ee1b67
|
||||
with:
|
||||
testbox_id: ${{ inputs.testbox_id }}
|
||||
|
||||
|
||||
@@ -88,13 +88,13 @@ jobs:
|
||||
|
||||
- name: Initialize CodeQL
|
||||
if: ${{ github.event_name != 'workflow_dispatch' || inputs.profile == 'all' || inputs.profile == matrix.category }}
|
||||
uses: github/codeql-action/init@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4
|
||||
uses: github/codeql-action/init@7211b7c8077ea37d8641b6271f6a365a22a5fbfa # v4
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
config-file: ${{ matrix.config_file }}
|
||||
|
||||
- name: Analyze
|
||||
if: ${{ github.event_name != 'workflow_dispatch' || inputs.profile == 'all' || inputs.profile == matrix.category }}
|
||||
uses: github/codeql-action/analyze@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4
|
||||
uses: github/codeql-action/analyze@7211b7c8077ea37d8641b6271f6a365a22a5fbfa # v4
|
||||
with:
|
||||
category: "/codeql-light/${{ matrix.category }}"
|
||||
|
||||
@@ -61,14 +61,11 @@ on:
|
||||
description: Optional source path inside the repository for monorepo package publishes.
|
||||
required: false
|
||||
type: string
|
||||
package_artifact_name:
|
||||
description: Optional Actions artifact name containing a prebuilt ClawPack .tgz to publish.
|
||||
required: false
|
||||
type: string
|
||||
package_artifact_path:
|
||||
description: Optional path to the .tgz inside package_artifact_name. Defaults to the only .tgz in the artifact.
|
||||
clawhub_version:
|
||||
description: Legacy npm CLI version input. Kept for compatibility; the workflow now runs the checked-out source.
|
||||
required: false
|
||||
type: string
|
||||
default: latest
|
||||
secrets:
|
||||
clawhub_token:
|
||||
required: false
|
||||
@@ -90,7 +87,6 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
id-token: write
|
||||
outputs:
|
||||
@@ -222,56 +218,7 @@ jobs:
|
||||
PY
|
||||
echo "CLAWHUB_CONFIG_PATH=$RUNNER_TEMP/clawhub-config.json" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Download prebuilt package artifact
|
||||
if: inputs.package_artifact_name != ''
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: ${{ inputs.package_artifact_name }}
|
||||
path: ${{ runner.temp }}/prebuilt-package-artifact
|
||||
|
||||
- name: Resolve prebuilt package artifact
|
||||
id: resolve_artifact
|
||||
env:
|
||||
INPUT_PACKAGE_ARTIFACT_NAME: ${{ inputs.package_artifact_name }}
|
||||
INPUT_PACKAGE_ARTIFACT_PATH: ${{ inputs.package_artifact_path }}
|
||||
run: |
|
||||
python3 - <<'PY'
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
artifact_name = os.environ["INPUT_PACKAGE_ARTIFACT_NAME"].strip()
|
||||
output_path = Path(os.environ["GITHUB_OUTPUT"])
|
||||
if not artifact_name:
|
||||
with output_path.open("a", encoding="utf-8") as fh:
|
||||
fh.write("package_artifact_path=\n")
|
||||
raise SystemExit(0)
|
||||
|
||||
artifact_root = Path(os.environ["RUNNER_TEMP"]) / "prebuilt-package-artifact"
|
||||
requested_path = os.environ["INPUT_PACKAGE_ARTIFACT_PATH"].strip()
|
||||
if requested_path:
|
||||
candidate = (artifact_root / requested_path).resolve()
|
||||
if artifact_root.resolve() not in candidate.parents and candidate != artifact_root.resolve():
|
||||
raise SystemExit(f"Prebuilt artifact path escapes downloaded artifact: {requested_path}")
|
||||
if not candidate.is_file():
|
||||
raise SystemExit(f"Prebuilt package artifact path not found: {requested_path}")
|
||||
else:
|
||||
candidates = sorted(path for path in artifact_root.rglob("*.tgz") if path.is_file())
|
||||
if not candidates:
|
||||
raise SystemExit(f"Prebuilt package artifact {artifact_name!r} did not contain a .tgz file.")
|
||||
if len(candidates) > 1:
|
||||
joined = ", ".join(str(path.relative_to(artifact_root)) for path in candidates)
|
||||
raise SystemExit(
|
||||
"Prebuilt package artifact contains multiple .tgz files; set package_artifact_path. "
|
||||
f"Found: {joined}"
|
||||
)
|
||||
candidate = candidates[0]
|
||||
|
||||
with output_path.open("a", encoding="utf-8") as fh:
|
||||
fh.write(f"package_artifact_path={candidate}\n")
|
||||
PY
|
||||
|
||||
- name: Resolve publish command
|
||||
id: resolve_publish
|
||||
env:
|
||||
INPUT_SOURCE: ${{ inputs.source }}
|
||||
INPUT_REF: ${{ inputs.ref }}
|
||||
@@ -283,11 +230,9 @@ jobs:
|
||||
INPUT_SOURCE_COMMIT: ${{ inputs.source_commit }}
|
||||
INPUT_SOURCE_REF: ${{ inputs.source_ref }}
|
||||
INPUT_SOURCE_PATH: ${{ inputs.source_path }}
|
||||
PREBUILT_PACKAGE_ARTIFACT_PATH: ${{ steps.resolve_artifact.outputs.package_artifact_path }}
|
||||
INPUT_SITE: ${{ inputs.site }}
|
||||
INPUT_REGISTRY: ${{ inputs.registry }}
|
||||
CLAWHUB_TOKEN: ${{ secrets.clawhub_token }}
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
GITHUB_EVENT_NAME: ${{ github.event_name }}
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
GITHUB_REF: ${{ github.ref }}
|
||||
@@ -298,78 +243,6 @@ jobs:
|
||||
import os
|
||||
import shlex
|
||||
from pathlib import Path
|
||||
from urllib.error import HTTPError
|
||||
from urllib.parse import quote, urlparse
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
def split_ref_path(value):
|
||||
if not value:
|
||||
return "", ""
|
||||
if ":" not in value:
|
||||
return value, ""
|
||||
ref, path = value.split(":", 1)
|
||||
return ref, path.strip("/")
|
||||
|
||||
def github_commit_exists(repo, ref):
|
||||
token = os.environ.get("GITHUB_TOKEN", "").strip()
|
||||
headers = {
|
||||
"Accept": "application/vnd.github+json",
|
||||
"User-Agent": "clawhub-package-publish",
|
||||
}
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
request = Request(
|
||||
f"https://api.github.com/repos/{repo}/commits/{quote(ref, safe='')}",
|
||||
headers=headers,
|
||||
)
|
||||
try:
|
||||
with urlopen(request, timeout=10) as response:
|
||||
return 200 <= response.status < 300
|
||||
except HTTPError as error:
|
||||
if error.code in (404, 422):
|
||||
return False
|
||||
raise
|
||||
|
||||
def resolve_github_url_ref_and_path(repo, kind, segments):
|
||||
min_path_segments = 1 if kind == "blob" else 0
|
||||
max_ref_segments = len(segments) - min_path_segments
|
||||
for ref_segment_count in range(max_ref_segments, 0, -1):
|
||||
ref = "/".join(segments[:ref_segment_count])
|
||||
path = "/".join(segments[ref_segment_count:]).strip("/")
|
||||
if kind == "blob" and not path:
|
||||
continue
|
||||
if not github_commit_exists(repo, ref):
|
||||
continue
|
||||
if kind == "blob":
|
||||
path = "/".join(path.split("/")[:-1]).strip("/")
|
||||
return ref, path
|
||||
raise SystemExit(f"GitHub ref not found in source URL for {repo}")
|
||||
|
||||
def parse_github_source(value):
|
||||
raw = value.strip()
|
||||
if raw.startswith("github:"):
|
||||
raw = raw[len("github:"):]
|
||||
if raw.startswith("https://") or raw.startswith("http://"):
|
||||
parsed = urlparse(raw)
|
||||
if parsed.netloc.lower() != "github.com":
|
||||
return None
|
||||
parts = [part for part in parsed.path.strip("/").split("/") if part]
|
||||
if len(parts) < 2:
|
||||
return None
|
||||
repo_name = parts[1][:-4] if parts[1].endswith(".git") else parts[1]
|
||||
repo = f"{parts[0]}/{repo_name}"
|
||||
if len(parts) >= 4 and parts[2] in {"tree", "blob"}:
|
||||
ref, path = resolve_github_url_ref_and_path(repo, parts[2], parts[3:])
|
||||
return {"repo": repo, "ref": ref, "path": path}
|
||||
return {"repo": repo, "ref": "", "path": ""}
|
||||
|
||||
source_part, at, ref_part = raw.partition("@")
|
||||
repo_parts = source_part.split("/")
|
||||
if len(repo_parts) != 2 or not repo_parts[0] or not repo_parts[1]:
|
||||
return None
|
||||
repo_name = repo_parts[1][:-4] if repo_parts[1].endswith(".git") else repo_parts[1]
|
||||
ref, path = split_ref_path(ref_part if at else "")
|
||||
return {"repo": f"{repo_parts[0]}/{repo_name}", "ref": ref, "path": path}
|
||||
|
||||
source = os.environ["INPUT_SOURCE"].strip()
|
||||
if not source:
|
||||
@@ -381,36 +254,6 @@ jobs:
|
||||
is_local_source = source.startswith(".") or source.startswith("/") or Path(source).exists()
|
||||
if ref and "@" not in source and not source.startswith("http") and not is_local_source:
|
||||
source = f"{source}@{ref}"
|
||||
source_path = os.environ["INPUT_SOURCE_PATH"].strip()
|
||||
prebuilt_artifact_path = os.environ["PREBUILT_PACKAGE_ARTIFACT_PATH"].strip()
|
||||
if source_path and prebuilt_artifact_path:
|
||||
raise SystemExit("Prebuilt artifact mode does not accept source_path; publish the already-packed artifact instead.")
|
||||
|
||||
inspect_checkout_repository = ""
|
||||
inspect_checkout_ref = ""
|
||||
inspect_local_root = str(Path(os.environ["GITHUB_WORKSPACE"]).resolve())
|
||||
inspect_subdir = source_path
|
||||
if prebuilt_artifact_path:
|
||||
inspect_local_root = str((Path(os.environ["RUNNER_TEMP"]) / "prebuilt-package-inspect").resolve())
|
||||
inspect_subdir = ""
|
||||
elif is_local_source:
|
||||
inspect_local_root = str(Path(source).resolve())
|
||||
else:
|
||||
github_source = parse_github_source(source)
|
||||
source_ref_differs_from_checkout = (
|
||||
bool(github_source and github_source["ref"])
|
||||
and github_source["ref"] != os.environ["GITHUB_SHA"]
|
||||
)
|
||||
if github_source and (
|
||||
github_source["repo"] != os.environ["GITHUB_REPOSITORY"]
|
||||
or source_ref_differs_from_checkout
|
||||
):
|
||||
inspect_checkout_repository = github_source["repo"]
|
||||
inspect_checkout_ref = github_source["ref"]
|
||||
inspect_local_root = str((Path(os.environ["GITHUB_WORKSPACE"]) / "clawhub-publish-source").resolve())
|
||||
inspect_subdir = source_path or github_source["path"]
|
||||
elif github_source:
|
||||
inspect_subdir = source_path or github_source["path"]
|
||||
|
||||
cli_entry = (
|
||||
Path(os.environ["GITHUB_WORKSPACE"])
|
||||
@@ -423,13 +266,12 @@ jobs:
|
||||
if not cli_entry.exists():
|
||||
raise SystemExit(f"Missing ClawHub CLI entrypoint at {cli_entry}")
|
||||
|
||||
cmd_source = prebuilt_artifact_path or source
|
||||
cmd = [
|
||||
"bun",
|
||||
str(cli_entry),
|
||||
"package",
|
||||
"publish",
|
||||
cmd_source,
|
||||
source,
|
||||
"--site",
|
||||
os.environ["INPUT_SITE"],
|
||||
"--registry",
|
||||
@@ -452,16 +294,7 @@ jobs:
|
||||
source_repo = os.environ["INPUT_SOURCE_REPO"].strip()
|
||||
source_commit = os.environ["INPUT_SOURCE_COMMIT"].strip()
|
||||
source_ref = os.environ["INPUT_SOURCE_REF"].strip()
|
||||
if prebuilt_artifact_path:
|
||||
if not source_repo and not source_commit:
|
||||
source_repo = os.environ["GITHUB_REPOSITORY"].strip()
|
||||
source_commit = os.environ["GITHUB_SHA"].strip()
|
||||
elif not source_repo or not source_commit:
|
||||
raise SystemExit(
|
||||
"Prebuilt artifact mode requires source_repo and source_commit together when overriding source attribution."
|
||||
)
|
||||
if not source_ref:
|
||||
source_ref = os.environ["GITHUB_REF"].strip()
|
||||
source_path = os.environ["INPUT_SOURCE_PATH"].strip()
|
||||
if source_repo:
|
||||
cmd += ["--source-repo", source_repo]
|
||||
if source_commit:
|
||||
@@ -485,65 +318,8 @@ jobs:
|
||||
path.write_text("#!/usr/bin/env bash\nset -euo pipefail\n" + shell_line + "\n", encoding="utf-8")
|
||||
path.chmod(0o755)
|
||||
print(shell_line)
|
||||
|
||||
output_path = Path(os.environ["GITHUB_OUTPUT"])
|
||||
with output_path.open("a", encoding="utf-8") as fh:
|
||||
fh.write(f"inspect_checkout_repository={inspect_checkout_repository}\n")
|
||||
fh.write(f"inspect_checkout_ref={inspect_checkout_ref}\n")
|
||||
fh.write(f"inspect_local_root={inspect_local_root}\n")
|
||||
fh.write(f"inspect_subdir={inspect_subdir}\n")
|
||||
PY
|
||||
|
||||
- name: Extract prebuilt package artifact for plugin validation
|
||||
if: steps.resolve_artifact.outputs.package_artifact_path != ''
|
||||
env:
|
||||
PREBUILT_PACKAGE_ARTIFACT_PATH: ${{ steps.resolve_artifact.outputs.package_artifact_path }}
|
||||
INSPECT_LOCAL_ROOT: ${{ steps.resolve_publish.outputs.inspect_local_root }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p "$INSPECT_LOCAL_ROOT"
|
||||
tar -xzf "$PREBUILT_PACKAGE_ARTIFACT_PATH" -C "$INSPECT_LOCAL_ROOT" --strip-components=1
|
||||
|
||||
- name: Checkout publish source for plugin inspector
|
||||
if: steps.resolve_publish.outputs.inspect_checkout_repository != ''
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
repository: ${{ steps.resolve_publish.outputs.inspect_checkout_repository }}
|
||||
ref: ${{ steps.resolve_publish.outputs.inspect_checkout_ref }}
|
||||
path: clawhub-publish-source
|
||||
|
||||
- name: Run plugin validation
|
||||
env:
|
||||
INSPECT_LOCAL_ROOT: ${{ steps.resolve_publish.outputs.inspect_local_root }}
|
||||
INSPECT_SUBDIR: ${{ steps.resolve_publish.outputs.inspect_subdir }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
inspect_root="$(python3 - <<'PY'
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
root = Path(os.environ["INSPECT_LOCAL_ROOT"]).resolve()
|
||||
subdir = os.environ["INSPECT_SUBDIR"].strip()
|
||||
inspect_root = (root / subdir).resolve() if subdir else root
|
||||
if inspect_root != root and root not in inspect_root.parents:
|
||||
raise SystemExit(f"Inspector source path escapes publish source: {subdir}")
|
||||
print(inspect_root)
|
||||
PY
|
||||
)"
|
||||
if [ ! -f "$inspect_root/package.json" ] && [ ! -f "$inspect_root/openclaw.plugin.json" ]; then
|
||||
echo "::warning::Plugin Inspector skipped because $inspect_root is not a plugin root."
|
||||
exit 0
|
||||
fi
|
||||
bun "$GITHUB_WORKSPACE/clawhub-source/packages/clawhub/src/cli.ts" package validate "$inspect_root" --out "$RUNNER_TEMP/plugin-inspector"
|
||||
|
||||
- name: Upload plugin inspector reports
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: plugin-inspector-report
|
||||
path: ${{ runner.temp }}/plugin-inspector
|
||||
if-no-files-found: ignore
|
||||
|
||||
- name: Run package publish
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
@@ -1,63 +0,0 @@
|
||||
name: Plugin Inspector Bulk Scan
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
batch_size:
|
||||
description: "Maximum plugin releases to scan"
|
||||
required: false
|
||||
default: "25"
|
||||
dry_run:
|
||||
description: "Preview impact without persisting findings or sending emails"
|
||||
required: false
|
||||
default: "false"
|
||||
type: boolean
|
||||
dry_run_max_batches:
|
||||
description: "Maximum preview batches to scan when dry_run is enabled"
|
||||
required: false
|
||||
default: "20"
|
||||
source_pr:
|
||||
description: "Merged PR number that triggered this scan, when dispatched automatically"
|
||||
required: false
|
||||
default: ""
|
||||
source_sha:
|
||||
description: "Merged commit SHA that triggered this scan, when dispatched automatically"
|
||||
required: false
|
||||
default: ""
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
scan:
|
||||
name: Scan published plugins
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Bun
|
||||
uses: oven-sh/setup-bun@v2
|
||||
|
||||
- name: Install dependencies
|
||||
run: bun install --frozen-lockfile
|
||||
|
||||
- name: Run plugin inspector bulk scan
|
||||
env:
|
||||
CLAWHUB_SITE_URL: ${{ vars.CLAWHUB_SITE_URL || 'https://clawhub.ai' }}
|
||||
CLAWHUB_PLUGIN_INSPECTOR_WORKER_TOKEN: ${{ secrets.CLAWHUB_PLUGIN_INSPECTOR_WORKER_TOKEN }}
|
||||
PLUGIN_INSPECTOR_BATCH_SIZE: ${{ inputs.batch_size || '25' }}
|
||||
PLUGIN_INSPECTOR_DRY_RUN: ${{ inputs.dry_run && '1' || '0' }}
|
||||
PLUGIN_INSPECTOR_DRY_RUN_MAX_BATCHES: ${{ inputs.dry_run_max_batches || '20' }}
|
||||
PLUGIN_INSPECTOR_SOURCE_PR: ${{ inputs.source_pr || '' }}
|
||||
PLUGIN_INSPECTOR_SOURCE_SHA: ${{ inputs.source_sha || '' }}
|
||||
PLUGIN_INSPECTOR_ARTIFACT_DIR: plugin-inspector-bulk-scan-reports
|
||||
run: bun scripts/package-inspector-nightly-scan.ts
|
||||
|
||||
- name: Upload inspector reports
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: plugin-inspector-bulk-scan-reports
|
||||
path: plugin-inspector-bulk-scan-reports
|
||||
if-no-files-found: warn
|
||||
@@ -1,55 +0,0 @@
|
||||
name: Plugin Inspector Pin Bump Dispatch
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- package.json
|
||||
- packages/clawhub/package.json
|
||||
- bun.lock
|
||||
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
dispatch-plugin-inspector-bulk-scan:
|
||||
name: Dispatch Plugin Inspector bulk scan
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout main commit
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.sha }}
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Fetch previous main commit
|
||||
env:
|
||||
BASE_SHA: ${{ github.event.before }}
|
||||
run: git fetch --no-tags --depth=1 origin "$BASE_SHA"
|
||||
|
||||
- name: Detect pinned Plugin Inspector change
|
||||
id: detect
|
||||
env:
|
||||
BASE_SHA: ${{ github.event.before }}
|
||||
HEAD_SHA: ${{ github.sha }}
|
||||
run: node scripts/github/plugin-inspector-pin-change.mjs --base "$BASE_SHA" --head "$HEAD_SHA"
|
||||
|
||||
- name: Dispatch Plugin Inspector bulk scan
|
||||
if: ${{ steps.detect.outputs.changed == 'true' }}
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
gh workflow run plugin-inspector-bulk-scan.yml \
|
||||
--ref main \
|
||||
-f batch_size=25 \
|
||||
-f dry_run=false \
|
||||
-f dry_run_max_batches=20 \
|
||||
-f source_sha=${{ github.sha }}
|
||||
|
||||
- name: Explain skipped dispatch
|
||||
if: ${{ steps.detect.outputs.changed != 'true' }}
|
||||
env:
|
||||
DISPATCH_SKIP_REASON: ${{ steps.detect.outputs.reason }}
|
||||
run: printf '%s\n' "$DISPATCH_SKIP_REASON"
|
||||
@@ -1,84 +0,0 @@
|
||||
name: ClawHub Scheduled Live Checks
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "17 5 * * *"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
github-repo:
|
||||
description: GitHub skills repo to use for the source-backed canary
|
||||
required: false
|
||||
default: openclaw/agent-skills
|
||||
github-skill:
|
||||
description: Skill slug to verify from the GitHub skills repo
|
||||
required: false
|
||||
default: handoff
|
||||
|
||||
concurrency:
|
||||
group: clawhub-scheduled-live-checks-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
VITE_CONVEX_URL: https://example.invalid
|
||||
|
||||
jobs:
|
||||
github-backed-skills:
|
||||
name: GitHub-backed skills canary
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- uses: ./.github/actions/setup-bun
|
||||
|
||||
- name: Run GitHub-backed skills live canary
|
||||
env:
|
||||
CLAWHUB_LIVE_GITHUB_CANARY: "1"
|
||||
CLAWHUB_LIVE_GITHUB_REPO: ${{ inputs.github-repo || 'openclaw/agent-skills' }}
|
||||
CLAWHUB_LIVE_GITHUB_SKILL: ${{ inputs.github-skill || 'handoff' }}
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: bunx vitest run convex/githubSkillSync.live.test.ts
|
||||
|
||||
open-failure-issue:
|
||||
name: Open failure issue
|
||||
needs: github-backed-skills
|
||||
if: ${{ always() && needs.github-backed-skills.result == 'failure' }}
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
issues: write
|
||||
steps:
|
||||
- name: Open or update failure issue
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
WORKFLOW_NAME: ${{ github.workflow }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
marker_token="clawhub-scheduled-live-checks-failure"
|
||||
marker="<!-- $marker_token -->"
|
||||
title="ClawHub scheduled live checks failing"
|
||||
issue_number="$(gh issue list \
|
||||
--repo "$GITHUB_REPOSITORY" \
|
||||
--state open \
|
||||
--search "$marker_token in:body" \
|
||||
--json number \
|
||||
--jq '.[0].number // empty')"
|
||||
|
||||
body_file="$(mktemp)"
|
||||
cat > "$body_file" <<EOF
|
||||
$marker
|
||||
The scheduled ClawHub live checks failed.
|
||||
|
||||
Workflow: $WORKFLOW_NAME
|
||||
Run: $RUN_URL
|
||||
EOF
|
||||
|
||||
if [[ -n "$issue_number" ]]; then
|
||||
gh issue comment "$issue_number" --repo "$GITHUB_REPOSITORY" --body-file "$body_file"
|
||||
else
|
||||
gh issue create --repo "$GITHUB_REPOSITORY" --title "$title" --body-file "$body_file"
|
||||
fi
|
||||
@@ -51,7 +51,7 @@ jobs:
|
||||
id: trufflehog
|
||||
# Use a concrete released ref that resolves in upstream action registry.
|
||||
# v3 (major tag) is not published by trufflesecurity/trufflehog.
|
||||
uses: trufflesecurity/trufflehog@v3.95.5
|
||||
uses: trufflesecurity/trufflehog@v3.95.3
|
||||
with:
|
||||
path: ./
|
||||
base: ${{ steps.scan_range.outputs.base }}
|
||||
|
||||
@@ -1,292 +0,0 @@
|
||||
name: Skill Publish
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
skill_path:
|
||||
description: Optional path to one skill folder. When set, only this skill is processed.
|
||||
required: false
|
||||
type: string
|
||||
default: ""
|
||||
root:
|
||||
description: Directory containing skill folders for bulk catalog publishing.
|
||||
required: false
|
||||
type: string
|
||||
default: skills
|
||||
dry_run:
|
||||
description: Preview only. When true, no publish mutation is performed.
|
||||
required: false
|
||||
type: boolean
|
||||
default: true
|
||||
owner:
|
||||
description: Optional owner/publisher handle for org publishing.
|
||||
required: false
|
||||
type: string
|
||||
default: ""
|
||||
tags:
|
||||
description: Optional comma-separated tags override.
|
||||
required: false
|
||||
type: string
|
||||
default: latest
|
||||
bump:
|
||||
description: Version bump for updated skills. One of patch, minor, or major.
|
||||
required: false
|
||||
type: string
|
||||
default: patch
|
||||
registry:
|
||||
description: ClawHub registry URL.
|
||||
required: false
|
||||
type: string
|
||||
default: https://clawhub.ai
|
||||
site:
|
||||
description: ClawHub site URL.
|
||||
required: false
|
||||
type: string
|
||||
default: https://clawhub.ai
|
||||
ref:
|
||||
description: Optional caller repository ref to check out.
|
||||
required: false
|
||||
type: string
|
||||
default: ""
|
||||
secrets:
|
||||
clawhub_token:
|
||||
required: false
|
||||
outputs:
|
||||
publish_json:
|
||||
description: Structured JSON output from clawhub sync.
|
||||
value: ${{ jobs.publish.outputs.publish_json }}
|
||||
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
outputs:
|
||||
publish_json: ${{ steps.capture.outputs.publish_json }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.sha }}
|
||||
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6
|
||||
with:
|
||||
bun-version: 1.3.10
|
||||
|
||||
- name: Resolve ClawHub workflow source
|
||||
id: clawhub_source
|
||||
run: |
|
||||
python3 - <<'PY'
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
request_token = os.environ.get("ACTIONS_ID_TOKEN_REQUEST_TOKEN", "").strip()
|
||||
request_url = os.environ.get("ACTIONS_ID_TOKEN_REQUEST_URL", "").strip()
|
||||
if not request_token or not request_url:
|
||||
raise SystemExit("GitHub OIDC token request env vars are missing; id-token: write is required.")
|
||||
|
||||
audience = "clawhub-workflow-source"
|
||||
joiner = "&" if "?" in request_url else "?"
|
||||
token_url = f"{request_url}{joiner}audience={audience}"
|
||||
request = Request(token_url, headers={"Authorization": f"Bearer {request_token}"})
|
||||
with urlopen(request) as response:
|
||||
payload = json.load(response)
|
||||
|
||||
token = str(payload.get("value", "")).strip()
|
||||
if not token:
|
||||
raise SystemExit("GitHub OIDC token response did not include a token value.")
|
||||
|
||||
try:
|
||||
encoded_payload = token.split(".")[1]
|
||||
except IndexError as exc:
|
||||
raise SystemExit("GitHub OIDC token was not a valid JWT.") from exc
|
||||
padding = "=" * (-len(encoded_payload) % 4)
|
||||
claims = json.loads(base64.urlsafe_b64decode(encoded_payload + padding).decode("utf-8"))
|
||||
|
||||
workflow_ref = str(claims.get("job_workflow_ref", "")).strip()
|
||||
workflow_sha = str(claims.get("job_workflow_sha", "")).strip()
|
||||
repo, marker, _ = workflow_ref.partition("/.github/workflows/")
|
||||
if not marker or not repo or not workflow_sha:
|
||||
raise SystemExit(
|
||||
"Unable to resolve reusable workflow source from GitHub OIDC claims: "
|
||||
f"job_workflow_ref={workflow_ref!r} job_workflow_sha={workflow_sha!r}"
|
||||
)
|
||||
|
||||
output_path = Path(os.environ["GITHUB_OUTPUT"])
|
||||
with output_path.open("a", encoding="utf-8") as fh:
|
||||
fh.write(f"repository={repo}\n")
|
||||
fh.write(f"ref={workflow_sha}\n")
|
||||
PY
|
||||
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
repository: ${{ steps.clawhub_source.outputs.repository }}
|
||||
ref: ${{ steps.clawhub_source.outputs.ref }}
|
||||
path: clawhub-source
|
||||
|
||||
- name: Install ClawHub CLI dependencies
|
||||
working-directory: clawhub-source
|
||||
run: bun install --frozen-lockfile
|
||||
|
||||
- name: Validate publish mode inputs
|
||||
env:
|
||||
DRY_RUN: ${{ inputs.dry_run }}
|
||||
CLAWHUB_TOKEN: ${{ secrets.clawhub_token }}
|
||||
run: |
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
exit 0
|
||||
fi
|
||||
if [[ -n "$CLAWHUB_TOKEN" ]]; then
|
||||
exit 0
|
||||
fi
|
||||
echo "::error::Real skill publishes need secrets.clawhub_token. GitHub OIDC trusted publishing for skills is not supported yet."
|
||||
exit 1
|
||||
|
||||
- name: Write ClawHub config
|
||||
env:
|
||||
CLAWHUB_TOKEN: ${{ secrets.clawhub_token }}
|
||||
CLAWHUB_REGISTRY: ${{ inputs.registry }}
|
||||
run: |
|
||||
if [[ -z "$CLAWHUB_TOKEN" ]]; then
|
||||
echo "No ClawHub token provided, skipping config file creation."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
path = Path(os.environ["RUNNER_TEMP"]) / "clawhub-config.json"
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"registry": os.environ["CLAWHUB_REGISTRY"],
|
||||
"token": os.environ["CLAWHUB_TOKEN"],
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(path)
|
||||
PY
|
||||
echo "CLAWHUB_CONFIG_PATH=$RUNNER_TEMP/clawhub-config.json" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Resolve sync command
|
||||
env:
|
||||
INPUT_SKILL_PATH: ${{ inputs.skill_path }}
|
||||
INPUT_ROOT: ${{ inputs.root }}
|
||||
INPUT_DRY_RUN: ${{ inputs.dry_run }}
|
||||
INPUT_OWNER: ${{ inputs.owner }}
|
||||
INPUT_TAGS: ${{ inputs.tags }}
|
||||
INPUT_BUMP: ${{ inputs.bump }}
|
||||
INPUT_SITE: ${{ inputs.site }}
|
||||
INPUT_REGISTRY: ${{ inputs.registry }}
|
||||
INPUT_REF: ${{ inputs.ref }}
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
GITHUB_REF: ${{ github.ref }}
|
||||
run: |
|
||||
python3 - <<'PY'
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
skill_path = os.environ["INPUT_SKILL_PATH"].strip()
|
||||
root = os.environ["INPUT_ROOT"].strip() or "skills"
|
||||
scan_root = skill_path or root
|
||||
source_commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
|
||||
source_ref = os.environ["INPUT_REF"].strip() or os.environ["GITHUB_REF"].strip()
|
||||
|
||||
cli_entry = (
|
||||
Path(os.environ["GITHUB_WORKSPACE"])
|
||||
/ "clawhub-source"
|
||||
/ "packages"
|
||||
/ "clawhub"
|
||||
/ "src"
|
||||
/ "cli.ts"
|
||||
)
|
||||
if not cli_entry.exists():
|
||||
raise SystemExit(f"Missing ClawHub CLI entrypoint at {cli_entry}")
|
||||
|
||||
cmd = [
|
||||
"bun",
|
||||
str(cli_entry),
|
||||
"--workdir",
|
||||
scan_root,
|
||||
"--dir",
|
||||
".",
|
||||
"sync",
|
||||
"--all",
|
||||
"--json",
|
||||
"--no-clawdbot-roots",
|
||||
"--site",
|
||||
os.environ["INPUT_SITE"],
|
||||
"--registry",
|
||||
os.environ["INPUT_REGISTRY"],
|
||||
"--bump",
|
||||
os.environ["INPUT_BUMP"].strip() or "patch",
|
||||
"--source-repo",
|
||||
os.environ["GITHUB_REPOSITORY"],
|
||||
"--source-commit",
|
||||
source_commit,
|
||||
]
|
||||
|
||||
if os.environ["INPUT_DRY_RUN"] == "true":
|
||||
cmd.append("--dry-run")
|
||||
owner = os.environ["INPUT_OWNER"].strip()
|
||||
tags = os.environ["INPUT_TAGS"].strip()
|
||||
if owner:
|
||||
cmd += ["--owner", owner]
|
||||
if tags:
|
||||
cmd += ["--tags", tags]
|
||||
if source_ref:
|
||||
cmd += ["--source-ref", source_ref]
|
||||
|
||||
path = Path(os.environ["RUNNER_TEMP"]) / "clawhub-skill-publish-command.sh"
|
||||
shell_line = " ".join(shlex.quote(part) for part in cmd)
|
||||
path.write_text("#!/usr/bin/env bash\nset -euo pipefail\n" + shell_line + "\n", encoding="utf-8")
|
||||
path.chmod(0o755)
|
||||
print(shell_line)
|
||||
PY
|
||||
|
||||
- name: Run skill sync
|
||||
run: |
|
||||
set -euo pipefail
|
||||
"$RUNNER_TEMP/clawhub-skill-publish-command.sh" | tee "$RUNNER_TEMP/skill-publish.json"
|
||||
|
||||
- name: Capture workflow outputs
|
||||
id: capture
|
||||
run: |
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
output_path = Path(os.environ["RUNNER_TEMP"]) / "skill-publish.json"
|
||||
raw = output_path.read_text(encoding="utf-8").strip()
|
||||
parsed = json.loads(raw)
|
||||
|
||||
github_output = Path(os.environ["GITHUB_OUTPUT"])
|
||||
with github_output.open("a", encoding="utf-8") as fh:
|
||||
fh.write("publish_json<<__CLAWHUB_JSON__\n")
|
||||
fh.write(json.dumps(parsed, indent=2))
|
||||
fh.write("\n__CLAWHUB_JSON__\n")
|
||||
PY
|
||||
|
||||
- name: Upload publish JSON artifact
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: clawhub-skill-publish-json
|
||||
path: ${{ runner.temp }}/skill-publish.json
|
||||
if-no-files-found: error
|
||||
+12
-1
@@ -3,7 +3,6 @@ node_modules
|
||||
.bun-build
|
||||
*.bun-build
|
||||
.artifacts/
|
||||
artifacts/
|
||||
.cache/
|
||||
.data/
|
||||
bin/docs-list
|
||||
@@ -38,6 +37,18 @@ convex/_generated/ai/*
|
||||
!convex/_generated/ai/guidelines.md
|
||||
!convex/_generated/ai/ai-files.state.json
|
||||
skills-lock.json
|
||||
*/skills/*
|
||||
!.agents/skills/
|
||||
!.agents/skills/convex*/
|
||||
!.agents/skills/convex*/**
|
||||
!.agents/skills/clawhub-ui-proof/
|
||||
!.agents/skills/clawhub-ui-proof/**
|
||||
!.agents/skills/clawhub-pr-maintainer/
|
||||
!.agents/skills/clawhub-pr-maintainer/**
|
||||
!.agents/skills/clawhub-moderation/
|
||||
!.agents/skills/clawhub-moderation/**
|
||||
!.agents/skills/autoreview/
|
||||
!.agents/skills/autoreview/**
|
||||
skills/*
|
||||
.codex/*
|
||||
!.codex/environments/
|
||||
|
||||
@@ -51,8 +51,6 @@ Specialized corpus, scanner, security-worker, UI proof, proof publishing, Crabbo
|
||||
- Tests live in `src/**` and `convex/lib/**`.
|
||||
- Coverage threshold: 80% global (lines/functions/branches/statements).
|
||||
- Example: `convex/lib/skills.test.ts`.
|
||||
- When adding or changing Convex functions, do not rely only on mocked `ctx` tests for behavior that depends on Convex runtime semantics such as pagination, indexes, validators, auth identity, internal/public function boundaries, scheduler/cron behavior, actions calling queries/mutations, HTTP actions, storage, or OCC/transaction behavior. Add or run a real Convex validation path, such as `convex dev --once`, `convex run`, an HTTP action smoke, or a local-auth Playwright flow, covering the changed behavior. Mocked `ctx.db` / `ctx.runQuery` tests are still fine for pure business logic, but they do not count as Convex runtime validation.
|
||||
- For local UI state testing, prefer creating realistic backend state through seed logic plus a DevPersonaFab entry for the associated test user. Avoid one-off manual DB edits when the state is likely to be reused, such as org membership, official publisher access, moderation holds, or publishing permissions.
|
||||
|
||||
## Commit & Pull Request Guidelines
|
||||
|
||||
@@ -62,9 +60,9 @@ Specialized corpus, scanner, security-worker, UI proof, proof publishing, Crabbo
|
||||
- Before commit/PR handoff for non-trivial code changes, use `$autoreview` until no accepted/actionable findings remain, unless equivalent manual review already happened, the change is trivial/docs-only, or the user opts out.
|
||||
- Before opening a PR for source or test changes, run the targeted tests for the touched behavior and `bun run ci:unit` (`VITE_CONVEX_URL=https://example.invalid bun run coverage`) unless the change is docs/config-only or the user explicitly asks to rely on CI. For runtime, build, or package changes, also run the matching broader gate when it covers the touched surface: `bun run ci:types-build`, `bun run ci:packages`, `bun run ci:e2e-http`, or `bun run ci:playwright-smoke`.
|
||||
- PRs: include summary + test commands run. Add screenshots for UI changes.
|
||||
- Screenshot proof MUST come from a real running ClawHub instance in a real browser. Do not use generated HTML mockups, synthetic terminal cards, or manually composed images as proof. For route/status/backend visibility bugs, run ClawHub locally with the relevant Convex code and fixture state, capture the actual browser page, and state the local URL and fixture used.
|
||||
- Before merging any PR, verify TypeScript cleanly with `bunx tsc -p packages/schema/tsconfig.json --noEmit` and `bunx tsc -p packages/clawhub/tsconfig.json --noEmit`; if Convex code changed, also run the repo typecheck path used by deploy so `bunx convex deploy` will not fail on `tsc`.
|
||||
- GitHub comments: for multiline `gh` comments/close messages, use `--body-file`, `--input`, or stdin/heredoc with real newlines; never pass literal `\\n` in shell strings.
|
||||
- Reject PRs that add skills into source code/repo content directly (for example under `skills/` or seed-only additions intended as published skills). Skills must be uploaded/published via CLI.
|
||||
- Repo-local developer skills under `.agents/skills/` are allowed only when they are ClawHub-specific, such as Convex, moderation, PR maintainer, or UI proof workflows. Keep generic shared skills such as `crabbox` and `autoreview` in the global `agent-skills` install, not this repo. Keep top-level `skills/` reserved for installed/published skill content and ignored by git.
|
||||
|
||||
## Production Release
|
||||
@@ -103,11 +101,6 @@ Specialized corpus, scanner, security-worker, UI proof, proof publishing, Crabbo
|
||||
- For non-interactive prod deploys, use `bunx convex deploy -y` to skip confirmation.
|
||||
- If `bunx convex run --env-file .env.local ...` returns `401 MissingAccessToken` despite `bunx convex login`, workaround: omit `--env-file` and use `--deployment-name <name>` / `--prod`.
|
||||
|
||||
## Convex Migrations & Backfills
|
||||
|
||||
- Any Convex production data migration, backfill, destructive cleanup, schema narrowing, or table reshaping must start with the `convex-migration-helper` skill. Default to `@convex-dev/migrations` for production data changes because it provides batching, dry runs, resume/progress tracking, and safer operator UX. Exceptions require an explicit note explaining why the component is unnecessary, plus equivalent dry-run support, cursor batching, resume/progress behavior, confirmation for destructive writes, and real Convex runtime validation.
|
||||
- After a migration or cleanup is verified complete, remove temporary migration functions/code in a follow-up PR unless they are intentionally retained as ongoing maintenance tooling.
|
||||
|
||||
## Convex Query & Bandwidth Rules
|
||||
|
||||
- **Always use `.withIndex()` instead of `.filter()` for fields that can be indexed.** `.filter()` causes full table scans — every doc is read and billed. Even a single `.filter()` on a 16K-row table reads ~16 MB per call.
|
||||
|
||||
+1
-31
@@ -1,39 +1,9 @@
|
||||
# Changelog
|
||||
|
||||
## 0.20.2 - 2026-06-11
|
||||
|
||||
### Changes
|
||||
|
||||
- CLI packages now require Node.js 22 or newer, dropping the EOL Node 20 runtime floor.
|
||||
- CLI: add `clawhub package validate <source>` for local plugin validation with author-facing Plugin Inspector findings, remediation text, and report artifacts.
|
||||
|
||||
## 0.20.0 - 2026-06-06
|
||||
|
||||
### Changes
|
||||
|
||||
- CLI/API: replace local `clawhub scan` uploads with stored submitted-version scan report downloads, including owner-authorized `clawhub scan download <name> --version <version>` support for blocked skill and plugin submissions.
|
||||
|
||||
## 0.19.2 - 2026-06-05
|
||||
## Unreleased
|
||||
|
||||
### Fixes
|
||||
|
||||
- CLI: accept the legacy `clawhub skill verify --json` flag as a hidden compatibility no-op while continuing to print JSON by default.
|
||||
|
||||
## 0.19.1 - 2026-06-05
|
||||
|
||||
### Fixes
|
||||
|
||||
- CLI: install source-backed GitHub skills from the deployed `/api/v1/skills/:slug/install` resolver so `clawhub install` works for skills without hosted ClawHub versions.
|
||||
|
||||
## 0.19.0 - 2026-06-03
|
||||
|
||||
### Changes
|
||||
|
||||
- CLI/API: add authenticated `clawhub scan` submit/poll support for ephemeral local skill bundles and owner-authorized published skill scans, including JSON output and report ZIP downloads (#2479).
|
||||
|
||||
### Fixes
|
||||
|
||||
- Auth/Ops: keep GitHub account-age lookups on immutable numeric IDs, retry without auth when a configured GitHub token is rejected, and add an operator backfill for missing cached account ages.
|
||||
- API/CLI: report Skill Card verification with flattened skill/version metadata, ClawScan verdict fields at `security.*`, and supporting scanner evidence under `security.signals`.
|
||||
|
||||
## 0.18.0 - 2026-05-25
|
||||
|
||||
@@ -103,24 +103,6 @@ CLAWHUB_WORKTREE_SOURCE=/path/to/source/worktree bun run setup:worktree
|
||||
|
||||
The detached server writes runtime state under `.codex/runtime/`. Stop it with `wt --yes stop` before removing the worktree.
|
||||
|
||||
### Local Codex workers
|
||||
|
||||
Local dev does not start Codex-backed workers by default, so `dev:worktree` does
|
||||
not spend Codex quota.
|
||||
|
||||
To process local ClawScan or Skill Card jobs, opt in for that shell:
|
||||
|
||||
```bash
|
||||
CLAWHUB_ALLOW_LOCAL_CODEX_SCAN=1 bun run dev:workers -- --workers security-scan --once
|
||||
CLAWHUB_ALLOW_LOCAL_CODEX_SCAN=1 bun run dev:workers -- --workers skill-card --once
|
||||
```
|
||||
|
||||
Opted-in local runs use an ignored worktree-local `CODEX_HOME` unless you provide
|
||||
one.
|
||||
|
||||
Without those workers, local ClawScan and Skill Card jobs stay pending until you
|
||||
opt in, seed/mock results, or use the production workflows.
|
||||
|
||||
### Seed the database
|
||||
|
||||
Populate local QA fixtures and the committed public corpus so the UI isn't empty:
|
||||
|
||||
@@ -74,7 +74,7 @@ Docs: [`docs/quickstart.md`](docs/quickstart.md), [`docs/cli.md`](docs/cli.md).
|
||||
|
||||
## Telemetry
|
||||
|
||||
ClawHub tracks minimal **install telemetry** (to compute install counts) when you run `clawhub install` while logged in.
|
||||
ClawHub tracks minimal **install telemetry** (to compute install counts) when you run `clawhub sync` while logged in.
|
||||
Disable via:
|
||||
|
||||
```bash
|
||||
|
||||
-13
@@ -9,19 +9,6 @@ Good ClawHub advisory reports include bugs in:
|
||||
- authentication, authorization, or API tokens
|
||||
- scanning, moderation, or report handling
|
||||
|
||||
Because ClawHub is a hosted cloud application, ClawHub service vulnerabilities
|
||||
are not publicly disclosed by default. They are publicly disclosed when there is
|
||||
evidence of real user impact or when users need to take action.
|
||||
|
||||
Examples of real user impact include confirmed exploitation, exposure of user
|
||||
data or secrets, malicious content reaching users because of a platform failure,
|
||||
or any issue that requires users to rotate credentials, update local software, or
|
||||
take other protective action.
|
||||
|
||||
Vulnerabilities in user-installed software are publicly disclosed, such as
|
||||
ClawHub CLI packages, binaries, libraries, or other release artifacts that users
|
||||
need to update locally.
|
||||
|
||||
Do not use ClawHub advisories for vulnerabilities in a third-party skill or
|
||||
plugin's own source code. Report those directly to the publisher or source
|
||||
repository linked from the ClawHub listing.
|
||||
|
||||
Vendored
+2
-34
@@ -17,19 +17,14 @@ import type * as crons from "../crons.js";
|
||||
import type * as depRegistryScan from "../depRegistryScan.js";
|
||||
import type * as devSeed from "../devSeed.js";
|
||||
import type * as devSeedExtra from "../devSeedExtra.js";
|
||||
import type * as downloadMetrics from "../downloadMetrics.js";
|
||||
import type * as downloads from "../downloads.js";
|
||||
import type * as emailsNode from "../emailsNode.js";
|
||||
import type * as functions from "../functions.js";
|
||||
import type * as githubAccountAgeBackfill from "../githubAccountAgeBackfill.js";
|
||||
import type * as githubBackups from "../githubBackups.js";
|
||||
import type * as githubBackupsNode from "../githubBackupsNode.js";
|
||||
import type * as githubIdentity from "../githubIdentity.js";
|
||||
import type * as githubImport from "../githubImport.js";
|
||||
import type * as githubRestore from "../githubRestore.js";
|
||||
import type * as githubRestoreMutations from "../githubRestoreMutations.js";
|
||||
import type * as githubSkillSources from "../githubSkillSources.js";
|
||||
import type * as githubSkillSync from "../githubSkillSync.js";
|
||||
import type * as githubSoulBackups from "../githubSoulBackups.js";
|
||||
import type * as githubSoulBackupsNode from "../githubSoulBackupsNode.js";
|
||||
import type * as http from "../http.js";
|
||||
@@ -57,32 +52,27 @@ import type * as lib_changelog from "../lib/changelog.js";
|
||||
import type * as lib_clawpack from "../lib/clawpack.js";
|
||||
import type * as lib_commentScamPrompt from "../lib/commentScamPrompt.js";
|
||||
import type * as lib_contentTypes from "../lib/contentTypes.js";
|
||||
import type * as lib_depRegistryScan from "../lib/depRegistryScan.js";
|
||||
import type * as lib_devAuth from "../lib/devAuth.js";
|
||||
import type * as lib_devSeed from "../lib/devSeed.js";
|
||||
import type * as lib_emails from "../lib/emails.js";
|
||||
import type * as lib_embeddingVisibility from "../lib/embeddingVisibility.js";
|
||||
import type * as lib_embeddings from "../lib/embeddings.js";
|
||||
import type * as lib_githubAccount from "../lib/githubAccount.js";
|
||||
import type * as lib_githubActionsOidc from "../lib/githubActionsOidc.js";
|
||||
import type * as lib_githubAuth from "../lib/githubAuth.js";
|
||||
import type * as lib_githubBackup from "../lib/githubBackup.js";
|
||||
import type * as lib_githubIdentity from "../lib/githubIdentity.js";
|
||||
import type * as lib_githubImport from "../lib/githubImport.js";
|
||||
import type * as lib_githubProfileSync from "../lib/githubProfileSync.js";
|
||||
import type * as lib_githubRestoreHelpers from "../lib/githubRestoreHelpers.js";
|
||||
import type * as lib_githubSkillSync from "../lib/githubSkillSync.js";
|
||||
import type * as lib_githubSoulBackup from "../lib/githubSoulBackup.js";
|
||||
import type * as lib_globalStats from "../lib/globalStats.js";
|
||||
import type * as lib_httpHeaders from "../lib/httpHeaders.js";
|
||||
import type * as lib_httpRateLimit from "../lib/httpRateLimit.js";
|
||||
import type * as lib_httpUtils from "../lib/httpUtils.js";
|
||||
import type * as lib_installResolver from "../lib/installResolver.js";
|
||||
import type * as lib_leaderboards from "../lib/leaderboards.js";
|
||||
import type * as lib_manualOverrides from "../lib/manualOverrides.js";
|
||||
import type * as lib_moderation from "../lib/moderation.js";
|
||||
import type * as lib_moderationEngine from "../lib/moderationEngine.js";
|
||||
import type * as lib_moderationReasonCodes from "../lib/moderationReasonCodes.js";
|
||||
import type * as lib_observabilityEvents from "../lib/observabilityEvents.js";
|
||||
import type * as lib_officialPublishers from "../lib/officialPublishers.js";
|
||||
import type * as lib_openaiResponse from "../lib/openaiResponse.js";
|
||||
import type * as lib_packageRegistry from "../lib/packageRegistry.js";
|
||||
@@ -93,7 +83,6 @@ import type * as lib_public from "../lib/public.js";
|
||||
import type * as lib_publicRouteReservations from "../lib/publicRouteReservations.js";
|
||||
import type * as lib_publishLimits from "../lib/publishLimits.js";
|
||||
import type * as lib_publisherAbuseScoring from "../lib/publisherAbuseScoring.js";
|
||||
import type * as lib_publisherCatalogDisplay from "../lib/publisherCatalogDisplay.js";
|
||||
import type * as lib_publisherStats from "../lib/publisherStats.js";
|
||||
import type * as lib_publishers from "../lib/publishers.js";
|
||||
import type * as lib_reporting from "../lib/reporting.js";
|
||||
@@ -104,7 +93,6 @@ import type * as lib_securityPrompt from "../lib/securityPrompt.js";
|
||||
import type * as lib_skillBackfill from "../lib/skillBackfill.js";
|
||||
import type * as lib_skillCapabilityTags from "../lib/skillCapabilityTags.js";
|
||||
import type * as lib_skillCards from "../lib/skillCards.js";
|
||||
import type * as lib_skillFileAccess from "../lib/skillFileAccess.js";
|
||||
import type * as lib_skillIcon from "../lib/skillIcon.js";
|
||||
import type * as lib_skillPublish from "../lib/skillPublish.js";
|
||||
import type * as lib_skillQuality from "../lib/skillQuality.js";
|
||||
@@ -124,13 +112,9 @@ import type * as lib_userSkillStats from "../lib/userSkillStats.js";
|
||||
import type * as lib_webhooks from "../lib/webhooks.js";
|
||||
import type * as llmEval from "../llmEval.js";
|
||||
import type * as maintenance from "../maintenance.js";
|
||||
import type * as managementDevSeed from "../managementDevSeed.js";
|
||||
import type * as packageInspectorHttp from "../packageInspectorHttp.js";
|
||||
import type * as packageInspectorNode from "../packageInspectorNode.js";
|
||||
import type * as packagePublishTokens from "../packagePublishTokens.js";
|
||||
import type * as packages from "../packages.js";
|
||||
import type * as publisherAbuse from "../publisherAbuse.js";
|
||||
import type * as publisherAbuseDevSeed from "../publisherAbuseDevSeed.js";
|
||||
import type * as publishers from "../publishers.js";
|
||||
import type * as rateLimits from "../rateLimits.js";
|
||||
import type * as search from "../search.js";
|
||||
@@ -172,19 +156,14 @@ declare const fullApi: ApiFromModules<{
|
||||
depRegistryScan: typeof depRegistryScan;
|
||||
devSeed: typeof devSeed;
|
||||
devSeedExtra: typeof devSeedExtra;
|
||||
downloadMetrics: typeof downloadMetrics;
|
||||
downloads: typeof downloads;
|
||||
emailsNode: typeof emailsNode;
|
||||
functions: typeof functions;
|
||||
githubAccountAgeBackfill: typeof githubAccountAgeBackfill;
|
||||
githubBackups: typeof githubBackups;
|
||||
githubBackupsNode: typeof githubBackupsNode;
|
||||
githubIdentity: typeof githubIdentity;
|
||||
githubImport: typeof githubImport;
|
||||
githubRestore: typeof githubRestore;
|
||||
githubRestoreMutations: typeof githubRestoreMutations;
|
||||
githubSkillSources: typeof githubSkillSources;
|
||||
githubSkillSync: typeof githubSkillSync;
|
||||
githubSoulBackups: typeof githubSoulBackups;
|
||||
githubSoulBackupsNode: typeof githubSoulBackupsNode;
|
||||
http: typeof http;
|
||||
@@ -212,32 +191,27 @@ declare const fullApi: ApiFromModules<{
|
||||
"lib/clawpack": typeof lib_clawpack;
|
||||
"lib/commentScamPrompt": typeof lib_commentScamPrompt;
|
||||
"lib/contentTypes": typeof lib_contentTypes;
|
||||
"lib/depRegistryScan": typeof lib_depRegistryScan;
|
||||
"lib/devAuth": typeof lib_devAuth;
|
||||
"lib/devSeed": typeof lib_devSeed;
|
||||
"lib/emails": typeof lib_emails;
|
||||
"lib/embeddingVisibility": typeof lib_embeddingVisibility;
|
||||
"lib/embeddings": typeof lib_embeddings;
|
||||
"lib/githubAccount": typeof lib_githubAccount;
|
||||
"lib/githubActionsOidc": typeof lib_githubActionsOidc;
|
||||
"lib/githubAuth": typeof lib_githubAuth;
|
||||
"lib/githubBackup": typeof lib_githubBackup;
|
||||
"lib/githubIdentity": typeof lib_githubIdentity;
|
||||
"lib/githubImport": typeof lib_githubImport;
|
||||
"lib/githubProfileSync": typeof lib_githubProfileSync;
|
||||
"lib/githubRestoreHelpers": typeof lib_githubRestoreHelpers;
|
||||
"lib/githubSkillSync": typeof lib_githubSkillSync;
|
||||
"lib/githubSoulBackup": typeof lib_githubSoulBackup;
|
||||
"lib/globalStats": typeof lib_globalStats;
|
||||
"lib/httpHeaders": typeof lib_httpHeaders;
|
||||
"lib/httpRateLimit": typeof lib_httpRateLimit;
|
||||
"lib/httpUtils": typeof lib_httpUtils;
|
||||
"lib/installResolver": typeof lib_installResolver;
|
||||
"lib/leaderboards": typeof lib_leaderboards;
|
||||
"lib/manualOverrides": typeof lib_manualOverrides;
|
||||
"lib/moderation": typeof lib_moderation;
|
||||
"lib/moderationEngine": typeof lib_moderationEngine;
|
||||
"lib/moderationReasonCodes": typeof lib_moderationReasonCodes;
|
||||
"lib/observabilityEvents": typeof lib_observabilityEvents;
|
||||
"lib/officialPublishers": typeof lib_officialPublishers;
|
||||
"lib/openaiResponse": typeof lib_openaiResponse;
|
||||
"lib/packageRegistry": typeof lib_packageRegistry;
|
||||
@@ -248,7 +222,6 @@ declare const fullApi: ApiFromModules<{
|
||||
"lib/publicRouteReservations": typeof lib_publicRouteReservations;
|
||||
"lib/publishLimits": typeof lib_publishLimits;
|
||||
"lib/publisherAbuseScoring": typeof lib_publisherAbuseScoring;
|
||||
"lib/publisherCatalogDisplay": typeof lib_publisherCatalogDisplay;
|
||||
"lib/publisherStats": typeof lib_publisherStats;
|
||||
"lib/publishers": typeof lib_publishers;
|
||||
"lib/reporting": typeof lib_reporting;
|
||||
@@ -259,7 +232,6 @@ declare const fullApi: ApiFromModules<{
|
||||
"lib/skillBackfill": typeof lib_skillBackfill;
|
||||
"lib/skillCapabilityTags": typeof lib_skillCapabilityTags;
|
||||
"lib/skillCards": typeof lib_skillCards;
|
||||
"lib/skillFileAccess": typeof lib_skillFileAccess;
|
||||
"lib/skillIcon": typeof lib_skillIcon;
|
||||
"lib/skillPublish": typeof lib_skillPublish;
|
||||
"lib/skillQuality": typeof lib_skillQuality;
|
||||
@@ -279,13 +251,9 @@ declare const fullApi: ApiFromModules<{
|
||||
"lib/webhooks": typeof lib_webhooks;
|
||||
llmEval: typeof llmEval;
|
||||
maintenance: typeof maintenance;
|
||||
managementDevSeed: typeof managementDevSeed;
|
||||
packageInspectorHttp: typeof packageInspectorHttp;
|
||||
packageInspectorNode: typeof packageInspectorNode;
|
||||
packagePublishTokens: typeof packagePublishTokens;
|
||||
packages: typeof packages;
|
||||
publisherAbuse: typeof publisherAbuse;
|
||||
publisherAbuseDevSeed: typeof publisherAbuseDevSeed;
|
||||
publishers: typeof publishers;
|
||||
rateLimits: typeof rateLimits;
|
||||
search: typeof search;
|
||||
|
||||
+2
-47
@@ -3,9 +3,7 @@ import type { Id } from "./_generated/dataModel";
|
||||
import {
|
||||
BANNED_REAUTH_MESSAGE,
|
||||
DELETED_ACCOUNT_REAUTH_MESSAGE,
|
||||
createGitHubAuthProvider,
|
||||
handleDeletedUserSignIn,
|
||||
normalizeGitHubProfileId,
|
||||
} from "./auth";
|
||||
|
||||
function makeCtx({
|
||||
@@ -130,7 +128,7 @@ describe("handleDeletedUserSignIn", () => {
|
||||
expect(ctx.db.patch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not leak the moderator ban reason in the sign-in error", async () => {
|
||||
it("includes the moderator ban reason in the sign-in error", async () => {
|
||||
const { ctx } = makeCtx({
|
||||
user: { deletedAt: 123, banReason: "Chargeback fraud" },
|
||||
banRecords: [{ action: "user.ban" }],
|
||||
@@ -138,49 +136,6 @@ describe("handleDeletedUserSignIn", () => {
|
||||
|
||||
await expect(
|
||||
handleDeletedUserSignIn(ctx as never, { userId, existingUserId: userId }),
|
||||
).rejects.toThrow(BANNED_REAUTH_MESSAGE);
|
||||
});
|
||||
});
|
||||
|
||||
describe("GitHub auth provider", () => {
|
||||
it("does not link ClawHub accounts by GitHub profile email", () => {
|
||||
const provider = createGitHubAuthProvider() as {
|
||||
options?: { allowDangerousEmailAccountLinking?: boolean };
|
||||
};
|
||||
|
||||
expect(provider.options?.allowDangerousEmailAccountLinking).toBe(false);
|
||||
});
|
||||
|
||||
it("normalizes numeric GitHub profile ids", () => {
|
||||
expect(normalizeGitHubProfileId(123456)).toBe("123456");
|
||||
expect(normalizeGitHubProfileId("789012")).toBe("789012");
|
||||
});
|
||||
|
||||
it("rejects missing or nonnumeric GitHub profile ids", () => {
|
||||
expect(() => normalizeGitHubProfileId(undefined)).toThrow(
|
||||
"GitHub OAuth profile is missing a valid numeric id",
|
||||
);
|
||||
expect(() => normalizeGitHubProfileId("undefined")).toThrow(
|
||||
"GitHub OAuth profile is missing a valid numeric id",
|
||||
);
|
||||
expect(() => normalizeGitHubProfileId("github-user")).toThrow(
|
||||
"GitHub OAuth profile is missing a valid numeric id",
|
||||
);
|
||||
});
|
||||
|
||||
it("fails closed when the GitHub provider receives a malformed profile", () => {
|
||||
const provider = createGitHubAuthProvider() as {
|
||||
options?: { profile?: (profile: Record<string, unknown>) => Record<string, unknown> };
|
||||
};
|
||||
|
||||
expect(() => provider.options?.profile?.({ message: "Bad credentials" })).toThrow(
|
||||
"GitHub OAuth profile is missing a valid numeric id",
|
||||
);
|
||||
expect(provider.options?.profile?.({ id: 123456, login: "fixture-user" })).toEqual({
|
||||
id: "123456",
|
||||
name: "fixture-user",
|
||||
email: undefined,
|
||||
image: undefined,
|
||||
});
|
||||
).rejects.toThrow(`${BANNED_REAUTH_MESSAGE} Reason: Chargeback fraud`);
|
||||
});
|
||||
});
|
||||
|
||||
+21
-44
@@ -9,48 +9,19 @@ import { isLocalDevAuthEnabled } from "./lib/devAuth";
|
||||
import { shouldScheduleGitHubProfileSync } from "./lib/githubProfileSync";
|
||||
|
||||
export const BANNED_REAUTH_MESSAGE =
|
||||
"This account has been banned and cannot sign in. If you believe this is a mistake, appeal this decision: https://appeals.openclaw.ai/.";
|
||||
"This account has been banned and cannot sign in. If you believe this is a mistake, please contact security@openclaw.ai and we will review it.";
|
||||
export const DELETED_ACCOUNT_REAUTH_MESSAGE =
|
||||
"This account has been permanently deleted and cannot be restored.";
|
||||
|
||||
const REAUTH_BLOCKING_BAN_ACTIONS = new Set(["user.ban", "user.autoban.malware"]);
|
||||
const DEV_PERSONAS = new Set(["owner", "user", "admin", "officialOrgMember", "abusePublisher"]);
|
||||
const DEV_PERSONAS = new Set(["owner", "user", "admin"]);
|
||||
|
||||
export function normalizeGitHubProfileId(profileId: unknown) {
|
||||
const id =
|
||||
typeof profileId === "number" && Number.isSafeInteger(profileId)
|
||||
? String(profileId)
|
||||
: typeof profileId === "string"
|
||||
? profileId.trim()
|
||||
: null;
|
||||
|
||||
if (!id || !/^\d+$/.test(id)) {
|
||||
throw new Error("GitHub OAuth profile is missing a valid numeric id");
|
||||
function getBannedReauthMessage(reason: string | undefined) {
|
||||
const normalizedReason = reason?.trim();
|
||||
if (!normalizedReason || normalizedReason.toLowerCase() === "malware auto-ban") {
|
||||
return BANNED_REAUTH_MESSAGE;
|
||||
}
|
||||
|
||||
return id;
|
||||
}
|
||||
|
||||
export function createGitHubAuthProvider() {
|
||||
return GitHub({
|
||||
clientId: process.env.AUTH_GITHUB_ID ?? "",
|
||||
clientSecret: process.env.AUTH_GITHUB_SECRET ?? "",
|
||||
// GitHub's OAuth email must not be treated as a ClawHub account key. The
|
||||
// immutable GitHub provider account id is the only account-linking key.
|
||||
allowDangerousEmailAccountLinking: false,
|
||||
profile(profile) {
|
||||
return {
|
||||
id: normalizeGitHubProfileId(profile.id),
|
||||
name: profile.login,
|
||||
email: profile.email ?? undefined,
|
||||
image: profile.avatar_url,
|
||||
};
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function getBannedReauthMessage(_reason: string | undefined) {
|
||||
return BANNED_REAUTH_MESSAGE;
|
||||
return `${BANNED_REAUTH_MESSAGE} Reason: ${normalizedReason}`;
|
||||
}
|
||||
|
||||
export async function handleDeletedUserSignIn(
|
||||
@@ -104,20 +75,26 @@ export async function handleDeletedUserSignIn(
|
||||
|
||||
export const { auth, signIn, signOut, store, isAuthenticated } = convexAuth({
|
||||
providers: [
|
||||
createGitHubAuthProvider(),
|
||||
GitHub({
|
||||
clientId: process.env.AUTH_GITHUB_ID ?? "",
|
||||
clientSecret: process.env.AUTH_GITHUB_SECRET ?? "",
|
||||
profile(profile) {
|
||||
return {
|
||||
id: String(profile.id),
|
||||
name: profile.login,
|
||||
email: profile.email ?? undefined,
|
||||
image: profile.avatar_url,
|
||||
};
|
||||
},
|
||||
}),
|
||||
ConvexCredentials({
|
||||
id: "dev-persona",
|
||||
authorize: async (credentials, ctx) => {
|
||||
const devAuthSecret =
|
||||
typeof credentials.devAuthSecret === "string" ? credentials.devAuthSecret : undefined;
|
||||
if (!isLocalDevAuthEnabled(process.env, devAuthSecret)) {
|
||||
throw new Error("Dev auth is disabled");
|
||||
}
|
||||
if (!isLocalDevAuthEnabled()) throw new Error("Dev auth is disabled");
|
||||
const persona = typeof credentials.persona === "string" ? credentials.persona : "";
|
||||
if (!DEV_PERSONAS.has(persona)) throw new Error("Unknown dev persona");
|
||||
const userId: Id<"users"> = await ctx.runMutation(internal.users.upsertDevPersonaInternal, {
|
||||
persona: persona as "owner" | "user" | "admin" | "officialOrgMember" | "abusePublisher",
|
||||
devAuthSecret,
|
||||
persona: persona as "owner" | "user" | "admin",
|
||||
});
|
||||
return { userId };
|
||||
},
|
||||
|
||||
@@ -1,263 +0,0 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("./lib/access", () => ({
|
||||
requireUser: vi.fn(),
|
||||
}));
|
||||
|
||||
const { requireUser } = await import("./lib/access");
|
||||
const { approve, deny } = await import("./cliDeviceAuth");
|
||||
|
||||
const approveHandler = (approve as unknown as { _handler: Function })._handler;
|
||||
const denyHandler = (deny as unknown as { _handler: Function })._handler;
|
||||
|
||||
function makeCtx(rows: Array<Record<string, unknown>>) {
|
||||
const take = vi.fn().mockResolvedValue(rows);
|
||||
const order = vi.fn().mockReturnValue({ take });
|
||||
const withIndex = vi.fn().mockReturnValue({ order });
|
||||
const query = vi.fn().mockReturnValue({ withIndex });
|
||||
const get = vi.fn().mockResolvedValue(null);
|
||||
const insert = vi.fn().mockResolvedValue("inserted:id");
|
||||
const patch = vi.fn().mockResolvedValue(undefined);
|
||||
const replace = vi.fn().mockResolvedValue(undefined);
|
||||
const delete_ = vi.fn().mockResolvedValue(undefined);
|
||||
const normalizeId = vi.fn().mockReturnValue(null);
|
||||
|
||||
return {
|
||||
ctx: { db: { get, insert, query, patch, replace, delete: delete_, normalizeId } },
|
||||
order,
|
||||
take,
|
||||
withIndex,
|
||||
query,
|
||||
patch,
|
||||
};
|
||||
}
|
||||
|
||||
describe("cliDeviceAuth approval", () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2026-06-10T12:00:00Z"));
|
||||
vi.mocked(requireUser).mockResolvedValue({
|
||||
userId: "users:approver",
|
||||
user: { _id: "users:approver" },
|
||||
} as never);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
vi.mocked(requireUser).mockReset();
|
||||
});
|
||||
|
||||
it("approves the newest active pending row when duplicate user codes exist", async () => {
|
||||
const now = Date.now();
|
||||
const { ctx, order, patch, take } = makeCtx([
|
||||
{
|
||||
_id: "cliDeviceCodes:new",
|
||||
_creationTime: now - 1_000,
|
||||
status: "pending",
|
||||
userCode: "Q639-NBSX",
|
||||
createdAt: now - 1_000,
|
||||
expiresAt: now + 60_000,
|
||||
},
|
||||
{
|
||||
_id: "cliDeviceCodes:old",
|
||||
_creationTime: now - 10_000,
|
||||
status: "pending",
|
||||
userCode: "Q639-NBSX",
|
||||
createdAt: now - 10_000,
|
||||
expiresAt: now + 60_000,
|
||||
},
|
||||
]);
|
||||
|
||||
const result = await approveHandler(ctx, { userCode: "q639-nbsx" });
|
||||
|
||||
expect(result).toEqual({
|
||||
ok: true,
|
||||
userCode: "Q639-NBSX",
|
||||
expiresAt: now + 60_000,
|
||||
});
|
||||
expect(order).toHaveBeenCalledWith("desc");
|
||||
expect(take).toHaveBeenCalledWith(50);
|
||||
expect(patch).toHaveBeenCalledTimes(1);
|
||||
expect(patch).toHaveBeenCalledWith("cliDeviceCodes:new", {
|
||||
status: "approved",
|
||||
approvedByUserId: "users:approver",
|
||||
approvedAt: now,
|
||||
});
|
||||
});
|
||||
|
||||
it("uses descending index order when duplicate creation timestamps disagree", async () => {
|
||||
const now = Date.now();
|
||||
const { ctx, patch } = makeCtx([
|
||||
{
|
||||
_id: "cliDeviceCodes:new",
|
||||
_creationTime: now - 1_000,
|
||||
status: "pending",
|
||||
userCode: "Q639-NBSX",
|
||||
createdAt: now - 20_000,
|
||||
expiresAt: now + 60_000,
|
||||
},
|
||||
{
|
||||
_id: "cliDeviceCodes:old",
|
||||
_creationTime: now - 10_000,
|
||||
status: "pending",
|
||||
userCode: "Q639-NBSX",
|
||||
createdAt: now - 500,
|
||||
expiresAt: now + 60_000,
|
||||
},
|
||||
]);
|
||||
|
||||
await approveHandler(ctx, { userCode: "Q639-NBSX" });
|
||||
|
||||
expect(patch).toHaveBeenCalledWith("cliDeviceCodes:new", {
|
||||
status: "approved",
|
||||
approvedByUserId: "users:approver",
|
||||
approvedAt: now,
|
||||
});
|
||||
});
|
||||
|
||||
it("expires stale pending rows before approving an active row", async () => {
|
||||
const now = Date.now();
|
||||
const { ctx, patch } = makeCtx([
|
||||
{
|
||||
_id: "cliDeviceCodes:expired",
|
||||
_creationTime: now - 100_000,
|
||||
status: "pending",
|
||||
userCode: "Q639-NBSX",
|
||||
createdAt: now - 100_000,
|
||||
expiresAt: now - 1,
|
||||
},
|
||||
{
|
||||
_id: "cliDeviceCodes:active",
|
||||
_creationTime: now - 1_000,
|
||||
status: "pending",
|
||||
userCode: "Q639-NBSX",
|
||||
createdAt: now - 1_000,
|
||||
expiresAt: now + 60_000,
|
||||
},
|
||||
]);
|
||||
|
||||
await approveHandler(ctx, { userCode: "Q639-NBSX" });
|
||||
|
||||
expect(patch).toHaveBeenNthCalledWith(1, "cliDeviceCodes:expired", { status: "expired" });
|
||||
expect(patch).toHaveBeenNthCalledWith(2, "cliDeviceCodes:active", {
|
||||
status: "approved",
|
||||
approvedByUserId: "users:approver",
|
||||
approvedAt: now,
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves stale terminal rows while expiring stale pending duplicates", async () => {
|
||||
const now = Date.now();
|
||||
const { ctx, patch } = makeCtx([
|
||||
{
|
||||
_id: "cliDeviceCodes:approved",
|
||||
_creationTime: now - 100_000,
|
||||
status: "approved",
|
||||
userCode: "Q639-NBSX",
|
||||
createdAt: now - 100_000,
|
||||
expiresAt: now - 1,
|
||||
approvedAt: now - 90_000,
|
||||
approvedByUserId: "users:previous",
|
||||
},
|
||||
{
|
||||
_id: "cliDeviceCodes:denied",
|
||||
_creationTime: now - 90_000,
|
||||
status: "denied",
|
||||
userCode: "Q639-NBSX",
|
||||
createdAt: now - 90_000,
|
||||
expiresAt: now - 1,
|
||||
deniedAt: now - 80_000,
|
||||
},
|
||||
{
|
||||
_id: "cliDeviceCodes:consumed",
|
||||
_creationTime: now - 80_000,
|
||||
status: "consumed",
|
||||
userCode: "Q639-NBSX",
|
||||
createdAt: now - 80_000,
|
||||
expiresAt: now - 1,
|
||||
consumedAt: now - 70_000,
|
||||
},
|
||||
{
|
||||
_id: "cliDeviceCodes:expired",
|
||||
_creationTime: now - 70_000,
|
||||
status: "pending",
|
||||
userCode: "Q639-NBSX",
|
||||
createdAt: now - 70_000,
|
||||
expiresAt: now - 1,
|
||||
},
|
||||
{
|
||||
_id: "cliDeviceCodes:active",
|
||||
_creationTime: now - 1_000,
|
||||
status: "pending",
|
||||
userCode: "Q639-NBSX",
|
||||
createdAt: now - 1_000,
|
||||
expiresAt: now + 60_000,
|
||||
},
|
||||
]);
|
||||
|
||||
await approveHandler(ctx, { userCode: "Q639-NBSX" });
|
||||
|
||||
expect(patch).toHaveBeenNthCalledWith(1, "cliDeviceCodes:expired", { status: "expired" });
|
||||
expect(patch).toHaveBeenNthCalledWith(2, "cliDeviceCodes:active", {
|
||||
status: "approved",
|
||||
approvedByUserId: "users:approver",
|
||||
approvedAt: now,
|
||||
});
|
||||
expect(patch).not.toHaveBeenCalledWith("cliDeviceCodes:approved", expect.anything());
|
||||
expect(patch).not.toHaveBeenCalledWith("cliDeviceCodes:denied", expect.anything());
|
||||
expect(patch).not.toHaveBeenCalledWith("cliDeviceCodes:consumed", expect.anything());
|
||||
});
|
||||
|
||||
it("denies the newest active pending row when duplicate user codes exist", async () => {
|
||||
const now = Date.now();
|
||||
const { ctx, patch } = makeCtx([
|
||||
{
|
||||
_id: "cliDeviceCodes:new",
|
||||
_creationTime: now - 1_000,
|
||||
status: "pending",
|
||||
userCode: "Q639-NBSX",
|
||||
createdAt: now - 1_000,
|
||||
expiresAt: now + 60_000,
|
||||
},
|
||||
{
|
||||
_id: "cliDeviceCodes:old",
|
||||
_creationTime: now - 10_000,
|
||||
status: "pending",
|
||||
userCode: "Q639-NBSX",
|
||||
createdAt: now - 10_000,
|
||||
expiresAt: now + 60_000,
|
||||
},
|
||||
]);
|
||||
|
||||
await expect(denyHandler(ctx, { userCode: "Q639-NBSX" })).resolves.toEqual({ ok: true });
|
||||
|
||||
expect(patch).toHaveBeenCalledTimes(1);
|
||||
expect(patch).toHaveBeenCalledWith("cliDeviceCodes:new", {
|
||||
status: "denied",
|
||||
deniedAt: now,
|
||||
});
|
||||
});
|
||||
|
||||
it("does not deny stale pending rows after expiring them", async () => {
|
||||
const now = Date.now();
|
||||
const { ctx, patch } = makeCtx([
|
||||
{
|
||||
_id: "cliDeviceCodes:expired",
|
||||
_creationTime: now - 100_000,
|
||||
status: "pending",
|
||||
userCode: "Q639-NBSX",
|
||||
createdAt: now - 100_000,
|
||||
expiresAt: now - 1,
|
||||
},
|
||||
]);
|
||||
|
||||
await expect(denyHandler(ctx, { userCode: "Q639-NBSX" })).resolves.toEqual({ ok: true });
|
||||
|
||||
expect(patch).toHaveBeenCalledTimes(1);
|
||||
expect(patch).toHaveBeenCalledWith("cliDeviceCodes:expired", { status: "expired" });
|
||||
expect(patch).not.toHaveBeenCalledWith("cliDeviceCodes:expired", {
|
||||
status: "denied",
|
||||
deniedAt: now,
|
||||
});
|
||||
});
|
||||
});
|
||||
+15
-45
@@ -1,13 +1,10 @@
|
||||
import { v } from "convex/values";
|
||||
import type { Doc } from "./_generated/dataModel";
|
||||
import type { MutationCtx } from "./_generated/server";
|
||||
import { internalMutation, mutation } from "./functions";
|
||||
import { requireUser } from "./lib/access";
|
||||
import { generateToken, hashToken } from "./lib/tokens";
|
||||
|
||||
const DEVICE_CODE_TTL_MS = 15 * 60_000;
|
||||
const DEVICE_POLL_INTERVAL_SECONDS = 5;
|
||||
const MAX_DUPLICATE_USER_CODE_ROWS = 50;
|
||||
const USER_CODE_ALPHABET = "ABCDEFGHJKLMNPQRSTUVWXYZ23456789";
|
||||
|
||||
export const createInternal = internalMutation({
|
||||
@@ -91,13 +88,17 @@ export const approve = mutation({
|
||||
if (!normalized) throw new Error("Code required");
|
||||
|
||||
const userCodeHash = await hashToken(normalized);
|
||||
const now = Date.now();
|
||||
const rows = await expireStaleRows(ctx, await getRowsByUserCodeHash(ctx, userCodeHash), now);
|
||||
const row =
|
||||
pickLatestRow(rows, now, "pending") ?? pickLatestRow(rows, now) ?? pickLatestRow(rows);
|
||||
const row = await ctx.db
|
||||
.query("cliDeviceCodes")
|
||||
.withIndex("by_user_code_hash", (q) => q.eq("userCodeHash", userCodeHash))
|
||||
.unique();
|
||||
if (!row) throw new Error("Device code not found");
|
||||
if (row.expiresAt <= now) throw new Error("Device code expired");
|
||||
if (row.status === "expired") throw new Error("Device code expired");
|
||||
|
||||
const now = Date.now();
|
||||
if (row.expiresAt <= now) {
|
||||
if (row.status !== "expired") await ctx.db.patch(row._id, { status: "expired" });
|
||||
throw new Error("Device code expired");
|
||||
}
|
||||
if (row.status === "consumed") throw new Error("Device code already used");
|
||||
if (row.status === "approved") throw new Error("Device code already authorized");
|
||||
if (row.status === "denied") throw new Error("Device code was denied");
|
||||
@@ -118,11 +119,12 @@ export const deny = mutation({
|
||||
const normalized = normalizeUserCode(args.userCode);
|
||||
if (!normalized) throw new Error("Code required");
|
||||
const userCodeHash = await hashToken(normalized);
|
||||
const now = Date.now();
|
||||
const rows = await expireStaleRows(ctx, await getRowsByUserCodeHash(ctx, userCodeHash), now);
|
||||
const row =
|
||||
pickLatestRow(rows, now, "pending") ?? pickLatestRow(rows, now) ?? pickLatestRow(rows);
|
||||
const row = await ctx.db
|
||||
.query("cliDeviceCodes")
|
||||
.withIndex("by_user_code_hash", (q) => q.eq("userCodeHash", userCodeHash))
|
||||
.unique();
|
||||
if (!row) throw new Error("Device code not found");
|
||||
const now = Date.now();
|
||||
if (row.status === "approved") throw new Error("Device code already authorized");
|
||||
if (row.status === "pending") {
|
||||
await ctx.db.patch(row._id, { status: "denied", deniedAt: now });
|
||||
@@ -138,38 +140,6 @@ function normalizeUserCode(value: string) {
|
||||
.replace(/[^A-Z0-9]/g, "");
|
||||
}
|
||||
|
||||
async function getRowsByUserCodeHash(ctx: MutationCtx, userCodeHash: string) {
|
||||
return await ctx.db
|
||||
.query("cliDeviceCodes")
|
||||
.withIndex("by_user_code_hash", (q) => q.eq("userCodeHash", userCodeHash))
|
||||
.order("desc")
|
||||
.take(MAX_DUPLICATE_USER_CODE_ROWS);
|
||||
}
|
||||
|
||||
async function expireStaleRows(ctx: MutationCtx, rows: Array<Doc<"cliDeviceCodes">>, now: number) {
|
||||
const nextRows: Array<Doc<"cliDeviceCodes">> = [];
|
||||
for (const row of rows) {
|
||||
if (row.status === "pending" && row.expiresAt <= now) {
|
||||
await ctx.db.patch(row._id, { status: "expired" });
|
||||
nextRows.push({ ...row, status: "expired" });
|
||||
} else {
|
||||
nextRows.push(row);
|
||||
}
|
||||
}
|
||||
return nextRows;
|
||||
}
|
||||
|
||||
function pickLatestRow(
|
||||
rows: Array<Doc<"cliDeviceCodes">>,
|
||||
now?: number,
|
||||
status?: Doc<"cliDeviceCodes">["status"],
|
||||
) {
|
||||
return rows.find(
|
||||
(row) =>
|
||||
(now === undefined || row.expiresAt > now) && (status === undefined || row.status === status),
|
||||
);
|
||||
}
|
||||
|
||||
function generateUserCode() {
|
||||
const bytes = new Uint8Array(8);
|
||||
crypto.getRandomValues(bytes);
|
||||
|
||||
@@ -1,58 +0,0 @@
|
||||
/* @vitest-environment node */
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
const mocks = vi.hoisted(() => {
|
||||
const interval = vi.fn();
|
||||
const githubSkillSyncRef = Symbol("github-skill-source-sync");
|
||||
return { interval, githubSkillSyncRef };
|
||||
});
|
||||
|
||||
vi.mock("convex/server", () => ({
|
||||
cronJobs: () => ({
|
||||
interval: mocks.interval,
|
||||
}),
|
||||
}));
|
||||
|
||||
vi.mock("./_generated/api", () => ({
|
||||
internal: {
|
||||
githubBackupsNode: { syncGitHubBackupsInternal: Symbol("github-backup-sync") },
|
||||
githubSkillSync: { syncGitHubSkillSourcesInternal: mocks.githubSkillSyncRef },
|
||||
leaderboards: { rebuildTrendingLeaderboardAction: Symbol("trending-leaderboard") },
|
||||
statsMaintenance: {
|
||||
runSkillStatBackfillInternal: Symbol("skill-stats-backfill"),
|
||||
updateGlobalStatsAction: Symbol("global-stats-update"),
|
||||
},
|
||||
skillStatEvents: { processSkillStatEventsAction: Symbol("skill-stat-events") },
|
||||
packages: {
|
||||
processPackageStatEventsInternal: Symbol("package-stat-events"),
|
||||
backfillPackageReleaseScansInternal: Symbol("package-scan-backfill"),
|
||||
},
|
||||
publisherAbuse: {
|
||||
runPublisherAbuseScoreRunInternal: Symbol("publisher-abuse-score-refresh"),
|
||||
},
|
||||
vt: {
|
||||
pollPendingScans: Symbol("vt-pending-scans"),
|
||||
backfillActiveSkillsVTCache: Symbol("vt-cache-backfill"),
|
||||
},
|
||||
securityScan: {
|
||||
pruneExpiredSkillScanRequestsInternal: Symbol("skill-scan-request-prune"),
|
||||
},
|
||||
downloads: { pruneDownloadDedupesInternal: Symbol("download-dedupe-prune") },
|
||||
downloadMetrics: {
|
||||
pruneDownloadMetricDedupesInternal: Symbol("download-metric-dedupe-prune"),
|
||||
},
|
||||
},
|
||||
}));
|
||||
|
||||
describe("crons", () => {
|
||||
it("runs GitHub skill source sync every 15 minutes", async () => {
|
||||
await import("./crons");
|
||||
|
||||
expect(mocks.interval).toHaveBeenCalledWith(
|
||||
"github-skill-source-sync",
|
||||
{ minutes: 15 },
|
||||
mocks.githubSkillSyncRef,
|
||||
{},
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -10,13 +10,6 @@ crons.interval(
|
||||
{ batchSize: 50, maxBatches: 5 },
|
||||
);
|
||||
|
||||
crons.interval(
|
||||
"github-skill-source-sync",
|
||||
{ minutes: 15 },
|
||||
internal.githubSkillSync.syncGitHubSkillSourcesInternal,
|
||||
{},
|
||||
);
|
||||
|
||||
crons.interval(
|
||||
"trending-leaderboard",
|
||||
{ minutes: 60 },
|
||||
@@ -71,20 +64,6 @@ crons.interval(
|
||||
{ batchSize: 250, maxPages: 5, trigger: "cron" },
|
||||
);
|
||||
|
||||
crons.interval(
|
||||
"publisher-temporal-abuse-scan",
|
||||
{ hours: 24 },
|
||||
internal.publisherAbuse.runTemporalPublisherAbuseScanInternal,
|
||||
{
|
||||
mode: "current",
|
||||
dryRun: false,
|
||||
candidateLimit: 1000,
|
||||
batchSize: 50,
|
||||
maxPages: 20,
|
||||
trigger: "cron",
|
||||
},
|
||||
);
|
||||
|
||||
crons.interval("vt-pending-scans", { minutes: 5 }, internal.vt.pollPendingScans, {
|
||||
batchSize: 100,
|
||||
});
|
||||
@@ -100,13 +79,6 @@ crons.interval(
|
||||
{ batchSize: 100 },
|
||||
);
|
||||
|
||||
crons.interval(
|
||||
"skill-scan-request-prune",
|
||||
{ hours: 6 },
|
||||
internal.securityScan.pruneExpiredSkillScanRequestsInternal,
|
||||
{ batchSize: 250 },
|
||||
);
|
||||
|
||||
crons.interval(
|
||||
"download-dedupe-prune",
|
||||
{ hours: 24 },
|
||||
@@ -114,11 +86,4 @@ crons.interval(
|
||||
{},
|
||||
);
|
||||
|
||||
crons.interval(
|
||||
"download-metric-dedupe-prune",
|
||||
{ hours: 24 },
|
||||
internal.downloadMetrics.pruneDownloadMetricDedupesInternal,
|
||||
{},
|
||||
);
|
||||
|
||||
export default crons;
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
/* @vitest-environment node */
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
const { checkDependencyRegistriesHandler } = await import("./depRegistryScan");
|
||||
|
||||
describe("dependency registry scan drain", () => {
|
||||
it("keeps legacy scheduled jobs harmless after the scanner is retired", async () => {
|
||||
await expect(checkDependencyRegistriesHandler()).resolves.toBeNull();
|
||||
});
|
||||
});
|
||||
+263
-4
@@ -1,11 +1,270 @@
|
||||
import { v } from "convex/values";
|
||||
import { internalAction } from "./_generated/server";
|
||||
import { internal } from "./_generated/api";
|
||||
import type { Doc, Id } from "./_generated/dataModel";
|
||||
import type { ActionCtx } from "./_generated/server";
|
||||
import { internalAction, internalMutation, internalQuery } from "./functions";
|
||||
import {
|
||||
dedupeDeps,
|
||||
depRegistryUrl,
|
||||
parseDependencyFile,
|
||||
SUPPORTED_DEP_REGISTRIES,
|
||||
summarizeDepRegistryChecks,
|
||||
type DepEntry,
|
||||
type DepRegistryResult,
|
||||
type DepRegistryUnresolved,
|
||||
type SupportedDepRegistry,
|
||||
} from "./lib/depRegistryScan";
|
||||
import { readStorageText } from "./lib/packageRegistry";
|
||||
|
||||
export async function checkDependencyRegistriesHandler(): Promise<null> {
|
||||
return null;
|
||||
const REQUEST_TIMEOUT_MS = 8_000;
|
||||
const MAX_RETRIES = 2;
|
||||
const BACKOFF_BASE_MS = 750;
|
||||
const INTER_REQUEST_DELAY_MS = 100;
|
||||
const MAX_DEPENDENCIES_PER_SCAN = 120;
|
||||
const CACHE_TTL_EXISTS_MS = 30 * 24 * 60 * 60 * 1_000;
|
||||
const CACHE_TTL_NOT_EXISTS_MS = 7 * 24 * 60 * 60 * 1_000;
|
||||
|
||||
const registryValidator = v.union(v.literal("pypi"), v.literal("npm"), v.literal("cargo"));
|
||||
|
||||
type RegistryCheck =
|
||||
| { kind: "found"; httpStatus: number }
|
||||
| { kind: "missing"; httpStatus: number }
|
||||
| { kind: "unresolved"; reason: string };
|
||||
|
||||
function isSupportedRegistry(value: string): value is SupportedDepRegistry {
|
||||
return (SUPPORTED_DEP_REGISTRIES as readonly string[]).includes(value);
|
||||
}
|
||||
|
||||
async function wait(ms: number) {
|
||||
await new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function checkRegistry(dep: DepEntry): Promise<RegistryCheck> {
|
||||
const headers: Record<string, string> = { Accept: "application/json" };
|
||||
if (dep.registry === "cargo") {
|
||||
headers["User-Agent"] = "ClawHub-DepRegistryScan/1.0 (https://clawhub.ai)";
|
||||
}
|
||||
|
||||
let lastStatus: number | undefined;
|
||||
for (let attempt = 0; attempt <= MAX_RETRIES; attempt += 1) {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
|
||||
try {
|
||||
const response = await fetch(depRegistryUrl(dep.registry, dep.name), {
|
||||
method: "GET",
|
||||
headers,
|
||||
signal: controller.signal,
|
||||
});
|
||||
clearTimeout(timeout);
|
||||
lastStatus = response.status;
|
||||
if (response.status === 200) return { kind: "found", httpStatus: response.status };
|
||||
if (response.status === 404) return { kind: "missing", httpStatus: response.status };
|
||||
if (response.status !== 429 && response.status < 500) {
|
||||
return {
|
||||
kind: "unresolved",
|
||||
reason: `unexpected HTTP ${response.status}`,
|
||||
};
|
||||
}
|
||||
} catch (error) {
|
||||
clearTimeout(timeout);
|
||||
if (attempt === MAX_RETRIES) {
|
||||
return {
|
||||
kind: "unresolved",
|
||||
reason: error instanceof Error ? error.message : "network error",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (attempt < MAX_RETRIES) {
|
||||
await wait(2 ** attempt * BACKOFF_BASE_MS);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
kind: "unresolved",
|
||||
reason: lastStatus ? `HTTP ${lastStatus}` : "network error",
|
||||
};
|
||||
}
|
||||
|
||||
async function extractDependencies(ctx: Pick<ActionCtx, "storage">, version: Doc<"skillVersions">) {
|
||||
const entries: DepEntry[] = [];
|
||||
for (const file of version.files) {
|
||||
const basename = file.path.split("/").pop()?.toLowerCase() ?? "";
|
||||
if (
|
||||
basename !== "requirements.txt" &&
|
||||
basename !== "requirements-dev.txt" &&
|
||||
basename !== "requirements_dev.txt" &&
|
||||
basename !== "requirements-test.txt" &&
|
||||
basename !== "requirements_test.txt" &&
|
||||
basename !== "package.json" &&
|
||||
basename !== "cargo.toml" &&
|
||||
basename !== "pyproject.toml"
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const content = await readStorageText(ctx, file.storageId);
|
||||
entries.push(...parseDependencyFile(file.path, content));
|
||||
}
|
||||
return dedupeDeps(entries);
|
||||
}
|
||||
|
||||
export const lookupCacheInternal = internalQuery({
|
||||
args: {
|
||||
registry: registryValidator,
|
||||
name: v.string(),
|
||||
},
|
||||
handler: async (ctx, args): Promise<Doc<"depRegistryCache"> | null> => {
|
||||
return ctx.db
|
||||
.query("depRegistryCache")
|
||||
.withIndex("by_registry_name", (q) => q.eq("registry", args.registry).eq("name", args.name))
|
||||
.unique();
|
||||
},
|
||||
});
|
||||
|
||||
export const upsertCacheInternal = internalMutation({
|
||||
args: {
|
||||
registry: registryValidator,
|
||||
name: v.string(),
|
||||
exists: v.boolean(),
|
||||
httpStatus: v.number(),
|
||||
checkedAt: v.number(),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const existing = await ctx.db
|
||||
.query("depRegistryCache")
|
||||
.withIndex("by_registry_name", (q) => q.eq("registry", args.registry).eq("name", args.name))
|
||||
.unique();
|
||||
const patch = {
|
||||
registry: args.registry,
|
||||
name: args.name,
|
||||
exists: args.exists,
|
||||
httpStatus: args.httpStatus,
|
||||
checkedAt: args.checkedAt,
|
||||
};
|
||||
if (existing) {
|
||||
await ctx.db.patch(existing._id, patch);
|
||||
} else {
|
||||
await ctx.db.insert("depRegistryCache", patch);
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
export const getRetryableVersionIdsInternal = internalQuery({
|
||||
args: {
|
||||
limit: v.optional(v.number()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const limit = Math.min(Math.max(args.limit ?? 25, 1), 100);
|
||||
const versions = await ctx.db
|
||||
.query("skillVersions")
|
||||
.withIndex("by_dep_registry_scan_status_and_created", (q) =>
|
||||
q.eq("depRegistryScanStatus", "error"),
|
||||
)
|
||||
.order("desc")
|
||||
.take(limit);
|
||||
return versions.map((version) => version._id);
|
||||
},
|
||||
});
|
||||
|
||||
async function checkWithCache(ctx: ActionCtx, dep: DepEntry) {
|
||||
const now = Date.now();
|
||||
const cached = (await ctx.runQuery(internal.depRegistryScan.lookupCacheInternal, {
|
||||
registry: dep.registry,
|
||||
name: dep.name,
|
||||
})) as Doc<"depRegistryCache"> | null;
|
||||
if (cached) {
|
||||
const ttl = cached.exists ? CACHE_TTL_EXISTS_MS : CACHE_TTL_NOT_EXISTS_MS;
|
||||
if (now - cached.checkedAt < ttl) {
|
||||
return cached.exists
|
||||
? ({ kind: "found", httpStatus: cached.httpStatus } as const)
|
||||
: ({ kind: "missing", httpStatus: cached.httpStatus } as const);
|
||||
}
|
||||
}
|
||||
|
||||
const check = await checkRegistry(dep);
|
||||
if (check.kind !== "unresolved") {
|
||||
await ctx.runMutation(internal.depRegistryScan.upsertCacheInternal, {
|
||||
registry: dep.registry,
|
||||
name: dep.name,
|
||||
exists: check.kind === "found",
|
||||
httpStatus: check.httpStatus,
|
||||
checkedAt: now,
|
||||
});
|
||||
}
|
||||
return check;
|
||||
}
|
||||
|
||||
export const checkDependencyRegistries = internalAction({
|
||||
args: { versionId: v.id("skillVersions") },
|
||||
handler: checkDependencyRegistriesHandler,
|
||||
handler: async (ctx, args) => {
|
||||
const version = (await ctx.runQuery(internal.skills.getVersionByIdInternal, {
|
||||
versionId: args.versionId,
|
||||
})) as Doc<"skillVersions"> | null;
|
||||
if (!version) return null;
|
||||
if (version.depRegistryAnalysis && version.depRegistryAnalysis.status !== "error") {
|
||||
return version.depRegistryAnalysis;
|
||||
}
|
||||
|
||||
const deps = await extractDependencies(ctx, version);
|
||||
const checkableDeps = deps.slice(0, MAX_DEPENDENCIES_PER_SCAN);
|
||||
const deferredDeps = deps.slice(MAX_DEPENDENCIES_PER_SCAN);
|
||||
const results: DepRegistryResult[] = [];
|
||||
const unresolved: DepRegistryUnresolved[] = deferredDeps.map((dep) => ({
|
||||
...dep,
|
||||
reason: "dependency scan limit reached",
|
||||
}));
|
||||
|
||||
for (const dep of checkableDeps) {
|
||||
if (!isSupportedRegistry(dep.registry)) continue;
|
||||
const check = await checkWithCache(ctx, dep);
|
||||
if (check.kind === "unresolved") {
|
||||
unresolved.push({ ...dep, reason: check.reason });
|
||||
} else {
|
||||
results.push({
|
||||
...dep,
|
||||
exists: check.kind === "found",
|
||||
httpStatus: check.httpStatus,
|
||||
});
|
||||
}
|
||||
await wait(INTER_REQUEST_DELAY_MS);
|
||||
}
|
||||
|
||||
const analysis = summarizeDepRegistryChecks({
|
||||
results,
|
||||
unresolved,
|
||||
checkedAt: Date.now(),
|
||||
});
|
||||
|
||||
await ctx.runMutation(internal.skills.updateVersionDepRegistryAnalysisInternal, {
|
||||
versionId: args.versionId,
|
||||
depRegistryAnalysis: analysis,
|
||||
});
|
||||
|
||||
return analysis;
|
||||
},
|
||||
});
|
||||
|
||||
export const rescanErrorDepRegistryVersions = internalAction({
|
||||
args: {
|
||||
batchSize: v.optional(v.number()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const versionIds = (await ctx.runQuery(
|
||||
internal.depRegistryScan.getRetryableVersionIdsInternal,
|
||||
{ limit: args.batchSize ?? 25 },
|
||||
)) as Id<"skillVersions">[];
|
||||
|
||||
let scheduled = 0;
|
||||
for (const versionId of versionIds) {
|
||||
await ctx.scheduler.runAfter(
|
||||
scheduled * 2_000,
|
||||
internal.depRegistryScan.checkDependencyRegistries,
|
||||
{
|
||||
versionId,
|
||||
},
|
||||
);
|
||||
scheduled += 1;
|
||||
}
|
||||
return { scheduled };
|
||||
},
|
||||
});
|
||||
|
||||
@@ -4,10 +4,7 @@ import {
|
||||
currentUserSeedPackageName,
|
||||
currentUserSeedSkillSlug,
|
||||
seedFeaturedPluginPackagesMutation,
|
||||
seedGitHubBackedSkillSourceMutation,
|
||||
seedLocalFixtures,
|
||||
seedLocalModerationFixturesHandler,
|
||||
seedPublicCorpusBatchMutation,
|
||||
seedSkillMutation,
|
||||
} from "./devSeed";
|
||||
|
||||
@@ -21,15 +18,6 @@ const seedSkillMutationHandler = (
|
||||
const seedFeaturedPluginPackagesHandler = (
|
||||
seedFeaturedPluginPackagesMutation as unknown as WrappedHandler<Record<string, unknown>>
|
||||
)._handler;
|
||||
const seedGitHubBackedSkillSourceHandler = (
|
||||
seedGitHubBackedSkillSourceMutation as unknown as WrappedHandler<Record<string, unknown>>
|
||||
)._handler;
|
||||
const seedLocalFixturesHandler = (
|
||||
seedLocalFixtures as unknown as WrappedHandler<{ reset?: boolean }>
|
||||
)._handler;
|
||||
const seedPublicCorpusBatchHandler = (
|
||||
seedPublicCorpusBatchMutation as unknown as WrappedHandler<Record<string, unknown>>
|
||||
)._handler;
|
||||
|
||||
function chainEq(constraints: Record<string, unknown>) {
|
||||
return {
|
||||
@@ -158,33 +146,6 @@ function seedSkillArgs(storageId: string) {
|
||||
}
|
||||
|
||||
describe("devSeed local fixtures", () => {
|
||||
it("does not preconfigure GitHub-backed source fixtures in the local seed action", async () => {
|
||||
const mutationCalls: Array<{ args: Record<string, unknown> }> = [];
|
||||
let storageCounter = 0;
|
||||
const ctx = {
|
||||
storage: {
|
||||
store: async () => `storage:${++storageCounter}`,
|
||||
},
|
||||
runMutation: async (_ref: unknown, args: Record<string, unknown>) => {
|
||||
mutationCalls.push({ args });
|
||||
return { ok: true, seeded: ["local-moderation-fixtures"], skipped: [] };
|
||||
},
|
||||
};
|
||||
|
||||
const result = await seedLocalFixturesHandler(ctx as never, { reset: true });
|
||||
|
||||
expect(mutationCalls).toHaveLength(1);
|
||||
expect(mutationCalls[0]?.args).toMatchObject({
|
||||
reset: true,
|
||||
});
|
||||
expect(result).toEqual(
|
||||
expect.objectContaining({
|
||||
ok: true,
|
||||
results: [expect.objectContaining({ slug: "local-moderation-fixtures" })],
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("seeds core skill fixtures for an explicit local user without creating @local", async () => {
|
||||
const { db, tables } = createDb();
|
||||
const userId = (await db.insert("users", {
|
||||
@@ -229,210 +190,6 @@ describe("devSeed local fixtures", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("does not copy publisher ownership onto public corpus skill embeddings", async () => {
|
||||
const { db, tables } = createDb();
|
||||
|
||||
await seedPublicCorpusBatchHandler(
|
||||
createMutationCtx(db) as never,
|
||||
{
|
||||
rows: [
|
||||
{
|
||||
kind: "skill",
|
||||
slug: "corpus-demo",
|
||||
displayName: "Corpus Demo",
|
||||
version: "0.1.0",
|
||||
skillMd: "---\ndescription: Corpus demo\n---\n# Corpus demo",
|
||||
storageId: "storage:corpus-demo",
|
||||
embedding: [0, 1, 2],
|
||||
dummyOwner: {
|
||||
handle: "corpus-owner",
|
||||
displayName: "Corpus Owner",
|
||||
image: "https://example.invalid/avatar.png",
|
||||
},
|
||||
},
|
||||
],
|
||||
} as never,
|
||||
);
|
||||
|
||||
expect(tables.skills?.[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
slug: "corpus-demo",
|
||||
ownerPublisherId: tables.publishers?.[0]?._id,
|
||||
}),
|
||||
);
|
||||
expect(tables.skillEmbeddings?.[0]).not.toHaveProperty("ownerPublisherId");
|
||||
});
|
||||
|
||||
it("seeds a GitHub-backed source and skills without creating mirrored versions", async () => {
|
||||
const { db, tables } = createDb();
|
||||
const userId = (await db.insert("users", {
|
||||
handle: "nvidia-dev",
|
||||
displayName: "NVIDIA Dev",
|
||||
role: "user",
|
||||
createdAt: 1,
|
||||
updatedAt: 1,
|
||||
})) as Id<"users">;
|
||||
|
||||
const result = await seedGitHubBackedSkillSourceHandler(
|
||||
createMutationCtx(db) as never,
|
||||
{
|
||||
ownerUserId: userId,
|
||||
repo: "NVIDIA/skills",
|
||||
defaultBranch: "main",
|
||||
displayManifestKind: "skills.sh",
|
||||
displayManifestHash: "manifest-sha256",
|
||||
displayManifestCommit: "0".repeat(40),
|
||||
displayManifestFetchedAt: 123,
|
||||
displayManifestStatus: "ok",
|
||||
displayManifest: {
|
||||
notGrouped: "bottom",
|
||||
groupings: [
|
||||
{
|
||||
title: "Agentic AI",
|
||||
description: "Agentic AI skills.",
|
||||
skills: ["aiq-deploy", "nemoclaw-user-configure-security"],
|
||||
},
|
||||
],
|
||||
},
|
||||
skills: [
|
||||
{
|
||||
slug: "aiq-deploy",
|
||||
displayName: "AIQ Deploy",
|
||||
summary: "Deploy AgentIQ workflows.",
|
||||
githubPath: "skills/aiq-deploy",
|
||||
githubCurrentCommit: "1".repeat(40),
|
||||
githubCurrentContentHash: "hash-aiq-deploy",
|
||||
githubScanStatus: "clean",
|
||||
githubCurrentCheckedAt: 456,
|
||||
},
|
||||
{
|
||||
slug: "nemoclaw-user-configure-security",
|
||||
displayName: "NeMoClaw User Configure Security",
|
||||
summary: "Configure NeMoClaw user security.",
|
||||
githubPath: "skills/nemoclaw-user-configure-security",
|
||||
githubCurrentCommit: "2".repeat(40),
|
||||
githubCurrentContentHash: "hash-nemoclaw",
|
||||
githubScanStatus: "clean",
|
||||
githubCurrentCheckedAt: 789,
|
||||
githubRemovedAt: 900,
|
||||
},
|
||||
],
|
||||
} as never,
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({
|
||||
ok: true,
|
||||
seeded: ["aiq-deploy", "nemoclaw-user-configure-security"],
|
||||
skipped: [],
|
||||
});
|
||||
expect(tables.githubSkillSources).toHaveLength(1);
|
||||
expect(tables.githubSkillSources?.[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
repo: "NVIDIA/skills",
|
||||
ownerPublisherId: tables.publishers?.[0]?._id,
|
||||
defaultBranch: "main",
|
||||
displayManifestKind: "skills.sh",
|
||||
displayManifestHash: "manifest-sha256",
|
||||
displayManifestCommit: "0".repeat(40),
|
||||
displayManifestFetchedAt: 123,
|
||||
displayManifestStatus: "ok",
|
||||
displayManifest: {
|
||||
notGrouped: "bottom",
|
||||
groupings: [
|
||||
{
|
||||
title: "Agentic AI",
|
||||
description: "Agentic AI skills.",
|
||||
skills: ["aiq-deploy", "nemoclaw-user-configure-security"],
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
);
|
||||
expect(tables.skills).toHaveLength(2);
|
||||
expect(tables.skills).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
slug: "aiq-deploy",
|
||||
installKind: "github",
|
||||
githubSourceId: tables.githubSkillSources?.[0]?._id,
|
||||
githubPath: "skills/aiq-deploy",
|
||||
githubCurrentCommit: "1".repeat(40),
|
||||
githubCurrentContentHash: "hash-aiq-deploy",
|
||||
githubScanStatus: "clean",
|
||||
githubCurrentCheckedAt: 456,
|
||||
latestVersionId: undefined,
|
||||
latestVersionSummary: undefined,
|
||||
tags: {},
|
||||
stats: expect.objectContaining({ versions: 0 }),
|
||||
}),
|
||||
expect.objectContaining({
|
||||
slug: "nemoclaw-user-configure-security",
|
||||
installKind: "github",
|
||||
githubSourceId: tables.githubSkillSources?.[0]?._id,
|
||||
githubPath: "skills/nemoclaw-user-configure-security",
|
||||
githubCurrentContentHash: "hash-nemoclaw",
|
||||
githubRemovedAt: 900,
|
||||
moderationStatus: "hidden",
|
||||
moderationReason: "github.upstream.removed",
|
||||
moderationVerdict: undefined,
|
||||
isSuspicious: false,
|
||||
latestVersionId: undefined,
|
||||
tags: {},
|
||||
}),
|
||||
]),
|
||||
);
|
||||
expect(tables.skillVersions ?? []).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("keeps unscanned GitHub-backed skills hidden from public listings", async () => {
|
||||
const { db, tables } = createDb();
|
||||
|
||||
await seedGitHubBackedSkillSourceHandler(
|
||||
createMutationCtx(db) as never,
|
||||
{
|
||||
repo: "NVIDIA/skills",
|
||||
displayManifestStatus: "ok",
|
||||
skills: [
|
||||
{
|
||||
slug: "pending-github-skill",
|
||||
displayName: "Pending GitHub Skill",
|
||||
githubPath: "skills/pending-github-skill",
|
||||
githubCurrentCommit: "1".repeat(40),
|
||||
githubCurrentContentHash: "hash-pending",
|
||||
githubScanStatus: "pending",
|
||||
},
|
||||
{
|
||||
slug: "failed-scan-github-skill",
|
||||
displayName: "Failed Scan GitHub Skill",
|
||||
githubPath: "skills/failed-scan-github-skill",
|
||||
githubCurrentCommit: "2".repeat(40),
|
||||
githubCurrentContentHash: "hash-failed-scan",
|
||||
githubScanStatus: "failed",
|
||||
},
|
||||
],
|
||||
} as never,
|
||||
);
|
||||
|
||||
expect(tables.skills).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
slug: "pending-github-skill",
|
||||
moderationStatus: "hidden",
|
||||
moderationReason: "pending.scan",
|
||||
moderationVerdict: undefined,
|
||||
isSuspicious: false,
|
||||
}),
|
||||
expect.objectContaining({
|
||||
slug: "failed-scan-github-skill",
|
||||
moderationStatus: "hidden",
|
||||
moderationReason: "scanner.failed",
|
||||
moderationVerdict: undefined,
|
||||
isSuspicious: false,
|
||||
}),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("seeds moderation and plugin fixtures for an explicit local user with scoped identifiers", async () => {
|
||||
const { db, tables } = createDb();
|
||||
const userId = (await db.insert("users", {
|
||||
@@ -501,21 +258,6 @@ describe("devSeed local fixtures", () => {
|
||||
scannedPluginName,
|
||||
]);
|
||||
expect(tables.packages?.every((pkg) => pkg.ownerUserId === userId)).toBe(true);
|
||||
expect(tables.packageInspectorWarnings).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
packageName: scannedPluginName,
|
||||
findingKind: "warning",
|
||||
code: "legacy-before-agent-start",
|
||||
}),
|
||||
expect.objectContaining({
|
||||
packageName: scannedPluginName,
|
||||
findingKind: "error",
|
||||
code: "missing-expected-seam",
|
||||
scanSource: "nightly",
|
||||
}),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("retires legacy @local-owner seed publishers so dev-auth users can claim the handle", async () => {
|
||||
|
||||
+2
-1034
File diff suppressed because it is too large
Load Diff
@@ -1,249 +0,0 @@
|
||||
/* @vitest-environment node */
|
||||
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
__test,
|
||||
pruneDownloadMetricDedupesInternal,
|
||||
recordDownloadMetricInternal,
|
||||
} from "./downloadMetrics";
|
||||
|
||||
type WrappedHandler<TArgs, TResult> = {
|
||||
_handler: (ctx: unknown, args: TArgs) => Promise<TResult>;
|
||||
};
|
||||
|
||||
const recordDownloadMetricHandler = (
|
||||
recordDownloadMetricInternal as unknown as WrappedHandler<
|
||||
{
|
||||
target: { kind: "skill"; id: string } | { kind: "package"; id: string };
|
||||
identityKind: "user" | "ip";
|
||||
identityHash: string;
|
||||
dayStart: number;
|
||||
occurredAt?: number;
|
||||
},
|
||||
void
|
||||
>
|
||||
)._handler;
|
||||
|
||||
const pruneDownloadMetricDedupesHandler = (
|
||||
pruneDownloadMetricDedupesInternal as unknown as WrappedHandler<
|
||||
Record<string, never>,
|
||||
{ deleted: number; hasMore: boolean }
|
||||
>
|
||||
)._handler;
|
||||
|
||||
function makeQueryBuilder() {
|
||||
const builder = {
|
||||
eq: vi.fn(() => builder),
|
||||
lt: vi.fn(() => builder),
|
||||
};
|
||||
return builder;
|
||||
}
|
||||
|
||||
type QueryBuilder = ReturnType<typeof makeQueryBuilder>;
|
||||
|
||||
function makeDb(
|
||||
existingByTable: Record<string, unknown> = {},
|
||||
rowsByTable: Record<string, Array<{ _id: string }>> = {},
|
||||
) {
|
||||
const indexCalls: Array<{ table: string; indexName: string; builder: QueryBuilder }> = [];
|
||||
const insert = vi.fn();
|
||||
const unique = vi.fn(async function uniqueForTable(this: { table: string }) {
|
||||
return existingByTable[this.table] ?? null;
|
||||
});
|
||||
const take = vi.fn(async function takeForTable(this: { table: string }, limit: number) {
|
||||
return (rowsByTable[this.table] ?? []).slice(0, limit);
|
||||
});
|
||||
const query = vi.fn((table: string) => ({
|
||||
withIndex: vi.fn((indexName: string, buildQuery: (q: unknown) => unknown) => {
|
||||
const builder = makeQueryBuilder();
|
||||
buildQuery(builder);
|
||||
indexCalls.push({ table, indexName, builder });
|
||||
return {
|
||||
unique: unique.bind({ table }),
|
||||
take: take.bind({ table }),
|
||||
};
|
||||
}),
|
||||
}));
|
||||
const delete_ = vi.fn();
|
||||
return {
|
||||
db: {
|
||||
query,
|
||||
get: vi.fn(),
|
||||
insert,
|
||||
patch: vi.fn(),
|
||||
replace: vi.fn(),
|
||||
delete: delete_,
|
||||
normalizeId: vi.fn(),
|
||||
system: {
|
||||
get: vi.fn(),
|
||||
query: vi.fn(),
|
||||
},
|
||||
},
|
||||
insert,
|
||||
delete_,
|
||||
take,
|
||||
indexCalls,
|
||||
};
|
||||
}
|
||||
|
||||
describe("download metric helpers", () => {
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("uses a day bucket for download dedupe", () => {
|
||||
expect(__test.getDayStart(86_400_000 - 1)).toBe(0);
|
||||
expect(__test.getDayStart(86_400_000)).toBe(86_400_000);
|
||||
});
|
||||
|
||||
it("prefers user identity and falls back to IP identity", () => {
|
||||
const request = new Request("https://example.com", {
|
||||
headers: { "cf-connecting-ip": "203.0.113.10" },
|
||||
});
|
||||
|
||||
expect(__test.getDownloadIdentity(request, "users:one")).toEqual({
|
||||
identityKind: "user",
|
||||
identityValue: "users:one",
|
||||
});
|
||||
expect(__test.getDownloadIdentity(request, null)).toEqual({
|
||||
identityKind: "ip",
|
||||
identityValue: "203.0.113.10",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not create a metering identity when user and IP are missing", () => {
|
||||
expect(__test.getDownloadIdentity(new Request("https://example.com"), null)).toBeNull();
|
||||
});
|
||||
|
||||
it("records one authenticated skill download and emits the existing skill stat event", async () => {
|
||||
const { db, insert, indexCalls } = makeDb();
|
||||
|
||||
await recordDownloadMetricHandler(
|
||||
{ db },
|
||||
{
|
||||
target: { kind: "skill", id: "skills:one" },
|
||||
identityKind: "user",
|
||||
identityHash: "hash-user",
|
||||
dayStart: 86_400_000,
|
||||
occurredAt: 86_500_000,
|
||||
},
|
||||
);
|
||||
|
||||
expect(indexCalls[0]?.table).toBe("downloadMetricDedupes");
|
||||
expect(indexCalls[0]?.indexName).toBe("by_target_identity_day");
|
||||
expect(indexCalls[0]?.builder.eq).toHaveBeenCalledWith("targetKind", "skill");
|
||||
expect(indexCalls[0]?.builder.eq).toHaveBeenCalledWith("targetId", "skills:one");
|
||||
expect(indexCalls[0]?.builder.eq).toHaveBeenCalledWith("identityKind", "user");
|
||||
expect(indexCalls[0]?.builder.eq).toHaveBeenCalledWith("identityHash", "hash-user");
|
||||
expect(indexCalls[0]?.builder.eq).toHaveBeenCalledWith("dayStart", 86_400_000);
|
||||
expect(insert).toHaveBeenCalledWith(
|
||||
"downloadMetricDedupes",
|
||||
expect.objectContaining({
|
||||
targetKind: "skill",
|
||||
targetId: "skills:one",
|
||||
identityKind: "user",
|
||||
identityHash: "hash-user",
|
||||
dayStart: 86_400_000,
|
||||
}),
|
||||
);
|
||||
expect(insert).toHaveBeenCalledWith(
|
||||
"skillStatEvents",
|
||||
expect.objectContaining({
|
||||
skillId: "skills:one",
|
||||
kind: "download",
|
||||
occurredAt: 86_500_000,
|
||||
}),
|
||||
);
|
||||
expect(insert).not.toHaveBeenCalledWith("packageStatEvents", expect.anything());
|
||||
});
|
||||
|
||||
it("records one anonymous package download and emits the existing package stat event", async () => {
|
||||
const { db, insert } = makeDb();
|
||||
|
||||
await recordDownloadMetricHandler(
|
||||
{ db },
|
||||
{
|
||||
target: { kind: "package", id: "packages:one" },
|
||||
identityKind: "ip",
|
||||
identityHash: "hash-ip",
|
||||
dayStart: 86_400_000,
|
||||
occurredAt: 86_500_000,
|
||||
},
|
||||
);
|
||||
|
||||
expect(insert).toHaveBeenCalledWith(
|
||||
"downloadMetricDedupes",
|
||||
expect.objectContaining({
|
||||
targetKind: "package",
|
||||
targetId: "packages:one",
|
||||
identityKind: "ip",
|
||||
identityHash: "hash-ip",
|
||||
dayStart: 86_400_000,
|
||||
}),
|
||||
);
|
||||
expect(insert).toHaveBeenCalledWith(
|
||||
"packageStatEvents",
|
||||
expect.objectContaining({
|
||||
packageId: "packages:one",
|
||||
kind: "download",
|
||||
occurredAt: 86_500_000,
|
||||
}),
|
||||
);
|
||||
expect(insert).not.toHaveBeenCalledWith("skillStatEvents", expect.anything());
|
||||
});
|
||||
|
||||
it("ignores duplicate identities in the same target/day bucket", async () => {
|
||||
const { db, insert } = makeDb({
|
||||
downloadMetricDedupes: { _id: "downloadMetricDedupes:existing" },
|
||||
});
|
||||
|
||||
await recordDownloadMetricHandler(
|
||||
{ db },
|
||||
{
|
||||
target: { kind: "skill", id: "skills:one" },
|
||||
identityKind: "ip",
|
||||
identityHash: "hash-ip",
|
||||
dayStart: 86_400_000,
|
||||
},
|
||||
);
|
||||
|
||||
expect(insert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("prunes stale dedupe rows by day bucket", async () => {
|
||||
vi.setSystemTime(30 * 86_400_000);
|
||||
const { db, delete_, take, indexCalls } = makeDb(
|
||||
{},
|
||||
{
|
||||
downloadMetricDedupes: [
|
||||
{ _id: "downloadMetricDedupes:one" },
|
||||
{ _id: "downloadMetricDedupes:two" },
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
const result = await pruneDownloadMetricDedupesHandler({ db }, {});
|
||||
|
||||
expect(result).toEqual({ deleted: 2, hasMore: false });
|
||||
expect(indexCalls[0]?.table).toBe("downloadMetricDedupes");
|
||||
expect(indexCalls[0]?.indexName).toBe("by_day");
|
||||
expect(take).toHaveBeenCalledWith(200);
|
||||
expect(delete_).toHaveBeenCalledWith("downloadMetricDedupes:one");
|
||||
expect(delete_).toHaveBeenCalledWith("downloadMetricDedupes:two");
|
||||
});
|
||||
|
||||
it("reschedules stale dedupe pruning when one bounded batch fills", async () => {
|
||||
vi.setSystemTime(30 * 86_400_000);
|
||||
const rows = Array.from({ length: 200 }, (_, index) => ({
|
||||
_id: `downloadMetricDedupes:${index}`,
|
||||
}));
|
||||
const { db, delete_ } = makeDb({}, { downloadMetricDedupes: rows });
|
||||
const runAfter = vi.fn();
|
||||
|
||||
const result = await pruneDownloadMetricDedupesHandler({ db, scheduler: { runAfter } }, {});
|
||||
|
||||
expect(result).toEqual({ deleted: 200, hasMore: true });
|
||||
expect(delete_).toHaveBeenCalledTimes(200);
|
||||
expect(runAfter).toHaveBeenCalledWith(0, expect.anything(), {});
|
||||
});
|
||||
});
|
||||
@@ -1,137 +0,0 @@
|
||||
import { v } from "convex/values";
|
||||
import { internal } from "./_generated/api";
|
||||
import type { Id } from "./_generated/dataModel";
|
||||
import { internalMutation } from "./functions";
|
||||
import { getClientIp } from "./lib/httpRateLimit";
|
||||
import { hashToken } from "./lib/tokens";
|
||||
import { insertStatEvent } from "./skillStatEvents";
|
||||
|
||||
const DAY_MS = 86_400_000;
|
||||
const DEDUPE_RETENTION_MS = 14 * DAY_MS;
|
||||
const PRUNE_BATCH_SIZE = 200;
|
||||
|
||||
const identityKindValidator = v.union(v.literal("user"), v.literal("ip"));
|
||||
|
||||
const targetValidator = v.union(
|
||||
v.object({ kind: v.literal("skill"), id: v.id("skills") }),
|
||||
v.object({ kind: v.literal("package"), id: v.id("packages") }),
|
||||
);
|
||||
|
||||
type DownloadIdentityKind = "user" | "ip";
|
||||
|
||||
type DownloadIdentity = {
|
||||
identityKind: DownloadIdentityKind;
|
||||
identityValue: string;
|
||||
};
|
||||
|
||||
export function getDownloadIdentity(
|
||||
request: Request,
|
||||
userId: string | null,
|
||||
): DownloadIdentity | null {
|
||||
if (userId) return { identityKind: "user", identityValue: userId };
|
||||
const ip = getClientIp(request);
|
||||
if (!ip) return null;
|
||||
return { identityKind: "ip", identityValue: ip };
|
||||
}
|
||||
|
||||
export async function buildDownloadMetricArgs(params: {
|
||||
target: { kind: "skill"; id: Id<"skills"> } | { kind: "package"; id: Id<"packages"> };
|
||||
identity: DownloadIdentity;
|
||||
now: number;
|
||||
}) {
|
||||
return {
|
||||
target: params.target,
|
||||
identityKind: params.identity.identityKind,
|
||||
identityHash: await hashToken(
|
||||
`${params.identity.identityKind}:${params.identity.identityValue}`,
|
||||
),
|
||||
dayStart: getDayStart(params.now),
|
||||
occurredAt: params.now,
|
||||
};
|
||||
}
|
||||
|
||||
export const recordDownloadMetricInternal = internalMutation({
|
||||
args: {
|
||||
target: targetValidator,
|
||||
identityKind: identityKindValidator,
|
||||
identityHash: v.string(),
|
||||
dayStart: v.number(),
|
||||
occurredAt: v.optional(v.number()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const targetId = args.target.id;
|
||||
const existing = await ctx.db
|
||||
.query("downloadMetricDedupes")
|
||||
.withIndex("by_target_identity_day", (q) =>
|
||||
q
|
||||
.eq("targetKind", args.target.kind)
|
||||
.eq("targetId", targetId)
|
||||
.eq("identityKind", args.identityKind)
|
||||
.eq("identityHash", args.identityHash)
|
||||
.eq("dayStart", args.dayStart),
|
||||
)
|
||||
.unique();
|
||||
if (existing) return;
|
||||
|
||||
const now = Date.now();
|
||||
await ctx.db.insert("downloadMetricDedupes", {
|
||||
targetKind: args.target.kind,
|
||||
targetId,
|
||||
identityKind: args.identityKind,
|
||||
identityHash: args.identityHash,
|
||||
dayStart: args.dayStart,
|
||||
createdAt: now,
|
||||
});
|
||||
|
||||
if (args.target.kind === "skill") {
|
||||
await insertStatEvent(ctx, {
|
||||
skillId: args.target.id,
|
||||
kind: "download",
|
||||
occurredAt: args.occurredAt,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
await ctx.db.insert("packageStatEvents", {
|
||||
packageId: args.target.id,
|
||||
kind: "download",
|
||||
occurredAt: args.occurredAt ?? now,
|
||||
processedAt: undefined,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
export const pruneDownloadMetricDedupesInternal = internalMutation({
|
||||
args: {},
|
||||
handler: async (ctx) => {
|
||||
const cutoffDayStart = getDayStart(Date.now() - DEDUPE_RETENTION_MS);
|
||||
const stale = await ctx.db
|
||||
.query("downloadMetricDedupes")
|
||||
.withIndex("by_day", (q) => q.lt("dayStart", cutoffDayStart))
|
||||
.take(PRUNE_BATCH_SIZE);
|
||||
|
||||
for (const entry of stale) {
|
||||
await ctx.db.delete(entry._id);
|
||||
}
|
||||
|
||||
const hasMore = stale.length === PRUNE_BATCH_SIZE;
|
||||
if (hasMore) {
|
||||
await ctx.scheduler.runAfter(
|
||||
0,
|
||||
internal.downloadMetrics.pruneDownloadMetricDedupesInternal,
|
||||
{},
|
||||
);
|
||||
}
|
||||
|
||||
return { deleted: stale.length, hasMore };
|
||||
},
|
||||
});
|
||||
|
||||
function getDayStart(timestamp: number) {
|
||||
return Math.floor(timestamp / DAY_MS) * DAY_MS;
|
||||
}
|
||||
|
||||
export const __test = {
|
||||
getDayStart,
|
||||
getDownloadIdentity,
|
||||
};
|
||||
+14
-150
@@ -21,19 +21,6 @@ const okRate = () => ({
|
||||
resetAt: Date.now() + 60_000,
|
||||
});
|
||||
|
||||
function stubZipResponse() {
|
||||
class MockResponse {
|
||||
status: number;
|
||||
headers: Headers;
|
||||
|
||||
constructor(_body?: BodyInit | null, init?: ResponseInit) {
|
||||
this.status = init?.status ?? 200;
|
||||
this.headers = new Headers(init?.headers);
|
||||
}
|
||||
}
|
||||
vi.stubGlobal("Response", MockResponse as unknown as typeof Response);
|
||||
}
|
||||
|
||||
describe("downloads helpers", () => {
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
@@ -76,7 +63,16 @@ describe("downloads helpers", () => {
|
||||
});
|
||||
|
||||
it("schedules zip download stats outside the response path", async () => {
|
||||
stubZipResponse();
|
||||
class MockResponse {
|
||||
status: number;
|
||||
headers: Headers;
|
||||
|
||||
constructor(_body?: BodyInit | null, init?: ResponseInit) {
|
||||
this.status = init?.status ?? 200;
|
||||
this.headers = new Headers(init?.headers);
|
||||
}
|
||||
}
|
||||
vi.stubGlobal("Response", MockResponse as unknown as typeof Response);
|
||||
|
||||
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
|
||||
if (isRateLimitArgs(args)) return okRate();
|
||||
@@ -131,10 +127,9 @@ describe("downloads helpers", () => {
|
||||
if (!args || typeof args !== "object") return false;
|
||||
const value = args as Record<string, unknown>;
|
||||
return (
|
||||
typeof value.target === "object" &&
|
||||
value.skillId === "skills:1" &&
|
||||
typeof value.identityHash === "string" &&
|
||||
value.identityKind === "ip" &&
|
||||
typeof value.dayStart === "number"
|
||||
typeof value.hourStart === "number"
|
||||
);
|
||||
});
|
||||
expect(recordCalls).toHaveLength(1);
|
||||
@@ -142,11 +137,9 @@ describe("downloads helpers", () => {
|
||||
expect(recordCalls[0]?.[0]).toBeGreaterThanOrEqual(0);
|
||||
expect(recordCalls[0]?.[0]).toBeLessThan(60_000);
|
||||
expect(recordCalls[0]?.[2]).toEqual({
|
||||
target: { kind: "skill", id: "skills:1" },
|
||||
identityKind: "ip",
|
||||
skillId: "skills:1",
|
||||
identityHash: expect.any(String),
|
||||
dayStart: expect.any(Number),
|
||||
occurredAt: expect.any(Number),
|
||||
hourStart: expect.any(Number),
|
||||
});
|
||||
});
|
||||
|
||||
@@ -209,133 +202,4 @@ describe("downloads helpers", () => {
|
||||
expect(await response.text()).toBe("Version not found");
|
||||
expect(storageGet).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("uses API token user identity for zip download stats when present", async () => {
|
||||
stubZipResponse();
|
||||
|
||||
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
|
||||
if (isRateLimitArgs(args)) return okRate();
|
||||
if ("tokenHash" in args) {
|
||||
return { _id: "apiTokens:1", revokedAt: undefined };
|
||||
}
|
||||
if ("tokenId" in args) {
|
||||
return { _id: "users:token", deletedAt: undefined, deactivatedAt: undefined };
|
||||
}
|
||||
if ("slug" in args) {
|
||||
return {
|
||||
skill: {
|
||||
_id: "skills:1",
|
||||
ownerUserId: "users:1",
|
||||
slug: "demo",
|
||||
tags: {},
|
||||
latestVersionId: "skillVersions:1",
|
||||
},
|
||||
moderationInfo: null,
|
||||
};
|
||||
}
|
||||
if ("versionId" in args) {
|
||||
return {
|
||||
_id: "skillVersions:1",
|
||||
skillId: "skills:1",
|
||||
version: "1.0.0",
|
||||
createdAt: 3,
|
||||
files: [{ path: "SKILL.md", storageId: "_storage:1" }],
|
||||
softDeletedAt: undefined,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
});
|
||||
const runMutation = vi.fn(async (_mutation: unknown, args: Record<string, unknown>) => {
|
||||
if (isRateLimitArgs(args)) return okRate();
|
||||
return { tokenTouched: "tokenId" in args };
|
||||
});
|
||||
const runAfter = vi.fn();
|
||||
const storageGet = vi.fn().mockResolvedValue(new Blob(["hello"], { type: "text/markdown" }));
|
||||
|
||||
const response = await downloadZipHandler(
|
||||
{
|
||||
runQuery,
|
||||
runMutation,
|
||||
scheduler: { runAfter },
|
||||
storage: { get: storageGet },
|
||||
} as unknown as ActionCtx,
|
||||
new Request("https://example.com/api/v1/download?slug=demo", {
|
||||
headers: {
|
||||
authorization: "Bearer clh_test",
|
||||
"cf-connecting-ip": "1.2.3.4",
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(runAfter).toHaveBeenCalledWith(
|
||||
expect.any(Number),
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
target: { kind: "skill", id: "skills:1" },
|
||||
identityKind: "user",
|
||||
identityHash: expect.stringMatching(/^[a-f0-9]{64}$/),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("returns zip downloads when download metering is scheduled", async () => {
|
||||
stubZipResponse();
|
||||
|
||||
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
|
||||
if (isRateLimitArgs(args)) return okRate();
|
||||
if ("slug" in args) {
|
||||
return {
|
||||
skill: {
|
||||
_id: "skills:1",
|
||||
ownerUserId: "users:1",
|
||||
slug: "demo",
|
||||
tags: {},
|
||||
latestVersionId: "skillVersions:1",
|
||||
},
|
||||
moderationInfo: null,
|
||||
};
|
||||
}
|
||||
if ("versionId" in args) {
|
||||
return {
|
||||
_id: "skillVersions:1",
|
||||
skillId: "skills:1",
|
||||
version: "1.0.0",
|
||||
createdAt: 3,
|
||||
files: [{ path: "SKILL.md", storageId: "_storage:1" }],
|
||||
softDeletedAt: undefined,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
});
|
||||
const runMutation = vi.fn(async (_mutation: unknown, args: Record<string, unknown>) => {
|
||||
if (isRateLimitArgs(args)) return okRate();
|
||||
return { mutationRecorded: true };
|
||||
});
|
||||
const runAfter = vi.fn();
|
||||
const storageGet = vi.fn().mockResolvedValue(new Blob(["hello"], { type: "text/markdown" }));
|
||||
|
||||
const response = await downloadZipHandler(
|
||||
{
|
||||
runQuery,
|
||||
runMutation,
|
||||
scheduler: { runAfter },
|
||||
storage: { get: storageGet },
|
||||
} as unknown as ActionCtx,
|
||||
new Request("https://example.com/api/v1/download?slug=demo", {
|
||||
headers: { "cf-connecting-ip": "1.2.3.4" },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(runAfter).toHaveBeenCalledWith(
|
||||
expect.any(Number),
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
target: { kind: "skill", id: "skills:1" },
|
||||
identityKind: "ip",
|
||||
identityHash: expect.stringMatching(/^[a-f0-9]{64}$/),
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
+9
-20
@@ -1,14 +1,12 @@
|
||||
import { v } from "convex/values";
|
||||
import { api, internal } from "./_generated/api";
|
||||
import type { Id } from "./_generated/dataModel";
|
||||
import { buildDownloadMetricArgs, getDownloadIdentity } from "./downloadMetrics";
|
||||
import { httpAction, internalMutation } from "./functions";
|
||||
import { getOptionalActiveAuthUserIdFromAction } from "./lib/access";
|
||||
import { getOptionalApiTokenUserId } from "./lib/apiTokenAuth";
|
||||
import { corsHeaders, mergeHeaders } from "./lib/httpHeaders";
|
||||
import { applyRateLimit, getClientIp } from "./lib/httpRateLimit";
|
||||
import { getPublicSkillFileAccessBlock, isSkillVersionForSkill } from "./lib/skillFileAccess";
|
||||
import { buildDeterministicZip } from "./lib/skillZip";
|
||||
import { hashToken } from "./lib/tokens";
|
||||
import { insertStatEvent } from "./skillStatEvents";
|
||||
|
||||
const HOUR_MS = 3_600_000;
|
||||
@@ -100,17 +98,17 @@ export async function downloadZipHandler(
|
||||
const zipBlob = new Blob([zipArray], { type: "application/zip" });
|
||||
|
||||
try {
|
||||
const userId = await getOptionalDownloadUserId(ctx, request);
|
||||
const identity = getDownloadIdentity(request, userId ? String(userId) : null);
|
||||
const userId = await getOptionalApiTokenUserId(ctx, request);
|
||||
const identity = getDownloadIdentityValue(request, userId ? String(userId) : null);
|
||||
if (identity) {
|
||||
await ctx.scheduler.runAfter(
|
||||
Math.floor(Math.random() * DOWNLOAD_STAT_JITTER_MS),
|
||||
internal.downloadMetrics.recordDownloadMetricInternal,
|
||||
await buildDownloadMetricArgs({
|
||||
target: { kind: "skill", id: skill._id },
|
||||
identity,
|
||||
now: Date.now(),
|
||||
}),
|
||||
internal.downloads.recordDownloadInternal,
|
||||
{
|
||||
skillId: skill._id,
|
||||
identityHash: await hashToken(identity),
|
||||
hourStart: getHourStart(Date.now()),
|
||||
},
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
@@ -198,15 +196,6 @@ export function getDownloadIdentityValue(request: Request, userId: string | null
|
||||
return `ip:${ip}`;
|
||||
}
|
||||
|
||||
async function getOptionalDownloadUserId(
|
||||
ctx: Parameters<Parameters<typeof httpAction>[0]>[0],
|
||||
request: Request,
|
||||
): Promise<Id<"users"> | null> {
|
||||
const apiTokenUserId = await getOptionalApiTokenUserId(ctx, request);
|
||||
if (apiTokenUserId) return apiTokenUserId;
|
||||
return (await getOptionalActiveAuthUserIdFromAction(ctx)) ?? null;
|
||||
}
|
||||
|
||||
export const __test = {
|
||||
getHourStart,
|
||||
getDownloadIdentityValue,
|
||||
|
||||
@@ -1,160 +0,0 @@
|
||||
"use node";
|
||||
|
||||
import { mkdir, appendFile } from "node:fs/promises";
|
||||
import { dirname } from "node:path";
|
||||
import { v } from "convex/values";
|
||||
import { Resend } from "resend";
|
||||
import { internalAction } from "./functions";
|
||||
import {
|
||||
buildBanNotificationEmail,
|
||||
buildMaliciousArtifactEmail,
|
||||
buildRestoredAccountEmail,
|
||||
type NotificationArtifact,
|
||||
} from "./lib/emails";
|
||||
|
||||
const DEFAULT_FROM = "ClawHub Security <noreply@notifications.openclaw.ai>";
|
||||
const DEFAULT_REPLY_TO = "security@notifications.openclaw.ai";
|
||||
|
||||
const notificationArtifactValidator = v.object({
|
||||
kind: v.union(v.literal("skill"), v.literal("plugin")),
|
||||
name: v.string(),
|
||||
});
|
||||
|
||||
type SendEmailArgs = {
|
||||
idempotencyKey: string;
|
||||
to: string;
|
||||
subject: string;
|
||||
text: string;
|
||||
html: string;
|
||||
};
|
||||
|
||||
function getEmailConfig() {
|
||||
return {
|
||||
apiKey: process.env.RESEND_API_KEY,
|
||||
from: process.env.CLAWHUB_SECURITY_EMAIL_FROM || DEFAULT_FROM,
|
||||
replyTo: process.env.CLAWHUB_SECURITY_EMAIL || DEFAULT_REPLY_TO,
|
||||
};
|
||||
}
|
||||
|
||||
async function sendTransactionalEmail(args: SendEmailArgs) {
|
||||
const captureFile = process.env.CLAWHUB_EMAIL_CAPTURE_FILE?.trim();
|
||||
if (captureFile) {
|
||||
await mkdir(dirname(captureFile), { recursive: true });
|
||||
await appendFile(
|
||||
captureFile,
|
||||
`${JSON.stringify({ ...args, capturedAt: Date.now() })}\n`,
|
||||
"utf8",
|
||||
);
|
||||
return { ok: true as const, id: "local-capture" };
|
||||
}
|
||||
|
||||
const config = getEmailConfig();
|
||||
if (!config.apiKey) {
|
||||
console.warn(`[emails] RESEND_API_KEY is not configured; skipped ${args.idempotencyKey}`);
|
||||
return { ok: false as const, reason: "missing_api_key" as const };
|
||||
}
|
||||
|
||||
try {
|
||||
const resend = new Resend(config.apiKey);
|
||||
const result = await resend.emails.send(
|
||||
{
|
||||
from: config.from,
|
||||
to: args.to,
|
||||
replyTo: config.replyTo,
|
||||
subject: args.subject,
|
||||
text: args.text,
|
||||
html: args.html,
|
||||
},
|
||||
{ idempotencyKey: args.idempotencyKey },
|
||||
);
|
||||
if (result.error) {
|
||||
console.error("[emails] Resend error", result.error);
|
||||
return { ok: false as const, reason: "resend_error" as const };
|
||||
}
|
||||
return { ok: true as const, id: result.data?.id ?? null };
|
||||
} catch (error) {
|
||||
console.error("[emails] Send failed", error);
|
||||
return { ok: false as const, reason: "send_error" as const };
|
||||
}
|
||||
}
|
||||
|
||||
export const sendBanNotificationInternal = internalAction({
|
||||
args: {
|
||||
userId: v.id("users"),
|
||||
bannedAt: v.number(),
|
||||
to: v.string(),
|
||||
handle: v.optional(v.string()),
|
||||
source: v.union(v.literal("manual"), v.literal("autoban")),
|
||||
reason: v.optional(v.string()),
|
||||
trigger: v.optional(v.string()),
|
||||
artifact: v.optional(notificationArtifactValidator),
|
||||
},
|
||||
handler: async (_ctx, args) => {
|
||||
const email = buildBanNotificationEmail({
|
||||
handle: args.handle,
|
||||
source: args.source,
|
||||
reason: args.reason,
|
||||
trigger: args.trigger,
|
||||
artifact: args.artifact as NotificationArtifact | undefined,
|
||||
});
|
||||
return await sendTransactionalEmail({
|
||||
idempotencyKey: `ban:${args.userId}:${args.bannedAt}`,
|
||||
to: args.to,
|
||||
subject: email.subject,
|
||||
text: email.text,
|
||||
html: email.html,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
export const sendRestoredAccountNotificationInternal = internalAction({
|
||||
args: {
|
||||
userId: v.id("users"),
|
||||
restoredAt: v.number(),
|
||||
to: v.string(),
|
||||
handle: v.optional(v.string()),
|
||||
restoredListings: v.optional(v.array(notificationArtifactValidator)),
|
||||
},
|
||||
handler: async (_ctx, args) => {
|
||||
const email = buildRestoredAccountEmail({
|
||||
handle: args.handle,
|
||||
restoredListings: args.restoredListings as NotificationArtifact[] | undefined,
|
||||
});
|
||||
return await sendTransactionalEmail({
|
||||
idempotencyKey: `account-restored:${args.userId}:${args.restoredAt}`,
|
||||
to: args.to,
|
||||
subject: email.subject,
|
||||
text: email.text,
|
||||
html: email.html,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
export const sendMaliciousArtifactNotificationInternal = internalAction({
|
||||
args: {
|
||||
userId: v.id("users"),
|
||||
findingAt: v.number(),
|
||||
to: v.string(),
|
||||
handle: v.optional(v.string()),
|
||||
artifact: notificationArtifactValidator,
|
||||
version: v.optional(v.string()),
|
||||
trigger: v.optional(v.string()),
|
||||
findingSummary: v.optional(v.string()),
|
||||
},
|
||||
handler: async (_ctx, args) => {
|
||||
const email = buildMaliciousArtifactEmail({
|
||||
handle: args.handle,
|
||||
artifact: args.artifact as NotificationArtifact,
|
||||
version: args.version,
|
||||
trigger: args.trigger,
|
||||
findingSummary: args.findingSummary,
|
||||
});
|
||||
return await sendTransactionalEmail({
|
||||
idempotencyKey: `malicious-artifact:${args.userId}:${args.findingAt}:${args.artifact.kind}:${args.artifact.name}:${args.version ?? ""}`,
|
||||
to: args.to,
|
||||
subject: email.subject,
|
||||
text: email.text,
|
||||
html: email.html,
|
||||
});
|
||||
},
|
||||
});
|
||||
@@ -2,50 +2,17 @@
|
||||
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { internal } from "./_generated/api";
|
||||
import type { Doc, Id, TableNames } from "./_generated/dataModel";
|
||||
import {
|
||||
internalMutation,
|
||||
isGitHubMirrorEligibleSkillDoc,
|
||||
repointPackageLatestRelease,
|
||||
scheduleGitHubBackupDeletionForSkill,
|
||||
scheduleOwnerPublisherDigestSync,
|
||||
shouldScheduleOwnerPublisherDigestSyncForPublisherChange,
|
||||
shouldScheduleOwnerUserPackageDigestSyncForUserChange,
|
||||
syncPackageSearchDigestForPackageId,
|
||||
syncPackageSearchDigestsForOwnerPublisherId,
|
||||
syncPackageSearchDigestsForOwnerUserId,
|
||||
syncSkillSearchDigestsForOwnerPublisherId,
|
||||
} from "./functions";
|
||||
|
||||
type WrappedHandler = {
|
||||
_handler: (ctx: unknown, args: Record<string, never>) => Promise<unknown>;
|
||||
};
|
||||
|
||||
function hasWrappedHandler(value: unknown): value is WrappedHandler {
|
||||
return typeof value === "function" && "_handler" in value && typeof value._handler === "function";
|
||||
}
|
||||
|
||||
function getWrappedHandler(value: unknown): WrappedHandler["_handler"] {
|
||||
if (!hasWrappedHandler(value)) {
|
||||
throw new Error("Expected a Convex function with a test-callable _handler");
|
||||
}
|
||||
return value._handler;
|
||||
}
|
||||
|
||||
function testId<TableName extends TableNames>(
|
||||
tableName: TableName,
|
||||
value: `${TableName}:${string}`,
|
||||
): Id<TableName> {
|
||||
if (!value.startsWith(`${tableName}:`)) {
|
||||
throw new Error(`Expected ${value} to be a ${tableName} id`);
|
||||
}
|
||||
return value as Id<TableName>;
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
describe("package digest sync", () => {
|
||||
it("identifies GitHub mirror eligibility from skill visibility fields", () => {
|
||||
expect(isGitHubMirrorEligibleSkillDoc({ softDeletedAt: undefined })).toBe(true);
|
||||
@@ -55,20 +22,6 @@ describe("package digest sync", () => {
|
||||
moderationStatus: "active",
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
isGitHubMirrorEligibleSkillDoc({
|
||||
softDeletedAt: undefined,
|
||||
moderationStatus: "active",
|
||||
moderationVerdict: "malicious",
|
||||
}),
|
||||
).toBe(false);
|
||||
expect(
|
||||
isGitHubMirrorEligibleSkillDoc({
|
||||
softDeletedAt: undefined,
|
||||
moderationStatus: "active",
|
||||
moderationFlags: ["blocked.malware"],
|
||||
}),
|
||||
).toBe(false);
|
||||
expect(
|
||||
isGitHubMirrorEligibleSkillDoc({
|
||||
softDeletedAt: undefined,
|
||||
@@ -630,92 +583,7 @@ describe("package digest sync", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("user package digest scheduling", () => {
|
||||
const user = {
|
||||
_id: "users:owner",
|
||||
handle: "owner",
|
||||
deletedAt: undefined,
|
||||
deactivatedAt: undefined,
|
||||
};
|
||||
|
||||
it("schedules package digest sync when an active user's handle changes", () => {
|
||||
expect(
|
||||
shouldScheduleOwnerUserPackageDigestSyncForUserChange({
|
||||
id: "users:owner",
|
||||
operation: "update",
|
||||
oldDoc: user,
|
||||
newDoc: { ...user, handle: "renamed" },
|
||||
} as never),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("skips redundant package digest sync when a user becomes deactivated", () => {
|
||||
expect(
|
||||
shouldScheduleOwnerUserPackageDigestSyncForUserChange({
|
||||
id: "users:owner",
|
||||
operation: "update",
|
||||
oldDoc: user,
|
||||
newDoc: { ...user, handle: null, deactivatedAt: 1_700_000_000_000 },
|
||||
} as never),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("skips unchanged user updates", () => {
|
||||
expect(
|
||||
shouldScheduleOwnerUserPackageDigestSyncForUserChange({
|
||||
id: "users:owner",
|
||||
operation: "update",
|
||||
oldDoc: user,
|
||||
newDoc: { ...user },
|
||||
} as never),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("publisher digest scheduling", () => {
|
||||
const publisherChangeDoc = {
|
||||
_id: "publishers:demo",
|
||||
kind: "org",
|
||||
handle: "demo",
|
||||
displayName: "Demo",
|
||||
image: null,
|
||||
deletedAt: undefined,
|
||||
deactivatedAt: undefined,
|
||||
};
|
||||
|
||||
it("schedules digest sync when an active publisher's profile changes", () => {
|
||||
expect(
|
||||
shouldScheduleOwnerPublisherDigestSyncForPublisherChange({
|
||||
id: "publishers:demo",
|
||||
operation: "update",
|
||||
oldDoc: publisherChangeDoc,
|
||||
newDoc: { ...publisherChangeDoc, displayName: "Renamed Demo" },
|
||||
} as never),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("skips redundant digest sync when a publisher becomes deactivated", () => {
|
||||
expect(
|
||||
shouldScheduleOwnerPublisherDigestSyncForPublisherChange({
|
||||
id: "publishers:demo",
|
||||
operation: "update",
|
||||
oldDoc: publisherChangeDoc,
|
||||
newDoc: { ...publisherChangeDoc, handle: null, deactivatedAt: 1_700_000_000_000 },
|
||||
} as never),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("skips unchanged publisher updates", () => {
|
||||
expect(
|
||||
shouldScheduleOwnerPublisherDigestSyncForPublisherChange({
|
||||
id: "publishers:demo",
|
||||
operation: "update",
|
||||
oldDoc: publisherChangeDoc,
|
||||
newDoc: { ...publisherChangeDoc },
|
||||
} as never),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("schedules package and skill digest sync in separate background mutations", async () => {
|
||||
const ctx = {
|
||||
scheduler: {
|
||||
@@ -809,159 +677,4 @@ describe("publisher digest scheduling", () => {
|
||||
{ ownerPublisherId: "publishers:demo", cursor: "next-skills" },
|
||||
);
|
||||
});
|
||||
|
||||
it("syncs recommended rank stats into the skill search digest after wrapped skill patches", async () => {
|
||||
const skillId = testId("skills", "skills:demo");
|
||||
const ownerUserId = testId("users", "users:owner");
|
||||
const publisherId = testId("publishers", "publishers:owner");
|
||||
const digestId = testId("skillSearchDigest", "skillSearchDigest:demo");
|
||||
|
||||
const skill = {
|
||||
_id: skillId,
|
||||
_creationTime: 1,
|
||||
slug: "demo-skill",
|
||||
displayName: "Demo Skill",
|
||||
summary: "Demo summary",
|
||||
ownerUserId,
|
||||
ownerPublisherId: publisherId,
|
||||
tags: {},
|
||||
statsDownloads: 3,
|
||||
statsStars: 2,
|
||||
statsInstallsCurrent: 4,
|
||||
statsInstallsAllTime: 5,
|
||||
stats: {
|
||||
downloads: 3,
|
||||
stars: 2,
|
||||
installsCurrent: 4,
|
||||
installsAllTime: 5,
|
||||
versions: 1,
|
||||
comments: 0,
|
||||
},
|
||||
createdAt: 10,
|
||||
updatedAt: 20,
|
||||
} satisfies Doc<"skills">;
|
||||
const publisher = {
|
||||
_id: publisherId,
|
||||
_creationTime: 2,
|
||||
kind: "user",
|
||||
handle: "owner",
|
||||
displayName: "Owner",
|
||||
linkedUserId: ownerUserId,
|
||||
publishedSkills: 1,
|
||||
publishedPackages: 0,
|
||||
totalInstalls: 5,
|
||||
totalDownloads: 3,
|
||||
totalStars: 2,
|
||||
skillTotalInstalls: 5,
|
||||
skillTotalDownloads: 3,
|
||||
skillTotalStars: 2,
|
||||
createdAt: 10,
|
||||
updatedAt: 20,
|
||||
} satisfies Doc<"publishers">;
|
||||
const digest = {
|
||||
_id: digestId,
|
||||
_creationTime: 3,
|
||||
skillId,
|
||||
slug: "demo-skill",
|
||||
displayName: "Demo Skill",
|
||||
summary: "Demo summary",
|
||||
ownerUserId,
|
||||
ownerPublisherId: publisherId,
|
||||
ownerHandle: "owner",
|
||||
ownerKind: "user",
|
||||
ownerDisplayName: "Owner",
|
||||
tags: {},
|
||||
statsDownloads: 3,
|
||||
statsStars: 2,
|
||||
statsInstallsCurrent: 4,
|
||||
statsInstallsAllTime: 5,
|
||||
stats: {
|
||||
downloads: 3,
|
||||
stars: 2,
|
||||
installsCurrent: 4,
|
||||
installsAllTime: 5,
|
||||
versions: 1,
|
||||
comments: 0,
|
||||
},
|
||||
createdAt: 10,
|
||||
updatedAt: 20,
|
||||
} satisfies Doc<"skillSearchDigest">;
|
||||
const docs = new Map<string, unknown>([
|
||||
[skillId, skill],
|
||||
[publisherId, publisher],
|
||||
[digestId, digest],
|
||||
]);
|
||||
const patchSkillRankStats = internalMutation({
|
||||
args: {},
|
||||
handler: async (ctx) => {
|
||||
await ctx.db.patch(skillId, {
|
||||
statsDownloads: 13,
|
||||
statsStars: 7,
|
||||
statsInstallsAllTime: 11,
|
||||
stats: {
|
||||
downloads: 13,
|
||||
stars: 7,
|
||||
installsCurrent: 4,
|
||||
installsAllTime: 11,
|
||||
versions: 1,
|
||||
comments: 0,
|
||||
},
|
||||
});
|
||||
},
|
||||
});
|
||||
const handler = getWrappedHandler(patchSkillRankStats);
|
||||
const db = {
|
||||
system: {},
|
||||
normalizeId: vi.fn((tableName: string, id: string) =>
|
||||
id.startsWith(`${tableName}:`) ? id : null,
|
||||
),
|
||||
get: vi.fn(async (first: string, second?: string) => docs.get(second ?? first) ?? null),
|
||||
insert: vi.fn(async (tableName: string, value: unknown) => {
|
||||
if (!isRecord(value))
|
||||
throw new Error(`Expected inserted ${tableName} value to be an object`);
|
||||
const insertedId = `${tableName}:inserted`;
|
||||
docs.set(insertedId, { ...value, _id: insertedId, _creationTime: 0 });
|
||||
return insertedId;
|
||||
}),
|
||||
patch: vi.fn(
|
||||
async (first: string, second: string | Record<string, unknown>, third?: unknown) => {
|
||||
const id = typeof second === "string" ? second : first;
|
||||
const patch = typeof second === "string" ? third : second;
|
||||
if (!isRecord(patch)) throw new Error(`Expected patch for ${id} to be an object`);
|
||||
const existing = docs.get(id);
|
||||
if (!isRecord(existing)) throw new Error(`Missing test doc ${id}`);
|
||||
docs.set(id, { ...existing, ...patch });
|
||||
},
|
||||
),
|
||||
delete: vi.fn(async (first: string, second?: string) => {
|
||||
docs.delete(second ?? first);
|
||||
}),
|
||||
query: vi.fn((tableName: string) => ({
|
||||
withIndex: vi.fn(() => ({
|
||||
unique: vi.fn(async () => {
|
||||
if (tableName === "skillSearchDigest") return docs.get(digestId) ?? null;
|
||||
return null;
|
||||
}),
|
||||
collect: vi.fn(async () => []),
|
||||
paginate: vi.fn(async () => ({ page: [], isDone: true, continueCursor: "" })),
|
||||
take: vi.fn(async () => []),
|
||||
})),
|
||||
})),
|
||||
};
|
||||
|
||||
await expect(handler({ db }, {})).resolves.toBeUndefined();
|
||||
|
||||
expect(docs.get(digestId)).toEqual(
|
||||
expect.objectContaining({
|
||||
statsDownloads: 13,
|
||||
statsStars: 7,
|
||||
statsInstallsAllTime: 11,
|
||||
stats: expect.objectContaining({
|
||||
downloads: 13,
|
||||
stars: 7,
|
||||
installsAllTime: 11,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
+41
-102
@@ -1,5 +1,5 @@
|
||||
import { customCtx, customMutation } from "convex-helpers/server/customFunctions";
|
||||
import { Triggers, type Change } from "convex-helpers/server/triggers";
|
||||
import { Triggers } from "convex-helpers/server/triggers";
|
||||
import { v } from "convex/values";
|
||||
import semver from "semver";
|
||||
import { internal } from "./_generated/api";
|
||||
@@ -14,7 +14,6 @@ import {
|
||||
httpAction,
|
||||
} from "./_generated/server";
|
||||
import type { MutationCtx } from "./_generated/server";
|
||||
import { isPublicSkillDoc } from "./lib/globalStats";
|
||||
import {
|
||||
deletePackageSearchDigests,
|
||||
extractPackageDigestFields,
|
||||
@@ -153,28 +152,22 @@ export async function syncPackageSearchDigestForPackageId(
|
||||
}
|
||||
|
||||
export async function syncPackageSearchDigestsForOwnerUserId(
|
||||
ctx: PackageDigestSyncCtx & OwnerPublisherDigestScheduleCtx,
|
||||
ctx: PackageDigestSyncCtx,
|
||||
ownerUserId: Id<"users"> | null | undefined,
|
||||
cursor: string | null = null,
|
||||
) {
|
||||
if (!ownerUserId) return;
|
||||
let cursor: string | null = null;
|
||||
try {
|
||||
const page = await ctx.db
|
||||
.query("packages")
|
||||
.withIndex("by_owner", (q) => q.eq("ownerUserId", ownerUserId))
|
||||
.paginate({ cursor, numItems: OWNER_PUBLISHER_DIGEST_PAGE_SIZE });
|
||||
for (const pkg of page.page) {
|
||||
await syncPackageSearchDigest(ctx, pkg);
|
||||
}
|
||||
if (!page.isDone && ctx.scheduler && page.continueCursor) {
|
||||
await ctx.scheduler.runAfter(
|
||||
0,
|
||||
internal.functions.syncPackageSearchDigestsForOwnerUserIdInternal,
|
||||
{
|
||||
ownerUserId,
|
||||
cursor: page.continueCursor,
|
||||
},
|
||||
);
|
||||
while (true) {
|
||||
const page = await ctx.db
|
||||
.query("packages")
|
||||
.withIndex("by_owner", (q) => q.eq("ownerUserId", ownerUserId))
|
||||
.paginate({ cursor, numItems: 100 });
|
||||
for (const pkg of page.page) {
|
||||
await syncPackageSearchDigest(ctx, pkg);
|
||||
}
|
||||
if (page.isDone) break;
|
||||
cursor = page.continueCursor;
|
||||
}
|
||||
} catch (error) {
|
||||
if (isMissingTableError(error, "packages")) return;
|
||||
@@ -230,28 +223,21 @@ async function syncSkillSearchDigestForSkill(
|
||||
}
|
||||
|
||||
export function isGitHubMirrorEligibleSkillDoc(
|
||||
skill:
|
||||
| Pick<
|
||||
Doc<"skills">,
|
||||
"softDeletedAt" | "moderationStatus" | "moderationFlags" | "moderationVerdict"
|
||||
>
|
||||
| null
|
||||
| undefined,
|
||||
skill: Pick<Doc<"skills">, "softDeletedAt" | "moderationStatus"> | null | undefined,
|
||||
) {
|
||||
return isPublicSkillDoc(skill);
|
||||
if (!skill || skill.softDeletedAt) return false;
|
||||
return (
|
||||
skill.moderationStatus === undefined ||
|
||||
skill.moderationStatus === null ||
|
||||
skill.moderationStatus === "active"
|
||||
);
|
||||
}
|
||||
|
||||
export async function scheduleGitHubBackupDeletionForSkill(
|
||||
ctx: GitHubBackupDeletionCtx,
|
||||
skill: Pick<
|
||||
Doc<"skills">,
|
||||
| "slug"
|
||||
| "ownerPublisherId"
|
||||
| "ownerUserId"
|
||||
| "softDeletedAt"
|
||||
| "moderationStatus"
|
||||
| "moderationFlags"
|
||||
| "moderationVerdict"
|
||||
"slug" | "ownerPublisherId" | "ownerUserId" | "softDeletedAt" | "moderationStatus"
|
||||
>,
|
||||
) {
|
||||
const owner = await getOwnerPublisher(ctx, {
|
||||
@@ -309,69 +295,6 @@ export async function scheduleOwnerPublisherDigestSync(
|
||||
);
|
||||
}
|
||||
|
||||
export async function scheduleOwnerUserPackageDigestSync(
|
||||
ctx: OwnerPublisherDigestScheduleCtx,
|
||||
ownerUserId: Id<"users"> | null | undefined,
|
||||
) {
|
||||
if (!ownerUserId || !ctx.scheduler) return;
|
||||
await ctx.scheduler.runAfter(
|
||||
0,
|
||||
internal.functions.syncPackageSearchDigestsForOwnerUserIdInternal,
|
||||
{
|
||||
ownerUserId,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
export function shouldScheduleOwnerUserPackageDigestSyncForUserChange(
|
||||
change: Change<DataModel, "users">,
|
||||
) {
|
||||
if (change.operation === "delete") return true;
|
||||
if (
|
||||
change.operation === "update" &&
|
||||
change.oldDoc.handle === change.newDoc.handle &&
|
||||
change.oldDoc.deletedAt === change.newDoc.deletedAt &&
|
||||
change.oldDoc.deactivatedAt === change.newDoc.deactivatedAt
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
if (change.operation === "update" && (change.newDoc.deletedAt || change.newDoc.deactivatedAt)) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
export function shouldScheduleOwnerPublisherDigestSyncForPublisherChange(
|
||||
change: Change<DataModel, "publishers">,
|
||||
) {
|
||||
if (change.operation === "delete") return true;
|
||||
if (
|
||||
change.operation === "update" &&
|
||||
change.oldDoc.handle === change.newDoc.handle &&
|
||||
change.oldDoc.kind === change.newDoc.kind &&
|
||||
change.oldDoc.displayName === change.newDoc.displayName &&
|
||||
change.oldDoc.image === change.newDoc.image &&
|
||||
change.oldDoc.deletedAt === change.newDoc.deletedAt &&
|
||||
change.oldDoc.deactivatedAt === change.newDoc.deactivatedAt
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
if (change.operation === "update" && (change.newDoc.deletedAt || change.newDoc.deactivatedAt)) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
export const syncPackageSearchDigestsForOwnerUserIdInternal = rawInternalMutation({
|
||||
args: {
|
||||
ownerUserId: v.id("users"),
|
||||
cursor: v.optional(v.union(v.string(), v.null())),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
await syncPackageSearchDigestsForOwnerUserId(ctx, args.ownerUserId, args.cursor ?? null);
|
||||
},
|
||||
});
|
||||
|
||||
export const syncPackageSearchDigestsForOwnerPublisherIdInternal = rawInternalMutation({
|
||||
args: {
|
||||
ownerPublisherId: v.id("publishers"),
|
||||
@@ -408,7 +331,6 @@ export async function repointPackageLatestRelease(
|
||||
if (!packageId || !affectedReleaseId) return;
|
||||
const pkg = await ctx.db.get(packageId);
|
||||
if (!pkg) return;
|
||||
if (pkg.softDeletedAt) return;
|
||||
|
||||
const nextTags = Object.fromEntries(
|
||||
Object.entries(pkg.tags).filter(([, releaseId]) => releaseId !== affectedReleaseId),
|
||||
@@ -509,13 +431,30 @@ triggers.register("packageReleases", async (ctx, change) => {
|
||||
});
|
||||
|
||||
triggers.register("users", async (ctx, change) => {
|
||||
if (!shouldScheduleOwnerUserPackageDigestSyncForUserChange(change)) return;
|
||||
if (
|
||||
change.operation === "update" &&
|
||||
change.oldDoc.handle === change.newDoc.handle &&
|
||||
change.oldDoc.deletedAt === change.newDoc.deletedAt &&
|
||||
change.oldDoc.deactivatedAt === change.newDoc.deactivatedAt
|
||||
) {
|
||||
return;
|
||||
}
|
||||
const ownerUserId = change.operation === "delete" ? change.id : change.newDoc._id;
|
||||
await scheduleOwnerUserPackageDigestSync(ctx, ownerUserId);
|
||||
await syncPackageSearchDigestsForOwnerUserId(ctx, ownerUserId);
|
||||
});
|
||||
|
||||
triggers.register("publishers", async (ctx, change) => {
|
||||
if (!shouldScheduleOwnerPublisherDigestSyncForPublisherChange(change)) return;
|
||||
if (
|
||||
change.operation === "update" &&
|
||||
change.oldDoc.handle === change.newDoc.handle &&
|
||||
change.oldDoc.kind === change.newDoc.kind &&
|
||||
change.oldDoc.displayName === change.newDoc.displayName &&
|
||||
change.oldDoc.image === change.newDoc.image &&
|
||||
change.oldDoc.deletedAt === change.newDoc.deletedAt &&
|
||||
change.oldDoc.deactivatedAt === change.newDoc.deactivatedAt
|
||||
) {
|
||||
return;
|
||||
}
|
||||
const ownerPublisherId = change.operation === "delete" ? change.id : change.newDoc._id;
|
||||
await scheduleOwnerPublisherDigestSync(ctx, ownerPublisherId);
|
||||
});
|
||||
|
||||
@@ -1,278 +0,0 @@
|
||||
import { ConvexError, v } from "convex/values";
|
||||
import { internal } from "./_generated/api";
|
||||
import type { Id } from "./_generated/dataModel";
|
||||
import type { ActionCtx } from "./_generated/server";
|
||||
import { internalAction, internalMutation, internalQuery } from "./functions";
|
||||
import { fetchGitHubCreatedAtByProviderAccountId } from "./lib/githubAccount";
|
||||
import { getGitHubProviderAccountId } from "./lib/githubIdentity";
|
||||
import { getUserByHandleOrPersonalPublisher } from "./lib/publishers";
|
||||
|
||||
const DEFAULT_BATCH_SIZE = 25;
|
||||
const MAX_ACTION_BATCH_SIZE = 50;
|
||||
const MAX_LIST_BATCH_SIZE = 500;
|
||||
const DEFAULT_MAX_PAGES = 1;
|
||||
const MAX_MAX_PAGES = 20;
|
||||
|
||||
type BackfillCandidate = {
|
||||
userId: Id<"users">;
|
||||
providerAccountId: string;
|
||||
handle: string | null;
|
||||
};
|
||||
|
||||
type BackfillStats = {
|
||||
scanned: number;
|
||||
candidates: number;
|
||||
fetched: number;
|
||||
patched: number;
|
||||
failed: number;
|
||||
missingHandles: string[];
|
||||
errors: Array<{ userId: string; handle: string | null; message: string }>;
|
||||
};
|
||||
|
||||
type BackfillPageResult = {
|
||||
candidates: BackfillCandidate[];
|
||||
scanned: number;
|
||||
cursor: string | null;
|
||||
isDone: boolean;
|
||||
};
|
||||
|
||||
type BackfillHandlesResult = {
|
||||
candidates: BackfillCandidate[];
|
||||
missingHandles: string[];
|
||||
};
|
||||
|
||||
type BackfillResult =
|
||||
| { ok: true; stats: BackfillStats; cursor: string | null; isDone: boolean }
|
||||
| { ok: false; rateLimited: true; stats: BackfillStats; cursor: string | null; isDone: false };
|
||||
|
||||
function clampPositiveInteger(value: number | undefined, fallback: number, max: number) {
|
||||
if (!value || !Number.isFinite(value)) return fallback;
|
||||
return Math.max(1, Math.min(max, Math.floor(value)));
|
||||
}
|
||||
|
||||
async function candidateForUser(
|
||||
ctx: Parameters<typeof getGitHubProviderAccountId>[0],
|
||||
userId: Id<"users">,
|
||||
): Promise<BackfillCandidate | null> {
|
||||
const user = await ctx.db.get(userId);
|
||||
if (!user || user.deletedAt || user.deactivatedAt || user.githubCreatedAt) return null;
|
||||
const providerAccountId = await getGitHubProviderAccountId(ctx, userId);
|
||||
if (!providerAccountId || !/^\d+$/.test(providerAccountId)) return null;
|
||||
return { userId, providerAccountId, handle: user.handle ?? null };
|
||||
}
|
||||
|
||||
export const listGitHubCreatedAtBackfillPageInternal = internalQuery({
|
||||
args: {
|
||||
cursor: v.optional(v.string()),
|
||||
batchSize: v.optional(v.number()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const batchSize = clampPositiveInteger(args.batchSize, DEFAULT_BATCH_SIZE, MAX_LIST_BATCH_SIZE);
|
||||
const page = await ctx.db
|
||||
.query("authAccounts")
|
||||
.withIndex("providerAndAccountId", (q) => q.eq("provider", "github"))
|
||||
.paginate({ cursor: args.cursor ?? null, numItems: batchSize });
|
||||
|
||||
const candidates: BackfillCandidate[] = [];
|
||||
for (const account of page.page) {
|
||||
if (!/^\d+$/.test(account.providerAccountId)) continue;
|
||||
const user = await ctx.db.get(account.userId);
|
||||
if (!user || user.deletedAt || user.deactivatedAt || user.githubCreatedAt) continue;
|
||||
candidates.push({
|
||||
userId: account.userId,
|
||||
providerAccountId: account.providerAccountId,
|
||||
handle: user.handle ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
candidates,
|
||||
scanned: page.page.length,
|
||||
cursor: page.continueCursor,
|
||||
isDone: page.isDone,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
export const listGitHubCreatedAtBackfillHandlesInternal = internalQuery({
|
||||
args: { handles: v.array(v.string()) },
|
||||
handler: async (ctx, args) => {
|
||||
const seen = new Set<string>();
|
||||
const candidates: BackfillCandidate[] = [];
|
||||
const missingHandles: string[] = [];
|
||||
for (const handle of args.handles) {
|
||||
const user = await getUserByHandleOrPersonalPublisher(ctx, handle);
|
||||
if (!user) {
|
||||
missingHandles.push(handle);
|
||||
continue;
|
||||
}
|
||||
if (seen.has(user._id)) continue;
|
||||
seen.add(user._id);
|
||||
const candidate = await candidateForUser(ctx, user._id);
|
||||
if (candidate) candidates.push(candidate);
|
||||
}
|
||||
return { candidates, missingHandles };
|
||||
},
|
||||
});
|
||||
|
||||
export const applyGitHubCreatedAtBackfillInternal = internalMutation({
|
||||
args: {
|
||||
userId: v.id("users"),
|
||||
githubCreatedAt: v.number(),
|
||||
fetchedAt: v.number(),
|
||||
dryRun: v.optional(v.boolean()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const user = await ctx.db.get(args.userId);
|
||||
if (!user || user.deletedAt || user.deactivatedAt || user.githubCreatedAt) {
|
||||
return { patched: false };
|
||||
}
|
||||
if (args.dryRun) return { patched: false };
|
||||
await ctx.db.patch(args.userId, {
|
||||
githubCreatedAt: args.githubCreatedAt,
|
||||
githubFetchedAt: args.fetchedAt,
|
||||
updatedAt: Date.now(),
|
||||
});
|
||||
return { patched: true };
|
||||
},
|
||||
});
|
||||
|
||||
export const applyGitHubCreatedAtBackfillBatchInternal = internalMutation({
|
||||
args: {
|
||||
items: v.array(
|
||||
v.object({
|
||||
userId: v.id("users"),
|
||||
githubCreatedAt: v.number(),
|
||||
}),
|
||||
),
|
||||
fetchedAt: v.number(),
|
||||
dryRun: v.optional(v.boolean()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
let patched = 0;
|
||||
let skipped = 0;
|
||||
for (const item of args.items) {
|
||||
const user = await ctx.db.get(item.userId);
|
||||
if (!user || user.deletedAt || user.deactivatedAt || user.githubCreatedAt) {
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
if (!args.dryRun) {
|
||||
await ctx.db.patch(item.userId, {
|
||||
githubCreatedAt: item.githubCreatedAt,
|
||||
githubFetchedAt: args.fetchedAt,
|
||||
updatedAt: Date.now(),
|
||||
});
|
||||
}
|
||||
patched += 1;
|
||||
}
|
||||
return { patched, skipped };
|
||||
},
|
||||
});
|
||||
|
||||
export const backfillGitHubCreatedAtInternal = internalAction({
|
||||
args: {
|
||||
cursor: v.optional(v.string()),
|
||||
batchSize: v.optional(v.number()),
|
||||
maxPages: v.optional(v.number()),
|
||||
dryRun: v.optional(v.boolean()),
|
||||
handles: v.optional(v.array(v.string())),
|
||||
},
|
||||
handler: async (ctx: ActionCtx, args): Promise<BackfillResult> => {
|
||||
const batchSize = clampPositiveInteger(
|
||||
args.batchSize,
|
||||
DEFAULT_BATCH_SIZE,
|
||||
MAX_ACTION_BATCH_SIZE,
|
||||
);
|
||||
const maxPages = clampPositiveInteger(args.maxPages, DEFAULT_MAX_PAGES, MAX_MAX_PAGES);
|
||||
const dryRun = args.dryRun ?? false;
|
||||
const fetchedAt = Date.now();
|
||||
const stats = {
|
||||
scanned: 0,
|
||||
candidates: 0,
|
||||
fetched: 0,
|
||||
patched: 0,
|
||||
failed: 0,
|
||||
missingHandles: [] as string[],
|
||||
errors: [] as Array<{ userId: string; handle: string | null; message: string }>,
|
||||
};
|
||||
|
||||
let cursor = args.cursor ?? null;
|
||||
let isDone = true;
|
||||
let pages = 0;
|
||||
|
||||
while (pages < maxPages) {
|
||||
pages += 1;
|
||||
const page: BackfillPageResult | BackfillHandlesResult = args.handles
|
||||
? ((await ctx.runQuery(
|
||||
internal.githubAccountAgeBackfill.listGitHubCreatedAtBackfillHandlesInternal,
|
||||
{
|
||||
handles: args.handles,
|
||||
},
|
||||
)) as BackfillHandlesResult)
|
||||
: ((await ctx.runQuery(
|
||||
internal.githubAccountAgeBackfill.listGitHubCreatedAtBackfillPageInternal,
|
||||
{
|
||||
cursor: cursor ?? undefined,
|
||||
batchSize,
|
||||
},
|
||||
)) as BackfillPageResult);
|
||||
|
||||
const candidates = page.candidates;
|
||||
stats.scanned += "scanned" in page ? page.scanned : (args.handles?.length ?? 0);
|
||||
if ("missingHandles" in page) stats.missingHandles.push(...page.missingHandles);
|
||||
stats.candidates += candidates.length;
|
||||
|
||||
for (const candidate of candidates) {
|
||||
try {
|
||||
const githubCreatedAt = await fetchGitHubCreatedAtByProviderAccountId(
|
||||
candidate.providerAccountId,
|
||||
);
|
||||
stats.fetched += 1;
|
||||
const result: { patched: boolean } = await ctx.runMutation(
|
||||
internal.githubAccountAgeBackfill.applyGitHubCreatedAtBackfillInternal,
|
||||
{
|
||||
userId: candidate.userId,
|
||||
githubCreatedAt,
|
||||
fetchedAt,
|
||||
dryRun,
|
||||
},
|
||||
);
|
||||
if (result.patched) stats.patched += 1;
|
||||
} catch (error) {
|
||||
stats.failed += 1;
|
||||
const message = error instanceof ConvexError ? String(error.data) : String(error);
|
||||
if (stats.errors.length < 10) {
|
||||
stats.errors.push({
|
||||
userId: candidate.userId,
|
||||
handle: candidate.handle,
|
||||
message,
|
||||
});
|
||||
}
|
||||
if (/rate limit/i.test(message)) {
|
||||
return { ok: false as const, rateLimited: true as const, stats, cursor, isDone: false };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (args.handles) return { ok: true as const, stats, cursor: null, isDone: true };
|
||||
cursor = "cursor" in page ? page.cursor : null;
|
||||
isDone = "isDone" in page ? page.isDone : true;
|
||||
if (isDone) break;
|
||||
}
|
||||
|
||||
if (!dryRun && !isDone && cursor) {
|
||||
await ctx.scheduler.runAfter(
|
||||
0,
|
||||
internal.githubAccountAgeBackfill.backfillGitHubCreatedAtInternal,
|
||||
{
|
||||
cursor,
|
||||
batchSize,
|
||||
maxPages,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
return { ok: true as const, stats, cursor, isDone };
|
||||
},
|
||||
});
|
||||
+11
-9
@@ -1,9 +1,8 @@
|
||||
import { v } from "convex/values";
|
||||
import { internal } from "./_generated/api";
|
||||
import type { Doc, Id } from "./_generated/dataModel";
|
||||
import type { Id } from "./_generated/dataModel";
|
||||
import { action, internalMutation, internalQuery } from "./functions";
|
||||
import { assertRole, requireUserFromAction } from "./lib/access";
|
||||
import { isPublicSkillDoc } from "./lib/globalStats";
|
||||
|
||||
const DEFAULT_BATCH_SIZE = 50;
|
||||
const MAX_BATCH_SIZE = 200;
|
||||
@@ -105,13 +104,16 @@ export const getGitHubBackupPageInternal = internalQuery({
|
||||
},
|
||||
});
|
||||
|
||||
function isPubliclyAvailableSkill(
|
||||
skill: Pick<
|
||||
Doc<"skillSearchDigest">,
|
||||
"softDeletedAt" | "moderationStatus" | "moderationFlags" | "moderationVerdict"
|
||||
>,
|
||||
) {
|
||||
return isPublicSkillDoc(skill);
|
||||
function isPubliclyAvailableSkill(skill: {
|
||||
softDeletedAt?: number;
|
||||
moderationStatus?: string | null;
|
||||
}) {
|
||||
if (skill.softDeletedAt) return false;
|
||||
return (
|
||||
skill.moderationStatus === undefined ||
|
||||
skill.moderationStatus === null ||
|
||||
skill.moderationStatus === "active"
|
||||
);
|
||||
}
|
||||
|
||||
function isStaleCursorError(error: unknown) {
|
||||
|
||||
@@ -14,7 +14,6 @@ import {
|
||||
listGitHubSkillBackupEntries,
|
||||
normalizeOwner,
|
||||
} from "./lib/githubBackup";
|
||||
import { isPublicSkillDoc } from "./lib/globalStats";
|
||||
|
||||
const DEFAULT_BATCH_SIZE = 50;
|
||||
const MAX_BATCH_SIZE = 200;
|
||||
@@ -274,7 +273,12 @@ async function pruneDeletedSkillBackups(
|
||||
}
|
||||
|
||||
function isMirrorEligibleSkill(skill: Doc<"skills"> | null): skill is Doc<"skills"> {
|
||||
return isPublicSkillDoc(skill);
|
||||
if (!skill || skill.softDeletedAt) return false;
|
||||
return (
|
||||
skill.moderationStatus === undefined ||
|
||||
skill.moderationStatus === null ||
|
||||
skill.moderationStatus === "active"
|
||||
);
|
||||
}
|
||||
|
||||
async function deleteBackupIfNeeded(
|
||||
|
||||
@@ -1,241 +0,0 @@
|
||||
import { ConvexError } from "convex/values";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("./lib/access", () => ({
|
||||
requireUser: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("./lib/publishers", async () => {
|
||||
const actual = await vi.importActual<typeof import("./lib/publishers")>("./lib/publishers");
|
||||
return {
|
||||
...actual,
|
||||
requirePublisherRole: vi.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
const { requireUser } = await import("./lib/access");
|
||||
const { requirePublisherRole } = await import("./lib/publishers");
|
||||
const { deleteForPublisherHandler } = await import("./githubSkillSources");
|
||||
const { buildSkillInstallResolution } = await import("./lib/installResolver");
|
||||
|
||||
type Row = Record<string, unknown> & { _id: string };
|
||||
|
||||
function chainEq(constraints: Record<string, unknown>) {
|
||||
return {
|
||||
eq(field: string, value: unknown) {
|
||||
constraints[field] = value;
|
||||
return chainEq(constraints);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function matches(doc: Row, constraints: Record<string, unknown>) {
|
||||
return Object.entries(constraints).every(([key, value]) => doc[key] === value);
|
||||
}
|
||||
|
||||
function createDb(initial: Record<string, Row[]> = {}) {
|
||||
const tables: Record<string, Row[]> = Object.fromEntries(
|
||||
Object.entries(initial).map(([table, rows]) => [table, [...rows]]),
|
||||
);
|
||||
const list = (table: string) => {
|
||||
tables[table] ??= [];
|
||||
return tables[table];
|
||||
};
|
||||
|
||||
const db = {
|
||||
get: async (id: string) => {
|
||||
const table = id.split(":")[0] ?? "";
|
||||
return list(table).find((row) => row._id === id) ?? null;
|
||||
},
|
||||
patch: async (id: string, patch: Record<string, unknown>) => {
|
||||
const table = id.split(":")[0] ?? "";
|
||||
const row = list(table).find((candidate) => candidate._id === id);
|
||||
if (!row) return;
|
||||
for (const [key, value] of Object.entries(patch)) {
|
||||
if (value === undefined) delete row[key];
|
||||
else row[key] = value;
|
||||
}
|
||||
},
|
||||
insert: async (table: string, doc: Record<string, unknown>) => {
|
||||
const id = `${table}:${list(table).length + 1}`;
|
||||
list(table).push({ _id: id, ...doc });
|
||||
return id;
|
||||
},
|
||||
delete: async (id: string) => {
|
||||
const table = id.split(":")[0] ?? "";
|
||||
const rows = list(table);
|
||||
const index = rows.findIndex((row) => row._id === id);
|
||||
if (index >= 0) rows.splice(index, 1);
|
||||
},
|
||||
query: (table: string) => ({
|
||||
withIndex: (_indexName: string, build: (q: ReturnType<typeof chainEq>) => unknown) => {
|
||||
const constraints: Record<string, unknown> = {};
|
||||
build(chainEq(constraints));
|
||||
const matched = () => list(table).filter((row) => matches(row, constraints));
|
||||
return {
|
||||
collect: async () => matched(),
|
||||
unique: async () => matched()[0] ?? null,
|
||||
};
|
||||
},
|
||||
}),
|
||||
};
|
||||
|
||||
return { db, tables };
|
||||
}
|
||||
|
||||
describe("githubSkillSources.deleteForPublisherHandler", () => {
|
||||
beforeEach(() => {
|
||||
vi.mocked(requireUser).mockResolvedValue({ userId: "users:owner" } as never);
|
||||
vi.mocked(requirePublisherRole).mockResolvedValue(undefined as never);
|
||||
});
|
||||
|
||||
it("deletes a source and removes only GitHub-backed skills from that source", async () => {
|
||||
const { db, tables } = createDb({
|
||||
githubSkillSources: [
|
||||
{
|
||||
_id: "githubSkillSources:matt",
|
||||
repo: "mattpocock/skills",
|
||||
ownerPublisherId: "publishers:openclaw",
|
||||
createdAt: 1,
|
||||
updatedAt: 2,
|
||||
},
|
||||
],
|
||||
githubSkillContents: [
|
||||
{
|
||||
_id: "githubSkillContents:one",
|
||||
skillId: "skills:github",
|
||||
githubSourceId: "githubSkillSources:matt",
|
||||
},
|
||||
],
|
||||
skills: [
|
||||
{
|
||||
_id: "skills:github",
|
||||
slug: "source-backed",
|
||||
displayName: "Source Backed",
|
||||
installKind: "github",
|
||||
githubSourceId: "githubSkillSources:matt",
|
||||
githubPath: "skills/source-backed",
|
||||
githubCurrentCommit: "a".repeat(40),
|
||||
githubCurrentContentHash: "hash-source-backed",
|
||||
githubCurrentStatus: "present",
|
||||
githubScanStatus: "clean",
|
||||
ownerUserId: "users:owner",
|
||||
ownerPublisherId: "publishers:openclaw",
|
||||
forkOf: undefined,
|
||||
tags: {},
|
||||
capabilityTags: undefined,
|
||||
badges: {},
|
||||
stats: {
|
||||
comments: 0,
|
||||
downloads: 0,
|
||||
installsAllTime: 0,
|
||||
installsCurrent: 0,
|
||||
stars: 0,
|
||||
versions: 0,
|
||||
},
|
||||
moderationStatus: "active",
|
||||
moderationFlags: [],
|
||||
isSuspicious: false,
|
||||
createdAt: 1,
|
||||
updatedAt: 2,
|
||||
softDeletedAt: undefined,
|
||||
},
|
||||
{
|
||||
_id: "skills:direct",
|
||||
slug: "direct-upload",
|
||||
displayName: "Direct Upload",
|
||||
ownerPublisherId: "publishers:openclaw",
|
||||
softDeletedAt: undefined,
|
||||
},
|
||||
{
|
||||
_id: "skills:other-source",
|
||||
slug: "other-source",
|
||||
displayName: "Other Source",
|
||||
installKind: "github",
|
||||
githubSourceId: "githubSkillSources:other",
|
||||
githubPath: "skills/other-source",
|
||||
githubCurrentCommit: "b".repeat(40),
|
||||
githubCurrentContentHash: "hash-other-source",
|
||||
githubCurrentStatus: "present",
|
||||
githubScanStatus: "clean",
|
||||
ownerPublisherId: "publishers:openclaw",
|
||||
softDeletedAt: undefined,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
await expect(
|
||||
deleteForPublisherHandler({ db } as never, {
|
||||
ownerPublisherId: "publishers:openclaw" as never,
|
||||
sourceId: "githubSkillSources:matt" as never,
|
||||
now: 123,
|
||||
}),
|
||||
).resolves.toEqual({ ok: true, deletedSkills: 1 });
|
||||
|
||||
expect(requirePublisherRole).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
publisherId: "publishers:openclaw",
|
||||
userId: "users:owner",
|
||||
allowed: ["admin"],
|
||||
}),
|
||||
);
|
||||
expect(tables.githubSkillSources).toHaveLength(0);
|
||||
expect(tables.githubSkillContents).toHaveLength(0);
|
||||
const deletedSkill = tables.skills.find((skill) => skill._id === "skills:github");
|
||||
expect(deletedSkill).toMatchObject({
|
||||
softDeletedAt: 123,
|
||||
githubRemovedAt: 123,
|
||||
githubCurrentStatus: "missing",
|
||||
updatedAt: 123,
|
||||
});
|
||||
expect(tables.skillSearchDigest).toEqual([
|
||||
expect.objectContaining({
|
||||
skillId: "skills:github",
|
||||
githubCurrentStatus: "missing",
|
||||
githubScanStatus: "clean",
|
||||
softDeletedAt: 123,
|
||||
}),
|
||||
]);
|
||||
expect(
|
||||
buildSkillInstallResolution({
|
||||
origin: "https://clawhub.ai",
|
||||
skill: deletedSkill as never,
|
||||
source: null,
|
||||
}),
|
||||
).toMatchObject({
|
||||
ok: false,
|
||||
reason: "github_upstream_removed",
|
||||
status: 410,
|
||||
});
|
||||
expect(tables.skills.find((skill) => skill._id === "skills:direct")).toMatchObject({
|
||||
softDeletedAt: undefined,
|
||||
});
|
||||
expect(tables.skills.find((skill) => skill._id === "skills:other-source")).toMatchObject({
|
||||
githubCurrentStatus: "present",
|
||||
softDeletedAt: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects deleting a source from another publisher", async () => {
|
||||
const { db } = createDb({
|
||||
githubSkillSources: [
|
||||
{
|
||||
_id: "githubSkillSources:matt",
|
||||
repo: "mattpocock/skills",
|
||||
ownerPublisherId: "publishers:other",
|
||||
createdAt: 1,
|
||||
updatedAt: 2,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
await expect(
|
||||
deleteForPublisherHandler({ db } as never, {
|
||||
ownerPublisherId: "publishers:openclaw" as never,
|
||||
sourceId: "githubSkillSources:matt" as never,
|
||||
now: 123,
|
||||
}),
|
||||
).rejects.toBeInstanceOf(ConvexError);
|
||||
});
|
||||
});
|
||||
@@ -1,206 +0,0 @@
|
||||
import { ConvexError, v } from "convex/values";
|
||||
import type { Doc, Id } from "./_generated/dataModel";
|
||||
import type { MutationCtx, QueryCtx } from "./_generated/server";
|
||||
import { internalQuery, mutation, query } from "./functions";
|
||||
import { requireUser } from "./lib/access";
|
||||
import { adjustGlobalPublicSkillsCount, getPublicSkillVisibilityDelta } from "./lib/globalStats";
|
||||
import { isOfficialPublisher } from "./lib/officialPublishers";
|
||||
import { isPublisherActive, isPublisherRoleAllowed, requirePublisherRole } from "./lib/publishers";
|
||||
import { syncSkillSearchDigestForSkill } from "./lib/skillSearchDigest";
|
||||
|
||||
type PublicGitHubSkillSource = Pick<
|
||||
Doc<"githubSkillSources">,
|
||||
| "_id"
|
||||
| "repo"
|
||||
| "defaultBranch"
|
||||
| "lastSyncStatus"
|
||||
| "lastSyncError"
|
||||
| "lastSyncErrorAt"
|
||||
| "displayManifestStatus"
|
||||
| "displayManifestFetchedAt"
|
||||
| "displayManifestCommit"
|
||||
| "lastSyncIssues"
|
||||
| "lastSyncInvalidSkills"
|
||||
| "createdAt"
|
||||
| "updatedAt"
|
||||
> & {
|
||||
ownerPublisher: Pick<Doc<"publishers">, "_id" | "handle" | "displayName"> | null;
|
||||
skills: Array<
|
||||
Pick<Doc<"skills">, "_id" | "slug" | "displayName" | "githubPath" | "githubCurrentStatus">
|
||||
>;
|
||||
};
|
||||
|
||||
export const getByIdInternal = internalQuery({
|
||||
args: { sourceId: v.id("githubSkillSources") },
|
||||
handler: async (ctx, args) => ctx.db.get(args.sourceId),
|
||||
});
|
||||
|
||||
async function toPublicGitHubSkillSource(
|
||||
ctx: Pick<QueryCtx, "db">,
|
||||
source: Doc<"githubSkillSources">,
|
||||
): Promise<PublicGitHubSkillSource> {
|
||||
const skills = await ctx.db
|
||||
.query("skills")
|
||||
.withIndex("by_github_source", (q) => q.eq("githubSourceId", source._id))
|
||||
.collect();
|
||||
const visibleGitHubSkills = skills
|
||||
.filter((skill) => skill.installKind === "github" && !skill.softDeletedAt)
|
||||
.sort((a, b) => a.displayName.localeCompare(b.displayName))
|
||||
.map((skill) => ({
|
||||
_id: skill._id,
|
||||
slug: skill.slug,
|
||||
displayName: skill.displayName,
|
||||
githubPath: skill.githubPath,
|
||||
githubCurrentStatus: skill.githubCurrentStatus,
|
||||
}));
|
||||
const ownerPublisher = source.ownerPublisherId ? await ctx.db.get(source.ownerPublisherId) : null;
|
||||
|
||||
return {
|
||||
_id: source._id as Id<"githubSkillSources">,
|
||||
repo: source.repo,
|
||||
ownerPublisher: ownerPublisher
|
||||
? {
|
||||
_id: ownerPublisher._id,
|
||||
handle: ownerPublisher.handle,
|
||||
displayName: ownerPublisher.displayName,
|
||||
}
|
||||
: null,
|
||||
defaultBranch: source.defaultBranch,
|
||||
lastSyncStatus: source.lastSyncStatus,
|
||||
lastSyncError: source.lastSyncError,
|
||||
lastSyncErrorAt: source.lastSyncErrorAt,
|
||||
displayManifestStatus: source.displayManifestStatus,
|
||||
displayManifestFetchedAt: source.displayManifestFetchedAt,
|
||||
displayManifestCommit: source.displayManifestCommit,
|
||||
lastSyncIssues: source.lastSyncIssues,
|
||||
lastSyncInvalidSkills: source.lastSyncInvalidSkills,
|
||||
createdAt: source.createdAt,
|
||||
updatedAt: source.updatedAt,
|
||||
skills: visibleGitHubSkills,
|
||||
};
|
||||
}
|
||||
|
||||
export const listForPublisher = query({
|
||||
args: { ownerPublisherId: v.id("publishers") },
|
||||
handler: async (ctx, args): Promise<PublicGitHubSkillSource[]> => {
|
||||
const { userId } = await requireUser(ctx);
|
||||
await requirePublisherRole(ctx, {
|
||||
publisherId: args.ownerPublisherId,
|
||||
userId,
|
||||
allowed: ["admin"],
|
||||
});
|
||||
const sources = await ctx.db
|
||||
.query("githubSkillSources")
|
||||
.withIndex("by_owner_publisher", (q) => q.eq("ownerPublisherId", args.ownerPublisherId))
|
||||
.collect();
|
||||
const sortedSources = sources.sort((a, b) => b.updatedAt - a.updatedAt);
|
||||
return await Promise.all(sortedSources.map((source) => toPublicGitHubSkillSource(ctx, source)));
|
||||
},
|
||||
});
|
||||
|
||||
export const listForManageableOfficialPublishers = query({
|
||||
args: {},
|
||||
handler: async (ctx): Promise<PublicGitHubSkillSource[]> => {
|
||||
const { userId } = await requireUser(ctx);
|
||||
const memberships = await ctx.db
|
||||
.query("publisherMembers")
|
||||
.withIndex("by_user", (q) => q.eq("userId", userId))
|
||||
.collect();
|
||||
const ownerPublisherIds: Id<"publishers">[] = [];
|
||||
for (const membership of memberships) {
|
||||
if (!isPublisherRoleAllowed(membership.role, ["admin"])) continue;
|
||||
const publisher = await ctx.db.get(membership.publisherId);
|
||||
if (
|
||||
!publisher ||
|
||||
publisher.kind !== "org" ||
|
||||
!isPublisherActive(publisher) ||
|
||||
!(await isOfficialPublisher(ctx, publisher))
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
ownerPublisherIds.push(publisher._id);
|
||||
}
|
||||
const sourceGroups = await Promise.all(
|
||||
ownerPublisherIds.map((ownerPublisherId) =>
|
||||
ctx.db
|
||||
.query("githubSkillSources")
|
||||
.withIndex("by_owner_publisher", (q) => q.eq("ownerPublisherId", ownerPublisherId))
|
||||
.collect(),
|
||||
),
|
||||
);
|
||||
const sortedSources = sourceGroups.flat().sort((a, b) => b.updatedAt - a.updatedAt);
|
||||
return await Promise.all(sortedSources.map((source) => toPublicGitHubSkillSource(ctx, source)));
|
||||
},
|
||||
});
|
||||
|
||||
export async function deleteForPublisherHandler(
|
||||
ctx: MutationCtx,
|
||||
args: {
|
||||
ownerPublisherId: Id<"publishers">;
|
||||
sourceId: Id<"githubSkillSources">;
|
||||
now?: number;
|
||||
},
|
||||
) {
|
||||
const { userId } = await requireUser(ctx);
|
||||
await requirePublisherRole(ctx, {
|
||||
publisherId: args.ownerPublisherId,
|
||||
userId,
|
||||
allowed: ["admin"],
|
||||
});
|
||||
|
||||
const source = await ctx.db.get(args.sourceId);
|
||||
if (!source || source.ownerPublisherId !== args.ownerPublisherId) {
|
||||
throw new ConvexError("GitHub source not found.");
|
||||
}
|
||||
|
||||
const now = args.now ?? Date.now();
|
||||
const contents = await ctx.db
|
||||
.query("githubSkillContents")
|
||||
.withIndex("by_github_source", (q) => q.eq("githubSourceId", args.sourceId))
|
||||
.collect();
|
||||
for (const content of contents) {
|
||||
await ctx.db.delete(content._id);
|
||||
}
|
||||
|
||||
const skills = await ctx.db
|
||||
.query("skills")
|
||||
.withIndex("by_github_source", (q) => q.eq("githubSourceId", args.sourceId))
|
||||
.collect();
|
||||
let deletedSkills = 0;
|
||||
let publicSkillDelta = 0;
|
||||
for (const skill of skills) {
|
||||
if (skill.installKind !== "github") continue;
|
||||
|
||||
const nextSkill: Doc<"skills"> = {
|
||||
...skill,
|
||||
softDeletedAt: skill.softDeletedAt ?? now,
|
||||
githubCurrentStatus: "missing",
|
||||
githubRemovedAt: skill.githubRemovedAt ?? now,
|
||||
updatedAt: now,
|
||||
};
|
||||
publicSkillDelta += getPublicSkillVisibilityDelta(skill, nextSkill);
|
||||
await ctx.db.patch(skill._id, {
|
||||
softDeletedAt: nextSkill.softDeletedAt,
|
||||
githubCurrentStatus: nextSkill.githubCurrentStatus,
|
||||
githubRemovedAt: nextSkill.githubRemovedAt,
|
||||
updatedAt: now,
|
||||
});
|
||||
await syncSkillSearchDigestForSkill(ctx, nextSkill);
|
||||
deletedSkills += 1;
|
||||
}
|
||||
|
||||
if (publicSkillDelta !== 0) {
|
||||
await adjustGlobalPublicSkillsCount(ctx, publicSkillDelta, now);
|
||||
}
|
||||
await ctx.db.delete(args.sourceId);
|
||||
|
||||
return { ok: true as const, deletedSkills };
|
||||
}
|
||||
|
||||
export const deleteForPublisher: ReturnType<typeof mutation> = mutation({
|
||||
args: {
|
||||
ownerPublisherId: v.id("publishers"),
|
||||
sourceId: v.id("githubSkillSources"),
|
||||
},
|
||||
handler: async (ctx, args) => deleteForPublisherHandler(ctx, args),
|
||||
});
|
||||
@@ -1,249 +0,0 @@
|
||||
/* @vitest-environment node */
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
applyGitHubSkillSourceSyncHandler,
|
||||
applyGitHubSkillVerificationResultHandler,
|
||||
configurePublicGitHubSkillSourceHandler,
|
||||
upsertGitHubSkillContentHandler,
|
||||
verifyGitHubSkillHandler,
|
||||
} from "./githubSkillSync";
|
||||
import { buildSkillInstallResolution } from "./lib/installResolver";
|
||||
|
||||
type Row = Record<string, unknown> & { _id: string };
|
||||
|
||||
function chainEq(constraints: Record<string, unknown>) {
|
||||
return {
|
||||
eq(field: string, value: unknown) {
|
||||
constraints[field] = value;
|
||||
return chainEq(constraints);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function matches(doc: Row, constraints: Record<string, unknown>) {
|
||||
return Object.entries(constraints).every(([key, value]) => doc[key] === value);
|
||||
}
|
||||
|
||||
function createDb(initial: Record<string, Row[]> = {}) {
|
||||
const tables: Record<string, Row[]> = Object.fromEntries(
|
||||
Object.entries(initial).map(([table, rows]) => [table, [...rows]]),
|
||||
);
|
||||
const counters: Record<string, number> = {};
|
||||
const list = (table: string) => {
|
||||
tables[table] ??= [];
|
||||
return tables[table];
|
||||
};
|
||||
|
||||
const db = {
|
||||
get: async (id: string) => {
|
||||
const table = id.split(":")[0] ?? "";
|
||||
return list(table).find((row) => row._id === id) ?? null;
|
||||
},
|
||||
insert: async (table: string, doc: Record<string, unknown>) => {
|
||||
counters[table] = (counters[table] ?? 0) + 1;
|
||||
const inserted = {
|
||||
_id: `${table}:new-${counters[table]}`,
|
||||
_creationTime: counters[table],
|
||||
...doc,
|
||||
};
|
||||
list(table).push(inserted);
|
||||
return inserted._id;
|
||||
},
|
||||
patch: async (id: string, patch: Record<string, unknown>) => {
|
||||
const table = id.split(":")[0] ?? "";
|
||||
const row = list(table).find((candidate) => candidate._id === id);
|
||||
if (!row) return;
|
||||
for (const [key, value] of Object.entries(patch)) {
|
||||
if (value === undefined) delete row[key];
|
||||
else row[key] = value;
|
||||
}
|
||||
},
|
||||
query: (table: string) => ({
|
||||
withIndex: (_indexName: string, build?: (q: ReturnType<typeof chainEq>) => unknown) => {
|
||||
const constraints: Record<string, unknown> = {};
|
||||
build?.(chainEq(constraints));
|
||||
const matched = () => list(table).filter((row) => matches(row, constraints));
|
||||
return {
|
||||
collect: async () => matched(),
|
||||
unique: async () => matched()[0] ?? null,
|
||||
};
|
||||
},
|
||||
}),
|
||||
};
|
||||
|
||||
return { db, tables };
|
||||
}
|
||||
|
||||
function getSkill(tables: Record<string, Row[]>, slug: string) {
|
||||
const skill = tables.skills?.find((row) => row.slug === slug);
|
||||
if (!skill) throw new Error(`Live GitHub canary did not discover skill: ${slug}`);
|
||||
return skill;
|
||||
}
|
||||
|
||||
function resolveInstallFromTables(tables: Record<string, Row[]>, slug: string) {
|
||||
const skill = getSkill(tables, slug);
|
||||
const source =
|
||||
typeof skill.githubSourceId === "string"
|
||||
? (tables.githubSkillSources?.find((row) => row._id === skill.githubSourceId) ?? null)
|
||||
: null;
|
||||
return buildSkillInstallResolution({
|
||||
origin: "https://clawhub.ai",
|
||||
skill: skill as never,
|
||||
source: source as never,
|
||||
});
|
||||
}
|
||||
|
||||
const liveCanaryEnabled = process.env.CLAWHUB_LIVE_GITHUB_CANARY === "1";
|
||||
const itIfLive = liveCanaryEnabled ? it : it.skip;
|
||||
|
||||
describe("GitHub-backed skills live canary", () => {
|
||||
itIfLive(
|
||||
"discovers and verifies an installable skill from a real GitHub repo",
|
||||
{ timeout: 45_000 },
|
||||
async () => {
|
||||
const repo = process.env.CLAWHUB_LIVE_GITHUB_REPO?.trim() || "openclaw/agent-skills";
|
||||
const skillSlug = process.env.CLAWHUB_LIVE_GITHUB_SKILL?.trim() || "handoff";
|
||||
const { db, tables } = createDb({
|
||||
globalStats: [
|
||||
{
|
||||
_id: "globalStats:default",
|
||||
key: "default",
|
||||
activeSkillsCount: 0,
|
||||
updatedAt: 1,
|
||||
},
|
||||
],
|
||||
});
|
||||
const scheduler = { runAfter: async () => undefined };
|
||||
let now = Date.now();
|
||||
const actionCtx = {
|
||||
runQuery: async (_query: unknown, args: Record<string, unknown>) => {
|
||||
if ("ownerPublisherId" in args && "actorUserId" in args) {
|
||||
return {
|
||||
ownerUserId: "users:live-owner",
|
||||
existingSource:
|
||||
tables.githubSkillSources?.find((source) => source.repo === repo) ?? null,
|
||||
official: true,
|
||||
};
|
||||
}
|
||||
if ("skillId" in args) {
|
||||
const skill = tables.skills?.find((row) => row._id === args.skillId);
|
||||
const source =
|
||||
skill && typeof skill.githubSourceId === "string"
|
||||
? tables.githubSkillSources?.find((row) => row._id === skill.githubSourceId)
|
||||
: null;
|
||||
return skill && source ? { skill, source } : null;
|
||||
}
|
||||
if ("sourceId" in args) {
|
||||
return (tables.skills ?? []).flatMap((skill) => {
|
||||
if (
|
||||
skill.githubSourceId !== args.sourceId ||
|
||||
skill.installKind !== "github" ||
|
||||
skill.githubCurrentStatus !== "present" ||
|
||||
typeof skill.githubPath !== "string" ||
|
||||
typeof skill.githubCurrentContentHash !== "string"
|
||||
) {
|
||||
return [];
|
||||
}
|
||||
return [
|
||||
{
|
||||
skillId: skill._id,
|
||||
githubPath: skill.githubPath,
|
||||
githubCurrentContentHash: skill.githubCurrentContentHash,
|
||||
},
|
||||
];
|
||||
});
|
||||
}
|
||||
throw new Error(`unexpected live canary query args: ${JSON.stringify(args)}`);
|
||||
},
|
||||
runMutation: async (_mutation: unknown, args: Record<string, unknown>) => {
|
||||
if ("snapshot" in args) {
|
||||
return await applyGitHubSkillSourceSyncHandler(
|
||||
{ db, scheduler } as never,
|
||||
{
|
||||
...args,
|
||||
now,
|
||||
} as never,
|
||||
);
|
||||
}
|
||||
if ("scanStatus" in args && "contentHash" in args) {
|
||||
return await applyGitHubSkillVerificationResultHandler(
|
||||
{ db } as never,
|
||||
{
|
||||
...args,
|
||||
now,
|
||||
} as never,
|
||||
);
|
||||
}
|
||||
if ("discovered" in args && "commit" in args) {
|
||||
return await upsertGitHubSkillContentHandler(
|
||||
{ db } as never,
|
||||
{
|
||||
...args,
|
||||
now,
|
||||
} as never,
|
||||
);
|
||||
}
|
||||
throw new Error(`unexpected live canary mutation args: ${JSON.stringify(args)}`);
|
||||
},
|
||||
auth: { getUserIdentity: async () => null },
|
||||
};
|
||||
|
||||
const configured = await configurePublicGitHubSkillSourceHandler(
|
||||
actionCtx as never,
|
||||
{
|
||||
ownerPublisherId: "publishers:live" as never,
|
||||
repo,
|
||||
},
|
||||
fetch,
|
||||
{ userId: "users:live-owner" as never },
|
||||
);
|
||||
|
||||
expect(configured.stats.discovered).toBeGreaterThan(0);
|
||||
expect(configured.manifestStatus === "missing" || configured.manifestStatus === "ok").toBe(
|
||||
true,
|
||||
);
|
||||
expect(configured.commit).toMatch(/^[a-f0-9]{40}$/);
|
||||
|
||||
let skill = getSkill(tables, skillSlug);
|
||||
expect(skill).toMatchObject({
|
||||
installKind: "github",
|
||||
githubPath: `skills/${skillSlug}`,
|
||||
githubCurrentCommit: configured.commit,
|
||||
githubCurrentStatus: "present",
|
||||
githubScanStatus: "pending",
|
||||
});
|
||||
expect(resolveInstallFromTables(tables, skillSlug)).toMatchObject({
|
||||
ok: false,
|
||||
reason: "github_verification_pending",
|
||||
});
|
||||
|
||||
now = Date.now();
|
||||
const verified = await verifyGitHubSkillHandler(
|
||||
actionCtx as never,
|
||||
{
|
||||
skillId: skill._id as never,
|
||||
contentHash: skill.githubCurrentContentHash as string,
|
||||
},
|
||||
fetch,
|
||||
);
|
||||
|
||||
expect(verified).toMatchObject({ ok: true, scanStatus: "clean" });
|
||||
skill = getSkill(tables, skillSlug);
|
||||
expect(skill).toMatchObject({
|
||||
githubCurrentCommit: configured.commit,
|
||||
githubScanStatus: "clean",
|
||||
moderationStatus: "active",
|
||||
});
|
||||
expect(resolveInstallFromTables(tables, skillSlug)).toMatchObject({
|
||||
ok: true,
|
||||
installKind: "github",
|
||||
github: {
|
||||
repo,
|
||||
path: `skills/${skillSlug}`,
|
||||
commit: configured.commit,
|
||||
contentHash: skill.githubCurrentContentHash,
|
||||
},
|
||||
});
|
||||
},
|
||||
);
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
+3
-68
@@ -8,7 +8,7 @@ import {
|
||||
cliDeviceTokenHttp,
|
||||
cliSkillDeleteHttp,
|
||||
cliSkillUndeleteHttp,
|
||||
cliTelemetryInstallHttp,
|
||||
cliTelemetrySyncHttp,
|
||||
cliUploadUrlHttp,
|
||||
cliWhoamiHttp,
|
||||
getSkillHttp,
|
||||
@@ -17,7 +17,6 @@ import {
|
||||
} from "./httpApi";
|
||||
import {
|
||||
exportSkillsV1Http,
|
||||
exportPluginsV1Http,
|
||||
listBundlePluginsV1Http,
|
||||
listCodePluginsV1Http,
|
||||
listPackagesV1Http,
|
||||
@@ -36,10 +35,6 @@ import {
|
||||
publishSoulV1Http,
|
||||
resolveSkillVersionV1Http,
|
||||
searchSkillsV1Http,
|
||||
skillScanBatchStatusV1Http,
|
||||
skillScanBatchSubmitV1Http,
|
||||
skillScanGetRouterV1Http,
|
||||
skillScanSubmitV1Http,
|
||||
skillSecurityVerdictsV1Http,
|
||||
skillsDeleteRouterV1Http,
|
||||
skillsGetRouterV1Http,
|
||||
@@ -51,18 +46,12 @@ import {
|
||||
starsPostRouterV1Http,
|
||||
transfersGetRouterV1Http,
|
||||
banAppealContextV1Http,
|
||||
usersGetRouterV1Http,
|
||||
usersListV1Http,
|
||||
usersPostRouterV1Http,
|
||||
verifyDocsSessionV1Http,
|
||||
whoamiV1Http,
|
||||
} from "./httpApiV1";
|
||||
import { preflightHandler } from "./httpPreflight";
|
||||
import {
|
||||
packageInspectorArtifactHttp,
|
||||
packageInspectorClaimHttp,
|
||||
packageInspectorResultsHttp,
|
||||
} from "./packageInspectorHttp";
|
||||
|
||||
const http = httpRouter();
|
||||
|
||||
@@ -98,12 +87,6 @@ http.route({
|
||||
handler: listSkillsV1Http,
|
||||
});
|
||||
|
||||
http.route({
|
||||
pathPrefix: `${ApiRoutes.skillScans}/`,
|
||||
method: "GET",
|
||||
handler: skillScanGetRouterV1Http,
|
||||
});
|
||||
|
||||
http.route({
|
||||
path: ApiRoutes.packages,
|
||||
method: "GET",
|
||||
@@ -116,12 +99,6 @@ http.route({
|
||||
handler: listPluginsV1Http,
|
||||
});
|
||||
|
||||
http.route({
|
||||
path: ApiRoutes.pluginsExport,
|
||||
method: "GET",
|
||||
handler: exportPluginsV1Http,
|
||||
});
|
||||
|
||||
http.route({
|
||||
path: ApiRoutes.codePlugins,
|
||||
method: "GET",
|
||||
@@ -164,24 +141,6 @@ http.route({
|
||||
handler: publishSkillV1Http,
|
||||
});
|
||||
|
||||
http.route({
|
||||
path: ApiRoutes.skillScans,
|
||||
method: "POST",
|
||||
handler: skillScanSubmitV1Http,
|
||||
});
|
||||
|
||||
http.route({
|
||||
path: `${ApiRoutes.skillScans}/batch`,
|
||||
method: "POST",
|
||||
handler: skillScanBatchSubmitV1Http,
|
||||
});
|
||||
|
||||
http.route({
|
||||
path: `${ApiRoutes.skillScans}/batch/status`,
|
||||
method: "POST",
|
||||
handler: skillScanBatchStatusV1Http,
|
||||
});
|
||||
|
||||
http.route({
|
||||
path: ApiRoutes.packages,
|
||||
method: "POST",
|
||||
@@ -194,24 +153,6 @@ http.route({
|
||||
handler: mintPublishTokenV1Http,
|
||||
});
|
||||
|
||||
http.route({
|
||||
path: "/api/v1/package-inspector/claim",
|
||||
method: "POST",
|
||||
handler: packageInspectorClaimHttp,
|
||||
});
|
||||
|
||||
http.route({
|
||||
path: "/api/v1/package-inspector/artifact",
|
||||
method: "GET",
|
||||
handler: packageInspectorArtifactHttp,
|
||||
});
|
||||
|
||||
http.route({
|
||||
path: "/api/v1/package-inspector/results",
|
||||
method: "POST",
|
||||
handler: packageInspectorResultsHttp,
|
||||
});
|
||||
|
||||
http.route({
|
||||
pathPrefix: `${ApiRoutes.packages}/`,
|
||||
method: "POST",
|
||||
@@ -302,12 +243,6 @@ http.route({
|
||||
handler: banAppealContextV1Http,
|
||||
});
|
||||
|
||||
http.route({
|
||||
pathPrefix: `${ApiRoutes.users}/`,
|
||||
method: "GET",
|
||||
handler: usersGetRouterV1Http,
|
||||
});
|
||||
|
||||
http.route({
|
||||
path: ApiRoutes.users,
|
||||
method: "GET",
|
||||
@@ -393,9 +328,9 @@ http.route({
|
||||
});
|
||||
|
||||
http.route({
|
||||
path: LegacyApiRoutes.cliTelemetryInstall,
|
||||
path: LegacyApiRoutes.cliTelemetrySync,
|
||||
method: "POST",
|
||||
handler: cliTelemetryInstallHttp,
|
||||
handler: cliTelemetrySyncHttp,
|
||||
});
|
||||
|
||||
http.route({
|
||||
|
||||
@@ -4,15 +4,13 @@ import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
vi.mock("./lib/apiTokenAuth", () => ({
|
||||
getOptionalApiTokenUser: vi.fn(),
|
||||
requireApiTokenUser: vi.fn(),
|
||||
requirePackagePublishAuth: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("./skills", () => ({
|
||||
publishVersionForUser: vi.fn(),
|
||||
}));
|
||||
|
||||
const { getOptionalApiTokenUser, requireApiTokenUser, requirePackagePublishAuth } =
|
||||
await import("./lib/apiTokenAuth");
|
||||
const { getOptionalApiTokenUser, requireApiTokenUser } = await import("./lib/apiTokenAuth");
|
||||
const { publishVersionForUser } = await import("./skills");
|
||||
const { __handlers } = await import("./httpApi");
|
||||
const { hashSkillFiles } = await import("./lib/skills");
|
||||
@@ -25,7 +23,6 @@ describe("httpApi handlers", () => {
|
||||
afterEach(() => {
|
||||
vi.mocked(getOptionalApiTokenUser).mockReset();
|
||||
vi.mocked(requireApiTokenUser).mockReset();
|
||||
vi.mocked(requirePackagePublishAuth).mockReset();
|
||||
vi.mocked(publishVersionForUser).mockReset();
|
||||
});
|
||||
|
||||
@@ -243,7 +240,7 @@ describe("httpApi handlers", () => {
|
||||
it("cliWhoamiHttp returns 401 on auth failure", async () => {
|
||||
vi.mocked(requireApiTokenUser).mockRejectedValueOnce(
|
||||
new Error(
|
||||
"Unauthorized: This ClawHub account is not in good standing and cannot use API tokens. If you believe this is a mistake, open a GitHub issue: https://github.com/openclaw/clawhub/issues/new.",
|
||||
"Unauthorized: This ClawHub account is not in good standing and cannot use API tokens. If you believe this is a mistake, contact security@openclaw.ai.",
|
||||
),
|
||||
);
|
||||
const response = await __handlers.cliWhoamiHandler(
|
||||
@@ -267,40 +264,49 @@ describe("httpApi handlers", () => {
|
||||
expect(json.user.handle).toBe("p");
|
||||
});
|
||||
|
||||
it("cliTelemetryInstallHttp forwards one install event and returns ok", async () => {
|
||||
it("cliTelemetrySyncHttp forwards roots and returns ok", async () => {
|
||||
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: "users:1" } as never);
|
||||
const runMutation = vi.fn().mockResolvedValue(null);
|
||||
const response = await __handlers.cliTelemetryInstallHandler(
|
||||
const response = await __handlers.cliTelemetrySyncHandler(
|
||||
makeCtx({ runMutation }),
|
||||
new Request("https://x/api/cli/telemetry/install", {
|
||||
new Request("https://x/api/cli/telemetry/sync", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
event: "install",
|
||||
slug: "weather",
|
||||
version: "1.0.0",
|
||||
rootId: "abc",
|
||||
rootLabel: "~/skills",
|
||||
roots: [
|
||||
{
|
||||
rootId: "abc",
|
||||
label: "~/skills",
|
||||
skills: [{ slug: "weather", version: null }],
|
||||
},
|
||||
],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(response.status).toBe(200);
|
||||
expect(await response.json()).toEqual({ ok: true });
|
||||
expect(runMutation).toHaveBeenCalledWith(expect.anything(), {
|
||||
userId: "users:1",
|
||||
slug: "weather",
|
||||
version: "1.0.0",
|
||||
rootId: "abc",
|
||||
rootLabel: "~/skills",
|
||||
});
|
||||
expect(runMutation).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("cliTelemetryInstallHttp rejects sync-shaped roots snapshots", async () => {
|
||||
it("cliTelemetrySyncHttp returns 400 on invalid payload", async () => {
|
||||
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: "users:1" } as never);
|
||||
const response = await __handlers.cliTelemetrySyncHandler(
|
||||
makeCtx({ runMutation: vi.fn() }),
|
||||
new Request("https://x/api/cli/telemetry/sync", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ roots: "nope" }),
|
||||
}),
|
||||
);
|
||||
expect(response.status).toBe(400);
|
||||
});
|
||||
|
||||
it("cliTelemetrySyncHttp forwards skill versions when provided", async () => {
|
||||
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: "users:1" } as never);
|
||||
const runMutation = vi.fn().mockResolvedValue(null);
|
||||
const response = await __handlers.cliTelemetryInstallHandler(
|
||||
await __handlers.cliTelemetrySyncHandler(
|
||||
makeCtx({ runMutation }),
|
||||
new Request("https://x/api/cli/telemetry/install", {
|
||||
new Request("https://x/api/cli/telemetry/sync", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
@@ -314,9 +320,31 @@ describe("httpApi handlers", () => {
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(runMutation).toHaveBeenCalledWith(expect.anything(), {
|
||||
userId: "users:1",
|
||||
roots: [
|
||||
{ rootId: "abc", label: "~/skills", skills: [{ slug: "weather", version: "1.0.0" }] },
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("cliTelemetrySyncHttp returns 400 on invalid json", async () => {
|
||||
const request = new Request("https://x/api/cli/telemetry/sync", { method: "POST", body: "{" });
|
||||
const response = await __handlers.cliTelemetrySyncHandler(makeCtx({}), request);
|
||||
expect(response.status).toBe(400);
|
||||
expect(runMutation).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("cliTelemetrySyncHttp returns 401 when unauthorized", async () => {
|
||||
vi.mocked(requireApiTokenUser).mockRejectedValueOnce(new Error("Unauthorized"));
|
||||
const response = await __handlers.cliTelemetrySyncHandler(
|
||||
makeCtx({}),
|
||||
new Request("https://x/api/cli/telemetry/sync", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ roots: [] }),
|
||||
}),
|
||||
);
|
||||
expect(response.status).toBe(401);
|
||||
});
|
||||
|
||||
it("cliDeviceCodeHttp rate limits and creates a device code", async () => {
|
||||
@@ -416,51 +444,18 @@ describe("httpApi handlers", () => {
|
||||
});
|
||||
|
||||
it("cliUploadUrlHttp returns uploadUrl", async () => {
|
||||
vi.mocked(requirePackagePublishAuth).mockResolvedValueOnce({
|
||||
kind: "user",
|
||||
userId: "user1",
|
||||
} as never);
|
||||
const runMutation = vi.fn().mockResolvedValue({
|
||||
uploadUrl: "https://upload.local",
|
||||
uploadTicket: "packagePublishUploadTickets:1",
|
||||
});
|
||||
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: "user1" } as never);
|
||||
const runMutation = vi.fn().mockResolvedValue("https://upload.local");
|
||||
const response = await __handlers.cliUploadUrlHandler(
|
||||
makeCtx({ runMutation }),
|
||||
new Request("https://x/api/cli/upload-url", { method: "POST" }),
|
||||
);
|
||||
expect(response.status).toBe(200);
|
||||
expect(await response.json()).toEqual({
|
||||
uploadUrl: "https://upload.local",
|
||||
uploadTicket: "packagePublishUploadTickets:1",
|
||||
});
|
||||
});
|
||||
|
||||
it("cliUploadUrlHttp accepts package publish tokens", async () => {
|
||||
vi.mocked(requirePackagePublishAuth).mockResolvedValueOnce({
|
||||
kind: "github-actions",
|
||||
publishToken: { _id: "packagePublishTokens:1" },
|
||||
} as never);
|
||||
const runMutation = vi.fn().mockResolvedValue({
|
||||
uploadUrl: "https://upload.local/package",
|
||||
uploadTicket: "packagePublishUploadTickets:2",
|
||||
});
|
||||
const response = await __handlers.cliUploadUrlHandler(
|
||||
makeCtx({ runMutation }),
|
||||
new Request("https://x/api/cli/upload-url", { method: "POST" }),
|
||||
);
|
||||
expect(response.status).toBe(200);
|
||||
expect(await response.json()).toEqual({
|
||||
uploadUrl: "https://upload.local/package",
|
||||
uploadTicket: "packagePublishUploadTickets:2",
|
||||
});
|
||||
expect(runMutation).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({ publishTokenId: "packagePublishTokens:1" }),
|
||||
);
|
||||
expect(await response.json()).toEqual({ uploadUrl: "https://upload.local" });
|
||||
});
|
||||
|
||||
it("cliUploadUrlHttp returns 401 when unauthorized", async () => {
|
||||
vi.mocked(requirePackagePublishAuth).mockRejectedValueOnce(new Error("Unauthorized"));
|
||||
vi.mocked(requireApiTokenUser).mockRejectedValueOnce(new Error("Unauthorized"));
|
||||
const response = await __handlers.cliUploadUrlHandler(
|
||||
makeCtx({}),
|
||||
new Request("https://x/api/cli/upload-url", { method: "POST" }),
|
||||
|
||||
+22
-23
@@ -1,16 +1,16 @@
|
||||
import {
|
||||
ApiCliSkillDeleteResponseSchema,
|
||||
ApiCliTelemetryInstallResponseSchema,
|
||||
CliTelemetryInstallRequestSchema,
|
||||
ApiCliTelemetrySyncResponseSchema,
|
||||
CliPublishRequestSchema,
|
||||
CliSkillDeleteRequestSchema,
|
||||
CliTelemetrySyncRequestSchema,
|
||||
parseArk,
|
||||
} from "clawhub-schema";
|
||||
import { api, internal } from "./_generated/api";
|
||||
import type { Id } from "./_generated/dataModel";
|
||||
import type { ActionCtx } from "./_generated/server";
|
||||
import { httpAction } from "./functions";
|
||||
import { requireApiTokenUser, requirePackagePublishAuth } from "./lib/apiTokenAuth";
|
||||
import { requireApiTokenUser } from "./lib/apiTokenAuth";
|
||||
import { corsHeaders, mergeHeaders } from "./lib/httpHeaders";
|
||||
import { applyRateLimit } from "./lib/httpRateLimit";
|
||||
import { parseBooleanQueryParam, resolveBooleanQueryParam } from "./lib/httpUtils";
|
||||
@@ -148,16 +148,11 @@ export const cliWhoamiHttp = httpAction(cliWhoamiHandler);
|
||||
|
||||
async function cliUploadUrlHandler(ctx: ActionCtx, request: Request) {
|
||||
try {
|
||||
const auth = await requirePackagePublishAuth(ctx, request);
|
||||
const upload =
|
||||
auth.kind === "user"
|
||||
? await ctx.runMutation(internal.uploads.createPackagePublishUploadForUserInternal, {
|
||||
userId: auth.userId,
|
||||
})
|
||||
: await ctx.runMutation(internal.uploads.createPackagePublishUploadForTokenInternal, {
|
||||
publishTokenId: auth.publishToken._id,
|
||||
});
|
||||
return json(upload);
|
||||
const { userId } = await requireApiTokenUser(ctx, request);
|
||||
const uploadUrl = await ctx.runMutation(internal.uploads.generateUploadUrlForUserInternal, {
|
||||
userId,
|
||||
});
|
||||
return json({ uploadUrl });
|
||||
} catch (error) {
|
||||
return text(formatAuthFailure(error), 401);
|
||||
}
|
||||
@@ -227,7 +222,7 @@ export const cliSkillUndeleteHttp = httpAction((ctx, request) =>
|
||||
cliSkillDeleteHandler(ctx, request, false),
|
||||
);
|
||||
|
||||
async function cliTelemetryInstallHandler(ctx: ActionCtx, request: Request) {
|
||||
async function cliTelemetrySyncHandler(ctx: ActionCtx, request: Request) {
|
||||
let body: unknown;
|
||||
try {
|
||||
body = await request.json();
|
||||
@@ -237,15 +232,19 @@ async function cliTelemetryInstallHandler(ctx: ActionCtx, request: Request) {
|
||||
|
||||
try {
|
||||
const { userId } = await requireApiTokenUser(ctx, request);
|
||||
const args = parseArk(CliTelemetryInstallRequestSchema, body, "Install telemetry payload");
|
||||
await ctx.runMutation(internal.telemetry.reportCliInstallInternal, {
|
||||
const args = parseArk(CliTelemetrySyncRequestSchema, body, "Telemetry payload");
|
||||
await ctx.runMutation(internal.telemetry.reportCliSyncInternal, {
|
||||
userId,
|
||||
slug: args.slug,
|
||||
version: args.version,
|
||||
rootId: args.rootId,
|
||||
rootLabel: args.rootLabel,
|
||||
roots: args.roots.map((root) => ({
|
||||
rootId: root.rootId,
|
||||
label: root.label,
|
||||
skills: root.skills.map((skill) => ({
|
||||
slug: skill.slug,
|
||||
version: skill.version ?? undefined,
|
||||
})),
|
||||
})),
|
||||
});
|
||||
const ok = parseArk(ApiCliTelemetryInstallResponseSchema, { ok: true }, "Telemetry response");
|
||||
const ok = parseArk(ApiCliTelemetrySyncResponseSchema, { ok: true }, "Telemetry response");
|
||||
return json(ok);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "Telemetry failed";
|
||||
@@ -254,7 +253,7 @@ async function cliTelemetryInstallHandler(ctx: ActionCtx, request: Request) {
|
||||
}
|
||||
}
|
||||
|
||||
export const cliTelemetryInstallHttp = httpAction(cliTelemetryInstallHandler);
|
||||
export const cliTelemetrySyncHttp = httpAction(cliTelemetrySyncHandler);
|
||||
|
||||
async function cliDeviceCodeHandler(ctx: ActionCtx, request: Request) {
|
||||
if (request.method !== "POST") return text("Method not allowed", 405);
|
||||
@@ -388,7 +387,7 @@ export const __handlers = {
|
||||
cliUploadUrlHandler,
|
||||
cliPublishHandler,
|
||||
cliSkillDeleteHandler,
|
||||
cliTelemetryInstallHandler,
|
||||
cliTelemetrySyncHandler,
|
||||
cliDeviceCodeHandler,
|
||||
cliDeviceTokenHandler,
|
||||
};
|
||||
|
||||
+170
-2004
File diff suppressed because it is too large
Load Diff
@@ -2,12 +2,7 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { Id } from "./_generated/dataModel";
|
||||
import type { ActionCtx } from "./_generated/server";
|
||||
import {
|
||||
formatUserFacingErrorMessage,
|
||||
parseMultipartSkillScan,
|
||||
resolveVersionTagsBatch,
|
||||
softDeleteErrorToResponse,
|
||||
} from "./httpApiV1/shared";
|
||||
import { formatUserFacingErrorMessage, resolveVersionTagsBatch } from "./httpApiV1/shared";
|
||||
|
||||
function makeCtx() {
|
||||
return {
|
||||
@@ -33,54 +28,6 @@ describe("http API v1 shared helpers", () => {
|
||||
).toBe("Publisher not found");
|
||||
});
|
||||
|
||||
it("maps soft-delete validation failures to 400 with cleaned messages", async () => {
|
||||
const response = softDeleteErrorToResponse(
|
||||
"package",
|
||||
new Error(
|
||||
"[CONVEX M] [Request ID: abc] Server Error Called by client Uncaught ConvexError: Package name must be lowercase and npm-safe (example: @scope/name or plugin-name)",
|
||||
),
|
||||
{},
|
||||
);
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
await expect(response.text()).resolves.toBe(
|
||||
"Package name must be lowercase and npm-safe (example: @scope/name or plugin-name)",
|
||||
);
|
||||
});
|
||||
|
||||
it("maps reserved package route validation failures to 400 with cleaned messages", async () => {
|
||||
const response = softDeleteErrorToResponse(
|
||||
"package",
|
||||
new Error(
|
||||
'[CONVEX M] [Request ID: abc] Server Error Called by client Uncaught ConvexError: Package name "publish" is reserved for ClawHub routes. Use a scoped name or choose a different package name.',
|
||||
),
|
||||
{},
|
||||
);
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
await expect(response.text()).resolves.toBe(
|
||||
'Package name "publish" is reserved for ClawHub routes. Use a scoped name or choose a different package name.',
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps unknown soft-delete failures generic 500s", async () => {
|
||||
const response = softDeleteErrorToResponse("soul", new Error("boom"), {});
|
||||
|
||||
expect(response.status).toBe(500);
|
||||
await expect(response.text()).resolves.toBe("Internal Server Error");
|
||||
});
|
||||
|
||||
it("keeps unrelated reserved-word failures generic 500s", async () => {
|
||||
const response = softDeleteErrorToResponse(
|
||||
"package",
|
||||
new Error("database reserved capacity exceeded"),
|
||||
{},
|
||||
);
|
||||
|
||||
expect(response.status).toBe(500);
|
||||
await expect(response.text()).resolves.toBe("Internal Server Error");
|
||||
});
|
||||
|
||||
it("resolves latest tags without reading version documents", async () => {
|
||||
const ctx = makeCtx();
|
||||
const versionId = "skillVersions:latest" as Id<"skillVersions">;
|
||||
@@ -135,28 +82,4 @@ describe("http API v1 shared helpers", () => {
|
||||
|
||||
expect(result).toEqual([{ stable: "1.5.0" }]);
|
||||
});
|
||||
|
||||
it("validates skill scan multipart payloads before storing uploaded files", async () => {
|
||||
const form = new FormData();
|
||||
form.set("payload", JSON.stringify({ source: { kind: "upload" }, update: true }));
|
||||
form.append("files", new Blob(["# Demo"], { type: "text/markdown" }), "SKILL.md");
|
||||
const request = new Request("https://clawhub.ai/api/v1/skills/-/scan", {
|
||||
method: "POST",
|
||||
body: form,
|
||||
});
|
||||
const store = vi.fn();
|
||||
const ctx = {
|
||||
storage: {
|
||||
store,
|
||||
delete: vi.fn(),
|
||||
},
|
||||
} as unknown as ActionCtx;
|
||||
|
||||
await expect(
|
||||
parseMultipartSkillScan(ctx, request, () => {
|
||||
throw new Error("update is not valid for uploaded scans");
|
||||
}),
|
||||
).rejects.toThrow("update is not valid for uploaded scans");
|
||||
expect(store).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import { httpAction } from "./functions";
|
||||
import { verifyDocsSessionV1Handler } from "./httpApiV1/docsSessionV1";
|
||||
import {
|
||||
exportPluginsV1Handler,
|
||||
listBundlePluginsV1Handler,
|
||||
listCodePluginsV1Handler,
|
||||
listPackagesV1Handler,
|
||||
@@ -21,10 +20,6 @@ import {
|
||||
publishSkillV1Handler,
|
||||
resolveSkillVersionV1Handler,
|
||||
searchSkillsV1Handler,
|
||||
skillScanBatchStatusV1Handler,
|
||||
skillScanBatchSubmitV1Handler,
|
||||
skillScanGetRouterV1Handler,
|
||||
skillScanSubmitV1Handler,
|
||||
skillSecurityVerdictsV1Handler,
|
||||
skillsDeleteRouterV1Handler,
|
||||
skillsGetRouterV1Handler,
|
||||
@@ -41,7 +36,6 @@ import { starsDeleteRouterV1Handler, starsPostRouterV1Handler } from "./httpApiV
|
||||
import { transfersGetRouterV1Handler } from "./httpApiV1/transfersV1";
|
||||
import {
|
||||
banAppealContextV1Handler,
|
||||
usersGetRouterV1Handler,
|
||||
usersListV1Handler,
|
||||
usersPostRouterV1Handler,
|
||||
} from "./httpApiV1/usersV1";
|
||||
@@ -49,7 +43,6 @@ import { whoamiV1Handler } from "./httpApiV1/whoamiV1";
|
||||
|
||||
export const listPackagesV1Http = httpAction(listPackagesV1Handler);
|
||||
export const listPluginsV1Http = httpAction(listPluginsV1Handler);
|
||||
export const exportPluginsV1Http = httpAction(exportPluginsV1Handler);
|
||||
export const packagesGetRouterV1Http = httpAction(packagesGetRouterV1Handler);
|
||||
export const packagesPostRouterV1Http = httpAction(packagesPostRouterV1Handler);
|
||||
export const packagesDeleteRouterV1Http = httpAction(packagesDeleteRouterV1Handler);
|
||||
@@ -68,10 +61,6 @@ export const listSkillsV1Http = httpAction(listSkillsV1Handler);
|
||||
export const skillsGetRouterV1Http = httpAction(skillsGetRouterV1Handler);
|
||||
export const publishSkillV1Http = httpAction(publishSkillV1Handler);
|
||||
export const skillSecurityVerdictsV1Http = httpAction(skillSecurityVerdictsV1Handler);
|
||||
export const skillScanSubmitV1Http = httpAction(skillScanSubmitV1Handler);
|
||||
export const skillScanGetRouterV1Http = httpAction(skillScanGetRouterV1Handler);
|
||||
export const skillScanBatchSubmitV1Http = httpAction(skillScanBatchSubmitV1Handler);
|
||||
export const skillScanBatchStatusV1Http = httpAction(skillScanBatchStatusV1Handler);
|
||||
export const skillsPostRouterV1Http = httpAction(skillsPostRouterV1Handler);
|
||||
export const skillsDeleteRouterV1Http = httpAction(skillsDeleteRouterV1Handler);
|
||||
export const exportSkillsV1Http = httpAction(exportSkillsV1Handler);
|
||||
@@ -87,7 +76,6 @@ export const starsDeleteRouterV1Http = httpAction(starsDeleteRouterV1Handler);
|
||||
export const transfersGetRouterV1Http = httpAction(transfersGetRouterV1Handler);
|
||||
|
||||
export const whoamiV1Http = httpAction(whoamiV1Handler);
|
||||
export const usersGetRouterV1Http = httpAction(usersGetRouterV1Handler);
|
||||
export const usersPostRouterV1Http = httpAction(usersPostRouterV1Handler);
|
||||
export const usersListV1Http = httpAction(usersListV1Handler);
|
||||
export const banAppealContextV1Http = httpAction(banAppealContextV1Handler);
|
||||
@@ -95,7 +83,6 @@ export const banAppealContextV1Http = httpAction(banAppealContextV1Handler);
|
||||
export const __handlers = {
|
||||
listPackagesV1Handler,
|
||||
listPluginsV1Handler,
|
||||
exportPluginsV1Handler,
|
||||
packagesGetRouterV1Handler,
|
||||
packagesPostRouterV1Handler,
|
||||
packagesDeleteRouterV1Handler,
|
||||
@@ -113,10 +100,6 @@ export const __handlers = {
|
||||
skillsGetRouterV1Handler,
|
||||
publishSkillV1Handler,
|
||||
skillSecurityVerdictsV1Handler,
|
||||
skillScanSubmitV1Handler,
|
||||
skillScanGetRouterV1Handler,
|
||||
skillScanBatchSubmitV1Handler,
|
||||
skillScanBatchStatusV1Handler,
|
||||
skillsPostRouterV1Handler,
|
||||
skillsDeleteRouterV1Handler,
|
||||
exportSkillsV1Handler,
|
||||
@@ -129,7 +112,6 @@ export const __handlers = {
|
||||
starsDeleteRouterV1Handler,
|
||||
transfersGetRouterV1Handler,
|
||||
whoamiV1Handler,
|
||||
usersGetRouterV1Handler,
|
||||
usersPostRouterV1Handler,
|
||||
usersListV1Handler,
|
||||
banAppealContextV1Handler,
|
||||
|
||||
+195
-821
File diff suppressed because it is too large
Load Diff
@@ -422,71 +422,6 @@ export async function parseMultipartPublish(
|
||||
return parsePublishBody(body);
|
||||
}
|
||||
|
||||
export async function parseMultipartSkillScan(
|
||||
ctx: ActionCtx,
|
||||
request: Request,
|
||||
validatePayload?: (payload: Record<string, unknown>) => Record<string, unknown>,
|
||||
): Promise<{
|
||||
payload: Record<string, unknown>;
|
||||
files: Array<{
|
||||
path: string;
|
||||
size: number;
|
||||
storageId: Id<"_storage">;
|
||||
sha256: string;
|
||||
contentType?: string;
|
||||
}>;
|
||||
}> {
|
||||
const form = await request.formData();
|
||||
const payloadRaw = form.get("payload");
|
||||
if (!payloadRaw || typeof payloadRaw !== "string") {
|
||||
throw new Error("Missing payload");
|
||||
}
|
||||
let payload: Record<string, unknown>;
|
||||
try {
|
||||
payload = JSON.parse(payloadRaw) as Record<string, unknown>;
|
||||
} catch {
|
||||
throw new Error("Invalid JSON payload");
|
||||
}
|
||||
const validatedPayload = validatePayload ? validatePayload(payload) : payload;
|
||||
|
||||
const fileEntries = form
|
||||
.getAll("files")
|
||||
.map((entry) => toFileLike(entry))
|
||||
.filter((file): file is FileLikeEntry => Boolean(file))
|
||||
.filter((file) => !isMacJunkPath(file.name));
|
||||
if (fileEntries.length === 0) throw new Error("files required");
|
||||
if (!fileEntries.some((file) => file.name.trim().toLowerCase() === "skill.md")) {
|
||||
throw new Error("SKILL.md required");
|
||||
}
|
||||
const oversized = fileEntries.find((file) => file.size > MAX_PUBLISH_FILE_BYTES);
|
||||
if (oversized) throw new Error(getPublishFileSizeError(oversized.name));
|
||||
|
||||
const files: Array<{
|
||||
path: string;
|
||||
size: number;
|
||||
storageId: Id<"_storage">;
|
||||
sha256: string;
|
||||
contentType?: string;
|
||||
}> = [];
|
||||
|
||||
try {
|
||||
for (const file of fileEntries) {
|
||||
const path = file.name;
|
||||
const size = file.size;
|
||||
const contentType = file.type || undefined;
|
||||
const buffer = new Uint8Array(await file.arrayBuffer());
|
||||
const sha256 = await sha256Hex(buffer);
|
||||
const storageId = await ctx.storage.store(file as Blob);
|
||||
files.push({ path, size, storageId, sha256, contentType });
|
||||
}
|
||||
} catch (error) {
|
||||
await Promise.allSettled(files.map((file) => ctx.storage.delete(file.storageId)));
|
||||
throw error;
|
||||
}
|
||||
|
||||
return { payload: validatedPayload, files };
|
||||
}
|
||||
|
||||
export function parsePublishBody(body: unknown) {
|
||||
const parsed = parseArk(CliPublishRequestSchema, body, "Publish payload");
|
||||
if (parsed.files.length === 0) throw new Error("files required");
|
||||
@@ -514,40 +449,22 @@ export function parsePublishBody(body: unknown) {
|
||||
};
|
||||
}
|
||||
|
||||
// Substrings that indicate user-input validation failures from the underlying
|
||||
// mutations (e.g. `normalizePackageName` ConvexErrors). These are surfaced as
|
||||
// 400s with the cleaned message so CLI/API clients can see the actual reason
|
||||
// instead of an opaque 500.
|
||||
const SOFT_DELETE_BAD_REQUEST_HINTS = [
|
||||
"slug required",
|
||||
"package name required",
|
||||
"package name must be",
|
||||
"must be lowercase",
|
||||
"npm-safe",
|
||||
"reserved for clawhub routes",
|
||||
"version required",
|
||||
] as const;
|
||||
|
||||
export function softDeleteErrorToResponse(
|
||||
entity: "skill" | "soul" | "package",
|
||||
error: unknown,
|
||||
headers: HeadersInit,
|
||||
) {
|
||||
const rawMessage = error instanceof Error ? error.message : `${entity} delete failed`;
|
||||
const cleaned = cleanUserFacingErrorMessage(rawMessage) || rawMessage;
|
||||
const lower = cleaned.toLowerCase();
|
||||
const message = error instanceof Error ? error.message : `${entity} delete failed`;
|
||||
const lower = message.toLowerCase();
|
||||
|
||||
if (lower.includes("unauthorized"))
|
||||
return text(formatAuthzMessage(error, "Unauthorized"), 401, headers);
|
||||
if (lower.includes("forbidden"))
|
||||
return text(formatAuthzMessage(error, "Forbidden"), 403, headers);
|
||||
if (lower.includes("not found")) return text(cleaned, 404, headers);
|
||||
if (SOFT_DELETE_BAD_REQUEST_HINTS.some((hint) => lower.includes(hint))) {
|
||||
return text(cleaned, 400, headers);
|
||||
}
|
||||
if (lower.includes("not found")) return text(message, 404, headers);
|
||||
if (lower.includes("slug required")) return text("Slug required", 400, headers);
|
||||
|
||||
// Unknown: server-side failure. Keep the body generic; only known
|
||||
// user-input validation failures above surface the cleaned mutation message.
|
||||
// Unknown: server-side failure. Keep body generic.
|
||||
return text("Internal Server Error", 500, headers);
|
||||
}
|
||||
|
||||
|
||||
+45
-510
@@ -1,11 +1,7 @@
|
||||
import {
|
||||
ApiRoutes,
|
||||
ApiV1SkillBulkRescanBatchRequestSchema,
|
||||
ApiV1SkillBulkRescanStatusRequestSchema,
|
||||
ApiV1SkillRepairVtPendingRequestSchema,
|
||||
ApiV1SkillScanBatchRequestSchema,
|
||||
ApiV1SkillScanBatchStatusRequestSchema,
|
||||
ApiV1SkillScanSubmitRequestSchema,
|
||||
SkillAppealRequestSchema,
|
||||
SkillAppealResolveRequestSchema,
|
||||
SkillReportTriageRequestSchema,
|
||||
@@ -21,26 +17,14 @@ import { getOptionalApiTokenUserId, requireApiTokenUser } from "../lib/apiTokenA
|
||||
import { mergeHeaders } from "../lib/httpHeaders";
|
||||
import { applyRateLimit } from "../lib/httpRateLimit";
|
||||
import { parseBooleanQueryParam, resolveBooleanQueryParam } from "../lib/httpUtils";
|
||||
import {
|
||||
buildSkillInstallResolution,
|
||||
type InstallResolverSkill,
|
||||
type InstallResolverSource,
|
||||
type SkillInstallResolution,
|
||||
} from "../lib/installResolver";
|
||||
import type {
|
||||
LlmAgenticRiskFinding,
|
||||
LlmEvalDimension,
|
||||
LlmRiskSummary,
|
||||
} from "../lib/securityPrompt";
|
||||
import { selectGeneratedSkillCardFile, sourceSkillVersionFiles } from "../lib/skillCards";
|
||||
import { getPublicSkillFileAccessBlock, isSkillVersionForSkill } from "../lib/skillFileAccess";
|
||||
import {
|
||||
getPublicSkillFileAccessBlock,
|
||||
getPublicSkillVersionAccessBlock,
|
||||
getSkillFileModerationInfoFromSkill,
|
||||
isSkillVersionForSkill,
|
||||
} from "../lib/skillFileAccess";
|
||||
import {
|
||||
buildDeterministicZip,
|
||||
buildMergedExportZip,
|
||||
type MergedExportManifestEntry,
|
||||
validateSlug,
|
||||
@@ -196,7 +180,6 @@ type GetBySlugResult = {
|
||||
summary?: string;
|
||||
engineVersion?: string;
|
||||
updatedAt?: number;
|
||||
sourceVersionId?: Id<"skillVersions"> | null;
|
||||
reason?: string;
|
||||
} | null;
|
||||
} | null;
|
||||
@@ -295,14 +278,8 @@ type SkillSecuritySnapshot = {
|
||||
};
|
||||
|
||||
const internalRefs = internal as unknown as {
|
||||
githubSkillSources: {
|
||||
getByIdInternal: unknown;
|
||||
};
|
||||
securityScan: {
|
||||
createPublishedSkillScanRequestInternal: unknown;
|
||||
enqueueBulkSkillRescanBatchForAdminInternal: unknown;
|
||||
getStoredScanReportForUserInternal: unknown;
|
||||
getSkillScanRequestForUserInternal: unknown;
|
||||
getBulkSkillRescanBatchStatusForAdminInternal: unknown;
|
||||
requestSkillRescanForUserInternal: unknown;
|
||||
};
|
||||
@@ -311,10 +288,6 @@ const internalRefs = internal as unknown as {
|
||||
};
|
||||
skills: {
|
||||
getSecurityVerdictTargetInternal: unknown;
|
||||
getVerifyTargetBySlugInternal: unknown;
|
||||
getSkillBySlugInternal: unknown;
|
||||
getVersionByIdInternal: unknown;
|
||||
getVersionBySkillAndVersionInternal: unknown;
|
||||
reportSkillForUserInternal: unknown;
|
||||
listSkillReportsInternal: unknown;
|
||||
triageSkillReportForUserInternal: unknown;
|
||||
@@ -336,156 +309,6 @@ async function runActionRef<T>(ctx: ActionCtx, ref: unknown, args: unknown): Pro
|
||||
return (await ctx.runAction(ref as never, args as never)) as T;
|
||||
}
|
||||
|
||||
function isMultipartRequest(request: Request) {
|
||||
return (
|
||||
request.headers.get("content-type")?.toLowerCase().includes("multipart/form-data") === true
|
||||
);
|
||||
}
|
||||
|
||||
function encodeJsonEntry(value: unknown) {
|
||||
return new TextEncoder().encode(`${JSON.stringify(value, null, 2)}\n`);
|
||||
}
|
||||
|
||||
function encodeTextEntry(value: string) {
|
||||
return new TextEncoder().encode(value);
|
||||
}
|
||||
|
||||
function scanReportPart(status: Record<string, unknown>, key: string) {
|
||||
const report = status.report;
|
||||
if (!report || typeof report !== "object" || Array.isArray(report)) return null;
|
||||
return (report as Record<string, unknown>)[key] ?? null;
|
||||
}
|
||||
|
||||
function buildSkillScanReportZip(status: Record<string, unknown>) {
|
||||
const manifest = {
|
||||
scanId: status.scanId,
|
||||
sourceKind: status.sourceKind,
|
||||
update: status.update,
|
||||
status: status.status,
|
||||
artifact: status.artifact ?? null,
|
||||
createdAt: status.createdAt,
|
||||
updatedAt: status.updatedAt,
|
||||
completedAt: status.completedAt ?? null,
|
||||
writtenBack: status.writtenBack === true,
|
||||
};
|
||||
const scanIdText = typeof status.scanId === "string" ? status.scanId : "";
|
||||
const statusText = typeof status.status === "string" ? status.status : "";
|
||||
const readme = [
|
||||
"# ClawHub Scan Report",
|
||||
"",
|
||||
`Scan ID: ${scanIdText}`,
|
||||
`Status: ${statusText}`,
|
||||
"",
|
||||
"This archive contains the stored security scan results for the submitted ClawHub version.",
|
||||
"",
|
||||
"## How to read this report",
|
||||
"",
|
||||
"Start with `clawscan.json`. ClawScan is the primary security verdict for the submitted artifact. Its `summary` field is the short explanation of what triggered the result, and `guidance` explains what to change before uploading a fixed version.",
|
||||
"",
|
||||
"- `malicious` means ClawHub blocked the submitted version from public install surfaces.",
|
||||
"- `suspicious` means ClawHub found behavior that needs review before users should rely on it.",
|
||||
"- `clean` means ClawHub did not find blocking security issues in this scan.",
|
||||
"",
|
||||
"VirusTotal results are supporting reputation telemetry. They can help explain a risk signal, but they are not the sole source of ClawHub's final verdict.",
|
||||
"",
|
||||
"## Files",
|
||||
"",
|
||||
"- `manifest.json`: artifact identity, scan status, timestamps, and writeback state.",
|
||||
"- `clawscan.json`: final ClawScan verdict, summary, guidance, and findings.",
|
||||
"- `skillspector.json`: SkillSpector structure and agentic-risk signals when available.",
|
||||
"- `static-analysis.json`: deterministic scanner findings, reason codes, and static summary.",
|
||||
"- `virustotal.json`: external reputation counts and status when available.",
|
||||
"- `README.md`: this interpretation guide.",
|
||||
"",
|
||||
].join("\n");
|
||||
|
||||
return buildDeterministicZip([
|
||||
{ path: "manifest.json", bytes: encodeJsonEntry(manifest) },
|
||||
{ path: "clawscan.json", bytes: encodeJsonEntry(scanReportPart(status, "clawscan")) },
|
||||
{ path: "skillspector.json", bytes: encodeJsonEntry(scanReportPart(status, "skillspector")) },
|
||||
{
|
||||
path: "static-analysis.json",
|
||||
bytes: encodeJsonEntry(scanReportPart(status, "staticAnalysis")),
|
||||
},
|
||||
{ path: "virustotal.json", bytes: encodeJsonEntry(scanReportPart(status, "virustotal")) },
|
||||
{ path: "README.md", bytes: encodeTextEntry(readme) },
|
||||
]);
|
||||
}
|
||||
|
||||
function safeScanReportFilenamePart(value: string) {
|
||||
return (
|
||||
value
|
||||
.replace(/^@/, "")
|
||||
.replaceAll("/", "-")
|
||||
.replaceAll(/[^a-zA-Z0-9._-]+/g, "-")
|
||||
.replace(/^-+|-+$/g, "") || "artifact"
|
||||
);
|
||||
}
|
||||
|
||||
async function handleSkillScanBatchSubmit(ctx: ActionCtx, request: Request, headers: HeadersInit) {
|
||||
const auth = await requireApiTokenUserOrResponse(ctx, request, headers);
|
||||
if (!auth.ok) return auth.response;
|
||||
const admin = requireAdminOrResponse(auth.user, headers);
|
||||
if (!admin.ok) return admin.response;
|
||||
try {
|
||||
const body = parseArk(
|
||||
ApiV1SkillScanBatchRequestSchema,
|
||||
await request.json(),
|
||||
"Skill scan batch payload",
|
||||
) as {
|
||||
mode?: "all-active-latest";
|
||||
cursor?: string | null;
|
||||
batchSize?: number;
|
||||
dryRun?: boolean;
|
||||
};
|
||||
const result = await runMutationRef(
|
||||
ctx,
|
||||
internalRefs.securityScan.enqueueBulkSkillRescanBatchForAdminInternal,
|
||||
{
|
||||
actorUserId: auth.userId,
|
||||
...(body.mode ? { mode: body.mode } : {}),
|
||||
cursor: body.cursor ?? null,
|
||||
...(body.batchSize !== undefined ? { batchSize: body.batchSize } : {}),
|
||||
...(body.dryRun !== undefined ? { dryRun: body.dryRun } : {}),
|
||||
},
|
||||
);
|
||||
return json(result, 200, headers);
|
||||
} catch (error) {
|
||||
if (error instanceof SyntaxError) return text("Invalid JSON", 400, headers);
|
||||
return text(error instanceof Error ? error.message : "Skill scan batch failed", 400, headers);
|
||||
}
|
||||
}
|
||||
|
||||
async function handleSkillScanBatchStatus(ctx: ActionCtx, request: Request, headers: HeadersInit) {
|
||||
const auth = await requireApiTokenUserOrResponse(ctx, request, headers);
|
||||
if (!auth.ok) return auth.response;
|
||||
const admin = requireAdminOrResponse(auth.user, headers);
|
||||
if (!admin.ok) return admin.response;
|
||||
try {
|
||||
const body = parseArk(
|
||||
ApiV1SkillScanBatchStatusRequestSchema,
|
||||
await request.json(),
|
||||
"Skill scan batch status payload",
|
||||
) as { jobIds: string[] };
|
||||
const result = await runQueryRef(
|
||||
ctx,
|
||||
internalRefs.securityScan.getBulkSkillRescanBatchStatusForAdminInternal,
|
||||
{
|
||||
actorUserId: auth.userId,
|
||||
jobIds: body.jobIds,
|
||||
},
|
||||
);
|
||||
return json(result, 200, headers);
|
||||
} catch (error) {
|
||||
if (error instanceof SyntaxError) return text("Invalid JSON", 400, headers);
|
||||
return text(
|
||||
error instanceof Error ? error.message : "Skill scan batch status failed",
|
||||
400,
|
||||
headers,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function isDefinitiveSecurityStatus(
|
||||
status: NormalizedSecurityStatus | null | undefined,
|
||||
): status is "clean" | "suspicious" | "malicious" {
|
||||
@@ -665,6 +488,16 @@ type VerifySecurityVersion = {
|
||||
| "checkedAt"
|
||||
> &
|
||||
Partial<Pick<NonNullable<Doc<"skillVersions">["skillSpectorAnalysis"]>, "summary" | "error">>;
|
||||
depRegistryAnalysis?: Pick<
|
||||
NonNullable<Doc<"skillVersions">["depRegistryAnalysis"]>,
|
||||
"status" | "summary" | "checkedAt"
|
||||
> &
|
||||
Partial<
|
||||
Pick<
|
||||
NonNullable<Doc<"skillVersions">["depRegistryAnalysis"]>,
|
||||
"notFoundPackages" | "unresolvedPackages"
|
||||
>
|
||||
>;
|
||||
};
|
||||
|
||||
type SecurityVerdictTargetResult = {
|
||||
@@ -714,6 +547,9 @@ function buildVerifySecurity(version: VerifySecurityVersion) {
|
||||
const skillSpectorStatus = version.skillSpectorAnalysis
|
||||
? normalizeVerificationStatus(version.skillSpectorAnalysis.status)
|
||||
: null;
|
||||
const depStatus = version.depRegistryAnalysis
|
||||
? normalizeVerificationStatus(version.depRegistryAnalysis.status)
|
||||
: null;
|
||||
const status = clawStatus;
|
||||
|
||||
return {
|
||||
@@ -769,7 +605,16 @@ function buildVerifySecurity(version: VerifySecurityVersion) {
|
||||
checkedAt: version.skillSpectorAnalysis.checkedAt ?? null,
|
||||
}
|
||||
: null,
|
||||
dependencyRegistry: null,
|
||||
dependencyRegistry: version.depRegistryAnalysis
|
||||
? {
|
||||
status: depStatus ?? "pending",
|
||||
rawStatus: version.depRegistryAnalysis.status,
|
||||
summary: version.depRegistryAnalysis.summary ?? null,
|
||||
notFoundPackages: version.depRegistryAnalysis.notFoundPackages ?? [],
|
||||
unresolvedPackages: version.depRegistryAnalysis.unresolvedPackages ?? [],
|
||||
checkedAt: version.depRegistryAnalysis.checkedAt ?? null,
|
||||
}
|
||||
: null,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -802,7 +647,7 @@ function buildVerifyReasons(args: {
|
||||
securityStatus: NormalizedSecurityStatus;
|
||||
}) {
|
||||
const reasons: string[] = [];
|
||||
if (!args.cardAvailable && !args.isMalwareBlocked) reasons.push("card.missing");
|
||||
if (!args.cardAvailable) reasons.push("card.missing");
|
||||
reasons.push(
|
||||
...buildSecurityVerdictReasons({
|
||||
isMalwareBlocked: args.isMalwareBlocked,
|
||||
@@ -836,6 +681,7 @@ function getVerifySecurityCheckedAt(security: ReturnType<typeof buildVerifySecur
|
||||
security.signals.staticScan?.checkedAt,
|
||||
security.signals.virusTotal?.checkedAt,
|
||||
security.signals.skillSpector?.checkedAt,
|
||||
security.signals.dependencyRegistry?.checkedAt,
|
||||
].filter((value): value is number => typeof value === "number");
|
||||
return candidates.length > 0 ? Math.max(...candidates) : null;
|
||||
}
|
||||
@@ -882,7 +728,14 @@ function buildSecurityVerdictSummary(security: ReturnType<typeof buildVerifySecu
|
||||
checkedAt: security.signals.skillSpector.checkedAt,
|
||||
}
|
||||
: null,
|
||||
dependencyRegistry: null,
|
||||
dependencyRegistry: security.signals.dependencyRegistry
|
||||
? {
|
||||
status: security.signals.dependencyRegistry.status,
|
||||
rawStatus: security.signals.dependencyRegistry.rawStatus,
|
||||
summary: security.signals.dependencyRegistry.summary,
|
||||
checkedAt: security.signals.dependencyRegistry.checkedAt,
|
||||
}
|
||||
: null,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -1111,133 +964,6 @@ export async function skillSecurityVerdictsV1Handler(ctx: ActionCtx, request: Re
|
||||
return json({ schema: "clawhub.skill.security-verdicts.v1", items }, 200, rate.headers);
|
||||
}
|
||||
|
||||
export async function skillScanSubmitV1Handler(ctx: ActionCtx, request: Request) {
|
||||
const rate = await applyRateLimit(ctx, request, "write");
|
||||
if (!rate.ok) return rate.response;
|
||||
const auth = await requireApiTokenUserOrResponse(ctx, request, rate.headers);
|
||||
if (!auth.ok) return auth.response;
|
||||
|
||||
try {
|
||||
if (isMultipartRequest(request)) {
|
||||
return text(
|
||||
"Local upload scans are no longer supported. Upload a version, then use `clawhub scan download <slug> --version <version>` to retrieve stored scan results.",
|
||||
410,
|
||||
rate.headers,
|
||||
);
|
||||
}
|
||||
|
||||
const body = parseArk(
|
||||
ApiV1SkillScanSubmitRequestSchema,
|
||||
await request.json(),
|
||||
"Skill scan payload",
|
||||
) as {
|
||||
source: { kind: "upload" } | { kind: "published"; slug: string; version?: string };
|
||||
update?: boolean;
|
||||
};
|
||||
if (body.source.kind === "upload") {
|
||||
return text(
|
||||
"Local upload scans are no longer supported. Upload a version, then use `clawhub scan download <slug> --version <version>` to retrieve stored scan results.",
|
||||
410,
|
||||
rate.headers,
|
||||
);
|
||||
}
|
||||
const result = await runMutationRef(
|
||||
ctx,
|
||||
internalRefs.securityScan.createPublishedSkillScanRequestInternal,
|
||||
{
|
||||
actorUserId: auth.userId,
|
||||
slug: body.source.slug,
|
||||
...(body.source.version ? { version: body.source.version } : {}),
|
||||
update: body.update === true,
|
||||
},
|
||||
);
|
||||
return json(result, 202, rate.headers);
|
||||
} catch (error) {
|
||||
if (error instanceof SyntaxError) return text("Invalid JSON", 400, rate.headers);
|
||||
return text(
|
||||
error instanceof Error ? error.message : "Skill scan submit failed",
|
||||
400,
|
||||
rate.headers,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export async function skillScanGetRouterV1Handler(ctx: ActionCtx, request: Request) {
|
||||
const rate = await applyRateLimit(ctx, request, "read");
|
||||
if (!rate.ok) return rate.response;
|
||||
const auth = await requireApiTokenUserOrResponse(ctx, request, rate.headers);
|
||||
if (!auth.ok) return auth.response;
|
||||
|
||||
const segments = getPathSegments(request, `${ApiRoutes.skillScans}/`);
|
||||
const scanId = segments[0];
|
||||
if (!scanId) return text("scanId required", 400, rate.headers);
|
||||
|
||||
try {
|
||||
if (segments.length === 2 && scanId === "download") {
|
||||
const name = (segments[1] ?? "").trim();
|
||||
const url = new URL(request.url);
|
||||
const version = url.searchParams.get("version")?.trim() ?? "";
|
||||
const kind = url.searchParams.get("kind")?.trim() === "plugin" ? "plugin" : "skill";
|
||||
if (!name) return text("name required", 400, rate.headers);
|
||||
if (!version) return text("version required", 400, rate.headers);
|
||||
|
||||
const status = (await runQueryRef(
|
||||
ctx,
|
||||
internalRefs.securityScan.getStoredScanReportForUserInternal,
|
||||
{
|
||||
actorUserId: auth.userId,
|
||||
kind,
|
||||
name,
|
||||
version,
|
||||
},
|
||||
)) as Record<string, unknown>;
|
||||
const zip = buildSkillScanReportZip(status);
|
||||
const headers = mergeHeaders(rate.headers, {
|
||||
"Content-Type": "application/zip",
|
||||
"Content-Disposition": `attachment; filename="clawhub-scan-${safeScanReportFilenamePart(name)}-${safeScanReportFilenamePart(version)}.zip"`,
|
||||
});
|
||||
return new Response(zip, { status: 200, headers });
|
||||
}
|
||||
|
||||
const status = (await runQueryRef(
|
||||
ctx,
|
||||
internalRefs.securityScan.getSkillScanRequestForUserInternal,
|
||||
{
|
||||
actorUserId: auth.userId,
|
||||
scanId: scanId as Id<"skillScanRequests">,
|
||||
},
|
||||
)) as Record<string, unknown>;
|
||||
|
||||
if (segments.length === 1) return json(status, 200, rate.headers);
|
||||
|
||||
if (segments.length === 2 && segments[1] === "download") {
|
||||
if (status.status !== "succeeded") return text("Scan is not complete", 409, rate.headers);
|
||||
const zip = buildSkillScanReportZip(status);
|
||||
const headers = mergeHeaders(rate.headers, {
|
||||
"Content-Type": "application/zip",
|
||||
"Content-Disposition": `attachment; filename="clawhub-scan-${scanId}.zip"`,
|
||||
});
|
||||
return new Response(zip, { status: 200, headers });
|
||||
}
|
||||
|
||||
return text("Not found", 404, rate.headers);
|
||||
} catch (error) {
|
||||
return text(error instanceof Error ? error.message : "Skill scan failed", 400, rate.headers);
|
||||
}
|
||||
}
|
||||
|
||||
export async function skillScanBatchSubmitV1Handler(ctx: ActionCtx, request: Request) {
|
||||
const rate = await applyRateLimit(ctx, request, "write");
|
||||
if (!rate.ok) return rate.response;
|
||||
return handleSkillScanBatchSubmit(ctx, request, rate.headers);
|
||||
}
|
||||
|
||||
export async function skillScanBatchStatusV1Handler(ctx: ActionCtx, request: Request) {
|
||||
const rate = await applyRateLimit(ctx, request, "write");
|
||||
if (!rate.ok) return rate.response;
|
||||
return handleSkillScanBatchStatus(ctx, request, rate.headers);
|
||||
}
|
||||
|
||||
export async function searchSkillsV1Handler(ctx: ActionCtx, request: Request) {
|
||||
const rate = await applyRateLimit(ctx, request, "read");
|
||||
if (!rate.ok) return rate.response;
|
||||
@@ -1308,7 +1034,6 @@ export async function resolveSkillVersionV1Handler(ctx: ActionCtx, request: Requ
|
||||
}
|
||||
|
||||
type SkillListSort =
|
||||
| "recommended"
|
||||
| "createdAt"
|
||||
| "updated"
|
||||
| "downloads"
|
||||
@@ -1317,14 +1042,11 @@ type SkillListSort =
|
||||
| "installsAllTime"
|
||||
| "trending";
|
||||
|
||||
type PublicListSort = "recommended" | "newest" | "updated" | "downloads" | "stars" | "installs";
|
||||
type PublicListSort = "newest" | "updated" | "downloads" | "stars" | "installs";
|
||||
|
||||
function parseListSort(value: string | null): SkillListSort | null {
|
||||
if (value === null) return "updated";
|
||||
const normalized = value.trim().toLowerCase();
|
||||
if (normalized === "default" || normalized === "recommended") {
|
||||
return "recommended";
|
||||
}
|
||||
const normalized = value?.trim().toLowerCase();
|
||||
if (normalized === "createdat" || normalized === "created-at" || normalized === "newest") {
|
||||
return "createdAt";
|
||||
}
|
||||
@@ -1347,7 +1069,6 @@ function parseListSort(value: string | null): SkillListSort | null {
|
||||
}
|
||||
|
||||
function toPublicListSort(sort: Exclude<SkillListSort, "trending">): PublicListSort {
|
||||
if (sort === "recommended") return "recommended";
|
||||
if (sort === "createdAt") return "newest";
|
||||
if (sort === "updated") return "updated";
|
||||
if (sort === "downloads" || sort === "stars") return sort;
|
||||
@@ -1478,80 +1199,6 @@ async function describeOwnerVisibleSkillState(
|
||||
return null;
|
||||
}
|
||||
|
||||
function shouldExposeHiddenGitHubInstallBlock(
|
||||
skill: InstallResolverSkill & {
|
||||
installKind?: "github";
|
||||
moderationStatus?: "active" | "hidden" | "removed";
|
||||
moderationReason?: string;
|
||||
},
|
||||
resolution: SkillInstallResolution,
|
||||
) {
|
||||
if (skill.installKind !== "github" || resolution.ok) return false;
|
||||
if (skill.moderationStatus !== "hidden") return false;
|
||||
const reason = skill.moderationReason ?? "";
|
||||
return (
|
||||
reason === "pending.scan" ||
|
||||
reason === "scanner.failed" ||
|
||||
reason === "scanner.llm.malicious" ||
|
||||
reason.startsWith("github.")
|
||||
);
|
||||
}
|
||||
|
||||
type ExactVersionModeratedSkill = Pick<
|
||||
Doc<"skills">,
|
||||
| "_id"
|
||||
| "softDeletedAt"
|
||||
| "latestVersionId"
|
||||
| "tags"
|
||||
| "moderationStatus"
|
||||
| "moderationReason"
|
||||
| "moderationFlags"
|
||||
| "moderationVerdict"
|
||||
| "moderationSourceVersionId"
|
||||
>;
|
||||
|
||||
async function getUnavailableSkillVersionBlock(
|
||||
ctx: ActionCtx,
|
||||
slug: string,
|
||||
selector?: { versionName?: string; tagName?: string },
|
||||
) {
|
||||
const skill = await runQueryRef<ExactVersionModeratedSkill | null>(
|
||||
ctx,
|
||||
internalRefs.skills.getSkillBySlugInternal,
|
||||
{ slug },
|
||||
);
|
||||
if (!skill || skill.softDeletedAt) return null;
|
||||
|
||||
const latestVersionId = skill.latestVersionId ?? skill.tags?.latest;
|
||||
const selectedVersionId = selector?.tagName ? skill.tags?.[selector.tagName] : latestVersionId;
|
||||
if (!selector?.versionName && !selectedVersionId) return null;
|
||||
|
||||
const version = selector?.versionName
|
||||
? await runQueryRef<PublicSkillVersionResponse | null>(
|
||||
ctx,
|
||||
internalRefs.skills.getVersionBySkillAndVersionInternal,
|
||||
{
|
||||
skillId: skill._id,
|
||||
version: selector.versionName,
|
||||
},
|
||||
)
|
||||
: await runQueryRef<PublicSkillVersionResponse | null>(
|
||||
ctx,
|
||||
internalRefs.skills.getVersionByIdInternal,
|
||||
{
|
||||
versionId: selectedVersionId,
|
||||
},
|
||||
);
|
||||
if (!version || !isSkillVersionForSkill(version, skill._id)) return null;
|
||||
if (version.softDeletedAt) return { status: 410, message: "Version not available" };
|
||||
|
||||
return getPublicSkillVersionAccessBlock(
|
||||
getSkillFileModerationInfoFromSkill(skill),
|
||||
version._id,
|
||||
skill.latestVersionId ?? skill.tags?.latest,
|
||||
);
|
||||
}
|
||||
|
||||
export async function skillsGetRouterV1Handler(ctx: ActionCtx, request: Request) {
|
||||
const rate = await applyRateLimit(ctx, request, "read");
|
||||
if (!rate.ok) return rate.response;
|
||||
@@ -1616,60 +1263,6 @@ export async function skillsGetRouterV1Handler(ctx: ActionCtx, request: Request)
|
||||
return json(result, 200, rate.headers);
|
||||
}
|
||||
|
||||
if (second === "install" && segments.length === 2) {
|
||||
const url = new URL(request.url);
|
||||
const forceInstall = parseBooleanQueryParam(url.searchParams.get("forceInstall"));
|
||||
const skill = (await runQueryRef<
|
||||
| (InstallResolverSkill & {
|
||||
_id: Id<"skills">;
|
||||
githubSourceId?: Id<"githubSkillSources">;
|
||||
softDeletedAt?: number;
|
||||
moderationStatus?: "active" | "hidden" | "removed";
|
||||
moderationReason?: string;
|
||||
moderationFlags?: string[];
|
||||
})
|
||||
| null
|
||||
>(ctx, internalRefs.skills.getSkillBySlugInternal, { slug })) as
|
||||
| (InstallResolverSkill & {
|
||||
_id: Id<"skills">;
|
||||
githubSourceId?: Id<"githubSkillSources">;
|
||||
softDeletedAt?: number;
|
||||
moderationStatus?: "active" | "hidden" | "removed";
|
||||
moderationReason?: string;
|
||||
moderationFlags?: string[];
|
||||
})
|
||||
| null;
|
||||
if (!skill || skill.softDeletedAt || skill.moderationStatus === "removed") {
|
||||
return text("Skill not found", 404, rate.headers);
|
||||
}
|
||||
|
||||
const source =
|
||||
skill.installKind === "github" && skill.githubSourceId
|
||||
? ((await runQueryRef(ctx, internalRefs.githubSkillSources.getByIdInternal, {
|
||||
sourceId: skill.githubSourceId,
|
||||
})) as InstallResolverSource | null)
|
||||
: null;
|
||||
const resolution = buildSkillInstallResolution({
|
||||
origin: publicApiOrigin(request),
|
||||
skill,
|
||||
source,
|
||||
forceInstall,
|
||||
});
|
||||
|
||||
const publicSkillResult = (await ctx.runQuery(api.skills.getBySlug, {
|
||||
slug,
|
||||
})) as GetBySlugResult;
|
||||
const publiclyVisible = publicSkillResult?.skill?._id === skill._id;
|
||||
if (!publiclyVisible) {
|
||||
if (!resolution.ok && shouldExposeHiddenGitHubInstallBlock(skill, resolution)) {
|
||||
return json(resolution, resolution.status, rate.headers);
|
||||
}
|
||||
return text("Skill not found", 404, rate.headers);
|
||||
}
|
||||
|
||||
return json(resolution, resolution.ok ? 200 : resolution.status, rate.headers);
|
||||
}
|
||||
|
||||
if (segments.length === 1) {
|
||||
const result = (await ctx.runQuery(api.skills.getBySlug, { slug })) as GetBySlugResult;
|
||||
if (!result?.skill) {
|
||||
@@ -1844,15 +1437,7 @@ export async function skillsGetRouterV1Handler(ctx: ActionCtx, request: Request)
|
||||
|
||||
if (second === "versions" && third && segments.length === 3) {
|
||||
const skillResult = (await ctx.runQuery(api.skills.getBySlug, { slug })) as GetBySlugResult;
|
||||
if (!skillResult?.skill) {
|
||||
const moderationBlock = await getUnavailableSkillVersionBlock(ctx, slug, {
|
||||
versionName: third,
|
||||
});
|
||||
if (moderationBlock) {
|
||||
return text(moderationBlock.message, moderationBlock.status, rate.headers);
|
||||
}
|
||||
return text("Skill not found", 404, rate.headers);
|
||||
}
|
||||
if (!skillResult?.skill) return text("Skill not found", 404, rate.headers);
|
||||
|
||||
const version = (await ctx.runQuery(api.skills.getVersionBySkillAndVersion, {
|
||||
skillId: skillResult.skill._id,
|
||||
@@ -1860,16 +1445,6 @@ export async function skillsGetRouterV1Handler(ctx: ActionCtx, request: Request)
|
||||
})) as PublicSkillVersionResponse | null;
|
||||
if (!version) return text("Version not found", 404, rate.headers);
|
||||
if (version.softDeletedAt) return text("Version not available", 410, rate.headers);
|
||||
const effectiveLatestVersionId =
|
||||
skillResult.skill.latestVersionId ?? skillResult.skill.tags?.latest;
|
||||
const moderationBlock = getPublicSkillVersionAccessBlock(
|
||||
skillResult.moderationInfo,
|
||||
version._id,
|
||||
effectiveLatestVersionId,
|
||||
);
|
||||
if (moderationBlock) {
|
||||
return text(moderationBlock.message, moderationBlock.status, rate.headers);
|
||||
}
|
||||
const security = buildSkillSecuritySnapshot(version);
|
||||
|
||||
return json(
|
||||
@@ -1902,13 +1477,6 @@ export async function skillsGetRouterV1Handler(ctx: ActionCtx, request: Request)
|
||||
|
||||
const result = (await ctx.runQuery(api.skills.getBySlug, { slug })) as GetBySlugResult;
|
||||
if (!result?.skill) {
|
||||
const moderationBlock = await getUnavailableSkillVersionBlock(ctx, slug, {
|
||||
versionName: versionParam,
|
||||
tagName: tagParam,
|
||||
});
|
||||
if (moderationBlock) {
|
||||
return text(moderationBlock.message, moderationBlock.status, rate.headers);
|
||||
}
|
||||
const hidden = await describeOwnerVisibleSkillState(ctx, request, slug);
|
||||
if (hidden) return text(hidden.message, hidden.status, rate.headers);
|
||||
return text("Skill not found", 404, rate.headers);
|
||||
@@ -1934,33 +1502,9 @@ export async function skillsGetRouterV1Handler(ctx: ActionCtx, request: Request)
|
||||
}
|
||||
if (version.softDeletedAt) return text("Version not available", 410, rate.headers);
|
||||
|
||||
const effectiveLatestVersionId = result.skill.latestVersionId ?? result.skill.tags?.latest;
|
||||
const moderationBlock = getPublicSkillVersionAccessBlock(
|
||||
result.moderationInfo,
|
||||
version._id,
|
||||
effectiveLatestVersionId,
|
||||
);
|
||||
if (moderationBlock) {
|
||||
return text(moderationBlock.message, moderationBlock.status, rate.headers);
|
||||
}
|
||||
|
||||
let moderationSourceVersion: PublicSkillVersionResponse | null = result.latestVersion;
|
||||
const moderationSourceVersionId = result.moderationInfo?.sourceVersionId;
|
||||
if (moderationSourceVersionId) {
|
||||
if (version._id === moderationSourceVersionId) {
|
||||
moderationSourceVersion = version;
|
||||
} else if (result.latestVersion?._id !== moderationSourceVersionId) {
|
||||
const sourceVersion = (await ctx.runQuery(api.skills.getVersionById, {
|
||||
versionId: moderationSourceVersionId,
|
||||
})) as PublicSkillVersionResponse | null;
|
||||
moderationSourceVersion = isSkillVersionForSkill(sourceVersion, result.skill._id)
|
||||
? sourceVersion
|
||||
: null;
|
||||
}
|
||||
}
|
||||
const security = buildSkillSecuritySnapshot(version);
|
||||
const moderationMatchesRequestedVersion = Boolean(
|
||||
moderationSourceVersion && moderationSourceVersion._id === version._id,
|
||||
result.latestVersion && result.latestVersion._id === version._id,
|
||||
);
|
||||
|
||||
return json(
|
||||
@@ -1977,10 +1521,10 @@ export async function skillsGetRouterV1Handler(ctx: ActionCtx, request: Request)
|
||||
moderation: result.moderationInfo
|
||||
? {
|
||||
scope: "skill",
|
||||
sourceVersion: moderationSourceVersion
|
||||
sourceVersion: result.latestVersion
|
||||
? {
|
||||
version: moderationSourceVersion.version,
|
||||
createdAt: moderationSourceVersion.createdAt,
|
||||
version: result.latestVersion.version,
|
||||
createdAt: result.latestVersion.createdAt,
|
||||
}
|
||||
: null,
|
||||
matchesRequestedVersion: moderationMatchesRequestedVersion,
|
||||
@@ -2004,11 +1548,7 @@ export async function skillsGetRouterV1Handler(ctx: ActionCtx, request: Request)
|
||||
const tagParam = url.searchParams.get("tag")?.trim();
|
||||
if (versionParam && tagParam) return text("Use either version or tag", 400, rate.headers);
|
||||
|
||||
const skillResult = (await runQueryRef<GetBySlugResult>(
|
||||
ctx,
|
||||
internalRefs.skills.getVerifyTargetBySlugInternal,
|
||||
{ slug },
|
||||
)) as GetBySlugResult;
|
||||
const skillResult = (await ctx.runQuery(api.skills.getBySlug, { slug })) as GetBySlugResult;
|
||||
if (!skillResult?.skill) {
|
||||
const hidden = await describeOwnerVisibleSkillState(ctx, request, slug);
|
||||
if (hidden) return text(hidden.message, hidden.status, rate.headers);
|
||||
@@ -2047,14 +1587,11 @@ export async function skillsGetRouterV1Handler(ctx: ActionCtx, request: Request)
|
||||
const bundleFingerprints = fingerprintEntries
|
||||
.filter((entry) => entry.kind === "generated-bundle")
|
||||
.map((entry) => entry.fingerprint);
|
||||
const isMalwareBlocked = skillResult.moderationInfo?.isMalwareBlocked ?? false;
|
||||
const generatedCardFile = isMalwareBlocked
|
||||
? null
|
||||
: await selectGeneratedSkillCardFile(version.files, bundleFingerprints);
|
||||
const generatedCardFile = await selectGeneratedSkillCardFile(version.files, bundleFingerprints);
|
||||
const security = buildVerifySecurity(version);
|
||||
const reasons = buildVerifyReasons({
|
||||
cardAvailable: Boolean(generatedCardFile),
|
||||
isMalwareBlocked,
|
||||
isMalwareBlocked: skillResult.moderationInfo?.isMalwareBlocked ?? false,
|
||||
securityPassed: security.passed,
|
||||
securityStatus: security.status,
|
||||
});
|
||||
@@ -2091,9 +1628,7 @@ export async function skillsGetRouterV1Handler(ctx: ActionCtx, request: Request)
|
||||
: {
|
||||
available: false,
|
||||
path: "skill-card.md",
|
||||
url: isMalwareBlocked
|
||||
? null
|
||||
: buildCardUrl(request, skillResult.skill.slug, version.version),
|
||||
url: buildCardUrl(request, skillResult.skill.slug, version.version),
|
||||
sha256: null,
|
||||
size: null,
|
||||
contentType: null,
|
||||
|
||||
@@ -14,17 +14,11 @@ import {
|
||||
|
||||
const usersV1InternalRefs = internal as unknown as {
|
||||
publishers: {
|
||||
addOfficialPublisherInternal: unknown;
|
||||
deleteEmptyOrgPublisherInternal: unknown;
|
||||
listOfficialPublishersInternal: unknown;
|
||||
removeOrgPublisherMemberInternal: unknown;
|
||||
removeOfficialPublisherInternal: unknown;
|
||||
};
|
||||
users: {
|
||||
getBanAppealContextByGitHubProviderAccountIdInternal: unknown;
|
||||
getByHandleInternal: unknown;
|
||||
recordStaffEmailAttemptAuditInternal: unknown;
|
||||
recordStaffEmailSentAuditInternal: unknown;
|
||||
remediateAutobansInternal: unknown;
|
||||
reclassifyBanInternal: unknown;
|
||||
unbanUserForBanAppealServiceInternal: unknown;
|
||||
@@ -89,10 +83,7 @@ export async function usersPostRouterV1Handler(ctx: ActionCtx, request: Request)
|
||||
action !== "ban-appeal-unban" &&
|
||||
action !== "reclaim" &&
|
||||
action !== "reserve" &&
|
||||
action !== "email" &&
|
||||
action !== "publisher" &&
|
||||
action !== "publisher-delete" &&
|
||||
action !== "publisher-official" &&
|
||||
action !== "publisher-member"
|
||||
) {
|
||||
return text("Not found", 404, rate.headers);
|
||||
@@ -142,30 +133,12 @@ export async function usersPostRouterV1Handler(ctx: ActionCtx, request: Request)
|
||||
return handleAdminReserve(ctx, payload, actorUserId, rate.headers);
|
||||
}
|
||||
|
||||
if (action === "email") {
|
||||
const admin = requireAdminOrResponse(actorUser, rate.headers);
|
||||
if (!admin.ok) return admin.response;
|
||||
return handleAdminStaffEmail(ctx, payload, actorUserId, rate.headers);
|
||||
}
|
||||
|
||||
if (action === "publisher") {
|
||||
const admin = requireAdminOrResponse(actorUser, rate.headers);
|
||||
if (!admin.ok) return admin.response;
|
||||
return handleAdminEnsurePublisher(ctx, payload, actorUserId, rate.headers);
|
||||
}
|
||||
|
||||
if (action === "publisher-delete") {
|
||||
const admin = requireAdminOrResponse(actorUser, rate.headers);
|
||||
if (!admin.ok) return admin.response;
|
||||
return handleAdminDeletePublisher(ctx, payload, actorUserId, rate.headers);
|
||||
}
|
||||
|
||||
if (action === "publisher-official") {
|
||||
const admin = requireAdminOrResponse(actorUser, rate.headers);
|
||||
if (!admin.ok) return admin.response;
|
||||
return handleAdminOfficialPublisherPost(ctx, payload, actorUserId, rate.headers);
|
||||
}
|
||||
|
||||
if (action === "publisher-member") {
|
||||
const admin = requireAdminOrResponse(actorUser, rate.headers);
|
||||
if (!admin.ok) return admin.response;
|
||||
@@ -268,159 +241,6 @@ export async function usersPostRouterV1Handler(ctx: ActionCtx, request: Request)
|
||||
}
|
||||
}
|
||||
|
||||
async function handleAdminDeletePublisher(
|
||||
ctx: ActionCtx,
|
||||
payload: Record<string, unknown>,
|
||||
actorUserId: Id<"users">,
|
||||
headers: HeadersInit,
|
||||
) {
|
||||
const handle = typeof payload.handle === "string" ? payload.handle.trim().toLowerCase() : "";
|
||||
const reason = typeof payload.reason === "string" ? payload.reason.trim() : "";
|
||||
const dryRun = payload.dryRun !== false;
|
||||
if (!handle) return text("Missing handle", 400, headers);
|
||||
if (!reason) return text("Missing reason", 400, headers);
|
||||
if (reason.length > 500) return text("Reason too long (max 500 chars)", 400, headers);
|
||||
|
||||
try {
|
||||
const result = await runUsersV1MutationRef(
|
||||
ctx,
|
||||
usersV1InternalRefs.publishers.deleteEmptyOrgPublisherInternal,
|
||||
{
|
||||
actorUserId,
|
||||
handle,
|
||||
reason,
|
||||
dryRun,
|
||||
},
|
||||
);
|
||||
return json(result, 200, headers);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "Publisher delete failed";
|
||||
if (message.toLowerCase().includes("forbidden")) {
|
||||
return text("Forbidden", 403, headers);
|
||||
}
|
||||
if (message.toLowerCase().includes("not found")) {
|
||||
return text(message, 404, headers);
|
||||
}
|
||||
return text(message, 400, headers);
|
||||
}
|
||||
}
|
||||
|
||||
async function handleAdminStaffEmail(
|
||||
ctx: ActionCtx,
|
||||
payload: Record<string, unknown>,
|
||||
actorUserId: Id<"users">,
|
||||
headers: HeadersInit,
|
||||
) {
|
||||
const toEmail = typeof payload.toEmail === "string" ? payload.toEmail.trim().toLowerCase() : "";
|
||||
const userHandle =
|
||||
typeof payload.userHandle === "string"
|
||||
? payload.userHandle.trim().replace(/^@+/, "").toLowerCase()
|
||||
: "";
|
||||
const subject = typeof payload.subject === "string" ? payload.subject.trim() : "";
|
||||
const body = typeof payload.body === "string" ? payload.body.trim() : "";
|
||||
const confirmUserRequest = payload.confirmUserRequest === true;
|
||||
const confirmUserSignoff = payload.confirmUserSignoff === true;
|
||||
|
||||
if (toEmail && userHandle) return text("Pass toEmail or userHandle, not both", 400, headers);
|
||||
if (!toEmail && !userHandle) return text("Missing toEmail or userHandle", 400, headers);
|
||||
if (toEmail && !/^[^@\s]+@[^@\s]+\.[^@\s]+$/.test(toEmail)) {
|
||||
return text("Invalid toEmail", 400, headers);
|
||||
}
|
||||
if (!subject) return text("Missing subject", 400, headers);
|
||||
if (subject.length > 200) return text("Subject too long (max 200 chars)", 400, headers);
|
||||
if (!body) return text("Missing body", 400, headers);
|
||||
if (body.length > 20_000) return text("Body too long (max 20000 chars)", 400, headers);
|
||||
if (!confirmUserRequest || !confirmUserSignoff) {
|
||||
return text(
|
||||
"Staff email requires explicit user request and user sign-off on final recipient, subject, and body.",
|
||||
400,
|
||||
headers,
|
||||
);
|
||||
}
|
||||
|
||||
let recipientEmail = toEmail;
|
||||
let recipientUserId: Id<"users"> | undefined;
|
||||
let recipientHandle: string | null | undefined;
|
||||
if (userHandle) {
|
||||
const user = await runUsersV1QueryRef<{
|
||||
_id?: Id<"users">;
|
||||
handle?: string | null;
|
||||
email?: string | null;
|
||||
} | null>(ctx, usersV1InternalRefs.users.getByHandleInternal, { handle: userHandle });
|
||||
if (!user?._id) return text("User not found", 404, headers);
|
||||
if (!user.email?.trim()) return text("User has no email address", 400, headers);
|
||||
recipientEmail = user.email.trim().toLowerCase();
|
||||
recipientUserId = user._id;
|
||||
recipientHandle = user.handle ?? userHandle;
|
||||
}
|
||||
|
||||
const apiKey = process.env.RESEND_API_KEY?.trim();
|
||||
if (!apiKey) return text("RESEND_API_KEY is not configured", 500, headers);
|
||||
const from =
|
||||
process.env.CLAWHUB_NOREPLY_FROM?.trim() ||
|
||||
process.env.NOREPLY_EMAIL_FROM?.trim() ||
|
||||
"ClawHub <noreply@clawhub.ai>";
|
||||
|
||||
const emailAudit = await runUsersV1MutationRef<{ auditLogId: Id<"auditLogs"> }>(
|
||||
ctx,
|
||||
usersV1InternalRefs.users.recordStaffEmailAttemptAuditInternal,
|
||||
{
|
||||
actorUserId,
|
||||
toEmail: recipientEmail,
|
||||
...(recipientUserId ? { recipientUserId } : {}),
|
||||
...(recipientHandle ? { recipientHandle } : {}),
|
||||
subject,
|
||||
},
|
||||
);
|
||||
|
||||
const response = await fetch("https://api.resend.com/emails", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
from,
|
||||
to: [recipientEmail],
|
||||
subject,
|
||||
text: body,
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => "");
|
||||
return text(
|
||||
`Email provider failed (${response.status})${errorText ? `: ${errorText}` : ""}`,
|
||||
502,
|
||||
headers,
|
||||
);
|
||||
}
|
||||
const providerJson = (await response.json().catch(() => null)) as { id?: unknown } | null;
|
||||
const providerId = typeof providerJson?.id === "string" ? providerJson.id : null;
|
||||
|
||||
await runUsersV1MutationRef(ctx, usersV1InternalRefs.users.recordStaffEmailSentAuditInternal, {
|
||||
actorUserId,
|
||||
auditLogId: emailAudit.auditLogId,
|
||||
providerId,
|
||||
});
|
||||
|
||||
return json(
|
||||
{
|
||||
ok: true,
|
||||
sent: true,
|
||||
recipient: {
|
||||
email: recipientEmail,
|
||||
...(recipientUserId ? { userId: recipientUserId } : {}),
|
||||
...(recipientHandle ? { handle: recipientHandle } : {}),
|
||||
},
|
||||
subject,
|
||||
providerId,
|
||||
},
|
||||
200,
|
||||
headers,
|
||||
);
|
||||
}
|
||||
|
||||
async function handleAdminReclassifyBan(
|
||||
ctx: ActionCtx,
|
||||
payload: unknown,
|
||||
@@ -532,37 +352,6 @@ async function handleAdminRemediateAutobans(
|
||||
}
|
||||
}
|
||||
|
||||
export async function usersGetRouterV1Handler(ctx: ActionCtx, request: Request) {
|
||||
const rate = await applyRateLimit(ctx, request, "read");
|
||||
if (!rate.ok) return rate.response;
|
||||
|
||||
const segments = getPathSegments(request, "/api/v1/users/");
|
||||
if (segments.length !== 1 || segments[0] !== "publisher-official") {
|
||||
return text("Not found", 404, rate.headers);
|
||||
}
|
||||
|
||||
const authResult = await requireApiTokenUserOrResponse(ctx, request, rate.headers);
|
||||
if (!authResult.ok) return authResult.response;
|
||||
const admin = requireAdminOrResponse(authResult.user, rate.headers);
|
||||
if (!admin.ok) return admin.response;
|
||||
|
||||
try {
|
||||
const result = await runUsersV1QueryRef(
|
||||
ctx,
|
||||
usersV1InternalRefs.publishers.listOfficialPublishersInternal,
|
||||
{ actorUserId: authResult.userId },
|
||||
);
|
||||
return json(result, 200, rate.headers);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "Official publisher list failed";
|
||||
if (message.toLowerCase().includes("forbidden")) return text("Forbidden", 403, rate.headers);
|
||||
if (message.toLowerCase().includes("unauthorized")) {
|
||||
return text("Unauthorized", 401, rate.headers);
|
||||
}
|
||||
return text(message, 400, rate.headers);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /api/v1/users/restore
|
||||
* Admin-only: restore skills from GitHub backup for a user.
|
||||
@@ -743,42 +532,6 @@ async function handleAdminReserve(
|
||||
return json({ ok: true, results, succeeded, failed }, 200, headers);
|
||||
}
|
||||
|
||||
async function handleAdminOfficialPublisherPost(
|
||||
ctx: ActionCtx,
|
||||
payload: Record<string, unknown>,
|
||||
actorUserId: Id<"users">,
|
||||
headers: HeadersInit,
|
||||
) {
|
||||
const action = typeof payload.action === "string" ? payload.action.trim().toLowerCase() : "";
|
||||
const handle = typeof payload.handle === "string" ? payload.handle.trim().toLowerCase() : "";
|
||||
const reason = typeof payload.reason === "string" ? payload.reason.trim() : "";
|
||||
if (action !== "add" && action !== "remove") return text("Invalid action", 400, headers);
|
||||
if (!handle) return text("Missing handle", 400, headers);
|
||||
if (!reason) return text("Missing reason", 400, headers);
|
||||
if (reason.length > 500) return text("Reason too long (max 500 chars)", 400, headers);
|
||||
|
||||
try {
|
||||
const result = await runUsersV1MutationRef(
|
||||
ctx,
|
||||
action === "add"
|
||||
? usersV1InternalRefs.publishers.addOfficialPublisherInternal
|
||||
: usersV1InternalRefs.publishers.removeOfficialPublisherInternal,
|
||||
{
|
||||
actorUserId,
|
||||
handle,
|
||||
reason,
|
||||
},
|
||||
);
|
||||
return json(result, 200, headers);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "Official publisher update failed";
|
||||
if (message.toLowerCase().includes("forbidden")) return text("Forbidden", 403, headers);
|
||||
if (message.toLowerCase().includes("unauthorized")) return text("Unauthorized", 401, headers);
|
||||
if (message.toLowerCase().includes("not found")) return text(message, 404, headers);
|
||||
return text(message, 400, headers);
|
||||
}
|
||||
}
|
||||
|
||||
async function handleAdminEnsurePublisher(
|
||||
ctx: ActionCtx,
|
||||
payload: Record<string, unknown>,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("@convex-dev/auth/server", () => ({
|
||||
getAuthUserId: vi.fn(),
|
||||
@@ -9,10 +9,6 @@ const { getAuthUserId } = await import("@convex-dev/auth/server");
|
||||
const { assertAdmin, assertModerator, assertRole, requireUser, requireUserFromAction } =
|
||||
await import("./access");
|
||||
|
||||
beforeEach(() => {
|
||||
vi.mocked(getAuthUserId).mockReset();
|
||||
});
|
||||
|
||||
describe("access.requireUser", () => {
|
||||
it("throws when auth is missing", async () => {
|
||||
vi.mocked(getAuthUserId).mockResolvedValue(null);
|
||||
@@ -63,67 +59,6 @@ describe("access.requireUser", () => {
|
||||
expect(dbGet).toHaveBeenCalledWith("users:2");
|
||||
expect(result).toEqual({ userId: "users:2", user });
|
||||
});
|
||||
|
||||
it("uses the local dev impersonation user before browser auth", async () => {
|
||||
const previousHandle = process.env.CLAW_HUB_DEV_IMPERSONATE_USER_HANDLE;
|
||||
const previousEnabled = process.env.CLAW_HUB_ENABLE_DEV_IMPERSONATION;
|
||||
try {
|
||||
process.env.CLAW_HUB_DEV_IMPERSONATE_USER_HANDLE = "local";
|
||||
process.env.CLAW_HUB_ENABLE_DEV_IMPERSONATION = "1";
|
||||
vi.mocked(getAuthUserId).mockResolvedValue("users:browser" as never);
|
||||
const user = { _id: "users:local", handle: "local", role: "admin" };
|
||||
const unique = vi.fn().mockResolvedValue(user as never);
|
||||
const withIndex = vi.fn().mockReturnValue({ unique });
|
||||
const query = vi.fn().mockReturnValue({ withIndex });
|
||||
const dbGet = vi.fn().mockResolvedValue(user as never);
|
||||
|
||||
const result = await requireUser({
|
||||
db: { get: dbGet, query },
|
||||
} as never);
|
||||
|
||||
expect(query).toHaveBeenCalledWith("users");
|
||||
expect(withIndex).toHaveBeenCalledWith("handle", expect.any(Function));
|
||||
expect(dbGet).toHaveBeenCalledWith("users:local");
|
||||
expect(getAuthUserId).not.toHaveBeenCalled();
|
||||
expect(result).toEqual({ userId: "users:local", user });
|
||||
} finally {
|
||||
if (previousHandle === undefined) delete process.env.CLAW_HUB_DEV_IMPERSONATE_USER_HANDLE;
|
||||
else process.env.CLAW_HUB_DEV_IMPERSONATE_USER_HANDLE = previousHandle;
|
||||
if (previousEnabled === undefined) delete process.env.CLAW_HUB_ENABLE_DEV_IMPERSONATION;
|
||||
else process.env.CLAW_HUB_ENABLE_DEV_IMPERSONATION = previousEnabled;
|
||||
}
|
||||
});
|
||||
|
||||
it("does not use local dev impersonation in production deployments", async () => {
|
||||
const previousHandle = process.env.CLAW_HUB_DEV_IMPERSONATE_USER_HANDLE;
|
||||
const previousEnabled = process.env.CLAW_HUB_ENABLE_DEV_IMPERSONATION;
|
||||
const previousDeployment = process.env.CONVEX_DEPLOYMENT;
|
||||
try {
|
||||
process.env.CLAW_HUB_DEV_IMPERSONATE_USER_HANDLE = "local";
|
||||
process.env.CLAW_HUB_ENABLE_DEV_IMPERSONATION = "1";
|
||||
process.env.CONVEX_DEPLOYMENT = "prod:wry-manatee-359";
|
||||
vi.mocked(getAuthUserId).mockResolvedValue("users:browser" as never);
|
||||
const user = { _id: "users:browser", handle: "browser", role: "user" };
|
||||
const query = vi.fn();
|
||||
const dbGet = vi.fn().mockResolvedValue(user as never);
|
||||
|
||||
const result = await requireUser({
|
||||
db: { get: dbGet, query },
|
||||
} as never);
|
||||
|
||||
expect(query).not.toHaveBeenCalled();
|
||||
expect(getAuthUserId).toHaveBeenCalled();
|
||||
expect(dbGet).toHaveBeenCalledWith("users:browser");
|
||||
expect(result).toEqual({ userId: "users:browser", user });
|
||||
} finally {
|
||||
if (previousHandle === undefined) delete process.env.CLAW_HUB_DEV_IMPERSONATE_USER_HANDLE;
|
||||
else process.env.CLAW_HUB_DEV_IMPERSONATE_USER_HANDLE = previousHandle;
|
||||
if (previousEnabled === undefined) delete process.env.CLAW_HUB_ENABLE_DEV_IMPERSONATION;
|
||||
else process.env.CLAW_HUB_ENABLE_DEV_IMPERSONATION = previousEnabled;
|
||||
if (previousDeployment === undefined) delete process.env.CONVEX_DEPLOYMENT;
|
||||
else process.env.CONVEX_DEPLOYMENT = previousDeployment;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("access.requireUserFromAction", () => {
|
||||
@@ -176,37 +111,6 @@ describe("access.requireUserFromAction", () => {
|
||||
expect(runQuery).toHaveBeenCalledTimes(1);
|
||||
expect(result).toEqual({ userId: "users:9", user });
|
||||
});
|
||||
|
||||
it("uses the local dev impersonation user before action auth", async () => {
|
||||
const previousHandle = process.env.CLAW_HUB_DEV_IMPERSONATE_USER_HANDLE;
|
||||
const previousEnabled = process.env.CLAW_HUB_ENABLE_DEV_IMPERSONATION;
|
||||
try {
|
||||
process.env.CLAW_HUB_DEV_IMPERSONATE_USER_HANDLE = "local";
|
||||
process.env.CLAW_HUB_ENABLE_DEV_IMPERSONATION = "1";
|
||||
vi.mocked(getAuthUserId).mockResolvedValue("users:browser" as never);
|
||||
const user = { _id: "users:local", handle: "local", role: "admin" };
|
||||
const runQuery = vi.fn(async (_query, args: { handle?: string; userId?: string }) => {
|
||||
if (args.handle === "local") return user;
|
||||
if (args.userId === "users:local") return user;
|
||||
return null;
|
||||
});
|
||||
|
||||
const result = await requireUserFromAction({ runQuery } as never);
|
||||
|
||||
expect(runQuery).toHaveBeenCalledTimes(2);
|
||||
expect(runQuery).toHaveBeenNthCalledWith(1, expect.anything(), { handle: "local" });
|
||||
expect(runQuery).toHaveBeenNthCalledWith(2, expect.anything(), {
|
||||
userId: "users:local",
|
||||
});
|
||||
expect(getAuthUserId).not.toHaveBeenCalled();
|
||||
expect(result).toEqual({ userId: "users:local", user });
|
||||
} finally {
|
||||
if (previousHandle === undefined) delete process.env.CLAW_HUB_DEV_IMPERSONATE_USER_HANDLE;
|
||||
else process.env.CLAW_HUB_DEV_IMPERSONATE_USER_HANDLE = previousHandle;
|
||||
if (previousEnabled === undefined) delete process.env.CLAW_HUB_ENABLE_DEV_IMPERSONATION;
|
||||
else process.env.CLAW_HUB_ENABLE_DEV_IMPERSONATION = previousEnabled;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("access role assertions", () => {
|
||||
|
||||
+7
-36
@@ -8,23 +8,10 @@ export type Role = "admin" | "moderator" | "user" | "mirror";
|
||||
const DEV_IMPERSONATE_LOCAL_HANDLE = "local";
|
||||
|
||||
function readEnv(name: string) {
|
||||
const value = readKnownEnv(name)?.trim();
|
||||
const value = process.env[name]?.trim();
|
||||
return value ? value : undefined;
|
||||
}
|
||||
|
||||
function readKnownEnv(name: string) {
|
||||
if (name === "CLAW_HUB_DEV_IMPERSONATE_USER_HANDLE") {
|
||||
return process.env.CLAW_HUB_DEV_IMPERSONATE_USER_HANDLE;
|
||||
}
|
||||
if (name === "CLAW_HUB_ENABLE_DEV_IMPERSONATION") {
|
||||
return process.env.CLAW_HUB_ENABLE_DEV_IMPERSONATION;
|
||||
}
|
||||
if (name === "CONVEX_DEPLOYMENT") {
|
||||
return process.env.CONVEX_DEPLOYMENT;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function isDevImpersonationAllowed() {
|
||||
const requestedHandle = readEnv("CLAW_HUB_DEV_IMPERSONATE_USER_HANDLE");
|
||||
if (requestedHandle !== DEV_IMPERSONATE_LOCAL_HANDLE) return false;
|
||||
@@ -65,49 +52,39 @@ async function getDevImpersonatedUserIdFromAction(
|
||||
export async function getOptionalActiveAuthUserId(
|
||||
ctx: MutationCtx | QueryCtx,
|
||||
): Promise<Id<"users"> | undefined> {
|
||||
const devUserId = await getDevImpersonatedUserId(ctx);
|
||||
if (devUserId) return devUserId;
|
||||
try {
|
||||
const userId = await getAuthUserId(ctx);
|
||||
if (!userId) return undefined;
|
||||
if (!userId) return await getDevImpersonatedUserId(ctx);
|
||||
const user = await ctx.db.get(userId);
|
||||
if (!user || user.deletedAt || user.deactivatedAt) return undefined;
|
||||
return userId;
|
||||
} catch {
|
||||
return undefined;
|
||||
return await getDevImpersonatedUserId(ctx);
|
||||
}
|
||||
}
|
||||
|
||||
export async function getOptionalActiveAuthUserIdFromAction(
|
||||
ctx: ActionCtx,
|
||||
): Promise<Id<"users"> | undefined> {
|
||||
const devUserId = await getDevImpersonatedUserIdFromAction(ctx);
|
||||
if (devUserId) return devUserId;
|
||||
try {
|
||||
const userId = await getAuthUserId(ctx);
|
||||
if (!userId) return undefined;
|
||||
if (!userId) return await getDevImpersonatedUserIdFromAction(ctx);
|
||||
const user = await ctx.runQuery(internal.users.getByIdInternal, { userId });
|
||||
if (!user || user.deletedAt || user.deactivatedAt) return undefined;
|
||||
return userId;
|
||||
} catch {
|
||||
return undefined;
|
||||
return await getDevImpersonatedUserIdFromAction(ctx);
|
||||
}
|
||||
}
|
||||
|
||||
export async function requireUser(ctx: MutationCtx | QueryCtx) {
|
||||
const devUserId = await getDevImpersonatedUserId(ctx);
|
||||
if (devUserId) {
|
||||
const devUser = await ctx.db.get(devUserId);
|
||||
if (!devUser || devUser.deletedAt || devUser.deactivatedAt) throw new Error("User not found");
|
||||
return { userId: devUserId, user: devUser };
|
||||
}
|
||||
|
||||
let userId: Id<"users"> | null | undefined = null;
|
||||
try {
|
||||
userId = await getAuthUserId(ctx);
|
||||
} catch {
|
||||
userId = null;
|
||||
}
|
||||
userId ??= await getDevImpersonatedUserId(ctx);
|
||||
if (!userId) throw new Error("Unauthorized");
|
||||
let user: Doc<"users"> | null;
|
||||
try {
|
||||
@@ -122,19 +99,13 @@ export async function requireUser(ctx: MutationCtx | QueryCtx) {
|
||||
export async function requireUserFromAction(
|
||||
ctx: ActionCtx,
|
||||
): Promise<{ userId: Id<"users">; user: Doc<"users"> }> {
|
||||
const devUserId = await getDevImpersonatedUserIdFromAction(ctx);
|
||||
if (devUserId) {
|
||||
const devUser = await ctx.runQuery(internal.users.getByIdInternal, { userId: devUserId });
|
||||
if (!devUser || devUser.deletedAt || devUser.deactivatedAt) throw new Error("User not found");
|
||||
return { userId: devUserId, user: devUser as Doc<"users"> };
|
||||
}
|
||||
|
||||
let userId: Id<"users"> | null | undefined = null;
|
||||
try {
|
||||
userId = await getAuthUserId(ctx);
|
||||
} catch {
|
||||
userId = null;
|
||||
}
|
||||
userId ??= await getDevImpersonatedUserIdFromAction(ctx);
|
||||
if (!userId) throw new Error("Unauthorized");
|
||||
let user: Doc<"users"> | null;
|
||||
try {
|
||||
|
||||
@@ -34,7 +34,7 @@ export const MISSING_API_TOKEN_MESSAGE =
|
||||
export const INVALID_API_TOKEN_MESSAGE =
|
||||
"Unauthorized: API token is invalid or revoked. Run `clawhub login` again.";
|
||||
export const BLOCKED_API_TOKEN_ACCOUNT_MESSAGE =
|
||||
"Unauthorized: This ClawHub account is not in good standing and cannot use API tokens. If you believe this is a mistake, open a GitHub issue: https://github.com/openclaw/clawhub/issues/new.";
|
||||
"Unauthorized: This ClawHub account is not in good standing and cannot use API tokens. If you believe this is a mistake, contact security@openclaw.ai.";
|
||||
|
||||
export async function requireApiTokenUser(
|
||||
ctx: ActionCtx,
|
||||
|
||||
@@ -37,9 +37,9 @@ function tarFile(path: string, content: string) {
|
||||
return [header, body];
|
||||
}
|
||||
|
||||
function npmPackFixtureEntries(files: Array<[string, string]>) {
|
||||
function npmPackFixture(files: Record<string, string>) {
|
||||
const parts: Uint8Array[] = [];
|
||||
for (const [path, content] of files) {
|
||||
for (const [path, content] of Object.entries(files)) {
|
||||
parts.push(...tarFile(path, content));
|
||||
}
|
||||
parts.push(new Uint8Array(BLOCK_SIZE), new Uint8Array(BLOCK_SIZE));
|
||||
@@ -53,10 +53,6 @@ function npmPackFixtureEntries(files: Array<[string, string]>) {
|
||||
return gzipSync(tar);
|
||||
}
|
||||
|
||||
function npmPackFixture(files: Record<string, string>) {
|
||||
return npmPackFixtureEntries(Object.entries(files));
|
||||
}
|
||||
|
||||
describe("clawpack", () => {
|
||||
it("parses npm pack tarballs and computes npm integrity fields", async () => {
|
||||
const pack = npmPackFixture({
|
||||
@@ -99,18 +95,6 @@ describe("clawpack", () => {
|
||||
await expect(parseClawPack(pack)).rejects.toThrow("rooted under package");
|
||||
});
|
||||
|
||||
it("rejects duplicate normalized archive paths", async () => {
|
||||
const pack = npmPackFixtureEntries([
|
||||
["package/package.json", JSON.stringify({ name: "demo", version: "1.0.0" })],
|
||||
["package/openclaw.plugin.json", JSON.stringify({ id: "demo" })],
|
||||
["package/package.json", JSON.stringify({ name: "other", version: "9.9.9" })],
|
||||
]);
|
||||
|
||||
await expect(parseClawPack(pack)).rejects.toThrow(
|
||||
"ClawPack contains duplicate path: package.json",
|
||||
);
|
||||
});
|
||||
|
||||
it("uses npm-style tarball names", () => {
|
||||
expect(npmTarballName("demo", "1.0.0")).toBe("demo-1.0.0.tgz");
|
||||
expect(npmTarballName("@scope/demo", "1.0.0")).toBe("scope-demo-1.0.0.tgz");
|
||||
|
||||
@@ -67,7 +67,6 @@ function isZeroBlock(block: Uint8Array) {
|
||||
|
||||
function parseTarEntries(bytes: Uint8Array): ClawPackEntry[] {
|
||||
const entries: ClawPackEntry[] = [];
|
||||
const paths = new Set<string>();
|
||||
let offset = 0;
|
||||
|
||||
while (offset + TAR_BLOCK_SIZE <= bytes.byteLength) {
|
||||
@@ -94,10 +93,6 @@ function parseTarEntries(bytes: Uint8Array): ClawPackEntry[] {
|
||||
offset = nextTarOffset(payloadOffset, size);
|
||||
continue;
|
||||
}
|
||||
if (paths.has(relPath)) {
|
||||
throw new Error(`ClawPack contains duplicate path: ${relPath}`);
|
||||
}
|
||||
paths.add(relPath);
|
||||
entries.push({
|
||||
path: relPath,
|
||||
bytes: Uint8Array.from(tarEntryPayload(bytes, payloadOffset, size)),
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
depRegistryUrl,
|
||||
mergeDepRegistryFinding,
|
||||
parseDependencyFile,
|
||||
summarizeDepRegistryChecks,
|
||||
} from "./depRegistryScan";
|
||||
import { summarizeReasonCodes, verdictFromCodes } from "./moderationReasonCodes";
|
||||
|
||||
describe("depRegistryScan", () => {
|
||||
it("parses registry dependency manifests and skips vendored or non-registry specs", () => {
|
||||
expect(
|
||||
parseDependencyFile(
|
||||
"package.json",
|
||||
JSON.stringify({
|
||||
dependencies: {
|
||||
"@types/node": "^24.0.0",
|
||||
local: "file:../local",
|
||||
remote: "github:owner/repo",
|
||||
},
|
||||
optionalDependencies: {
|
||||
undici: "^7.0.0",
|
||||
},
|
||||
}),
|
||||
),
|
||||
).toEqual([
|
||||
{ name: "@types/node", registry: "npm", source: "package.json" },
|
||||
{ name: "undici", registry: "npm", source: "package.json" },
|
||||
]);
|
||||
|
||||
expect(
|
||||
parseDependencyFile("vendor/package.json", '{"dependencies":{"phantom":"1.0.0"}}'),
|
||||
).toEqual([]);
|
||||
expect(
|
||||
parseDependencyFile(
|
||||
"requirements.txt",
|
||||
["requests>=2", "demo @ git+https://example.test/demo.git", "-r dev.txt"].join("\n"),
|
||||
),
|
||||
).toEqual([{ name: "requests", registry: "pypi", source: "requirements.txt" }]);
|
||||
});
|
||||
|
||||
it("keeps npm scope names compatible with registry URL lookup", () => {
|
||||
expect(depRegistryUrl("npm", "@types/node")).toBe("https://registry.npmjs.org/@types%2Fnode");
|
||||
});
|
||||
|
||||
it("does not produce clean status when registry lookups are unresolved", () => {
|
||||
const analysis = summarizeDepRegistryChecks({
|
||||
checkedAt: 123,
|
||||
results: [{ name: "requests", registry: "pypi", source: "requirements.txt", exists: true }],
|
||||
unresolved: [
|
||||
{
|
||||
name: "maybe-real",
|
||||
registry: "npm",
|
||||
source: "package.json",
|
||||
reason: "network error",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(analysis.status).toBe("error");
|
||||
expect(analysis.notFoundPackages).toEqual([]);
|
||||
expect(analysis.unresolvedPackages).toEqual(["maybe-real (npm)"]);
|
||||
});
|
||||
|
||||
it("injects a static finding only for confirmed missing packages", () => {
|
||||
const suspicious = summarizeDepRegistryChecks({
|
||||
checkedAt: 456,
|
||||
results: [
|
||||
{
|
||||
name: "phantom-package-xyz",
|
||||
registry: "npm",
|
||||
source: "package.json",
|
||||
exists: false,
|
||||
httpStatus: 404,
|
||||
},
|
||||
],
|
||||
unresolved: [],
|
||||
});
|
||||
|
||||
const merged = mergeDepRegistryFinding({
|
||||
staticScan: undefined,
|
||||
analysis: suspicious,
|
||||
statusFromCodes: verdictFromCodes,
|
||||
summarizeCodes: summarizeReasonCodes,
|
||||
});
|
||||
|
||||
expect(merged.status).toBe("suspicious");
|
||||
expect(merged.reasonCodes).toEqual(["suspicious.dep_not_found_on_registry"]);
|
||||
expect(merged.findings[0]?.file).toBe("Dependency manifests");
|
||||
|
||||
const cleanAgain = mergeDepRegistryFinding({
|
||||
staticScan: merged,
|
||||
analysis: summarizeDepRegistryChecks({ checkedAt: 789, results: [], unresolved: [] }),
|
||||
statusFromCodes: verdictFromCodes,
|
||||
summarizeCodes: summarizeReasonCodes,
|
||||
});
|
||||
expect(cleanAgain.status).toBe("clean");
|
||||
expect(cleanAgain.findings).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,321 @@
|
||||
import {
|
||||
MODERATION_ENGINE_VERSION,
|
||||
REASON_CODES,
|
||||
type ModerationFinding,
|
||||
type ModerationVerdict,
|
||||
} from "./moderationReasonCodes";
|
||||
|
||||
export const SUPPORTED_DEP_REGISTRIES = ["pypi", "npm", "cargo"] as const;
|
||||
|
||||
export type SupportedDepRegistry = (typeof SUPPORTED_DEP_REGISTRIES)[number];
|
||||
|
||||
export type DepRegistryStatus = "clean" | "suspicious" | "error";
|
||||
|
||||
export type DepEntry = {
|
||||
name: string;
|
||||
registry: SupportedDepRegistry;
|
||||
source: string;
|
||||
};
|
||||
|
||||
export type DepRegistryResult = DepEntry & {
|
||||
exists: boolean;
|
||||
httpStatus?: number;
|
||||
};
|
||||
|
||||
export type DepRegistryUnresolved = DepEntry & {
|
||||
reason: string;
|
||||
};
|
||||
|
||||
export type DepRegistryAnalysis = {
|
||||
status: DepRegistryStatus;
|
||||
results: DepRegistryResult[];
|
||||
notFoundPackages: string[];
|
||||
unresolvedPackages: string[];
|
||||
summary: string;
|
||||
checkedAt: number;
|
||||
};
|
||||
|
||||
const DEP_FILE_PARSERS: Record<string, (content: string, path: string) => DepEntry[]> = {
|
||||
"requirements.txt": parseRequirementsTxt,
|
||||
"requirements-dev.txt": parseRequirementsTxt,
|
||||
"requirements_dev.txt": parseRequirementsTxt,
|
||||
"requirements-test.txt": parseRequirementsTxt,
|
||||
"requirements_test.txt": parseRequirementsTxt,
|
||||
"package.json": parsePackageJson,
|
||||
"cargo.toml": parseCargoToml,
|
||||
"pyproject.toml": parsePyprojectToml,
|
||||
};
|
||||
|
||||
const NON_REGISTRY_NPM_SPEC_PREFIXES = [
|
||||
"file:",
|
||||
"link:",
|
||||
"git+",
|
||||
"git://",
|
||||
"github:",
|
||||
"bitbucket:",
|
||||
"gist:",
|
||||
"http:",
|
||||
"https:",
|
||||
"workspace:",
|
||||
"npm:",
|
||||
];
|
||||
|
||||
const VENDORED_PATH_PATTERNS = [
|
||||
/(^|\/)node_modules\//,
|
||||
/(^|\/)vendor\//,
|
||||
/(^|\/)__pycache__\//,
|
||||
/(^|\/)\.venv\//,
|
||||
/(^|\/)venv\//,
|
||||
/(^|\/)target\//,
|
||||
/(^|\/)\.cargo\//,
|
||||
/(^|\/)dist\//,
|
||||
/(^|\/)build\//,
|
||||
];
|
||||
|
||||
function normalizeName(name: string, registry: SupportedDepRegistry) {
|
||||
const normalized = name.trim().toLowerCase();
|
||||
return registry === "cargo" ? normalized.replaceAll("_", "-") : normalized;
|
||||
}
|
||||
|
||||
export function isVendoredDependencyPath(path: string) {
|
||||
return VENDORED_PATH_PATTERNS.some((pattern) => pattern.test(path));
|
||||
}
|
||||
|
||||
export function parseDependencyFile(path: string, content: string): DepEntry[] {
|
||||
if (isVendoredDependencyPath(path)) return [];
|
||||
const basename = path.split("/").pop()?.toLowerCase() ?? "";
|
||||
const parser = DEP_FILE_PARSERS[basename];
|
||||
return parser ? dedupeDeps(parser(content, path)) : [];
|
||||
}
|
||||
|
||||
export function dedupeDeps(entries: DepEntry[]) {
|
||||
const seen = new Set<string>();
|
||||
return entries.filter((entry) => {
|
||||
const key = `${entry.registry}:${entry.name}`;
|
||||
if (seen.has(key)) return false;
|
||||
seen.add(key);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
function stripInlineComment(line: string) {
|
||||
return line.replace(/\s+#.*$/, "").trim();
|
||||
}
|
||||
|
||||
function parseRequirementsTxt(content: string, path: string): DepEntry[] {
|
||||
const entries: DepEntry[] = [];
|
||||
for (const rawLine of content.split("\n")) {
|
||||
const line = stripInlineComment(rawLine);
|
||||
if (!line || line.startsWith("-")) continue;
|
||||
if (/^(?:git\+|https?:|file:|\.{0,2}\/)/i.test(line)) continue;
|
||||
if (/\s@\s/.test(line)) continue;
|
||||
const match = line.match(/^([a-zA-Z0-9_][a-zA-Z0-9._-]*)/);
|
||||
if (!match) continue;
|
||||
entries.push({ name: normalizeName(match[1], "pypi"), registry: "pypi", source: path });
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
function parsePackageJson(content: string, path: string): DepEntry[] {
|
||||
const entries: DepEntry[] = [];
|
||||
let pkg: Record<string, unknown>;
|
||||
try {
|
||||
pkg = JSON.parse(content) as Record<string, unknown>;
|
||||
} catch {
|
||||
return entries;
|
||||
}
|
||||
|
||||
for (const field of ["dependencies", "devDependencies", "optionalDependencies"]) {
|
||||
const deps = pkg[field];
|
||||
if (!deps || typeof deps !== "object" || Array.isArray(deps)) continue;
|
||||
for (const [rawName, rawSpec] of Object.entries(deps as Record<string, unknown>)) {
|
||||
const spec = typeof rawSpec === "string" ? rawSpec.trim().toLowerCase() : "";
|
||||
if (NON_REGISTRY_NPM_SPEC_PREFIXES.some((prefix) => spec.startsWith(prefix))) continue;
|
||||
entries.push({ name: normalizeName(rawName, "npm"), registry: "npm", source: path });
|
||||
}
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
function parseCargoToml(content: string, path: string): DepEntry[] {
|
||||
const entries: DepEntry[] = [];
|
||||
let inDepSection = false;
|
||||
for (const rawLine of content.split("\n")) {
|
||||
const line = stripInlineComment(rawLine);
|
||||
if (/^\[.*\]$/.test(line)) {
|
||||
const section = line.replace(/[[\]\s]/g, "").toLowerCase();
|
||||
inDepSection =
|
||||
section === "dependencies" ||
|
||||
section === "dev-dependencies" ||
|
||||
section === "build-dependencies";
|
||||
continue;
|
||||
}
|
||||
if (!inDepSection || !line) continue;
|
||||
const match = line.match(/^([a-zA-Z0-9_][a-zA-Z0-9_-]*)\s*=/);
|
||||
if (!match) continue;
|
||||
entries.push({ name: normalizeName(match[1], "cargo"), registry: "cargo", source: path });
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
function parsePyprojectToml(content: string, path: string): DepEntry[] {
|
||||
const entries: DepEntry[] = [];
|
||||
let inDepArray = false;
|
||||
let inPoetryDepTable = false;
|
||||
for (const rawLine of content.split("\n")) {
|
||||
const line = stripInlineComment(rawLine);
|
||||
if (/^\[.*\]$/.test(line)) {
|
||||
inDepArray = false;
|
||||
const section = line.replace(/[[\]\s]/g, "").toLowerCase();
|
||||
inPoetryDepTable =
|
||||
section === "tool.poetry.dependencies" ||
|
||||
section === "tool.poetry.dev-dependencies" ||
|
||||
section === "tool.poetry.group.dev.dependencies";
|
||||
continue;
|
||||
}
|
||||
if (/^dependencies\s*=\s*\[/.test(line)) {
|
||||
inDepArray = true;
|
||||
const inline = line.match(/\[\s*(.*)\s*\]/);
|
||||
if (inline) {
|
||||
for (const item of extractQuotedStrings(inline[1])) addPyPiDependency(entries, item, path);
|
||||
inDepArray = false;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (inDepArray) {
|
||||
if (line === "]") {
|
||||
inDepArray = false;
|
||||
continue;
|
||||
}
|
||||
const quoted = line.match(/^["']([^"']+)["']/);
|
||||
if (quoted) addPyPiDependency(entries, quoted[1], path);
|
||||
continue;
|
||||
}
|
||||
if (!inPoetryDepTable || !line) continue;
|
||||
const match = line.match(/^([a-zA-Z0-9_][a-zA-Z0-9._-]*)\s*=/);
|
||||
if (!match || match[1].toLowerCase() === "python") continue;
|
||||
entries.push({ name: normalizeName(match[1], "pypi"), registry: "pypi", source: path });
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
function addPyPiDependency(entries: DepEntry[], spec: string, path: string) {
|
||||
if (/\s@\s/.test(spec)) return;
|
||||
const match = spec.match(/^([a-zA-Z0-9_][a-zA-Z0-9._-]*)/);
|
||||
if (!match) return;
|
||||
entries.push({ name: normalizeName(match[1], "pypi"), registry: "pypi", source: path });
|
||||
}
|
||||
|
||||
function extractQuotedStrings(s: string) {
|
||||
return [...s.matchAll(/["']([^"']+)["']/g)].map((match) => match[1]);
|
||||
}
|
||||
|
||||
export function depRegistryUrl(registry: SupportedDepRegistry, packageName: string) {
|
||||
const encoded =
|
||||
registry === "npm" && packageName.startsWith("@")
|
||||
? `@${encodeURIComponent(packageName.slice(1))}`
|
||||
: encodeURIComponent(packageName);
|
||||
if (registry === "pypi") return `https://pypi.org/pypi/${encoded}/json`;
|
||||
if (registry === "npm") return `https://registry.npmjs.org/${encoded}`;
|
||||
return `https://crates.io/api/v1/crates/${encoded}`;
|
||||
}
|
||||
|
||||
export function summarizeDepRegistryChecks(params: {
|
||||
results: DepRegistryResult[];
|
||||
unresolved: DepRegistryUnresolved[];
|
||||
checkedAt?: number;
|
||||
}): DepRegistryAnalysis {
|
||||
const notFound = params.results.filter((result) => !result.exists);
|
||||
const notFoundPackages = notFound.map((result) => `${result.name} (${result.registry})`);
|
||||
const unresolvedPackages = params.unresolved.map(
|
||||
(result) => `${result.name} (${result.registry})`,
|
||||
);
|
||||
const checkedAt = params.checkedAt ?? Date.now();
|
||||
|
||||
if (notFoundPackages.length > 0) {
|
||||
const partial =
|
||||
unresolvedPackages.length > 0
|
||||
? ` ${unresolvedPackages.length} package(s) could not be checked and will be retried.`
|
||||
: "";
|
||||
return {
|
||||
status: "suspicious",
|
||||
results: params.results,
|
||||
notFoundPackages,
|
||||
unresolvedPackages,
|
||||
summary: `${notFoundPackages.length} declared dependency package(s) were not found on their public registry: ${notFoundPackages.join(", ")}.${partial}`,
|
||||
checkedAt,
|
||||
};
|
||||
}
|
||||
|
||||
if (unresolvedPackages.length > 0) {
|
||||
return {
|
||||
status: "error",
|
||||
results: params.results,
|
||||
notFoundPackages: [],
|
||||
unresolvedPackages,
|
||||
summary: `${unresolvedPackages.length} dependency package(s) could not be verified due to registry lookup errors. The scan will be retried.`,
|
||||
checkedAt,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
status: "clean",
|
||||
results: params.results,
|
||||
notFoundPackages: [],
|
||||
unresolvedPackages: [],
|
||||
summary: `All ${params.results.length} declared dependency package(s) verified as present on their public registries.`,
|
||||
checkedAt,
|
||||
};
|
||||
}
|
||||
|
||||
export function buildDepRegistryFinding(analysis: DepRegistryAnalysis): ModerationFinding | null {
|
||||
if (analysis.status !== "suspicious" || analysis.notFoundPackages.length === 0) return null;
|
||||
return {
|
||||
code: REASON_CODES.DEP_NOT_FOUND,
|
||||
severity: "critical",
|
||||
file: "Dependency manifests",
|
||||
line: 1,
|
||||
message: `${analysis.notFoundPackages.length} package(s) referenced in dependency files do not exist on their public registries: ${analysis.notFoundPackages.join(", ")}`,
|
||||
evidence:
|
||||
"An attacker could register these phantom package names and inject malicious install-time code through dependency confusion.",
|
||||
};
|
||||
}
|
||||
|
||||
export function mergeDepRegistryFinding(params: {
|
||||
staticScan:
|
||||
| {
|
||||
status: ModerationVerdict;
|
||||
reasonCodes: string[];
|
||||
findings: ModerationFinding[];
|
||||
summary: string;
|
||||
engineVersion: string;
|
||||
checkedAt: number;
|
||||
}
|
||||
| undefined;
|
||||
analysis: DepRegistryAnalysis;
|
||||
statusFromCodes: (codes: string[]) => ModerationVerdict;
|
||||
summarizeCodes: (codes: string[]) => string;
|
||||
}) {
|
||||
const base = params.staticScan ?? {
|
||||
status: "clean" as ModerationVerdict,
|
||||
reasonCodes: [],
|
||||
findings: [],
|
||||
summary: "No suspicious patterns detected.",
|
||||
engineVersion: MODERATION_ENGINE_VERSION,
|
||||
checkedAt: params.analysis.checkedAt,
|
||||
};
|
||||
const findings = base.findings.filter((finding) => finding.code !== REASON_CODES.DEP_NOT_FOUND);
|
||||
const depFinding = buildDepRegistryFinding(params.analysis);
|
||||
if (depFinding) findings.push(depFinding);
|
||||
const reasonCodes = Array.from(new Set(findings.map((finding) => finding.code))).sort((a, b) =>
|
||||
a.localeCompare(b),
|
||||
);
|
||||
return {
|
||||
...base,
|
||||
status: params.statusFromCodes(reasonCodes),
|
||||
reasonCodes,
|
||||
findings,
|
||||
summary: params.summarizeCodes(reasonCodes),
|
||||
checkedAt: params.analysis.checkedAt,
|
||||
};
|
||||
}
|
||||
@@ -1,8 +1,6 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { isLocalDevAuthEnabled } from "./devAuth";
|
||||
|
||||
const CLOUD_DEV_AUTH_SECRET = "dev-auth-secret-with-enough-entropy-123";
|
||||
|
||||
describe("isLocalDevAuthEnabled", () => {
|
||||
it("requires the explicit dev auth flag", () => {
|
||||
expect(
|
||||
@@ -43,78 +41,16 @@ describe("isLocalDevAuthEnabled", () => {
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("allows cloud dev deployments with an explicit localhost site and matching secret", () => {
|
||||
expect(
|
||||
isLocalDevAuthEnabled(
|
||||
{
|
||||
CONVEX_SITE_URL: "https://clever-rabbit-123.convex.cloud",
|
||||
DEV_AUTH_CONVEX_DEPLOYMENT: "dev:clever-rabbit-123",
|
||||
DEV_AUTH_ENABLED: "1",
|
||||
DEV_AUTH_SECRET: CLOUD_DEV_AUTH_SECRET,
|
||||
DEV_AUTH_SITE_URL: "http://127.0.0.1:3211",
|
||||
},
|
||||
CLOUD_DEV_AUTH_SECRET,
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("allows cloud dev deployments from the fallback marker when Convex deployment is blank", () => {
|
||||
expect(
|
||||
isLocalDevAuthEnabled(
|
||||
{
|
||||
CONVEX_DEPLOYMENT: "",
|
||||
CONVEX_SITE_URL: "https://clever-rabbit-123.convex.cloud",
|
||||
DEV_AUTH_CONVEX_DEPLOYMENT: "dev:clever-rabbit-123",
|
||||
DEV_AUTH_ENABLED: "1",
|
||||
DEV_AUTH_SECRET: CLOUD_DEV_AUTH_SECRET,
|
||||
DEV_AUTH_SITE_URL: "http://127.0.0.1:3211",
|
||||
},
|
||||
CLOUD_DEV_AUTH_SECRET,
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects cloud dev deployments when the secret is missing", () => {
|
||||
it("rejects cloud dev deployments even when the dev auth flag is set", () => {
|
||||
expect(
|
||||
isLocalDevAuthEnabled({
|
||||
CONVEX_SITE_URL: "https://clever-rabbit-123.convex.cloud",
|
||||
DEV_AUTH_CONVEX_DEPLOYMENT: "dev:clever-rabbit-123",
|
||||
CONVEX_SITE_URL: "http://127.0.0.1:3211",
|
||||
DEV_AUTH_ENABLED: "1",
|
||||
DEV_AUTH_SECRET: CLOUD_DEV_AUTH_SECRET,
|
||||
DEV_AUTH_SITE_URL: "http://127.0.0.1:3211",
|
||||
CONVEX_DEPLOYMENT: "dev:clever-rabbit-123",
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects cloud dev deployments when the configured secret is too short", () => {
|
||||
expect(
|
||||
isLocalDevAuthEnabled(
|
||||
{
|
||||
CONVEX_SITE_URL: "https://clever-rabbit-123.convex.cloud",
|
||||
DEV_AUTH_CONVEX_DEPLOYMENT: "dev:clever-rabbit-123",
|
||||
DEV_AUTH_ENABLED: "1",
|
||||
DEV_AUTH_SECRET: "short",
|
||||
DEV_AUTH_SITE_URL: "http://127.0.0.1:3211",
|
||||
},
|
||||
"short",
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects cloud dev deployments without an explicit localhost dev auth site", () => {
|
||||
expect(
|
||||
isLocalDevAuthEnabled(
|
||||
{
|
||||
CONVEX_SITE_URL: "http://127.0.0.1:3211",
|
||||
DEV_AUTH_ENABLED: "1",
|
||||
DEV_AUTH_SECRET: CLOUD_DEV_AUTH_SECRET,
|
||||
CONVEX_DEPLOYMENT: "dev:clever-rabbit-123",
|
||||
},
|
||||
CLOUD_DEV_AUTH_SECRET,
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects localhost site URLs without a local deployment marker", () => {
|
||||
expect(
|
||||
isLocalDevAuthEnabled({
|
||||
|
||||
+3
-33
@@ -3,48 +3,18 @@ type DevAuthEnv = {
|
||||
CONVEX_SITE_URL?: string;
|
||||
DEV_AUTH_CONVEX_DEPLOYMENT?: string;
|
||||
DEV_AUTH_ENABLED?: string;
|
||||
DEV_AUTH_SECRET?: string;
|
||||
DEV_AUTH_SITE_URL?: string;
|
||||
};
|
||||
|
||||
const MIN_CLOUD_DEV_AUTH_SECRET_LENGTH = 32;
|
||||
|
||||
export function isLocalDevAuthEnabled(env: DevAuthEnv = process.env, suppliedSecret?: string) {
|
||||
export function isLocalDevAuthEnabled(env: DevAuthEnv = process.env) {
|
||||
if (env.DEV_AUTH_ENABLED !== "1") return false;
|
||||
const convexDeployment = env.CONVEX_DEPLOYMENT?.trim();
|
||||
const devAuthDeployment = env.DEV_AUTH_CONVEX_DEPLOYMENT?.trim();
|
||||
const deployment = convexDeployment || devAuthDeployment || "";
|
||||
|
||||
if (isLocalConvexDeployment(deployment)) {
|
||||
return isLocalhostUrl(env.CONVEX_SITE_URL);
|
||||
}
|
||||
|
||||
if (isDevConvexDeployment(deployment)) {
|
||||
return isLocalhostUrl(env.DEV_AUTH_SITE_URL) && hasValidCloudDevAuthSecret(env, suppliedSecret);
|
||||
}
|
||||
|
||||
return false;
|
||||
const deployment = env.CONVEX_DEPLOYMENT?.trim() || env.DEV_AUTH_CONVEX_DEPLOYMENT?.trim() || "";
|
||||
return isLocalConvexDeployment(deployment) && isLocalhostUrl(env.CONVEX_SITE_URL);
|
||||
}
|
||||
|
||||
function isLocalConvexDeployment(deployment: string) {
|
||||
return deployment.startsWith("local:") || deployment.startsWith("anonymous:");
|
||||
}
|
||||
|
||||
function isDevConvexDeployment(deployment: string) {
|
||||
return deployment.startsWith("dev:");
|
||||
}
|
||||
|
||||
function hasValidCloudDevAuthSecret(env: DevAuthEnv, suppliedSecret: string | undefined) {
|
||||
const expected = env.DEV_AUTH_SECRET?.trim();
|
||||
const actual = suppliedSecret?.trim();
|
||||
return Boolean(
|
||||
expected &&
|
||||
actual &&
|
||||
expected.length >= MIN_CLOUD_DEV_AUTH_SECRET_LENGTH &&
|
||||
actual === expected,
|
||||
);
|
||||
}
|
||||
|
||||
function isLocalhostUrl(value: string | undefined) {
|
||||
if (!value) return false;
|
||||
try {
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
export function assertLocalDevSeedAllowed(seedName: string): void {
|
||||
const deployment =
|
||||
process.env.CONVEX_DEPLOYMENT?.trim() || process.env.DEV_AUTH_CONVEX_DEPLOYMENT?.trim() || "";
|
||||
if (
|
||||
deployment.startsWith("dev:") ||
|
||||
deployment.startsWith("local:") ||
|
||||
deployment.startsWith("anonymous:")
|
||||
) {
|
||||
return;
|
||||
}
|
||||
if (
|
||||
!deployment &&
|
||||
(process.env.DEV_AUTH_ENABLED === "1" || process.env.CLAW_HUB_ENABLE_DEV_IMPERSONATION === "1")
|
||||
) {
|
||||
return;
|
||||
}
|
||||
throw new Error(`${seedName} dev seed is disabled outside local/dev deployments`);
|
||||
}
|
||||
@@ -1,253 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
APPEALS_URL,
|
||||
buildMaliciousArtifactEmail,
|
||||
buildBanNotificationEmail,
|
||||
buildPackageInspectorFindingsEmail,
|
||||
buildRestoredAccountEmail,
|
||||
} from "./emails";
|
||||
|
||||
describe("moderation notification email copy", () => {
|
||||
it("builds public-safe malicious skill context with appeal but no local scan guidance", () => {
|
||||
const email = buildBanNotificationEmail({
|
||||
handle: "gingiris",
|
||||
source: "autoban",
|
||||
reason: "malicious.llm_malicious",
|
||||
artifact: { kind: "skill", name: "gingiris-launch" },
|
||||
trigger: "scanner.llm.malicious",
|
||||
});
|
||||
|
||||
expect(email.subject).toBe("Your ClawHub account was disabled");
|
||||
expect(email.context).toMatchObject({
|
||||
appealUrl: APPEALS_URL,
|
||||
artifact: { kind: "skill", name: "gingiris-launch" },
|
||||
scannerLabel: "ClawScan",
|
||||
findingSummary: "ClawScan classified the uploaded skill as malicious.",
|
||||
});
|
||||
expect(email.text).toContain("Skill: gingiris-launch");
|
||||
expect(email.text).not.toContain("Scanner:");
|
||||
expect(email.html).not.toContain("<strong>Scanner:</strong>");
|
||||
expect(email.text).not.toContain("republishing");
|
||||
expect(email.html).not.toContain("republishing");
|
||||
expect(email.text).not.toContain("To support your appeal, include scan results");
|
||||
expect(email.html).not.toContain("Include scan results with your appeal");
|
||||
expect(email.text).toContain("Appeal: https://appeals.openclaw.ai/");
|
||||
expect(email.html).not.toContain("If you already appealed");
|
||||
expect(email.html).not.toContain("separate support email");
|
||||
expect(email.text).not.toContain("clawhub scan ./my-skill --output clawhub-scan.zip");
|
||||
expect(email.text).not.toContain("https://docs.openclaw.ai/clawhub/cli#scan-path");
|
||||
});
|
||||
|
||||
it("does not leak raw manual moderator notes into outbound email", () => {
|
||||
const email = buildBanNotificationEmail({
|
||||
handle: "target",
|
||||
source: "manual",
|
||||
reason: "internal reviewer note: reporter=user_123 secret finding id=abc",
|
||||
});
|
||||
|
||||
expect(email.context.findingSummary).toBe(
|
||||
"ClawHub staff disabled the account after a security review.",
|
||||
);
|
||||
expect(email.text).not.toContain("internal reviewer note");
|
||||
expect(email.text).not.toContain("reporter=user_123");
|
||||
expect(email.html).not.toContain("secret finding id");
|
||||
});
|
||||
|
||||
it("uses rate-limit copy without scan remediation guidance", () => {
|
||||
const email = buildBanNotificationEmail({
|
||||
handle: "publish-loop",
|
||||
source: "manual",
|
||||
reason: "rate limit triggered by automated CLI publishing",
|
||||
});
|
||||
|
||||
expect(email.context).toMatchObject({
|
||||
scannerLabel: null,
|
||||
findingSummary: "Publishing automation triggered ClawHub rate-limit abuse controls.",
|
||||
});
|
||||
expect(email.text).toContain("Publishing automation");
|
||||
expect(email.text).not.toContain("clawhub scan");
|
||||
expect(email.text).not.toContain("Include scan results");
|
||||
expect(email.html).not.toContain("Include scan results");
|
||||
expect(email.html).not.toContain("fixed local copy");
|
||||
});
|
||||
|
||||
it("builds restored-account copy that explains tokens stay revoked", () => {
|
||||
const email = buildRestoredAccountEmail({
|
||||
handle: "restored",
|
||||
restoredListings: [
|
||||
{ kind: "skill", name: "safe-one" },
|
||||
{ kind: "plugin", name: "@scope/demo" },
|
||||
],
|
||||
});
|
||||
|
||||
expect(email.subject).toBe("Your ClawHub account was restored");
|
||||
expect(email.text).toContain("Your ClawHub account can sign in again.");
|
||||
expect(email.text).toContain("Skill: safe-one");
|
||||
expect(email.text).toContain("Plugin: @scope/demo");
|
||||
expect(email.text).toContain("Previously revoked API tokens stay revoked.");
|
||||
});
|
||||
|
||||
it("builds malicious artifact copy without account appeal language", () => {
|
||||
const email = buildMaliciousArtifactEmail({
|
||||
handle: "publisher",
|
||||
artifact: { kind: "skill", name: "demo-skill" },
|
||||
version: "1.2.3",
|
||||
trigger: "malicious.llm_malicious",
|
||||
findingSummary: "Attempts to exfiltrate credentials.",
|
||||
});
|
||||
|
||||
expect(email.subject).toBe("ClawHub blocked a skill version");
|
||||
expect(email.text).toContain("Reason: Attempts to exfiltrate credentials.");
|
||||
expect(email.html).toContain("Attempts to exfiltrate credentials.");
|
||||
expect(email.text).toContain("Skill: demo-skill");
|
||||
expect(email.text).toContain("Version: 1.2.3");
|
||||
expect(email.text).toContain("clawhub scan download demo-skill --version 1.2.3");
|
||||
expect(email.text).toContain("Increment the version number before uploading the fixed skill.");
|
||||
expect(email.text).toContain("https://docs.openclaw.ai/clawhub/moderation");
|
||||
expect(email.text).not.toContain("clawhub scan ./my-skill --output clawhub-scan.zip");
|
||||
expect(email.text).not.toContain("fixed local copy");
|
||||
expect(email.text).toContain("Repeated malicious rejections may lead to account disablement");
|
||||
expect(email.html).toContain("Repeated malicious rejections may lead to account disablement");
|
||||
expect(email.text).not.toContain(APPEALS_URL);
|
||||
expect(email.html).not.toContain(APPEALS_URL);
|
||||
expect(email.html).not.toContain("appeal this decision");
|
||||
});
|
||||
|
||||
it("falls back to generic malicious artifact copy when no ClawScan summary is available", () => {
|
||||
const email = buildMaliciousArtifactEmail({
|
||||
handle: "publisher",
|
||||
artifact: { kind: "skill", name: "demo-skill" },
|
||||
version: "1.2.3",
|
||||
trigger: "malicious.llm_malicious",
|
||||
});
|
||||
|
||||
expect(email.text).toContain("Reason: ClawScan classified the uploaded artifact as malicious.");
|
||||
});
|
||||
|
||||
it("keeps supplied ClawScan summaries to one email-safe line", () => {
|
||||
const email = buildMaliciousArtifactEmail({
|
||||
handle: "publisher",
|
||||
artifact: { kind: "skill", name: "demo-skill" },
|
||||
version: "1.2.3",
|
||||
findingSummary: ` ${"credential exfiltration ".repeat(30)}\nwith hidden tooling `,
|
||||
});
|
||||
|
||||
const reasonLine = email.text.split("\n").find((line) => line.startsWith("Reason: "));
|
||||
expect(reasonLine).toBeDefined();
|
||||
expect(reasonLine).not.toContain("\n");
|
||||
expect(reasonLine?.length).toBeLessThanOrEqual("Reason: ".length + 280);
|
||||
expect(reasonLine).toContain("...");
|
||||
});
|
||||
|
||||
it("builds plugin scan download copy with an explicit artifact kind", () => {
|
||||
const email = buildMaliciousArtifactEmail({
|
||||
handle: "publisher",
|
||||
artifact: { kind: "plugin", name: "@scope/demo" },
|
||||
version: "2.0.0",
|
||||
trigger: "malicious.static",
|
||||
});
|
||||
|
||||
expect(email.text).toContain("Plugin: @scope/demo");
|
||||
expect(email.text).toContain("clawhub scan download @scope/demo --version 2.0.0 --kind plugin");
|
||||
expect(email.text).toContain("Increment the version number before uploading the fixed plugin.");
|
||||
});
|
||||
|
||||
it("builds plugin inspector warning copy with local validation guidance", () => {
|
||||
const email = buildPackageInspectorFindingsEmail({
|
||||
handle: "octocat",
|
||||
packageName: "demo-plugin",
|
||||
version: "1.0.0",
|
||||
findings: [
|
||||
{
|
||||
findingKind: "warning",
|
||||
code: "legacy-before-agent-start",
|
||||
issueClass: "deprecation-warning",
|
||||
severity: "P2",
|
||||
message: "legacy before_agent_start hook is deprecated",
|
||||
inspectorVersion: "0.4.0",
|
||||
targetOpenClawVersion: "0.9.0",
|
||||
scanSource: "publish",
|
||||
authorRemediation: {
|
||||
summary: "Replace the legacy before_agent_start hook with current prompt hooks.",
|
||||
docsUrl:
|
||||
"https://docs.openclaw.ai/clawhub/plugin-validation-fixes#legacy-before-agent-start",
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(email.subject).toBe("Plugin Inspector findings for demo-plugin@1.0.0");
|
||||
expect(email.text).toContain("Hi octocat,");
|
||||
expect(email.text).toContain("We found 1 issue with version 1.0.0 of demo-plugin.");
|
||||
expect(email.text).toContain("OpenClaw Version: 0.9.0");
|
||||
expect(email.text).toContain("Address the findings below in your plugin package.");
|
||||
expect(email.text).toContain("Run the validation command locally against your changes.");
|
||||
expect(email.text).toContain("clawhub package validate <path-to-plugin>");
|
||||
expect(email.text).toContain(
|
||||
"- **WARNING** `legacy-before-agent-start` (deprecation-warning, P2)",
|
||||
);
|
||||
expect(email.text).toContain(" legacy before_agent_start hook is deprecated");
|
||||
expect(email.text).toContain(" Fix:");
|
||||
expect(email.text).toContain(
|
||||
" Replace the legacy before_agent_start hook with current prompt hooks.",
|
||||
);
|
||||
expect(email.text).toContain(" Docs:");
|
||||
expect(email.text).toContain(
|
||||
" https://docs.openclaw.ai/clawhub/plugin-validation-fixes#legacy-before-agent-start",
|
||||
);
|
||||
expect(email.text).not.toContain("ClawHub Security");
|
||||
expect(email.html).toContain("Validate a local fix");
|
||||
expect(email.html).toContain("Hi octocat,");
|
||||
expect(email.html).toContain("<strong>OpenClaw Version:</strong> 0.9.0");
|
||||
expect(email.html).toContain("clawhub package validate <path-to-plugin>");
|
||||
expect(email.html).toContain("legacy-before-agent-start");
|
||||
expect(email.html).toContain("deprecation-warning · P2");
|
||||
expect(email.html).toContain("Fix");
|
||||
expect(email.html).toContain("Replace the legacy before_agent_start hook");
|
||||
expect(email.html).toContain("<strong>Docs</strong>");
|
||||
expect(email.html).toContain("plugin-validation-fixes#legacy-before-agent-start");
|
||||
expect(email.html).not.toContain("plugin validation fix docs");
|
||||
expect(email.html).not.toContain("ClawHub Security");
|
||||
expect(email.text).not.toContain("Plugin Inspector: 0.4.0");
|
||||
expect(email.text).not.toContain("Target OpenClaw:");
|
||||
expect(email.html).not.toContain("<strong>Plugin Inspector:</strong>");
|
||||
expect(email.html).not.toContain("<strong>Target OpenClaw:</strong>");
|
||||
expect(email.html).not.toContain("Review:");
|
||||
expect(email.html).not.toContain("plugin validation findings");
|
||||
expect(email.html).not.toContain("https://clawhub.ai/plugins/demo-plugin#validation");
|
||||
expect(email.html).not.toContain("Your plugin was published");
|
||||
expect(email.html).not.toContain("published successfully");
|
||||
});
|
||||
|
||||
it("builds plugin inspector error copy without publish-time wording", () => {
|
||||
const email = buildPackageInspectorFindingsEmail({
|
||||
packageName: "demo-plugin",
|
||||
version: "1.0.1",
|
||||
findings: [
|
||||
{
|
||||
findingKind: "error",
|
||||
code: "missing-expected-seam",
|
||||
issueClass: "compatibility-error",
|
||||
severity: "P0",
|
||||
level: "breakage",
|
||||
message: "registerTool is no longer available",
|
||||
inspectorVersion: "0.5.0",
|
||||
targetOpenClawVersion: "0.10.0",
|
||||
scanSource: "nightly",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(email.text).toContain("We found 1 issue with version 1.0.1 of demo-plugin.");
|
||||
expect(email.text).toContain("Address the findings below in your plugin package.");
|
||||
expect(email.text).toContain("Run the validation command locally against your changes.");
|
||||
expect(email.text).toContain("clawhub package validate <path-to-plugin>");
|
||||
expect(email.text).toContain("- **ERROR** `missing-expected-seam` (compatibility-error, P0)");
|
||||
expect(email.text).not.toContain("Your plugin was published");
|
||||
expect(email.text).not.toContain("was published, but");
|
||||
expect(email.text).not.toContain("Some findings are errors");
|
||||
expect(email.text).not.toContain("nightly");
|
||||
expect(email.html).toContain("missing-expected-seam");
|
||||
expect(email.html).toContain("compatibility-error · P0");
|
||||
});
|
||||
});
|
||||
@@ -1,504 +0,0 @@
|
||||
export const APPEALS_URL = "https://appeals.openclaw.ai/";
|
||||
export const MODERATION_GUIDELINES_URL = "https://docs.openclaw.ai/clawhub/moderation";
|
||||
export const MALICIOUS_REJECTION_ACCOUNT_WARNING =
|
||||
"Repeated malicious rejections may lead to account disablement.";
|
||||
const MAX_EMAIL_FINDING_SUMMARY_LENGTH = 280;
|
||||
|
||||
export type NotificationArtifact = {
|
||||
kind: "skill" | "plugin";
|
||||
name: string;
|
||||
};
|
||||
|
||||
export type BanNotificationSource = "manual" | "autoban";
|
||||
|
||||
export type BanNotificationEmailArgs = {
|
||||
handle?: string;
|
||||
source: BanNotificationSource;
|
||||
reason?: string;
|
||||
trigger?: string;
|
||||
artifact?: NotificationArtifact;
|
||||
};
|
||||
|
||||
export type BanNotificationEmailContext = {
|
||||
appealUrl: typeof APPEALS_URL;
|
||||
artifact: NotificationArtifact | null;
|
||||
scannerLabel: string | null;
|
||||
findingSummary: string;
|
||||
};
|
||||
|
||||
export type TransactionalEmail = {
|
||||
subject: string;
|
||||
context: BanNotificationEmailContext;
|
||||
text: string;
|
||||
html: string;
|
||||
};
|
||||
|
||||
export type RestoredAccountEmailArgs = {
|
||||
handle?: string;
|
||||
restoredListings?: NotificationArtifact[];
|
||||
};
|
||||
|
||||
export type MaliciousArtifactEmailArgs = {
|
||||
handle?: string;
|
||||
artifact: NotificationArtifact;
|
||||
version?: string;
|
||||
trigger?: string;
|
||||
findingSummary?: string;
|
||||
};
|
||||
|
||||
export type PackageInspectorEmailFinding = {
|
||||
findingKind: "warning" | "error";
|
||||
code: string;
|
||||
issueClass?: string;
|
||||
level?: string;
|
||||
severity?: string;
|
||||
message: string;
|
||||
authorRemediation?: {
|
||||
summary: string;
|
||||
docsUrl?: string;
|
||||
};
|
||||
inspectorVersion?: string;
|
||||
targetOpenClawVersion?: string;
|
||||
scanSource?: "publish" | "nightly";
|
||||
};
|
||||
|
||||
export type PackageInspectorFindingsEmailArgs = {
|
||||
handle?: string;
|
||||
packageName: string;
|
||||
version: string;
|
||||
findings: PackageInspectorEmailFinding[];
|
||||
};
|
||||
|
||||
type BanReasonSummary = {
|
||||
scannerLabel: string | null;
|
||||
findingSummary: string;
|
||||
};
|
||||
|
||||
function escapeHtml(value: string) {
|
||||
return value
|
||||
.replaceAll("&", "&")
|
||||
.replaceAll("<", "<")
|
||||
.replaceAll(">", ">")
|
||||
.replaceAll('"', """)
|
||||
.replaceAll("'", "'");
|
||||
}
|
||||
|
||||
function normalizeReasonInput(args: Pick<BanNotificationEmailArgs, "reason" | "trigger">) {
|
||||
return `${args.reason ?? ""} ${args.trigger ?? ""}`.trim().toLowerCase();
|
||||
}
|
||||
|
||||
function summarizeBanReason(args: BanNotificationEmailArgs): BanReasonSummary {
|
||||
const normalized = normalizeReasonInput(args);
|
||||
|
||||
if (args.source === "autoban") {
|
||||
if (normalized.includes("virustotal") || normalized.includes("virus_total")) {
|
||||
return {
|
||||
scannerLabel: "VirusTotal",
|
||||
findingSummary: "VirusTotal telemetry contributed to a malicious upload finding.",
|
||||
};
|
||||
}
|
||||
if (normalized.includes("static")) {
|
||||
return {
|
||||
scannerLabel: "Static analysis",
|
||||
findingSummary: "Static analysis flagged malicious upload patterns.",
|
||||
};
|
||||
}
|
||||
if (
|
||||
normalized.includes("clawscan") ||
|
||||
normalized.includes("llm") ||
|
||||
normalized.includes("malicious")
|
||||
) {
|
||||
return {
|
||||
scannerLabel: "ClawScan",
|
||||
findingSummary: "ClawScan classified the uploaded skill as malicious.",
|
||||
};
|
||||
}
|
||||
return {
|
||||
scannerLabel: "ClawHub security checks",
|
||||
findingSummary: "ClawHub security checks classified the uploaded skill as malicious.",
|
||||
};
|
||||
}
|
||||
|
||||
if (/rate[-\s]?limit|publishing automation|automated(?: cli)? publishing/.test(normalized)) {
|
||||
return {
|
||||
scannerLabel: null,
|
||||
findingSummary: "Publishing automation triggered ClawHub rate-limit abuse controls.",
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
scannerLabel: null,
|
||||
findingSummary: "ClawHub staff disabled the account after a security review.",
|
||||
};
|
||||
}
|
||||
|
||||
function artifactLabel(artifact: NotificationArtifact) {
|
||||
return `${artifact.kind === "skill" ? "Skill" : "Plugin"}: ${artifact.name}`;
|
||||
}
|
||||
|
||||
function greeting(handle: string | undefined) {
|
||||
return handle?.trim() ? `Hi ${handle.trim()},` : "Hi,";
|
||||
}
|
||||
|
||||
function emailShell(args: { preheader: string; title: string; body: string }) {
|
||||
return `<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
|
||||
<title>${escapeHtml(args.title)}</title>
|
||||
</head>
|
||||
<body style="margin:0;background:#ffffff;color:#1f2328;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Helvetica,Arial,sans-serif;">
|
||||
<span style="display:none!important;visibility:hidden;opacity:0;color:transparent;height:0;width:0;overflow:hidden;">${escapeHtml(
|
||||
args.preheader,
|
||||
)}</span>
|
||||
<table role="presentation" width="100%" cellspacing="0" cellpadding="0" style="background:#ffffff;margin:0;padding:24px 16px;">
|
||||
<tr>
|
||||
<td align="center">
|
||||
<table role="presentation" width="100%" cellspacing="0" cellpadding="0" style="max-width:640px;background:#ffffff;">
|
||||
<tr>
|
||||
<td style="padding:0;font-size:15px;line-height:22px;color:#1f2328;">
|
||||
${args.body}
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</body>
|
||||
</html>`;
|
||||
}
|
||||
|
||||
function textLink(href: string, label: string) {
|
||||
return `<a href="${escapeHtml(href)}" style="color:#0969da;text-decoration:underline;">${escapeHtml(label)}</a>`;
|
||||
}
|
||||
|
||||
function detailLine(label: string, value: string) {
|
||||
return `<p style="margin:0 0 6px;font-size:15px;line-height:22px;color:#1f2328;"><strong>${escapeHtml(label)}:</strong> ${escapeHtml(value)}</p>`;
|
||||
}
|
||||
|
||||
function sectionHeading(label: string) {
|
||||
return `<p style="margin:18px 0 8px;font-size:15px;line-height:22px;color:#1f2328;"><strong>${escapeHtml(label)}</strong></p>`;
|
||||
}
|
||||
|
||||
function paragraph(value: string) {
|
||||
return `<p style="margin:0 0 14px;font-size:15px;line-height:22px;color:#1f2328;">${escapeHtml(value)}</p>`;
|
||||
}
|
||||
|
||||
function bulletList(items: string[]) {
|
||||
return `<ul style="margin:0 0 14px;padding-left:22px;font-size:15px;line-height:22px;color:#1f2328;">${items
|
||||
.map((item) => `<li style="margin:0 0 6px;">${escapeHtml(item)}</li>`)
|
||||
.join("")}</ul>`;
|
||||
}
|
||||
|
||||
function commandBlock(command: string) {
|
||||
return `<pre style="margin:8px 0 14px;padding:10px 12px;background:#f6f8fa;border:1px solid #d8dee4;border-radius:6px;white-space:pre-wrap;color:#1f2328;font-family:ui-monospace,SFMono-Regular,Consolas,'Liberation Mono',monospace;font-size:13px;line-height:20px;"><code>${escapeHtml(command)}</code></pre>`;
|
||||
}
|
||||
|
||||
function buildScanDownloadCommand(args: MaliciousArtifactEmailArgs) {
|
||||
const version = args.version?.trim() || "<version>";
|
||||
const kindFlag = args.artifact.kind === "plugin" ? " --kind plugin" : "";
|
||||
return `clawhub scan download ${args.artifact.name} --version ${version}${kindFlag}`;
|
||||
}
|
||||
|
||||
function buildPluginValidateCommand() {
|
||||
return "clawhub package validate <path-to-plugin>";
|
||||
}
|
||||
|
||||
function normalizeEmailFindingSummary(value: string | undefined) {
|
||||
const normalized = value?.replace(/\s+/g, " ").trim();
|
||||
if (!normalized) return undefined;
|
||||
if (normalized.length <= MAX_EMAIL_FINDING_SUMMARY_LENGTH) return normalized;
|
||||
return `${normalized.slice(0, MAX_EMAIL_FINDING_SUMMARY_LENGTH - 3).trimEnd()}...`;
|
||||
}
|
||||
|
||||
export function buildBanNotificationEmail(args: BanNotificationEmailArgs): TransactionalEmail {
|
||||
const summary = summarizeBanReason(args);
|
||||
const artifact = args.artifact ?? null;
|
||||
const context: BanNotificationEmailContext = {
|
||||
appealUrl: APPEALS_URL,
|
||||
artifact,
|
||||
scannerLabel: summary.scannerLabel,
|
||||
findingSummary: summary.findingSummary,
|
||||
};
|
||||
|
||||
const lines = [
|
||||
greeting(args.handle),
|
||||
"",
|
||||
"Your ClawHub account was disabled.",
|
||||
`Reason: ${context.findingSummary}`,
|
||||
];
|
||||
if (artifact) lines.push(artifactLabel(artifact));
|
||||
|
||||
lines.push(
|
||||
"",
|
||||
"What changed:",
|
||||
"- Your ClawHub account cannot sign in.",
|
||||
"- Existing API tokens for the account have been revoked.",
|
||||
"- Published listings owned by the account may be hidden from public view.",
|
||||
"",
|
||||
`Appeal: ${APPEALS_URL}`,
|
||||
);
|
||||
|
||||
lines.push("", "ClawHub Security");
|
||||
|
||||
const impactItems = [
|
||||
"Your ClawHub account cannot sign in.",
|
||||
"Existing API tokens for the account have been revoked.",
|
||||
"Published listings owned by the account may be hidden from public view.",
|
||||
];
|
||||
const detailLines = [
|
||||
detailLine("Reason", context.findingSummary),
|
||||
...(artifact
|
||||
? [detailLine(artifact.kind === "skill" ? "Skill" : "Plugin", artifact.name)]
|
||||
: []),
|
||||
].join("");
|
||||
|
||||
const html = emailShell({
|
||||
title: "Your ClawHub account was disabled",
|
||||
preheader: context.findingSummary,
|
||||
body: [
|
||||
paragraph(greeting(args.handle)),
|
||||
paragraph("We disabled your ClawHub account after an account-safety review."),
|
||||
detailLines,
|
||||
sectionHeading("What changed"),
|
||||
bulletList(impactItems),
|
||||
`<p style="margin:0 0 14px;font-size:15px;line-height:22px;color:#1f2328;">You can ${textLink(APPEALS_URL, "appeal this decision")} if you believe this was a mistake.</p>`,
|
||||
paragraph("ClawHub Security"),
|
||||
].join(""),
|
||||
});
|
||||
|
||||
return {
|
||||
subject: "Your ClawHub account was disabled",
|
||||
context,
|
||||
text: lines.join("\n"),
|
||||
html,
|
||||
};
|
||||
}
|
||||
|
||||
export function buildRestoredAccountEmail(args: RestoredAccountEmailArgs) {
|
||||
const restoredListings = args.restoredListings ?? [];
|
||||
const listingLines = restoredListings.map(artifactLabel);
|
||||
const lines = [
|
||||
greeting(args.handle),
|
||||
"",
|
||||
"Your ClawHub account can sign in again.",
|
||||
"Previously revoked API tokens stay revoked. Create a new token before using the CLI or API again.",
|
||||
];
|
||||
if (listingLines.length > 0) {
|
||||
lines.push("", "Restored listings:", ...listingLines);
|
||||
}
|
||||
lines.push("", "ClawHub Security");
|
||||
|
||||
const html = emailShell({
|
||||
title: "Your ClawHub account was restored",
|
||||
preheader: "Your ClawHub account can sign in again.",
|
||||
body: [
|
||||
paragraph(greeting(args.handle)),
|
||||
paragraph("Your ClawHub account can sign in again."),
|
||||
paragraph(
|
||||
"Previously revoked API tokens stay revoked. Create a new token before using the CLI or API again.",
|
||||
),
|
||||
listingLines.length > 0
|
||||
? `${sectionHeading("Restored listings")}${bulletList(listingLines)}`
|
||||
: "",
|
||||
`<p style="margin:0 0 14px;font-size:15px;line-height:22px;color:#1f2328;">Settings: ${textLink("https://clawhub.ai/settings", "open ClawHub settings")}</p>`,
|
||||
paragraph("ClawHub Security"),
|
||||
].join(""),
|
||||
});
|
||||
|
||||
return {
|
||||
subject: "Your ClawHub account was restored",
|
||||
text: lines.join("\n"),
|
||||
html,
|
||||
};
|
||||
}
|
||||
|
||||
export function buildMaliciousArtifactEmail(args: MaliciousArtifactEmailArgs) {
|
||||
const artifactKind = args.artifact.kind === "skill" ? "skill" : "plugin";
|
||||
const artifactLabelText = artifactLabel(args.artifact);
|
||||
const scanDownloadCommand = buildScanDownloadCommand(args);
|
||||
const findingSummary =
|
||||
normalizeEmailFindingSummary(args.findingSummary) ??
|
||||
(args.trigger?.includes("static") === true
|
||||
? "Static analysis flagged malicious upload patterns."
|
||||
: args.trigger?.includes("virustotal") === true || args.trigger?.includes("vt_") === true
|
||||
? "VirusTotal telemetry contributed to a malicious upload finding."
|
||||
: "ClawScan classified the uploaded artifact as malicious.");
|
||||
const subject = `ClawHub blocked a ${artifactKind} version`;
|
||||
|
||||
const lines = [
|
||||
greeting(args.handle),
|
||||
"",
|
||||
`ClawHub blocked a ${artifactKind} version after a security scan.`,
|
||||
`Reason: ${findingSummary}`,
|
||||
artifactLabelText,
|
||||
];
|
||||
if (args.version?.trim()) lines.push(`Version: ${args.version.trim()}`);
|
||||
lines.push(
|
||||
"",
|
||||
"What changed:",
|
||||
"- This version was not made public.",
|
||||
"- Your account can still sign in.",
|
||||
`- You can upload a fixed version of this ${artifactKind}.`,
|
||||
`- ${MALICIOUS_REJECTION_ACCOUNT_WARNING}`,
|
||||
"",
|
||||
"Download the scan results for the blocked submitted version:",
|
||||
scanDownloadCommand,
|
||||
`Docs: ${MODERATION_GUIDELINES_URL}`,
|
||||
`Increment the version number before uploading the fixed ${artifactKind}.`,
|
||||
"",
|
||||
"ClawHub Security",
|
||||
);
|
||||
|
||||
const detailLines = [
|
||||
detailLine("Reason", findingSummary),
|
||||
detailLine(args.artifact.kind === "skill" ? "Skill" : "Plugin", args.artifact.name),
|
||||
...(args.version?.trim() ? [detailLine("Version", args.version.trim())] : []),
|
||||
].join("");
|
||||
|
||||
const html = emailShell({
|
||||
title: subject,
|
||||
preheader: `${artifactLabelText} was blocked by ClawHub security scans.`,
|
||||
body: [
|
||||
paragraph(greeting(args.handle)),
|
||||
paragraph(`ClawHub blocked a ${artifactKind} version after a security scan.`),
|
||||
detailLines,
|
||||
sectionHeading("What changed"),
|
||||
bulletList([
|
||||
"This version was not made public.",
|
||||
"Your account can still sign in.",
|
||||
`You can upload a fixed version of this ${artifactKind}.`,
|
||||
MALICIOUS_REJECTION_ACCOUNT_WARNING,
|
||||
]),
|
||||
sectionHeading("Review the blocked-version scan results"),
|
||||
paragraph("Download the scan results for the blocked submitted version."),
|
||||
commandBlock(scanDownloadCommand),
|
||||
paragraph(`Increment the version number before uploading the fixed ${artifactKind}.`),
|
||||
`<p style="margin:0 0 14px;font-size:15px;line-height:22px;color:#1f2328;">Docs: ${textLink(MODERATION_GUIDELINES_URL, "moderation and account safety")}</p>`,
|
||||
paragraph("ClawHub Security"),
|
||||
].join(""),
|
||||
});
|
||||
|
||||
return {
|
||||
subject,
|
||||
text: lines.join("\n"),
|
||||
html,
|
||||
};
|
||||
}
|
||||
|
||||
export function buildPackageInspectorFindingsEmail(args: PackageInspectorFindingsEmailArgs) {
|
||||
const targetOpenClawVersion = args.findings.find(
|
||||
(finding) => finding.targetOpenClawVersion,
|
||||
)?.targetOpenClawVersion;
|
||||
const validateCommand = buildPluginValidateCommand();
|
||||
const subject = `Plugin Inspector findings for ${args.packageName}@${args.version}`;
|
||||
const findingCount = args.findings.length;
|
||||
const intro = `We found ${findingCount} ${findingCount === 1 ? "issue" : "issues"} with version ${args.version} of ${args.packageName}.`;
|
||||
const nextSteps = [
|
||||
"Address the findings below in your plugin package.",
|
||||
"Run the validation command locally against your changes.",
|
||||
"When validation passes, upload a new version.",
|
||||
];
|
||||
const findingLines = formatPackageInspectorFindingsText(args.findings);
|
||||
const metadataLines = [
|
||||
`Plugin: ${args.packageName}@${args.version}`,
|
||||
targetOpenClawVersion ? `OpenClaw Version: ${targetOpenClawVersion}` : null,
|
||||
].filter((line): line is string => line !== null);
|
||||
const lines = [
|
||||
greeting(args.handle),
|
||||
"",
|
||||
intro,
|
||||
"",
|
||||
...metadataLines,
|
||||
"",
|
||||
"Next steps:",
|
||||
...nextSteps.map((item) => `- ${item}`),
|
||||
"",
|
||||
"Findings:",
|
||||
...findingLines,
|
||||
"",
|
||||
"Validate a local fix:",
|
||||
validateCommand,
|
||||
];
|
||||
|
||||
const detailLines = [
|
||||
detailLine("Plugin", `${args.packageName}@${args.version}`),
|
||||
...(targetOpenClawVersion ? [detailLine("OpenClaw Version", targetOpenClawVersion)] : []),
|
||||
].join("");
|
||||
const html = emailShell({
|
||||
title: "Plugin Inspector findings",
|
||||
preheader: intro,
|
||||
body: [
|
||||
paragraph(greeting(args.handle)),
|
||||
paragraph(intro),
|
||||
detailLines,
|
||||
sectionHeading("Next steps"),
|
||||
bulletList(nextSteps),
|
||||
sectionHeading("Findings"),
|
||||
formatPackageInspectorFindingsHtml(args.findings),
|
||||
sectionHeading("Validate a local fix"),
|
||||
commandBlock(validateCommand),
|
||||
].join(""),
|
||||
});
|
||||
|
||||
return {
|
||||
subject,
|
||||
text: lines.join("\n"),
|
||||
html,
|
||||
};
|
||||
}
|
||||
|
||||
function formatPackageInspectorFindingsText(findings: PackageInspectorEmailFinding[]) {
|
||||
if (findings.length === 0) return ["- No findings were included."];
|
||||
return findings.flatMap((finding) => {
|
||||
const lines = [
|
||||
`- **${finding.findingKind.toUpperCase()}** \`${finding.code}\`${formatFindingMetaText(finding)}`,
|
||||
` ${finding.message}`,
|
||||
];
|
||||
if (finding.authorRemediation?.summary) {
|
||||
lines.push(" Fix:");
|
||||
lines.push(` ${finding.authorRemediation.summary}`);
|
||||
if (finding.authorRemediation.docsUrl) {
|
||||
lines.push(" Docs:");
|
||||
lines.push(` ${finding.authorRemediation.docsUrl}`);
|
||||
}
|
||||
}
|
||||
return lines;
|
||||
});
|
||||
}
|
||||
|
||||
function formatFindingMetaText(finding: PackageInspectorEmailFinding) {
|
||||
const meta = [finding.issueClass, finding.severity].filter(Boolean).join(", ");
|
||||
return meta ? ` (${meta})` : "";
|
||||
}
|
||||
|
||||
function formatPackageInspectorFindingsHtml(findings: PackageInspectorEmailFinding[]) {
|
||||
if (findings.length === 0) return paragraph("No findings were included.");
|
||||
return findings
|
||||
.map((finding) => {
|
||||
const meta = [finding.issueClass, finding.severity].filter(Boolean).join(" · ");
|
||||
const remediation = finding.authorRemediation?.summary
|
||||
? `<div style="margin:10px 0 0;padding-top:10px;border-top:1px solid #d8dee4;">
|
||||
<p style="margin:0 0 6px;font-size:15px;line-height:22px;color:#1f2328;"><strong>Fix</strong></p>
|
||||
<p style="margin:0 0 6px;font-size:15px;line-height:22px;color:#1f2328;">${escapeHtml(finding.authorRemediation.summary)}</p>
|
||||
${
|
||||
finding.authorRemediation.docsUrl
|
||||
? `<p style="margin:10px 0 6px;font-size:15px;line-height:22px;color:#1f2328;"><strong>Docs</strong></p>
|
||||
<p style="margin:0;font-size:15px;line-height:22px;color:#1f2328;word-break:break-word;">${textLink(finding.authorRemediation.docsUrl, finding.authorRemediation.docsUrl)}</p>`
|
||||
: ""
|
||||
}
|
||||
</div>`
|
||||
: "";
|
||||
return `<div style="margin:0 0 10px;padding:10px 12px;border:1px solid #d8dee4;border-radius:6px;background:#ffffff;">
|
||||
<p style="margin:0 0 6px;font-size:15px;line-height:22px;color:#1f2328;">
|
||||
<strong>${escapeHtml(finding.findingKind.toUpperCase())}</strong>
|
||||
<code style="font-family:ui-monospace,SFMono-Regular,Consolas,'Liberation Mono',monospace;font-size:13px;background:#f6f8fa;border:1px solid #d8dee4;border-radius:4px;padding:1px 4px;">${escapeHtml(finding.code)}</code>
|
||||
${meta ? `<span style="color:#57606a;">${escapeHtml(meta)}</span>` : ""}
|
||||
</p>
|
||||
<p style="margin:0;font-size:15px;line-height:22px;color:#1f2328;">${escapeHtml(finding.message)}</p>
|
||||
${remediation}
|
||||
</div>`;
|
||||
})
|
||||
.join("");
|
||||
}
|
||||
@@ -282,128 +282,13 @@ describe("requireGitHubAccountAge", () => {
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
"https://api.github.com/user/12345",
|
||||
expect.objectContaining({
|
||||
headers: expect.objectContaining({
|
||||
headers: {
|
||||
"User-Agent": "clawhub",
|
||||
Authorization: "Bearer ghp_test123",
|
||||
}),
|
||||
},
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("omits Authorization header when GITHUB_TOKEN is blank", async () => {
|
||||
vi.useFakeTimers();
|
||||
const now = new Date("2026-02-02T12:00:00Z");
|
||||
vi.setSystemTime(now);
|
||||
|
||||
vi.stubEnv("GITHUB_TOKEN", " ");
|
||||
|
||||
const runQuery = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({
|
||||
_id: "users:1",
|
||||
githubCreatedAt: undefined,
|
||||
})
|
||||
.mockResolvedValueOnce("12345");
|
||||
const runMutation = vi.fn();
|
||||
const fetchMock = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
created_at: "2020-01-01T00:00:00Z",
|
||||
}),
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
await requireGitHubAccountAge({ runQuery, runMutation } as never, "users:1" as never);
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
"https://api.github.com/user/12345",
|
||||
expect.objectContaining({
|
||||
headers: expect.objectContaining({ "User-Agent": "clawhub" }),
|
||||
}),
|
||||
);
|
||||
expect(fetchMock.mock.calls[0]?.[1]?.headers).not.toHaveProperty("Authorization");
|
||||
});
|
||||
|
||||
it("retries without Authorization when GITHUB_TOKEN is rejected", async () => {
|
||||
vi.useFakeTimers();
|
||||
const now = new Date("2026-02-02T12:00:00Z");
|
||||
vi.setSystemTime(now);
|
||||
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
|
||||
vi.stubEnv("GITHUB_TOKEN", "ghp_expired");
|
||||
|
||||
const runQuery = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({
|
||||
_id: "users:1",
|
||||
githubCreatedAt: undefined,
|
||||
})
|
||||
.mockResolvedValueOnce("12345");
|
||||
const runMutation = vi.fn();
|
||||
const fetchMock = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({ ok: false, status: 401 })
|
||||
.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
created_at: "2020-01-01T00:00:00Z",
|
||||
}),
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
await requireGitHubAccountAge({ runQuery, runMutation } as never, "users:1" as never);
|
||||
|
||||
expect(fetchMock).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
"https://api.github.com/user/12345",
|
||||
expect.objectContaining({
|
||||
headers: expect.objectContaining({
|
||||
"User-Agent": "clawhub",
|
||||
Authorization: "Bearer ghp_expired",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(fetchMock).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
"https://api.github.com/user/12345",
|
||||
expect.objectContaining({
|
||||
headers: { "User-Agent": "clawhub" },
|
||||
}),
|
||||
);
|
||||
expect(runMutation).toHaveBeenCalledWith(internal.users.setGitHubCreatedAtInternal, {
|
||||
userId: "users:1",
|
||||
githubCreatedAt: Date.parse("2020-01-01T00:00:00Z"),
|
||||
});
|
||||
expect(warnSpy).toHaveBeenCalledWith(
|
||||
"[githubAccount] GitHub API auth was rejected; retrying lookup without auth",
|
||||
);
|
||||
});
|
||||
|
||||
it("does not retry unauthenticated 401 responses", async () => {
|
||||
const runQuery = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({
|
||||
_id: "users:1",
|
||||
githubCreatedAt: undefined,
|
||||
})
|
||||
.mockResolvedValueOnce("12345");
|
||||
const runMutation = vi.fn();
|
||||
const fetchMock = vi.fn().mockResolvedValue({ ok: false, status: 401 });
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
await expect(
|
||||
requireGitHubAccountAge({ runQuery, runMutation } as never, "users:1" as never),
|
||||
).rejects.toThrow(/GitHub account lookup failed/i);
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
"https://api.github.com/user/12345",
|
||||
expect.objectContaining({
|
||||
headers: expect.objectContaining({ "User-Agent": "clawhub" }),
|
||||
}),
|
||||
);
|
||||
expect(fetchMock.mock.calls[0]?.[1]?.headers).not.toHaveProperty("Authorization");
|
||||
});
|
||||
});
|
||||
|
||||
describe("syncGitHubProfile", () => {
|
||||
|
||||
+26
-30
@@ -2,7 +2,6 @@ import { ConvexError } from "convex/values";
|
||||
import { internal } from "../_generated/api";
|
||||
import type { Id } from "../_generated/dataModel";
|
||||
import type { ActionCtx } from "../_generated/server";
|
||||
import { buildGitHubApiHeaders } from "./githubAuth";
|
||||
import { GITHUB_PROFILE_SYNC_WINDOW_MS } from "./githubProfileSync";
|
||||
|
||||
const GITHUB_API = "https://api.github.com";
|
||||
@@ -23,34 +22,13 @@ function assertGitHubNumericId(providerAccountId: string) {
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchGitHubUserByNumericId(providerAccountId: string) {
|
||||
assertGitHubNumericId(providerAccountId);
|
||||
const url = `${GITHUB_API}/user/${providerAccountId}`;
|
||||
const headers = await buildGitHubApiHeaders({ userAgent: "clawhub" });
|
||||
const response = await fetch(url, {
|
||||
headers,
|
||||
});
|
||||
if (response.status !== 401 || !headers.Authorization) return response;
|
||||
|
||||
console.warn("[githubAccount] GitHub API auth was rejected; retrying lookup without auth");
|
||||
return await fetch(url, {
|
||||
headers: { "User-Agent": "clawhub" },
|
||||
});
|
||||
}
|
||||
|
||||
export async function fetchGitHubCreatedAtByProviderAccountId(providerAccountId: string) {
|
||||
const response = await fetchGitHubUserByNumericId(providerAccountId);
|
||||
if (!response.ok) {
|
||||
if (response.status === 403 || response.status === 429) {
|
||||
throw new ConvexError("GitHub API rate limit exceeded — please try again in a few minutes");
|
||||
}
|
||||
throw new ConvexError("GitHub account lookup failed");
|
||||
function buildGitHubHeaders() {
|
||||
const headers: Record<string, string> = { "User-Agent": "clawhub" };
|
||||
const token = process.env.GITHUB_TOKEN;
|
||||
if (token) {
|
||||
headers.Authorization = `Bearer ${token}`;
|
||||
}
|
||||
|
||||
const payload = (await response.json()) as GitHubUser;
|
||||
const parsed = payload.created_at ? Date.parse(payload.created_at) : Number.NaN;
|
||||
if (!Number.isFinite(parsed)) throw new ConvexError("GitHub account lookup failed");
|
||||
return parsed;
|
||||
return headers;
|
||||
}
|
||||
|
||||
export async function requireGitHubAccountAge(ctx: GitHubAccountGateCtx, userId: Id<"users">) {
|
||||
@@ -70,8 +48,24 @@ export async function requireGitHubAccountAge(ctx: GitHubAccountGateCtx, userId:
|
||||
// Invariant: GitHub is our only auth provider, so this should never happen.
|
||||
throw new ConvexError("GitHub account required");
|
||||
}
|
||||
assertGitHubNumericId(providerAccountId);
|
||||
|
||||
createdAt = await fetchGitHubCreatedAtByProviderAccountId(providerAccountId);
|
||||
// Fetch by immutable GitHub numeric ID to avoid username swap attacks entirely.
|
||||
const response = await fetch(`${GITHUB_API}/user/${providerAccountId}`, {
|
||||
headers: buildGitHubHeaders(),
|
||||
});
|
||||
if (!response.ok) {
|
||||
if (response.status === 403 || response.status === 429) {
|
||||
throw new ConvexError("GitHub API rate limit exceeded — please try again in a few minutes");
|
||||
}
|
||||
throw new ConvexError("GitHub account lookup failed");
|
||||
}
|
||||
|
||||
const payload = (await response.json()) as GitHubUser;
|
||||
const parsed = payload.created_at ? Date.parse(payload.created_at) : Number.NaN;
|
||||
if (!Number.isFinite(parsed)) throw new ConvexError("GitHub account lookup failed");
|
||||
|
||||
createdAt = parsed;
|
||||
await ctx.runMutation(internal.users.setGitHubCreatedAtInternal, {
|
||||
userId,
|
||||
githubCreatedAt: createdAt,
|
||||
@@ -113,7 +107,9 @@ export async function syncGitHubProfile(ctx: ActionCtx, userId: Id<"users">) {
|
||||
|
||||
assertGitHubNumericId(providerAccountId);
|
||||
|
||||
const response = await fetchGitHubUserByNumericId(providerAccountId);
|
||||
const response = await fetch(`${GITHUB_API}/user/${providerAccountId}`, {
|
||||
headers: buildGitHubHeaders(),
|
||||
});
|
||||
if (!response.ok) {
|
||||
// Silently fail - this is a best-effort sync, not critical path
|
||||
console.warn(`[syncGitHubProfile] GitHub API error for user ${userId}: ${response.status}`);
|
||||
|
||||
@@ -76,34 +76,6 @@ describe("fetchGitHubRepositoryIdentity", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("does not use GitHub App auth for arbitrary repository lookup", async () => {
|
||||
vi.stubEnv("GITHUB_APP_ID", "123");
|
||||
vi.stubEnv("GITHUB_APP_INSTALLATION_ID", "456");
|
||||
vi.stubEnv("GITHUB_APP_PRIVATE_KEY", "not-needed-for-this-test");
|
||||
vi.stubEnv("GITHUB_TOKEN", "ghs_test_token");
|
||||
const fetchMock = vi.fn(async () =>
|
||||
Response.json({
|
||||
id: 123,
|
||||
full_name: "openclaw/clawhub",
|
||||
owner: { login: "openclaw", id: 456 },
|
||||
}),
|
||||
);
|
||||
|
||||
await fetchGitHubRepositoryIdentity("openclaw/clawhub", fetchMock);
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
"https://api.github.com/repos/openclaw/clawhub",
|
||||
expect.objectContaining({
|
||||
headers: expect.objectContaining({
|
||||
Accept: "application/vnd.github+json",
|
||||
Authorization: "Bearer ghs_test_token",
|
||||
"User-Agent": "clawhub/package-trusted-publisher",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("omits Authorization for repository lookup when GITHUB_TOKEN is blank", async () => {
|
||||
vi.stubEnv("GITHUB_TOKEN", " ");
|
||||
const fetchMock = vi.fn(async () =>
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
import { buildGitHubApiHeaders } from "./githubAuth";
|
||||
|
||||
type JwtHeader = {
|
||||
alg?: unknown;
|
||||
kid?: unknown;
|
||||
@@ -219,7 +217,7 @@ export async function fetchGitHubRepositoryIdentity(
|
||||
throw new Error(`Invalid GitHub repository: ${repository}`);
|
||||
}
|
||||
const response = await fetchImpl(`https://api.github.com/repos/${normalizedRepository}`, {
|
||||
headers: await buildGitHubRepositoryLookupHeaders(fetchImpl),
|
||||
headers: buildGitHubRepositoryLookupHeaders(),
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
@@ -241,16 +239,16 @@ export async function fetchGitHubRepositoryIdentity(
|
||||
};
|
||||
}
|
||||
|
||||
async function buildGitHubRepositoryLookupHeaders(fetchImpl: typeof fetch) {
|
||||
return await buildGitHubApiHeaders({
|
||||
accept: "application/vnd.github+json",
|
||||
fetchImpl,
|
||||
userAgent: "clawhub/package-trusted-publisher",
|
||||
// This lookup accepts arbitrary public repositories. GitHub App installation
|
||||
// tokens only see repositories where the App is installed, so prefer PAT or
|
||||
// anonymous auth here.
|
||||
useGitHubApp: false,
|
||||
});
|
||||
function buildGitHubRepositoryLookupHeaders() {
|
||||
const headers: Record<string, string> = {
|
||||
Accept: "application/vnd.github+json",
|
||||
"User-Agent": "clawhub/package-trusted-publisher",
|
||||
};
|
||||
const token = process.env.GITHUB_TOKEN?.trim();
|
||||
if (token) {
|
||||
headers.Authorization = `Bearer ${token}`;
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
|
||||
export function normalizeGitHubRepository(repository: string) {
|
||||
|
||||
@@ -1,100 +0,0 @@
|
||||
/* @vitest-environment node */
|
||||
|
||||
import { generateKeyPairSync } from "node:crypto";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { buildGitHubApiHeaders, createGitHubAppInstallationToken } from "./githubAuth";
|
||||
|
||||
function stubGitHubAppEnv() {
|
||||
const { privateKey } = generateKeyPairSync("rsa", {
|
||||
modulusLength: 2048,
|
||||
privateKeyEncoding: { type: "pkcs1", format: "pem" },
|
||||
publicKeyEncoding: { type: "spki", format: "pem" },
|
||||
});
|
||||
vi.stubEnv("GITHUB_APP_ID", "3536245");
|
||||
vi.stubEnv("GITHUB_APP_INSTALLATION_ID", "987654");
|
||||
vi.stubEnv("GITHUB_APP_PRIVATE_KEY", privateKey);
|
||||
}
|
||||
|
||||
describe("githubAuth", () => {
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
it("mints a GitHub App installation token from app credentials", async () => {
|
||||
stubGitHubAppEnv();
|
||||
const fetchMock = vi.fn(async () =>
|
||||
Response.json({
|
||||
token: "ghs_app_token",
|
||||
expires_at: "2026-02-02T13:00:00Z",
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(
|
||||
createGitHubAppInstallationToken({ fetchImpl: fetchMock, userAgent: "clawhub/test" }),
|
||||
).resolves.toEqual({
|
||||
token: "ghs_app_token",
|
||||
expiresAt: Date.parse("2026-02-02T13:00:00Z"),
|
||||
});
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
"https://api.github.com/app/installations/987654/access_tokens",
|
||||
expect.objectContaining({
|
||||
method: "POST",
|
||||
headers: expect.objectContaining({
|
||||
Accept: "application/vnd.github+json",
|
||||
Authorization: expect.stringMatching(/^Bearer [^.]+\.[^.]+\.[^.]+$/),
|
||||
"User-Agent": "clawhub/test",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("builds API headers with GitHub App auth before PAT fallback", async () => {
|
||||
stubGitHubAppEnv();
|
||||
vi.stubEnv("GITHUB_TOKEN", "ghp_pat_token");
|
||||
const fetchMock = vi.fn(async () =>
|
||||
Response.json({
|
||||
token: "ghs_app_token",
|
||||
expires_at: "2026-02-02T13:00:00Z",
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(
|
||||
buildGitHubApiHeaders({ fetchImpl: fetchMock, userAgent: "clawhub/test" }),
|
||||
).resolves.toEqual({
|
||||
Accept: "application/vnd.github+json",
|
||||
Authorization: "Bearer ghs_app_token",
|
||||
"User-Agent": "clawhub/test",
|
||||
});
|
||||
});
|
||||
|
||||
it("falls back to GITHUB_TOKEN when GitHub App credentials are absent", async () => {
|
||||
vi.stubEnv("GITHUB_TOKEN", "ghp_pat_token");
|
||||
|
||||
await expect(buildGitHubApiHeaders({ userAgent: "clawhub/test" })).resolves.toEqual({
|
||||
Accept: "application/vnd.github+json",
|
||||
Authorization: "Bearer ghp_pat_token",
|
||||
"User-Agent": "clawhub/test",
|
||||
});
|
||||
});
|
||||
|
||||
it("can skip GitHub App auth for arbitrary public resources", async () => {
|
||||
stubGitHubAppEnv();
|
||||
vi.stubEnv("GITHUB_TOKEN", "ghp_pat_token");
|
||||
const fetchMock = vi.fn();
|
||||
|
||||
await expect(
|
||||
buildGitHubApiHeaders({
|
||||
fetchImpl: fetchMock,
|
||||
userAgent: "clawhub/test",
|
||||
useGitHubApp: false,
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
Accept: "application/vnd.github+json",
|
||||
Authorization: "Bearer ghp_pat_token",
|
||||
"User-Agent": "clawhub/test",
|
||||
});
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -1,272 +0,0 @@
|
||||
const GITHUB_API = "https://api.github.com";
|
||||
const DEFAULT_ACCEPT = "application/vnd.github+json";
|
||||
const DEFAULT_USER_AGENT = "clawhub/github-api";
|
||||
const APP_TOKEN_CACHE_BUFFER_MS = 60 * 1000;
|
||||
|
||||
type FetchImpl = typeof fetch;
|
||||
|
||||
type GitHubAppConfig = {
|
||||
appId: string;
|
||||
installationId: string;
|
||||
privateKey: string;
|
||||
};
|
||||
|
||||
type InstallationToken = {
|
||||
token: string;
|
||||
expiresAt: number;
|
||||
};
|
||||
|
||||
type CachedInstallationToken = InstallationToken & {
|
||||
cacheKey: string;
|
||||
};
|
||||
|
||||
let cachedInstallationToken: CachedInstallationToken | null = null;
|
||||
|
||||
export function isGitHubAppConfigured(env: NodeJS.ProcessEnv = process.env) {
|
||||
return Boolean(readGitHubAppConfig(env));
|
||||
}
|
||||
|
||||
export async function buildGitHubApiHeaders(options: {
|
||||
userAgent: string;
|
||||
accept?: string;
|
||||
fetchImpl?: FetchImpl;
|
||||
allowAnonymous?: boolean;
|
||||
useGitHubApp?: boolean;
|
||||
}): Promise<Record<string, string>> {
|
||||
const headers = buildGitHubHeaders({
|
||||
userAgent: options.userAgent,
|
||||
accept: options.accept,
|
||||
});
|
||||
|
||||
if (options.useGitHubApp !== false) {
|
||||
const appToken = await getCachedGitHubAppInstallationToken({
|
||||
fetchImpl: options.fetchImpl,
|
||||
userAgent: options.userAgent,
|
||||
});
|
||||
if (appToken) {
|
||||
headers.Authorization = `Bearer ${appToken}`;
|
||||
return headers;
|
||||
}
|
||||
}
|
||||
|
||||
const token = process.env.GITHUB_TOKEN?.trim();
|
||||
if (token) {
|
||||
headers.Authorization = `Bearer ${token}`;
|
||||
return headers;
|
||||
}
|
||||
|
||||
if (options.allowAnonymous === false) {
|
||||
throw new Error("GitHub API authentication is not configured");
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
|
||||
export function buildGitHubHeaders(options: {
|
||||
userAgent: string;
|
||||
accept?: string;
|
||||
token?: string;
|
||||
isAppJwt?: boolean;
|
||||
}) {
|
||||
const headers: Record<string, string> = {
|
||||
Accept: options.accept ?? DEFAULT_ACCEPT,
|
||||
"User-Agent": options.userAgent,
|
||||
};
|
||||
if (options.token) {
|
||||
headers.Authorization = `Bearer ${options.token}`;
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
|
||||
export async function createGitHubAppInstallationToken(
|
||||
options: {
|
||||
fetchImpl?: FetchImpl;
|
||||
userAgent?: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
now?: number;
|
||||
} = {},
|
||||
): Promise<InstallationToken> {
|
||||
const env = options.env ?? process.env;
|
||||
const config = readGitHubAppConfig(env);
|
||||
if (!config) throw new Error("GitHub App credentials missing");
|
||||
|
||||
const jwt = await createGitHubAppJwt(config.appId, config.privateKey, options.now ?? Date.now());
|
||||
const response = await (options.fetchImpl ?? fetch)(
|
||||
`${GITHUB_API}/app/installations/${config.installationId}/access_tokens`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: buildGitHubHeaders({
|
||||
userAgent: options.userAgent ?? DEFAULT_USER_AGENT,
|
||||
token: jwt,
|
||||
isAppJwt: true,
|
||||
}),
|
||||
},
|
||||
);
|
||||
if (!response.ok) {
|
||||
const message = await response.text();
|
||||
throw new Error(`GitHub App token failed: ${message}`);
|
||||
}
|
||||
|
||||
const payload = (await response.json()) as { token?: string; expires_at?: string };
|
||||
const token = payload.token?.trim();
|
||||
if (!token) throw new Error("GitHub App token missing");
|
||||
const expiresAt = payload.expires_at ? Date.parse(payload.expires_at) : Number.NaN;
|
||||
if (!Number.isFinite(expiresAt)) throw new Error("GitHub App token expiry missing");
|
||||
return { token, expiresAt };
|
||||
}
|
||||
|
||||
async function getCachedGitHubAppInstallationToken(options: {
|
||||
fetchImpl?: FetchImpl;
|
||||
userAgent: string;
|
||||
}) {
|
||||
const config = readGitHubAppConfig(process.env);
|
||||
if (!config) return null;
|
||||
|
||||
const now = Date.now();
|
||||
const cacheKey = `${config.appId}:${config.installationId}:${hashCacheKey(config.privateKey)}`;
|
||||
if (
|
||||
cachedInstallationToken?.cacheKey === cacheKey &&
|
||||
cachedInstallationToken.expiresAt - APP_TOKEN_CACHE_BUFFER_MS > now
|
||||
) {
|
||||
return cachedInstallationToken.token;
|
||||
}
|
||||
|
||||
try {
|
||||
const next = await createGitHubAppInstallationToken({
|
||||
fetchImpl: options.fetchImpl,
|
||||
userAgent: options.userAgent,
|
||||
now,
|
||||
});
|
||||
cachedInstallationToken = { ...next, cacheKey };
|
||||
return next.token;
|
||||
} catch (error) {
|
||||
console.warn(`[githubAuth] GitHub App token unavailable: ${errorMessage(error)}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function readGitHubAppConfig(env: NodeJS.ProcessEnv): GitHubAppConfig | null {
|
||||
const appId = env.GITHUB_APP_ID?.trim();
|
||||
const installationId = env.GITHUB_APP_INSTALLATION_ID?.trim();
|
||||
const privateKey = env.GITHUB_APP_PRIVATE_KEY?.trim();
|
||||
if (!appId || !installationId || !privateKey) return null;
|
||||
return { appId, installationId, privateKey };
|
||||
}
|
||||
|
||||
async function createGitHubAppJwt(appId: string, rawPrivateKey: string, nowMs: number) {
|
||||
const now = Math.floor(nowMs / 1000);
|
||||
const header = { alg: "RS256", typ: "JWT" };
|
||||
const payload = { iat: now - 60, exp: now + 9 * 60, iss: appId };
|
||||
const signingInput = `${base64UrlString(JSON.stringify(header))}.${base64UrlString(
|
||||
JSON.stringify(payload),
|
||||
)}`;
|
||||
const key = await importPrivateKey(rawPrivateKey);
|
||||
const signature = await crypto.subtle.sign(
|
||||
"RSASSA-PKCS1-v1_5",
|
||||
key,
|
||||
new TextEncoder().encode(signingInput),
|
||||
);
|
||||
return `${signingInput}.${base64UrlBytes(new Uint8Array(signature))}`;
|
||||
}
|
||||
|
||||
async function importPrivateKey(rawPrivateKey: string) {
|
||||
const { label, der } = parsePem(rawPrivateKey);
|
||||
const pkcs8 = label === "RSA PRIVATE KEY" ? wrapPkcs1PrivateKeyAsPkcs8(der) : der;
|
||||
return await crypto.subtle.importKey(
|
||||
"pkcs8",
|
||||
pkcs8,
|
||||
{ name: "RSASSA-PKCS1-v1_5", hash: "SHA-256" },
|
||||
false,
|
||||
["sign"],
|
||||
);
|
||||
}
|
||||
|
||||
function parsePem(raw: string) {
|
||||
const normalized = raw.replace(/\\n/g, "\n").trim();
|
||||
const match = /^-----BEGIN ([A-Z0-9 ]+)-----\s*([A-Za-z0-9+/=\s]+)\s*-----END \1-----$/m.exec(
|
||||
normalized,
|
||||
);
|
||||
if (!match) throw new Error("Invalid GitHub App private key");
|
||||
const label = match[1];
|
||||
if (label !== "PRIVATE KEY" && label !== "RSA PRIVATE KEY") {
|
||||
throw new Error(`Unsupported GitHub App private key type: ${label}`);
|
||||
}
|
||||
return { label, der: base64ToBytes(match[2]) };
|
||||
}
|
||||
|
||||
function wrapPkcs1PrivateKeyAsPkcs8(pkcs1: Uint8Array) {
|
||||
const version = derInteger(0);
|
||||
const rsaEncryptionAlgorithm = derSequence(
|
||||
new Uint8Array([0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01]),
|
||||
new Uint8Array([0x05, 0x00]),
|
||||
);
|
||||
return derSequence(version, rsaEncryptionAlgorithm, derOctetString(pkcs1));
|
||||
}
|
||||
|
||||
function derSequence(...parts: Uint8Array[]) {
|
||||
return derTagged(0x30, concatBytes(parts));
|
||||
}
|
||||
|
||||
function derInteger(value: number) {
|
||||
return derTagged(0x02, new Uint8Array([value]));
|
||||
}
|
||||
|
||||
function derOctetString(value: Uint8Array) {
|
||||
return derTagged(0x04, value);
|
||||
}
|
||||
|
||||
function derTagged(tag: number, value: Uint8Array) {
|
||||
return concatBytes([new Uint8Array([tag]), derLength(value.length), value]);
|
||||
}
|
||||
|
||||
function derLength(length: number) {
|
||||
if (length < 0x80) return new Uint8Array([length]);
|
||||
const bytes: number[] = [];
|
||||
let remaining = length;
|
||||
while (remaining > 0) {
|
||||
bytes.unshift(remaining & 0xff);
|
||||
remaining >>= 8;
|
||||
}
|
||||
return new Uint8Array([0x80 | bytes.length, ...bytes]);
|
||||
}
|
||||
|
||||
function concatBytes(parts: Uint8Array[]) {
|
||||
const total = parts.reduce((sum, part) => sum + part.length, 0);
|
||||
const out = new Uint8Array(total);
|
||||
let offset = 0;
|
||||
for (const part of parts) {
|
||||
out.set(part, offset);
|
||||
offset += part.length;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function base64UrlString(value: string) {
|
||||
return base64UrlBytes(new TextEncoder().encode(value));
|
||||
}
|
||||
|
||||
function base64UrlBytes(value: Uint8Array) {
|
||||
let binary = "";
|
||||
for (const byte of value) binary += String.fromCharCode(byte);
|
||||
return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/g, "");
|
||||
}
|
||||
|
||||
function base64ToBytes(value: string) {
|
||||
const binary = atob(value.replace(/\s/g, ""));
|
||||
const bytes = new Uint8Array(binary.length);
|
||||
for (let i = 0; i < binary.length; i += 1) {
|
||||
bytes[i] = binary.charCodeAt(i);
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
function hashCacheKey(value: string) {
|
||||
let hash = 0;
|
||||
for (let i = 0; i < value.length; i += 1) {
|
||||
hash = (hash * 31 + value.charCodeAt(i)) | 0;
|
||||
}
|
||||
return String(hash);
|
||||
}
|
||||
|
||||
function errorMessage(error: unknown) {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
@@ -1,8 +1,8 @@
|
||||
"use node";
|
||||
|
||||
import { createPrivateKey, createSign } from "node:crypto";
|
||||
import type { Id } from "../_generated/dataModel";
|
||||
import type { ActionCtx } from "../_generated/server";
|
||||
import { buildGitHubHeaders, createGitHubAppInstallationToken } from "./githubAuth";
|
||||
|
||||
const GITHUB_API = "https://api.github.com";
|
||||
const DEFAULT_REPO = "clawdbot/skills";
|
||||
@@ -93,7 +93,7 @@ export async function getGitHubBackupContext(): Promise<GitHubBackupContext> {
|
||||
const repo = process.env.GITHUB_SKILLS_REPO ?? DEFAULT_REPO;
|
||||
const root = process.env.GITHUB_SKILLS_ROOT ?? DEFAULT_ROOT;
|
||||
const [repoOwner, repoName] = parseRepo(repo);
|
||||
const { token } = await createGitHubAppInstallationToken({ userAgent: USER_AGENT });
|
||||
const token = await createInstallationToken();
|
||||
const repoInfo = await githubGet<RepoInfo>(token, `/repos/${repoOwner}/${repoName}`);
|
||||
const branch = repoInfo.default_branch ?? "main";
|
||||
|
||||
@@ -439,6 +439,48 @@ async function fetchStorageBase64(ctx: ActionCtx, storageId: Id<"_storage">) {
|
||||
return buffer.toString("base64");
|
||||
}
|
||||
|
||||
async function createInstallationToken() {
|
||||
const appId = process.env.GITHUB_APP_ID;
|
||||
const installationId = process.env.GITHUB_APP_INSTALLATION_ID;
|
||||
if (!appId || !installationId) {
|
||||
throw new Error("GitHub App credentials missing");
|
||||
}
|
||||
const jwt = createAppJwt(appId);
|
||||
const response = await fetch(`${GITHUB_API}/app/installations/${installationId}/access_tokens`, {
|
||||
method: "POST",
|
||||
headers: buildHeaders(jwt, true),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const message = await response.text();
|
||||
throw new Error(`GitHub App token failed: ${message}`);
|
||||
}
|
||||
const payload = (await response.json()) as { token?: string };
|
||||
if (!payload.token) throw new Error("GitHub App token missing");
|
||||
return payload.token;
|
||||
}
|
||||
|
||||
function createAppJwt(appId: string) {
|
||||
const privateKey = loadPrivateKey();
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const header = { alg: "RS256", typ: "JWT" };
|
||||
const payload = { iat: now - 60, exp: now + 9 * 60, iss: appId };
|
||||
const encodedHeader = base64Url(JSON.stringify(header));
|
||||
const encodedPayload = base64Url(JSON.stringify(payload));
|
||||
const signingInput = `${encodedHeader}.${encodedPayload}`;
|
||||
const sign = createSign("RSA-SHA256");
|
||||
sign.update(signingInput);
|
||||
sign.end();
|
||||
const signature = sign.sign(privateKey);
|
||||
return `${signingInput}.${base64Url(signature)}`;
|
||||
}
|
||||
|
||||
function loadPrivateKey() {
|
||||
const raw = process.env.GITHUB_APP_PRIVATE_KEY;
|
||||
if (!raw) throw new Error("GITHUB_APP_PRIVATE_KEY is not configured");
|
||||
const normalized = raw.replace(/\\n/g, "\n");
|
||||
return createPrivateKey(normalized);
|
||||
}
|
||||
|
||||
async function createBlob(token: string, repoOwner: string, repoName: string, content: string) {
|
||||
const result = await githubPost<{ sha: string }>(
|
||||
token,
|
||||
@@ -489,7 +531,11 @@ async function githubPatch(token: string, path: string, body: unknown) {
|
||||
}
|
||||
|
||||
function buildHeaders(token: string, isAppJwt = false) {
|
||||
return buildGitHubHeaders({ token, isAppJwt, userAgent: USER_AGENT });
|
||||
return {
|
||||
Authorization: `${isAppJwt ? "Bearer" : "token"} ${token}`,
|
||||
Accept: "application/vnd.github+json",
|
||||
"User-Agent": USER_AGENT,
|
||||
};
|
||||
}
|
||||
|
||||
function parseRepo(repo: string) {
|
||||
@@ -524,6 +570,11 @@ function encodePath(path: string) {
|
||||
.join("/");
|
||||
}
|
||||
|
||||
function base64Url(value: string | Uint8Array) {
|
||||
const buffer = typeof value === "string" ? Buffer.from(value) : Buffer.from(value);
|
||||
return buffer.toString("base64").replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/g, "");
|
||||
}
|
||||
|
||||
function toBase64(value: string) {
|
||||
return Buffer.from(value).toString("base64");
|
||||
}
|
||||
|
||||
@@ -91,7 +91,12 @@ export function parseGitHubImportUrl(input: string): GitHubImportUrl {
|
||||
}
|
||||
|
||||
function canonicalGitHubImportUrl(url: URL) {
|
||||
return `https://${url.hostname}${url.pathname}`;
|
||||
const canonical = new URL(url.toString());
|
||||
canonical.username = "";
|
||||
canonical.password = "";
|
||||
canonical.search = "";
|
||||
canonical.hash = "";
|
||||
return `${canonical.origin}${canonical.pathname}`;
|
||||
}
|
||||
|
||||
export async function resolveGitHubCommit(
|
||||
|
||||
@@ -1,566 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
buildGitHubSkillSourceSnapshot,
|
||||
buildGitHubSkillSyncPlan,
|
||||
parseSkillsShDisplayManifest,
|
||||
} from "./githubSkillSync";
|
||||
|
||||
const encoder = new TextEncoder();
|
||||
|
||||
function bytes(text: string) {
|
||||
return encoder.encode(text);
|
||||
}
|
||||
|
||||
function repoEntries(entries: Record<string, string>) {
|
||||
return Object.fromEntries(Object.entries(entries).map(([path, text]) => [path, bytes(text)]));
|
||||
}
|
||||
|
||||
describe("parseSkillsShDisplayManifest", () => {
|
||||
it("keeps the supported skills.sh rendering fields and drops invalid groups", () => {
|
||||
const result = parseSkillsShDisplayManifest(
|
||||
JSON.stringify({
|
||||
notGrouped: "top",
|
||||
groupings: [
|
||||
{
|
||||
title: "Agentic AI",
|
||||
description: "Agentic workflows.",
|
||||
skills: ["aiq-deploy", "nemoclaw-user-configure-security"],
|
||||
},
|
||||
{ title: "Broken", skills: [123] },
|
||||
{ description: "Missing title", skills: ["ignored"] },
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
expect(result).toEqual({
|
||||
status: "ok",
|
||||
manifest: {
|
||||
notGrouped: "top",
|
||||
groupings: [
|
||||
{
|
||||
title: "Agentic AI",
|
||||
description: "Agentic workflows.",
|
||||
skills: ["aiq-deploy", "nemoclaw-user-configure-security"],
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("marks missing and invalid manifests so the UI can fall back", () => {
|
||||
expect(parseSkillsShDisplayManifest(undefined)).toEqual({
|
||||
status: "missing",
|
||||
manifest: undefined,
|
||||
});
|
||||
expect(parseSkillsShDisplayManifest("{nope")).toEqual({
|
||||
status: "invalid",
|
||||
manifest: undefined,
|
||||
});
|
||||
expect(parseSkillsShDisplayManifest(JSON.stringify({ groupings: [] }))).toEqual({
|
||||
status: "invalid",
|
||||
manifest: undefined,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildGitHubSkillSourceSnapshot", () => {
|
||||
it("discovers skill folders, parses SKILL.md metadata, and hashes exact folder bytes", async () => {
|
||||
const baseEntries = repoEntries({
|
||||
"skills/aiq-deploy/SKILL.md":
|
||||
"---\nname: AIQ Deploy\nversion: 0.2.0\ndescription: Deploy AgentIQ workflows.\n---\n# AIQ Deploy\n",
|
||||
"skills/aiq-deploy/skill-card.md": "# Card\n",
|
||||
"skills/vision-helper/SKILL.md": "# Vision Helper\n",
|
||||
"skills.sh.json": JSON.stringify({
|
||||
groupings: [{ title: "Agentic AI", skills: ["aiq-deploy"] }],
|
||||
}),
|
||||
});
|
||||
const changedEntries = {
|
||||
...baseEntries,
|
||||
"skills/aiq-deploy/skill-card.md": bytes("# Card changed\n"),
|
||||
};
|
||||
|
||||
const base = await buildGitHubSkillSourceSnapshot({
|
||||
repo: "NVIDIA/skills",
|
||||
defaultBranch: "main",
|
||||
commit: "1".repeat(40),
|
||||
entries: baseEntries,
|
||||
});
|
||||
const changed = await buildGitHubSkillSourceSnapshot({
|
||||
repo: "NVIDIA/skills",
|
||||
defaultBranch: "main",
|
||||
commit: "2".repeat(40),
|
||||
entries: changedEntries,
|
||||
});
|
||||
|
||||
expect(base.manifestStatus).toBe("ok");
|
||||
expect(base.manifest).toEqual({
|
||||
groupings: [{ title: "Agentic AI", skills: ["aiq-deploy"] }],
|
||||
});
|
||||
expect(base.skills).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
slug: "aiq-deploy",
|
||||
displayName: "AIQ Deploy",
|
||||
summary: "Deploy AgentIQ workflows.",
|
||||
upstreamVersion: "0.2.0",
|
||||
path: "skills/aiq-deploy",
|
||||
skillMarkdownPath: "skills/aiq-deploy/SKILL.md",
|
||||
skillMarkdown:
|
||||
"---\nname: AIQ Deploy\nversion: 0.2.0\ndescription: Deploy AgentIQ workflows.\n---\n# AIQ Deploy\n",
|
||||
skillCardMarkdownPath: "skills/aiq-deploy/skill-card.md",
|
||||
skillCardMarkdown: "# Card\n",
|
||||
}),
|
||||
expect.objectContaining({
|
||||
slug: "vision-helper",
|
||||
displayName: "Vision Helper",
|
||||
path: "skills/vision-helper",
|
||||
skillMarkdownPath: "skills/vision-helper/SKILL.md",
|
||||
skillMarkdown: "# Vision Helper\n",
|
||||
}),
|
||||
]),
|
||||
);
|
||||
expect(changed.skills.find((skill) => skill.slug === "aiq-deploy")?.contentHash).not.toBe(
|
||||
base.skills.find((skill) => skill.slug === "aiq-deploy")?.contentHash,
|
||||
);
|
||||
expect(changed.skills.find((skill) => skill.slug === "vision-helper")?.contentHash).toBe(
|
||||
base.skills.find((skill) => skill.slug === "vision-helper")?.contentHash,
|
||||
);
|
||||
});
|
||||
|
||||
it("includes valid filenames containing dot-dot text in folder hashes", async () => {
|
||||
const base = await buildGitHubSkillSourceSnapshot({
|
||||
repo: "NVIDIA/skills",
|
||||
defaultBranch: "main",
|
||||
commit: "1".repeat(40),
|
||||
entries: repoEntries({
|
||||
"skills/aiq-deploy/SKILL.md": "# AIQ Deploy\n",
|
||||
"skills/aiq-deploy/payload..sh": "echo safe\n",
|
||||
}),
|
||||
});
|
||||
const changed = await buildGitHubSkillSourceSnapshot({
|
||||
repo: "NVIDIA/skills",
|
||||
defaultBranch: "main",
|
||||
commit: "2".repeat(40),
|
||||
entries: repoEntries({
|
||||
"skills/aiq-deploy/SKILL.md": "# AIQ Deploy\n",
|
||||
"skills/aiq-deploy/payload..sh": "echo changed\n",
|
||||
}),
|
||||
});
|
||||
|
||||
expect(changed.skills[0]?.contentHash).not.toBe(base.skills[0]?.contentHash);
|
||||
});
|
||||
|
||||
it("rejects duplicate normalized skill slugs before syncing content", async () => {
|
||||
await expect(
|
||||
buildGitHubSkillSourceSnapshot({
|
||||
repo: "NVIDIA/skills",
|
||||
defaultBranch: "main",
|
||||
commit: "1".repeat(40),
|
||||
entries: repoEntries({
|
||||
"skills/aiq_deploy/SKILL.md": "# AIQ Deploy A\n",
|
||||
"skills/aiq-deploy/SKILL.md": "# AIQ Deploy B\n",
|
||||
}),
|
||||
}),
|
||||
).rejects.toThrow(/duplicate normalized slug/i);
|
||||
});
|
||||
|
||||
it("prefers the top-level skills catalog folder over duplicate plugin copies", async () => {
|
||||
const snapshot = await buildGitHubSkillSourceSnapshot({
|
||||
repo: "NVIDIA/skills",
|
||||
defaultBranch: "main",
|
||||
commit: "1".repeat(40),
|
||||
entries: repoEntries({
|
||||
"skills/aiq-deploy/SKILL.md": "# AIQ Deploy\n",
|
||||
"plugins/nvidia-skills/skills/aiq-deploy/SKILL.md": "# Plugin Copy\n",
|
||||
"skills.sh.json": JSON.stringify({
|
||||
groupings: [{ title: "Agentic AI", skills: ["aiq-deploy"] }],
|
||||
}),
|
||||
}),
|
||||
});
|
||||
|
||||
expect(snapshot.skills.map((skill) => skill.path)).toEqual(["skills/aiq-deploy"]);
|
||||
expect(snapshot.skills[0]?.displayName).toBe("AIQ Deploy");
|
||||
});
|
||||
|
||||
it("rejects oversized cached markdown before writing Convex content docs", async () => {
|
||||
await expect(
|
||||
buildGitHubSkillSourceSnapshot({
|
||||
repo: "NVIDIA/skills",
|
||||
defaultBranch: "main",
|
||||
commit: "1".repeat(40),
|
||||
entries: repoEntries({
|
||||
"skills/aiq-deploy/SKILL.md": `# AIQ Deploy\n${"x".repeat(513 * 1024)}`,
|
||||
}),
|
||||
}),
|
||||
).rejects.toThrow(/too large to cache/i);
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildGitHubSkillSyncPlan", () => {
|
||||
it("marks changed upstream content pending", async () => {
|
||||
const snapshot = await buildGitHubSkillSourceSnapshot({
|
||||
repo: "NVIDIA/skills",
|
||||
defaultBranch: "main",
|
||||
commit: "2".repeat(40),
|
||||
entries: repoEntries({
|
||||
"skills/aiq-deploy/SKILL.md": "# AIQ Deploy v2\n",
|
||||
"skills.sh.json": JSON.stringify({
|
||||
groupings: [{ title: "Agentic AI", skills: ["aiq-deploy"] }],
|
||||
}),
|
||||
}),
|
||||
});
|
||||
|
||||
const plan = buildGitHubSkillSyncPlan({
|
||||
sourceId: "githubSkillSources:nvidia",
|
||||
ownerUserId: "users:nvidia",
|
||||
ownerPublisherId: "publishers:nvidia",
|
||||
existingSkills: [
|
||||
{
|
||||
_id: "skills:aiq-deploy",
|
||||
slug: "aiq-deploy",
|
||||
displayName: "AIQ Deploy",
|
||||
githubPath: "skills/aiq-deploy",
|
||||
githubCurrentStatus: "present",
|
||||
githubCurrentContentHash: "old-hash",
|
||||
githubScanStatus: "clean",
|
||||
},
|
||||
],
|
||||
snapshot,
|
||||
now: 123,
|
||||
});
|
||||
|
||||
expect(plan.skillPatches).toEqual([
|
||||
expect.objectContaining({
|
||||
skillId: "skills:aiq-deploy",
|
||||
slug: "aiq-deploy",
|
||||
patch: expect.objectContaining({
|
||||
githubCurrentCommit: "2".repeat(40),
|
||||
githubCurrentContentHash: snapshot.skills[0]?.contentHash,
|
||||
githubScanStatus: "pending",
|
||||
moderationStatus: "active",
|
||||
moderationReason: "pending.scan",
|
||||
}),
|
||||
}),
|
||||
]);
|
||||
expect(plan.skillInserts).toEqual([]);
|
||||
expect(plan.stats.changed).toBe(1);
|
||||
});
|
||||
|
||||
it("keeps clean scan status when only the repo commit changes", async () => {
|
||||
const snapshot = await buildGitHubSkillSourceSnapshot({
|
||||
repo: "NVIDIA/skills",
|
||||
defaultBranch: "main",
|
||||
commit: "2".repeat(40),
|
||||
entries: repoEntries({
|
||||
"skills/aiq-deploy/SKILL.md": "---\nversion: 0.2.0\n---\n# AIQ Deploy\n",
|
||||
}),
|
||||
});
|
||||
const contentHash = snapshot.skills[0]?.contentHash ?? "";
|
||||
|
||||
const plan = buildGitHubSkillSyncPlan({
|
||||
sourceId: "githubSkillSources:nvidia",
|
||||
ownerUserId: "users:nvidia",
|
||||
ownerPublisherId: "publishers:nvidia",
|
||||
existingSkills: [
|
||||
{
|
||||
_id: "skills:aiq-deploy",
|
||||
slug: "aiq-deploy",
|
||||
displayName: "AIQ Deploy",
|
||||
latestVersionSummary: {
|
||||
version: "0.2.0",
|
||||
createdAt: 7,
|
||||
},
|
||||
githubPath: "skills/aiq-deploy",
|
||||
githubCurrentStatus: "present",
|
||||
githubCurrentContentHash: contentHash,
|
||||
githubScanStatus: "clean",
|
||||
},
|
||||
],
|
||||
snapshot,
|
||||
now: 123,
|
||||
});
|
||||
|
||||
expect(plan.skillPatches[0]?.patch).toMatchObject({
|
||||
githubCurrentCommit: "2".repeat(40),
|
||||
githubCurrentContentHash: contentHash,
|
||||
githubScanStatus: "clean",
|
||||
moderationStatus: "active",
|
||||
moderationVerdict: "clean",
|
||||
});
|
||||
expect(plan.skillPatches[0]?.patch).not.toHaveProperty("updatedAt");
|
||||
expect(plan.skillPatches[0]?.patch).not.toHaveProperty("latestVersionSummary");
|
||||
expect(plan.stats.unchanged).toBe(1);
|
||||
});
|
||||
|
||||
it("updates existing skill ownership when a source is reassigned", async () => {
|
||||
const snapshot = await buildGitHubSkillSourceSnapshot({
|
||||
repo: "NVIDIA/skills",
|
||||
defaultBranch: "main",
|
||||
commit: "2".repeat(40),
|
||||
entries: repoEntries({
|
||||
"skills/aiq-deploy/SKILL.md": "# AIQ Deploy\n",
|
||||
}),
|
||||
});
|
||||
|
||||
const plan = buildGitHubSkillSyncPlan({
|
||||
sourceId: "githubSkillSources:nvidia",
|
||||
ownerUserId: "users:new-owner",
|
||||
ownerPublisherId: "publishers:new-owner",
|
||||
existingSkills: [
|
||||
{
|
||||
_id: "skills:aiq-deploy",
|
||||
slug: "aiq-deploy",
|
||||
displayName: "AIQ Deploy",
|
||||
githubPath: "skills/aiq-deploy",
|
||||
githubCurrentStatus: "present",
|
||||
githubCurrentContentHash: snapshot.skills[0]?.contentHash ?? "",
|
||||
githubScanStatus: "clean",
|
||||
},
|
||||
],
|
||||
snapshot,
|
||||
now: 123,
|
||||
});
|
||||
|
||||
expect(plan.skillPatches[0]?.patch).toMatchObject({
|
||||
ownerUserId: "users:new-owner",
|
||||
ownerPublisherId: "publishers:new-owner",
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves pending scan status for unchanged pending content", async () => {
|
||||
const snapshot = await buildGitHubSkillSourceSnapshot({
|
||||
repo: "NVIDIA/skills",
|
||||
defaultBranch: "main",
|
||||
commit: "3".repeat(40),
|
||||
entries: repoEntries({
|
||||
"skills/aiq-deploy/SKILL.md": "# AIQ Deploy\n",
|
||||
}),
|
||||
});
|
||||
const contentHash = snapshot.skills[0]?.contentHash ?? "";
|
||||
|
||||
const plan = buildGitHubSkillSyncPlan({
|
||||
sourceId: "githubSkillSources:nvidia",
|
||||
ownerUserId: "users:nvidia",
|
||||
ownerPublisherId: "publishers:nvidia",
|
||||
existingSkills: [
|
||||
{
|
||||
_id: "skills:aiq-deploy",
|
||||
slug: "aiq-deploy",
|
||||
displayName: "AIQ Deploy",
|
||||
githubPath: "skills/aiq-deploy",
|
||||
githubCurrentStatus: "present",
|
||||
githubCurrentContentHash: contentHash,
|
||||
githubScanStatus: "pending",
|
||||
},
|
||||
],
|
||||
snapshot,
|
||||
now: 123,
|
||||
});
|
||||
|
||||
expect(plan.skillPatches[0]?.patch).toMatchObject({
|
||||
githubCurrentCommit: "3".repeat(40),
|
||||
githubCurrentContentHash: contentHash,
|
||||
githubScanStatus: "pending",
|
||||
moderationStatus: "active",
|
||||
moderationReason: "pending.scan",
|
||||
});
|
||||
expect(plan.stats.unchanged).toBe(1);
|
||||
});
|
||||
|
||||
it("preserves terminal scan status for unchanged current content", async () => {
|
||||
const snapshot = await buildGitHubSkillSourceSnapshot({
|
||||
repo: "NVIDIA/skills",
|
||||
defaultBranch: "main",
|
||||
commit: "3".repeat(40),
|
||||
entries: repoEntries({
|
||||
"skills/aiq-deploy/SKILL.md": "# AIQ Deploy\n",
|
||||
}),
|
||||
});
|
||||
const contentHash = snapshot.skills[0]?.contentHash ?? "";
|
||||
|
||||
const plan = buildGitHubSkillSyncPlan({
|
||||
sourceId: "githubSkillSources:nvidia",
|
||||
ownerUserId: "users:nvidia",
|
||||
ownerPublisherId: "publishers:nvidia",
|
||||
existingSkills: [
|
||||
{
|
||||
_id: "skills:aiq-deploy",
|
||||
slug: "aiq-deploy",
|
||||
displayName: "AIQ Deploy",
|
||||
githubPath: "skills/aiq-deploy",
|
||||
githubCurrentStatus: "present",
|
||||
githubCurrentContentHash: contentHash,
|
||||
githubScanStatus: "malicious",
|
||||
},
|
||||
],
|
||||
snapshot,
|
||||
now: 123,
|
||||
});
|
||||
|
||||
expect(plan.skillPatches[0]?.patch).toMatchObject({
|
||||
githubCurrentContentHash: contentHash,
|
||||
githubScanStatus: "malicious",
|
||||
moderationStatus: "hidden",
|
||||
moderationReason: "scanner.llm.malicious",
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves terminal scan status for unchanged current bytes", async () => {
|
||||
const snapshot = await buildGitHubSkillSourceSnapshot({
|
||||
repo: "NVIDIA/skills",
|
||||
defaultBranch: "main",
|
||||
commit: "3".repeat(40),
|
||||
entries: repoEntries({
|
||||
"skills/aiq-deploy/SKILL.md": "# AIQ Deploy\n",
|
||||
}),
|
||||
});
|
||||
const contentHash = snapshot.skills[0]?.contentHash ?? "";
|
||||
|
||||
const plan = buildGitHubSkillSyncPlan({
|
||||
sourceId: "githubSkillSources:nvidia",
|
||||
ownerUserId: "users:nvidia",
|
||||
ownerPublisherId: "publishers:nvidia",
|
||||
existingSkills: [
|
||||
{
|
||||
_id: "skills:aiq-deploy",
|
||||
slug: "aiq-deploy",
|
||||
displayName: "AIQ Deploy",
|
||||
githubPath: "skills/aiq-deploy",
|
||||
githubCurrentStatus: "present",
|
||||
githubCurrentContentHash: contentHash,
|
||||
githubScanStatus: "malicious",
|
||||
},
|
||||
],
|
||||
snapshot,
|
||||
now: 123,
|
||||
});
|
||||
|
||||
expect(plan.skillPatches[0]?.patch).toMatchObject({
|
||||
githubCurrentContentHash: contentHash,
|
||||
githubScanStatus: "malicious",
|
||||
moderationStatus: "hidden",
|
||||
moderationReason: "scanner.llm.malicious",
|
||||
});
|
||||
expect(plan.skillPatches[0]?.patch).not.toHaveProperty("updatedAt");
|
||||
expect(plan.stats.unchanged).toBe(1);
|
||||
});
|
||||
|
||||
it("revives soft-deleted skills when a configured repo is synced again", async () => {
|
||||
const snapshot = await buildGitHubSkillSourceSnapshot({
|
||||
repo: "mattpocock/skills",
|
||||
defaultBranch: "main",
|
||||
commit: "4".repeat(40),
|
||||
entries: repoEntries({
|
||||
"skills/engineering/tdd/SKILL.md": "# TDD\n",
|
||||
}),
|
||||
});
|
||||
|
||||
const plan = buildGitHubSkillSyncPlan({
|
||||
sourceId: "githubSkillSources:matt",
|
||||
ownerUserId: "users:matt",
|
||||
ownerPublisherId: "publishers:matt",
|
||||
existingSkills: [
|
||||
{
|
||||
_id: "skills:tdd",
|
||||
slug: "tdd",
|
||||
displayName: "TDD",
|
||||
githubPath: "skills/engineering/tdd",
|
||||
githubCurrentStatus: "missing",
|
||||
},
|
||||
],
|
||||
snapshot,
|
||||
now: 123,
|
||||
});
|
||||
|
||||
expect(plan.skillPatches[0]?.patch).toMatchObject({
|
||||
githubCurrentStatus: "present",
|
||||
githubRemovedAt: undefined,
|
||||
softDeletedAt: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it("tombstones upstream removals instead of leaving stale installs active", async () => {
|
||||
const snapshot = await buildGitHubSkillSourceSnapshot({
|
||||
repo: "NVIDIA/skills",
|
||||
defaultBranch: "main",
|
||||
commit: "2".repeat(40),
|
||||
entries: repoEntries({
|
||||
"skills/vision-helper/SKILL.md": "# Vision Helper\n",
|
||||
}),
|
||||
});
|
||||
|
||||
const plan = buildGitHubSkillSyncPlan({
|
||||
sourceId: "githubSkillSources:nvidia",
|
||||
ownerUserId: "users:nvidia",
|
||||
ownerPublisherId: "publishers:nvidia",
|
||||
existingSkills: [
|
||||
{
|
||||
_id: "skills:aiq-deploy",
|
||||
slug: "aiq-deploy",
|
||||
displayName: "AIQ Deploy",
|
||||
githubPath: "skills/aiq-deploy",
|
||||
githubScanStatus: "clean",
|
||||
},
|
||||
],
|
||||
snapshot,
|
||||
now: 123,
|
||||
});
|
||||
|
||||
expect(plan.skillPatches).toEqual([
|
||||
expect.objectContaining({
|
||||
skillId: "skills:aiq-deploy",
|
||||
slug: "aiq-deploy",
|
||||
patch: expect.objectContaining({
|
||||
githubCurrentCommit: "2".repeat(40),
|
||||
githubCurrentStatus: "missing",
|
||||
githubRemovedAt: 123,
|
||||
softDeletedAt: 123,
|
||||
moderationStatus: "hidden",
|
||||
moderationReason: "github.upstream.removed",
|
||||
}),
|
||||
}),
|
||||
]);
|
||||
expect(plan.skillInserts).toHaveLength(1);
|
||||
expect(plan.stats.removed).toBe(1);
|
||||
});
|
||||
|
||||
it("preserves first upstream removal time on later syncs", async () => {
|
||||
const snapshot = await buildGitHubSkillSourceSnapshot({
|
||||
repo: "NVIDIA/skills",
|
||||
defaultBranch: "main",
|
||||
commit: "3".repeat(40),
|
||||
entries: repoEntries({
|
||||
"skills/vision-helper/SKILL.md": "# Vision Helper\n",
|
||||
}),
|
||||
});
|
||||
|
||||
const plan = buildGitHubSkillSyncPlan({
|
||||
sourceId: "githubSkillSources:nvidia",
|
||||
ownerUserId: "users:nvidia",
|
||||
ownerPublisherId: "publishers:nvidia",
|
||||
existingSkills: [
|
||||
{
|
||||
_id: "skills:aiq-deploy",
|
||||
slug: "aiq-deploy",
|
||||
displayName: "AIQ Deploy",
|
||||
githubPath: "skills/aiq-deploy",
|
||||
githubCurrentStatus: "missing",
|
||||
githubRemovedAt: 77,
|
||||
githubScanStatus: "clean",
|
||||
},
|
||||
],
|
||||
snapshot,
|
||||
now: 123,
|
||||
});
|
||||
|
||||
expect(plan.skillPatches[0]?.patch).toMatchObject({
|
||||
githubCurrentCommit: "3".repeat(40),
|
||||
githubCurrentStatus: "missing",
|
||||
githubCurrentCheckedAt: 123,
|
||||
githubRemovedAt: 77,
|
||||
moderationStatus: "hidden",
|
||||
moderationReason: "github.upstream.removed",
|
||||
});
|
||||
expect(plan.skillPatches[0]?.patch).not.toHaveProperty("updatedAt");
|
||||
});
|
||||
});
|
||||
@@ -1,670 +0,0 @@
|
||||
import { getFrontmatterValue, parseFrontmatter } from "./skills";
|
||||
|
||||
export type GitHubSkillScanStatus = "clean" | "suspicious" | "malicious" | "pending" | "failed";
|
||||
export type GitHubCurrentStatus = "present" | "missing" | "unknown";
|
||||
export type DisplayManifestStatus = "ok" | "missing" | "invalid" | "failed";
|
||||
|
||||
export type DisplayManifest = {
|
||||
notGrouped?: "top" | "bottom";
|
||||
groupings: Array<{
|
||||
title: string;
|
||||
description?: string;
|
||||
skills: string[];
|
||||
}>;
|
||||
};
|
||||
|
||||
export type GitHubSkillSourceSnapshot = {
|
||||
repo: string;
|
||||
defaultBranch: string;
|
||||
commit: string;
|
||||
manifestStatus: DisplayManifestStatus;
|
||||
manifestHash?: string;
|
||||
manifest?: DisplayManifest;
|
||||
skills: DiscoveredGitHubSkill[];
|
||||
};
|
||||
|
||||
export type GitHubSkillSourceMetadataSnapshot = Omit<GitHubSkillSourceSnapshot, "skills"> & {
|
||||
skills: DiscoveredGitHubSkillMetadata[];
|
||||
};
|
||||
|
||||
export type DiscoveredGitHubSkill = {
|
||||
slug: string;
|
||||
displayName: string;
|
||||
summary?: string;
|
||||
upstreamVersion?: string;
|
||||
path: string;
|
||||
skillMarkdownPath: string;
|
||||
skillMarkdown: string;
|
||||
skillCardMarkdownPath?: string;
|
||||
skillCardMarkdown?: string;
|
||||
contentHash: string;
|
||||
};
|
||||
|
||||
export type DiscoveredGitHubSkillMetadata = Omit<
|
||||
DiscoveredGitHubSkill,
|
||||
"skillMarkdown" | "skillCardMarkdown"
|
||||
>;
|
||||
|
||||
export type ExistingGitHubSkillForSync = {
|
||||
_id: string;
|
||||
slug: string;
|
||||
displayName: string;
|
||||
summary?: string;
|
||||
latestVersionSummary?: {
|
||||
version: string;
|
||||
createdAt: number;
|
||||
};
|
||||
githubPath?: string;
|
||||
githubCurrentCommit?: string;
|
||||
githubCurrentContentHash?: string;
|
||||
githubCurrentStatus?: GitHubCurrentStatus;
|
||||
githubScanStatus?: GitHubSkillScanStatus;
|
||||
githubRemovedAt?: number;
|
||||
softDeletedAt?: number;
|
||||
};
|
||||
|
||||
export type GitHubBackedSkillModeration = {
|
||||
moderationStatus: "active" | "hidden";
|
||||
moderationReason?: string;
|
||||
moderationVerdict?: "clean" | "suspicious" | "malicious";
|
||||
moderationFlags: string[];
|
||||
isSuspicious: boolean;
|
||||
};
|
||||
|
||||
export type GitHubSkillPatchForSync = {
|
||||
skillId: string;
|
||||
slug: string;
|
||||
patch: Record<string, unknown>;
|
||||
};
|
||||
|
||||
export type GitHubSkillInsertForSync = {
|
||||
slug: string;
|
||||
doc: Record<string, unknown>;
|
||||
};
|
||||
|
||||
export type GitHubSkillSyncPlan = {
|
||||
sourcePatch: Record<string, unknown>;
|
||||
skillPatches: GitHubSkillPatchForSync[];
|
||||
skillInserts: GitHubSkillInsertForSync[];
|
||||
stats: {
|
||||
discovered: number;
|
||||
inserted: number;
|
||||
changed: number;
|
||||
unchanged: number;
|
||||
removed: number;
|
||||
};
|
||||
};
|
||||
|
||||
const SKILL_MARKDOWN_BASENAME = "skill.md";
|
||||
const SKILL_CARD_MARKDOWN_BASENAME = "skill-card.md";
|
||||
const MAX_STORED_MARKDOWN_BYTES = 512 * 1024;
|
||||
const MAX_STORED_SKILL_CONTENT_BYTES = 768 * 1024;
|
||||
|
||||
export function parseSkillsShDisplayManifest(raw: string | undefined | null): {
|
||||
status: DisplayManifestStatus;
|
||||
manifest?: DisplayManifest;
|
||||
} {
|
||||
if (raw === undefined || raw === null) return { status: "missing", manifest: undefined };
|
||||
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(raw);
|
||||
} catch {
|
||||
return { status: "invalid", manifest: undefined };
|
||||
}
|
||||
|
||||
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
||||
return { status: "invalid", manifest: undefined };
|
||||
}
|
||||
|
||||
const record = parsed as Record<string, unknown>;
|
||||
const rawGroups = record.groupings;
|
||||
if (!Array.isArray(rawGroups)) return { status: "invalid", manifest: undefined };
|
||||
|
||||
const groupings = rawGroups.flatMap((group): DisplayManifest["groupings"] => {
|
||||
if (!group || typeof group !== "object" || Array.isArray(group)) return [];
|
||||
const groupRecord = group as Record<string, unknown>;
|
||||
const title = typeof groupRecord.title === "string" ? groupRecord.title.trim() : "";
|
||||
const description =
|
||||
typeof groupRecord.description === "string" ? groupRecord.description.trim() : "";
|
||||
const skills = Array.isArray(groupRecord.skills)
|
||||
? groupRecord.skills
|
||||
.filter((value): value is string => typeof value === "string")
|
||||
.map((value) => value.trim())
|
||||
.filter(Boolean)
|
||||
: [];
|
||||
if (!title || skills.length === 0) return [];
|
||||
return [
|
||||
{
|
||||
title,
|
||||
...(description ? { description } : {}),
|
||||
skills,
|
||||
},
|
||||
];
|
||||
});
|
||||
|
||||
if (groupings.length === 0) return { status: "invalid", manifest: undefined };
|
||||
|
||||
const notGrouped =
|
||||
record.notGrouped === "top" || record.notGrouped === "bottom" ? record.notGrouped : undefined;
|
||||
return {
|
||||
status: "ok",
|
||||
manifest: {
|
||||
...(notGrouped ? { notGrouped } : {}),
|
||||
groupings,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function buildGitHubSkillSourceSnapshot({
|
||||
repo,
|
||||
defaultBranch,
|
||||
commit,
|
||||
entries,
|
||||
}: {
|
||||
repo: string;
|
||||
defaultBranch: string;
|
||||
commit: string;
|
||||
entries: Record<string, Uint8Array>;
|
||||
}): Promise<GitHubSkillSourceSnapshot> {
|
||||
const normalizedEntries = normalizeEntryMap(entries);
|
||||
const manifestBytes = normalizedEntries["skills.sh.json"];
|
||||
const manifestText = manifestBytes ? decodeUtf8(manifestBytes) : undefined;
|
||||
const parsedManifest = parseSkillsShDisplayManifest(manifestText);
|
||||
const manifestHash = manifestBytes ? await sha256Hex(manifestBytes) : undefined;
|
||||
const skillPaths = discoverSkillPaths(normalizedEntries);
|
||||
const skills: DiscoveredGitHubSkill[] = [];
|
||||
|
||||
for (const skillMdPath of skillPaths) {
|
||||
const path = parentPath(skillMdPath);
|
||||
const markdownBytes = normalizedEntries[skillMdPath] ?? new Uint8Array();
|
||||
assertStoredMarkdownSize(skillMdPath, markdownBytes);
|
||||
const markdown = decodeUtf8(markdownBytes);
|
||||
const frontmatter = parseFrontmatter(markdown);
|
||||
const folderName = path.split("/").filter(Boolean).at(-1) ?? "";
|
||||
const slug = slugFromPathSegment(folderName);
|
||||
if (!slug) continue;
|
||||
const frontmatterName = getFrontmatterValue(frontmatter, "name")?.trim();
|
||||
const frontmatterDescription = getFrontmatterValue(frontmatter, "description")?.trim();
|
||||
const frontmatterVersion = getFrontmatterValue(frontmatter, "version")?.trim();
|
||||
const heading = firstMarkdownHeading(markdown);
|
||||
const skillCardMarkdownPath = findFolderFilePath(
|
||||
normalizedEntries,
|
||||
path,
|
||||
SKILL_CARD_MARKDOWN_BASENAME,
|
||||
);
|
||||
const skillCardBytes = skillCardMarkdownPath
|
||||
? normalizedEntries[skillCardMarkdownPath]
|
||||
: undefined;
|
||||
if (skillCardMarkdownPath && skillCardBytes) {
|
||||
assertStoredMarkdownSize(skillCardMarkdownPath, skillCardBytes);
|
||||
assertStoredSkillContentSize(markdownBytes.byteLength + skillCardBytes.byteLength);
|
||||
} else {
|
||||
assertStoredSkillContentSize(markdownBytes.byteLength);
|
||||
}
|
||||
const skillCardMarkdown = skillCardBytes ? decodeUtf8(skillCardBytes) : undefined;
|
||||
|
||||
skills.push({
|
||||
slug,
|
||||
displayName: frontmatterName || heading || titleizeSlug(slug),
|
||||
...(frontmatterDescription ? { summary: frontmatterDescription } : {}),
|
||||
...(frontmatterVersion ? { upstreamVersion: frontmatterVersion } : {}),
|
||||
path,
|
||||
skillMarkdownPath: skillMdPath,
|
||||
skillMarkdown: markdown,
|
||||
...(skillCardMarkdownPath ? { skillCardMarkdownPath } : {}),
|
||||
...(skillCardMarkdown !== undefined ? { skillCardMarkdown } : {}),
|
||||
contentHash: await computeGitHubSkillFolderContentHash(normalizedEntries, path),
|
||||
});
|
||||
}
|
||||
|
||||
const sortedSkills = skills.sort((a, b) => a.path.localeCompare(b.path));
|
||||
assertUniqueDiscoveredSlugs(sortedSkills);
|
||||
|
||||
return {
|
||||
repo,
|
||||
defaultBranch,
|
||||
commit,
|
||||
manifestStatus: parsedManifest.status,
|
||||
...(manifestHash ? { manifestHash } : {}),
|
||||
...(parsedManifest.manifest ? { manifest: parsedManifest.manifest } : {}),
|
||||
skills: sortedSkills,
|
||||
};
|
||||
}
|
||||
|
||||
export async function computeGitHubSkillFolderContentHash(
|
||||
entries: Record<string, Uint8Array>,
|
||||
folderPath: string,
|
||||
) {
|
||||
const normalizedEntries = normalizeEntryMap(entries);
|
||||
const root = folderPath ? `${folderPath}/` : "";
|
||||
const lines: string[] = [];
|
||||
for (const [path, content] of Object.entries(normalizedEntries).sort(([a], [b]) =>
|
||||
a.localeCompare(b),
|
||||
)) {
|
||||
if (root && path !== folderPath && !path.startsWith(root)) continue;
|
||||
if (!root && path.includes("/")) continue;
|
||||
const relativePath = root ? path.slice(root.length) : path;
|
||||
if (!relativePath) continue;
|
||||
const fileHash = await sha256Hex(content);
|
||||
lines.push(`${relativePath}\0${content.byteLength}\0${fileHash}`);
|
||||
}
|
||||
return sha256Hex(new TextEncoder().encode(lines.join("\n")));
|
||||
}
|
||||
|
||||
export function buildGitHubSkillSyncPlan({
|
||||
sourceId,
|
||||
ownerUserId,
|
||||
ownerPublisherId,
|
||||
existingSkills,
|
||||
snapshot,
|
||||
now,
|
||||
}: {
|
||||
sourceId: string;
|
||||
ownerUserId: string;
|
||||
ownerPublisherId?: string;
|
||||
existingSkills: ExistingGitHubSkillForSync[];
|
||||
snapshot: GitHubSkillSourceSnapshot | GitHubSkillSourceMetadataSnapshot;
|
||||
now: number;
|
||||
}): GitHubSkillSyncPlan {
|
||||
const sourcePatch = {
|
||||
repo: snapshot.repo,
|
||||
defaultBranch: snapshot.defaultBranch,
|
||||
lastSyncStatus: "ok",
|
||||
lastSyncError: undefined,
|
||||
lastSyncErrorAt: undefined,
|
||||
displayManifestKind: "skills.sh",
|
||||
displayManifestHash: snapshot.manifestHash,
|
||||
displayManifestCommit: snapshot.commit,
|
||||
displayManifestFetchedAt: now,
|
||||
displayManifestStatus: snapshot.manifestStatus,
|
||||
displayManifest: snapshot.manifest,
|
||||
...(ownerPublisherId ? { ownerPublisherId } : {}),
|
||||
updatedAt: now,
|
||||
};
|
||||
const existingByPath = new Map(
|
||||
existingSkills
|
||||
.filter((skill) => skill.githubPath)
|
||||
.map((skill) => [skill.githubPath as string, skill]),
|
||||
);
|
||||
const existingBySlug = new Map(existingSkills.map((skill) => [skill.slug, skill]));
|
||||
const matchedSkillIds = new Set<string>();
|
||||
const skillPatches: GitHubSkillPatchForSync[] = [];
|
||||
const skillInserts: GitHubSkillInsertForSync[] = [];
|
||||
const stats = {
|
||||
discovered: snapshot.skills.length,
|
||||
inserted: 0,
|
||||
changed: 0,
|
||||
unchanged: 0,
|
||||
removed: 0,
|
||||
};
|
||||
|
||||
for (const discovered of snapshot.skills) {
|
||||
const existing = existingByPath.get(discovered.path) ?? existingBySlug.get(discovered.slug);
|
||||
if (!existing) {
|
||||
const scanStatus: GitHubSkillScanStatus = "pending";
|
||||
const moderation = githubBackedSkillModeration(scanStatus);
|
||||
skillInserts.push({
|
||||
slug: discovered.slug,
|
||||
doc: {
|
||||
slug: discovered.slug,
|
||||
displayName: discovered.displayName,
|
||||
summary: discovered.summary,
|
||||
ownerUserId,
|
||||
ownerPublisherId,
|
||||
installKind: "github",
|
||||
githubSourceId: sourceId,
|
||||
githubPath: discovered.path,
|
||||
githubHasSkillCard: Boolean(discovered.skillCardMarkdownPath),
|
||||
githubCurrentCommit: snapshot.commit,
|
||||
githubCurrentContentHash: discovered.contentHash,
|
||||
githubCurrentStatus: "present",
|
||||
githubCurrentCheckedAt: now,
|
||||
githubScanStatus: scanStatus,
|
||||
githubRemovedAt: undefined,
|
||||
latestVersionId: undefined,
|
||||
latestVersionSummary: latestVersionSummary(discovered.upstreamVersion, now),
|
||||
tags: {},
|
||||
capabilityTags: [],
|
||||
softDeletedAt: undefined,
|
||||
badges: undefined,
|
||||
statsDownloads: 0,
|
||||
statsStars: 0,
|
||||
statsInstallsCurrent: 0,
|
||||
statsInstallsAllTime: 0,
|
||||
stats: {
|
||||
downloads: 0,
|
||||
stars: 0,
|
||||
installsCurrent: 0,
|
||||
installsAllTime: 0,
|
||||
versions: 0,
|
||||
comments: 0,
|
||||
},
|
||||
...moderation,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
},
|
||||
});
|
||||
stats.inserted += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
matchedSkillIds.add(existing._id);
|
||||
const currentContentUnchanged =
|
||||
existing.githubCurrentStatus === "present" &&
|
||||
existing.githubCurrentContentHash === discovered.contentHash;
|
||||
const scanStatus: GitHubSkillScanStatus = currentContentUnchanged
|
||||
? githubScanStatusForUnchangedContent(existing.githubScanStatus)
|
||||
: "pending";
|
||||
const moderation = githubBackedSkillModeration(scanStatus);
|
||||
const nextLatestVersionSummary = latestVersionSummary(
|
||||
discovered.upstreamVersion,
|
||||
existing.latestVersionSummary?.createdAt ?? now,
|
||||
);
|
||||
const materialChanged =
|
||||
!currentContentUnchanged ||
|
||||
existing.displayName !== discovered.displayName ||
|
||||
(existing.summary ?? undefined) !== (discovered.summary ?? undefined) ||
|
||||
(existing.githubPath ?? undefined) !== discovered.path ||
|
||||
!sameLatestVersionSummary(existing.latestVersionSummary, nextLatestVersionSummary);
|
||||
const patch = {
|
||||
displayName: discovered.displayName,
|
||||
summary: discovered.summary,
|
||||
ownerUserId,
|
||||
...(ownerPublisherId ? { ownerPublisherId } : {}),
|
||||
githubSourceId: sourceId,
|
||||
githubPath: discovered.path,
|
||||
githubHasSkillCard: Boolean(discovered.skillCardMarkdownPath),
|
||||
githubCurrentCommit: snapshot.commit,
|
||||
githubCurrentContentHash: discovered.contentHash,
|
||||
githubCurrentStatus: "present",
|
||||
githubCurrentCheckedAt: now,
|
||||
githubScanStatus: scanStatus,
|
||||
githubRemovedAt: undefined,
|
||||
softDeletedAt: undefined,
|
||||
...(materialChanged
|
||||
? {
|
||||
latestVersionSummary: latestVersionSummary(discovered.upstreamVersion, now),
|
||||
updatedAt: now,
|
||||
}
|
||||
: {}),
|
||||
...moderation,
|
||||
};
|
||||
skillPatches.push({ skillId: existing._id, slug: existing.slug, patch });
|
||||
if (materialChanged) stats.changed += 1;
|
||||
else stats.unchanged += 1;
|
||||
}
|
||||
|
||||
for (const existing of existingSkills) {
|
||||
if (matchedSkillIds.has(existing._id)) continue;
|
||||
const removedAt = existing.githubRemovedAt ?? now;
|
||||
const moderation = githubBackedSkillModeration(
|
||||
existing.githubScanStatus ?? "pending",
|
||||
removedAt,
|
||||
);
|
||||
const wasAlreadyRemoved =
|
||||
existing.githubCurrentStatus === "missing" && existing.githubRemovedAt !== undefined;
|
||||
skillPatches.push({
|
||||
skillId: existing._id,
|
||||
slug: existing.slug,
|
||||
patch: {
|
||||
githubCurrentCommit: snapshot.commit,
|
||||
githubCurrentStatus: "missing",
|
||||
githubCurrentCheckedAt: now,
|
||||
githubRemovedAt: removedAt,
|
||||
softDeletedAt: existing.softDeletedAt ?? removedAt,
|
||||
...(wasAlreadyRemoved ? {} : { updatedAt: now }),
|
||||
...moderation,
|
||||
},
|
||||
});
|
||||
stats.removed += 1;
|
||||
}
|
||||
|
||||
return { sourcePatch, skillPatches, skillInserts, stats };
|
||||
}
|
||||
|
||||
function githubScanStatusForUnchangedContent(
|
||||
status: GitHubSkillScanStatus | undefined,
|
||||
): GitHubSkillScanStatus {
|
||||
if (
|
||||
status === "clean" ||
|
||||
status === "failed" ||
|
||||
status === "malicious" ||
|
||||
status === "suspicious"
|
||||
) {
|
||||
return status;
|
||||
}
|
||||
return "pending";
|
||||
}
|
||||
|
||||
export function githubBackedSkillModeration(
|
||||
scanStatus: GitHubSkillScanStatus,
|
||||
removedAt?: number,
|
||||
): GitHubBackedSkillModeration {
|
||||
if (typeof removedAt === "number") {
|
||||
return {
|
||||
moderationStatus: "hidden",
|
||||
moderationReason: "github.upstream.removed",
|
||||
moderationVerdict: undefined,
|
||||
moderationFlags: [],
|
||||
isSuspicious: false,
|
||||
};
|
||||
}
|
||||
if (scanStatus === "pending") {
|
||||
return {
|
||||
moderationStatus: "active",
|
||||
moderationReason: "pending.scan",
|
||||
moderationVerdict: undefined,
|
||||
moderationFlags: [],
|
||||
isSuspicious: false,
|
||||
};
|
||||
}
|
||||
if (scanStatus === "failed") {
|
||||
return {
|
||||
moderationStatus: "hidden",
|
||||
moderationReason: "scanner.failed",
|
||||
moderationVerdict: undefined,
|
||||
moderationFlags: [],
|
||||
isSuspicious: false,
|
||||
};
|
||||
}
|
||||
if (scanStatus === "malicious") {
|
||||
return {
|
||||
moderationStatus: "hidden",
|
||||
moderationReason: "scanner.llm.malicious",
|
||||
moderationVerdict: "malicious",
|
||||
moderationFlags: ["blocked.malware"],
|
||||
isSuspicious: true,
|
||||
};
|
||||
}
|
||||
if (scanStatus === "suspicious") {
|
||||
return {
|
||||
moderationStatus: "active",
|
||||
moderationReason: "scanner.llm.suspicious",
|
||||
moderationVerdict: "suspicious",
|
||||
moderationFlags: ["flagged.suspicious"],
|
||||
isSuspicious: true,
|
||||
};
|
||||
}
|
||||
return {
|
||||
moderationStatus: "active",
|
||||
moderationReason: undefined,
|
||||
moderationVerdict: "clean",
|
||||
moderationFlags: [],
|
||||
isSuspicious: false,
|
||||
};
|
||||
}
|
||||
|
||||
function latestVersionSummary(version: string | undefined, now: number) {
|
||||
if (!version) return undefined;
|
||||
return {
|
||||
version,
|
||||
createdAt: now,
|
||||
changelog: "Synced from GitHub source.",
|
||||
changelogSource: "auto" as const,
|
||||
};
|
||||
}
|
||||
|
||||
function sameLatestVersionSummary(
|
||||
a: ExistingGitHubSkillForSync["latestVersionSummary"] | undefined,
|
||||
b: ReturnType<typeof latestVersionSummary>,
|
||||
) {
|
||||
if (!a && !b) return true;
|
||||
if (!a || !b) return false;
|
||||
return a.version === b.version;
|
||||
}
|
||||
|
||||
function assertStoredMarkdownSize(path: string, bytes: Uint8Array) {
|
||||
if (bytes.byteLength > MAX_STORED_MARKDOWN_BYTES) {
|
||||
throw new Error(`GitHub skill markdown file is too large to cache: ${path}`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertStoredSkillContentSize(totalBytes: number) {
|
||||
if (totalBytes > MAX_STORED_SKILL_CONTENT_BYTES) {
|
||||
throw new Error("GitHub skill cached markdown is too large");
|
||||
}
|
||||
}
|
||||
|
||||
function assertUniqueDiscoveredSlugs(skills: DiscoveredGitHubSkill[]) {
|
||||
const firstPathBySlug = new Map<string, string>();
|
||||
for (const skill of skills) {
|
||||
const firstPath = firstPathBySlug.get(skill.slug);
|
||||
if (firstPath) {
|
||||
throw duplicateSkillSlugError(skill.slug, firstPath, skill.path);
|
||||
}
|
||||
firstPathBySlug.set(skill.slug, skill.path);
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeEntryMap(entries: Record<string, Uint8Array>) {
|
||||
const out: Record<string, Uint8Array> = {};
|
||||
for (const [rawPath, bytes] of Object.entries(entries)) {
|
||||
const normalized = normalizeRepoPath(rawPath);
|
||||
if (!normalized) continue;
|
||||
out[normalized] = new Uint8Array(bytes);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function discoverSkillPaths(entries: Record<string, Uint8Array>) {
|
||||
const candidates = Object.keys(entries)
|
||||
.filter((path) => path.split("/").at(-1)?.toLowerCase() === SKILL_MARKDOWN_BASENAME)
|
||||
.sort((a, b) => a.localeCompare(b));
|
||||
const pathsBySlug = new Map<string, string[]>();
|
||||
|
||||
for (const skillMdPath of candidates) {
|
||||
const path = parentPath(skillMdPath);
|
||||
const folderName = path.split("/").filter(Boolean).at(-1) ?? "";
|
||||
const slug = slugFromPathSegment(folderName);
|
||||
if (!slug) continue;
|
||||
const paths = pathsBySlug.get(slug) ?? [];
|
||||
paths.push(skillMdPath);
|
||||
pathsBySlug.set(slug, paths);
|
||||
}
|
||||
|
||||
const selected: string[] = [];
|
||||
for (const [slug, paths] of pathsBySlug) {
|
||||
if (paths.length === 1) {
|
||||
selected.push(paths[0] as string);
|
||||
continue;
|
||||
}
|
||||
|
||||
const canonicalPath = `skills/${slug}/${SKILL_MARKDOWN_BASENAME}`;
|
||||
const exactTopLevelMatches = paths.filter((path) => path.toLowerCase() === canonicalPath);
|
||||
const topLevelSkillMatches = paths.filter((path) => path.toLowerCase().startsWith("skills/"));
|
||||
if (exactTopLevelMatches.length === 1 && topLevelSkillMatches.length === 1) {
|
||||
selected.push(exactTopLevelMatches[0] as string);
|
||||
continue;
|
||||
}
|
||||
|
||||
throw duplicateSkillSlugError(
|
||||
slug,
|
||||
parentPath(paths[0] as string),
|
||||
parentPath(paths[1] as string),
|
||||
);
|
||||
}
|
||||
|
||||
return selected.sort((a, b) => a.localeCompare(b));
|
||||
}
|
||||
|
||||
function duplicateSkillSlugError(slug: string, firstPath: string, secondPath: string) {
|
||||
return new Error(
|
||||
`GitHub skill source has duplicate normalized slug "${slug}" at ${firstPath} and ${secondPath}`,
|
||||
);
|
||||
}
|
||||
|
||||
function findFolderFilePath(
|
||||
entries: Record<string, Uint8Array>,
|
||||
folderPath: string,
|
||||
basename: string,
|
||||
) {
|
||||
const prefix = folderPath ? `${folderPath}/` : "";
|
||||
return Object.keys(entries).find((entryPath) => {
|
||||
if (prefix) {
|
||||
if (!entryPath.startsWith(prefix)) return false;
|
||||
const relativePath = entryPath.slice(prefix.length);
|
||||
return !relativePath.includes("/") && relativePath.toLowerCase() === basename;
|
||||
}
|
||||
return !entryPath.includes("/") && entryPath.toLowerCase() === basename;
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeRepoPath(path: string) {
|
||||
if (path.includes("\u0000")) return "";
|
||||
const normalized = path
|
||||
.replaceAll("\\", "/")
|
||||
.trim()
|
||||
.replace(/^\.\/+/, "")
|
||||
.replace(/^\/+/, "");
|
||||
if (!normalized) return "";
|
||||
const segments = normalized.split("/").filter(Boolean);
|
||||
if (segments.some((segment) => segment === "." || segment === "..")) return "";
|
||||
return segments.join("/");
|
||||
}
|
||||
|
||||
function parentPath(path: string) {
|
||||
return path.split("/").slice(0, -1).join("/");
|
||||
}
|
||||
|
||||
function decodeUtf8(bytes: Uint8Array) {
|
||||
return new TextDecoder().decode(bytes);
|
||||
}
|
||||
|
||||
async function sha256Hex(bytes: Uint8Array) {
|
||||
const safe = new Uint8Array(bytes);
|
||||
const digest = await crypto.subtle.digest("SHA-256", safe);
|
||||
return toHex(new Uint8Array(digest));
|
||||
}
|
||||
|
||||
function toHex(bytes: Uint8Array) {
|
||||
let out = "";
|
||||
for (const byte of bytes) out += byte.toString(16).padStart(2, "0");
|
||||
return out;
|
||||
}
|
||||
|
||||
function slugFromPathSegment(value: string) {
|
||||
return value
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[\s_]+/g, "-")
|
||||
.replace(/[^a-z0-9-]+/g, "-")
|
||||
.replace(/-+/g, "-")
|
||||
.replace(/^-|-$/g, "");
|
||||
}
|
||||
|
||||
function titleizeSlug(slug: string) {
|
||||
return slug
|
||||
.split("-")
|
||||
.filter(Boolean)
|
||||
.map((part) => part.charAt(0).toUpperCase() + part.slice(1))
|
||||
.join(" ");
|
||||
}
|
||||
|
||||
function firstMarkdownHeading(markdown: string) {
|
||||
for (const line of markdown.split(/\r?\n/)) {
|
||||
const match = /^#\s+(.+)$/.exec(line.trim());
|
||||
if (match?.[1]) return match[1].trim();
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
@@ -1,8 +1,8 @@
|
||||
"use node";
|
||||
|
||||
import { createPrivateKey, createSign } from "node:crypto";
|
||||
import type { Id } from "../_generated/dataModel";
|
||||
import type { ActionCtx } from "../_generated/server";
|
||||
import { buildGitHubHeaders, createGitHubAppInstallationToken } from "./githubAuth";
|
||||
|
||||
const GITHUB_API = "https://api.github.com";
|
||||
const DEFAULT_REPO = "clawdbot/souls";
|
||||
@@ -86,7 +86,7 @@ export async function getGitHubSoulBackupContext(): Promise<GitHubBackupContext>
|
||||
const repo = process.env.GITHUB_SOULS_REPO ?? DEFAULT_REPO;
|
||||
const root = process.env.GITHUB_SOULS_ROOT ?? DEFAULT_ROOT;
|
||||
const [repoOwner, repoName] = parseRepo(repo);
|
||||
const { token } = await createGitHubAppInstallationToken({ userAgent: USER_AGENT });
|
||||
const token = await createInstallationToken();
|
||||
const repoInfo = await githubGet<RepoInfo>(token, `/repos/${repoOwner}/${repoName}`);
|
||||
const branch = repoInfo.default_branch ?? "main";
|
||||
|
||||
@@ -297,6 +297,48 @@ async function fetchStorageBase64(ctx: ActionCtx, storageId: Id<"_storage">) {
|
||||
return buffer.toString("base64");
|
||||
}
|
||||
|
||||
async function createInstallationToken() {
|
||||
const appId = process.env.GITHUB_APP_ID;
|
||||
const installationId = process.env.GITHUB_APP_INSTALLATION_ID;
|
||||
if (!appId || !installationId) {
|
||||
throw new Error("GitHub App credentials missing");
|
||||
}
|
||||
const jwt = createAppJwt(appId);
|
||||
const response = await fetch(`${GITHUB_API}/app/installations/${installationId}/access_tokens`, {
|
||||
method: "POST",
|
||||
headers: buildHeaders(jwt, true),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const message = await response.text();
|
||||
throw new Error(`GitHub App token failed: ${message}`);
|
||||
}
|
||||
const payload = (await response.json()) as { token?: string };
|
||||
if (!payload.token) throw new Error("GitHub App token missing");
|
||||
return payload.token;
|
||||
}
|
||||
|
||||
function createAppJwt(appId: string) {
|
||||
const privateKey = loadPrivateKey();
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const header = { alg: "RS256", typ: "JWT" };
|
||||
const payload = { iat: now - 60, exp: now + 9 * 60, iss: appId };
|
||||
const encodedHeader = base64Url(JSON.stringify(header));
|
||||
const encodedPayload = base64Url(JSON.stringify(payload));
|
||||
const signingInput = `${encodedHeader}.${encodedPayload}`;
|
||||
const sign = createSign("RSA-SHA256");
|
||||
sign.update(signingInput);
|
||||
sign.end();
|
||||
const signature = sign.sign(privateKey);
|
||||
return `${signingInput}.${base64Url(signature)}`;
|
||||
}
|
||||
|
||||
function loadPrivateKey() {
|
||||
const raw = process.env.GITHUB_APP_PRIVATE_KEY;
|
||||
if (!raw) throw new Error("GITHUB_APP_PRIVATE_KEY is not configured");
|
||||
const normalized = raw.replace(/\\n/g, "\n");
|
||||
return createPrivateKey(normalized);
|
||||
}
|
||||
|
||||
async function createBlob(token: string, repoOwner: string, repoName: string, content: string) {
|
||||
const result = await githubPost<{ sha: string }>(
|
||||
token,
|
||||
@@ -347,7 +389,11 @@ async function githubPatch(token: string, path: string, body: unknown) {
|
||||
}
|
||||
|
||||
function buildHeaders(token: string, isAppJwt = false) {
|
||||
return buildGitHubHeaders({ token, isAppJwt, userAgent: USER_AGENT });
|
||||
return {
|
||||
Authorization: `${isAppJwt ? "Bearer" : "token"} ${token}`,
|
||||
Accept: "application/vnd.github+json",
|
||||
"User-Agent": USER_AGENT,
|
||||
};
|
||||
}
|
||||
|
||||
function parseRepo(repo: string) {
|
||||
@@ -382,6 +428,11 @@ function encodePath(path: string) {
|
||||
.join("/");
|
||||
}
|
||||
|
||||
function base64Url(value: string | Uint8Array) {
|
||||
const buffer = typeof value === "string" ? Buffer.from(value) : Buffer.from(value);
|
||||
return buffer.toString("base64").replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/g, "");
|
||||
}
|
||||
|
||||
function toBase64(value: string) {
|
||||
return Buffer.from(value).toString("base64");
|
||||
}
|
||||
|
||||
@@ -6,8 +6,7 @@ export const GLOBAL_STATS_KEY = "default";
|
||||
type SkillVisibilityFields = Pick<
|
||||
Doc<"skills">,
|
||||
"softDeletedAt" | "moderationStatus" | "moderationFlags"
|
||||
> &
|
||||
Partial<Pick<Doc<"skills">, "moderationVerdict">>;
|
||||
>;
|
||||
|
||||
type GlobalStatsReadCtx = Pick<MutationCtx | QueryCtx, "db">;
|
||||
type GlobalStatsWriteCtx = Pick<MutationCtx, "db">;
|
||||
@@ -17,7 +16,6 @@ export function isPublicSkillDoc<T extends SkillVisibilityFields>(
|
||||
): skill is T {
|
||||
if (!skill || skill.softDeletedAt) return false;
|
||||
if (skill.moderationStatus && skill.moderationStatus !== "active") return false;
|
||||
if (skill.moderationVerdict === "malicious") return false;
|
||||
if (skill.moderationFlags?.includes("blocked.malware")) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -1,214 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildSkillInstallResolution } from "./installResolver";
|
||||
|
||||
const baseSkill = {
|
||||
slug: "aiq-deploy",
|
||||
displayName: "AIQ Deploy",
|
||||
latestVersionSummary: null,
|
||||
installKind: "github" as const,
|
||||
githubPath: "skills/aiq-deploy",
|
||||
githubCurrentCommit: "1".repeat(40),
|
||||
githubCurrentContentHash: "hash-aiq-deploy",
|
||||
githubCurrentStatus: "present" as const,
|
||||
githubScanStatus: "clean" as const,
|
||||
githubRemovedAt: undefined,
|
||||
};
|
||||
|
||||
const source = {
|
||||
repo: "NVIDIA/skills",
|
||||
defaultBranch: "main",
|
||||
};
|
||||
|
||||
describe("buildSkillInstallResolution", () => {
|
||||
it("returns an archive descriptor for hosted direct uploads", () => {
|
||||
const resolution = buildSkillInstallResolution({
|
||||
origin: "https://clawhub.ai",
|
||||
skill: {
|
||||
slug: "direct-skill",
|
||||
displayName: "Direct Skill",
|
||||
latestVersionSummary: { version: "1.2.3" },
|
||||
},
|
||||
source: null,
|
||||
});
|
||||
|
||||
expect(resolution).toEqual({
|
||||
ok: true,
|
||||
slug: "direct-skill",
|
||||
installKind: "archive",
|
||||
archive: {
|
||||
version: "1.2.3",
|
||||
downloadUrl: "https://clawhub.ai/api/v1/download?slug=direct-skill&version=1.2.3",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("returns a pinned GitHub descriptor when current upstream state is scan-clean", () => {
|
||||
const resolution = buildSkillInstallResolution({
|
||||
origin: "https://clawhub.ai",
|
||||
skill: baseSkill,
|
||||
source,
|
||||
});
|
||||
|
||||
expect(resolution).toEqual({
|
||||
ok: true,
|
||||
slug: "aiq-deploy",
|
||||
installKind: "github",
|
||||
github: {
|
||||
repo: "NVIDIA/skills",
|
||||
path: "skills/aiq-deploy",
|
||||
commit: "1".repeat(40),
|
||||
contentHash: "hash-aiq-deploy",
|
||||
sourceUrl: `https://github.com/NVIDIA/skills/tree/${"1".repeat(40)}/skills/aiq-deploy`,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("allows GitHub-backed installs when upstream content changed and the current hash is clean", () => {
|
||||
const resolution = buildSkillInstallResolution({
|
||||
origin: "https://clawhub.ai",
|
||||
skill: {
|
||||
...baseSkill,
|
||||
githubCurrentCommit: "2".repeat(40),
|
||||
githubCurrentContentHash: "hash-aiq-deploy-v2",
|
||||
},
|
||||
source,
|
||||
});
|
||||
|
||||
expect(resolution).toMatchObject({
|
||||
ok: true,
|
||||
installKind: "github",
|
||||
github: {
|
||||
commit: "2".repeat(40),
|
||||
contentHash: "hash-aiq-deploy-v2",
|
||||
sourceUrl: `https://github.com/NVIDIA/skills/tree/${"2".repeat(40)}/skills/aiq-deploy`,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("allows GitHub-backed installs when only unrelated repository content changed", () => {
|
||||
const resolution = buildSkillInstallResolution({
|
||||
origin: "https://clawhub.ai",
|
||||
skill: {
|
||||
...baseSkill,
|
||||
githubCurrentCommit: "2".repeat(40),
|
||||
githubCurrentContentHash: baseSkill.githubCurrentContentHash,
|
||||
},
|
||||
source,
|
||||
});
|
||||
|
||||
expect(resolution).toMatchObject({
|
||||
ok: true,
|
||||
installKind: "github",
|
||||
github: {
|
||||
commit: "2".repeat(40),
|
||||
contentHash: "hash-aiq-deploy",
|
||||
sourceUrl: `https://github.com/NVIDIA/skills/tree/${"2".repeat(40)}/skills/aiq-deploy`,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
name: "upstream path is missing",
|
||||
patch: { githubCurrentStatus: "missing" as const },
|
||||
reason: "github_upstream_missing",
|
||||
status: 410,
|
||||
},
|
||||
{
|
||||
name: "skill was pulled upstream",
|
||||
patch: { githubRemovedAt: 456 },
|
||||
reason: "github_upstream_removed",
|
||||
status: 410,
|
||||
},
|
||||
{
|
||||
name: "scan is pending",
|
||||
patch: { githubScanStatus: "pending" as const },
|
||||
reason: "github_verification_pending",
|
||||
status: 423,
|
||||
},
|
||||
{
|
||||
name: "scan failed",
|
||||
patch: { githubScanStatus: "failed" as const },
|
||||
reason: "github_scan_failed",
|
||||
status: 403,
|
||||
},
|
||||
{
|
||||
name: "scan is suspicious",
|
||||
patch: { githubScanStatus: "suspicious" as const },
|
||||
reason: "github_scan_failed",
|
||||
status: 403,
|
||||
},
|
||||
])("blocks GitHub-backed installs when $name", ({ patch, reason, status }) => {
|
||||
const resolution = buildSkillInstallResolution({
|
||||
origin: "https://clawhub.ai",
|
||||
skill: { ...baseSkill, ...patch },
|
||||
source,
|
||||
});
|
||||
|
||||
expect(resolution).toMatchObject({
|
||||
ok: false,
|
||||
slug: "aiq-deploy",
|
||||
reason,
|
||||
status,
|
||||
});
|
||||
});
|
||||
|
||||
it("explains pending GitHub-backed verification clearly", () => {
|
||||
const resolution = buildSkillInstallResolution({
|
||||
origin: "https://clawhub.ai",
|
||||
skill: {
|
||||
...baseSkill,
|
||||
githubScanStatus: "pending",
|
||||
},
|
||||
source,
|
||||
});
|
||||
|
||||
expect(resolution).toMatchObject({
|
||||
ok: false,
|
||||
slug: "aiq-deploy",
|
||||
reason: "github_verification_pending",
|
||||
status: 423,
|
||||
message:
|
||||
"GitHub-backed skill security scan is in progress. Try again shortly, or rerun with --force-install to install the unverified upstream commit.",
|
||||
});
|
||||
});
|
||||
|
||||
it("allows force-install for pending GitHub-backed verification", () => {
|
||||
const resolution = buildSkillInstallResolution({
|
||||
origin: "https://clawhub.ai",
|
||||
skill: {
|
||||
...baseSkill,
|
||||
githubScanStatus: "pending",
|
||||
},
|
||||
source,
|
||||
forceInstall: true,
|
||||
});
|
||||
|
||||
expect(resolution).toMatchObject({
|
||||
ok: true,
|
||||
installKind: "github",
|
||||
github: {
|
||||
commit: "1".repeat(40),
|
||||
contentHash: "hash-aiq-deploy",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("does not force-install failed GitHub-backed scans", () => {
|
||||
const resolution = buildSkillInstallResolution({
|
||||
origin: "https://clawhub.ai",
|
||||
skill: {
|
||||
...baseSkill,
|
||||
githubScanStatus: "failed",
|
||||
},
|
||||
source,
|
||||
forceInstall: true,
|
||||
});
|
||||
|
||||
expect(resolution).toMatchObject({
|
||||
ok: false,
|
||||
reason: "github_scan_failed",
|
||||
status: 403,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,180 +0,0 @@
|
||||
export type GitHubSkillScanStatus = "clean" | "suspicious" | "malicious" | "pending" | "failed";
|
||||
export type GitHubCurrentStatus = "present" | "missing" | "unknown";
|
||||
|
||||
export type InstallResolverSkill = {
|
||||
slug: string;
|
||||
displayName: string;
|
||||
latestVersionSummary?: { version: string } | null;
|
||||
installKind?: "github";
|
||||
githubPath?: string;
|
||||
githubCurrentCommit?: string;
|
||||
githubCurrentContentHash?: string;
|
||||
githubCurrentStatus?: GitHubCurrentStatus;
|
||||
githubScanStatus?: GitHubSkillScanStatus;
|
||||
githubRemovedAt?: number;
|
||||
};
|
||||
|
||||
export type InstallResolverSource = {
|
||||
repo: string;
|
||||
defaultBranch?: string | null;
|
||||
};
|
||||
|
||||
export type SkillInstallResolution =
|
||||
| {
|
||||
ok: true;
|
||||
slug: string;
|
||||
installKind: "archive";
|
||||
archive: {
|
||||
version: string;
|
||||
downloadUrl: string;
|
||||
};
|
||||
}
|
||||
| {
|
||||
ok: true;
|
||||
slug: string;
|
||||
installKind: "github";
|
||||
github: {
|
||||
repo: string;
|
||||
path: string;
|
||||
commit: string;
|
||||
contentHash: string;
|
||||
sourceUrl: string;
|
||||
};
|
||||
}
|
||||
| {
|
||||
ok: false;
|
||||
slug: string;
|
||||
reason:
|
||||
| "archive_version_missing"
|
||||
| "github_source_missing"
|
||||
| "github_upstream_removed"
|
||||
| "github_upstream_missing"
|
||||
| "github_upstream_unknown"
|
||||
| "github_verification_pending"
|
||||
| "github_scan_failed";
|
||||
message: string;
|
||||
status: 403 | 409 | 410 | 423;
|
||||
};
|
||||
|
||||
export function buildSkillInstallResolution({
|
||||
origin,
|
||||
skill,
|
||||
source,
|
||||
forceInstall = false,
|
||||
}: {
|
||||
origin: string;
|
||||
skill: InstallResolverSkill;
|
||||
source: InstallResolverSource | null;
|
||||
forceInstall?: boolean;
|
||||
}): SkillInstallResolution {
|
||||
if (skill.installKind !== "github") {
|
||||
const version = skill.latestVersionSummary?.version;
|
||||
if (!version) {
|
||||
return block(skill.slug, "archive_version_missing", 409);
|
||||
}
|
||||
|
||||
const url = new URL("/api/v1/download", origin);
|
||||
url.searchParams.set("slug", skill.slug);
|
||||
url.searchParams.set("version", version);
|
||||
return {
|
||||
ok: true,
|
||||
slug: skill.slug,
|
||||
installKind: "archive",
|
||||
archive: {
|
||||
version,
|
||||
downloadUrl: url.toString(),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
if (skill.githubRemovedAt) {
|
||||
return block(skill.slug, "github_upstream_removed", 410);
|
||||
}
|
||||
if (skill.githubCurrentStatus === "missing") {
|
||||
return block(skill.slug, "github_upstream_missing", 410);
|
||||
}
|
||||
if (
|
||||
skill.githubScanStatus === "failed" ||
|
||||
skill.githubScanStatus === "malicious" ||
|
||||
skill.githubScanStatus === "suspicious"
|
||||
) {
|
||||
return block(skill.slug, "github_scan_failed", 403);
|
||||
}
|
||||
if (!source || !skill.githubPath) {
|
||||
return block(skill.slug, "github_source_missing", 409);
|
||||
}
|
||||
if (
|
||||
skill.githubCurrentStatus !== "present" ||
|
||||
!skill.githubCurrentCommit ||
|
||||
!skill.githubCurrentContentHash
|
||||
) {
|
||||
return block(skill.slug, "github_upstream_unknown", 423);
|
||||
}
|
||||
if (
|
||||
skill.githubScanStatus !== "clean" &&
|
||||
!(forceInstall && skill.githubScanStatus === "pending")
|
||||
) {
|
||||
return block(skill.slug, "github_verification_pending", 423);
|
||||
}
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
slug: skill.slug,
|
||||
installKind: "github",
|
||||
github: {
|
||||
repo: source.repo,
|
||||
path: skill.githubPath,
|
||||
commit: skill.githubCurrentCommit,
|
||||
contentHash: skill.githubCurrentContentHash,
|
||||
sourceUrl: buildGitHubTreeUrl(source.repo, skill.githubCurrentCommit, skill.githubPath),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function block(
|
||||
slug: string,
|
||||
reason: Extract<SkillInstallResolution, { ok: false }>["reason"],
|
||||
status: Extract<SkillInstallResolution, { ok: false }>["status"],
|
||||
): SkillInstallResolution {
|
||||
return {
|
||||
ok: false,
|
||||
slug,
|
||||
reason,
|
||||
status,
|
||||
message: INSTALL_BLOCK_MESSAGES[reason],
|
||||
};
|
||||
}
|
||||
|
||||
const INSTALL_BLOCK_MESSAGES: Record<
|
||||
Extract<SkillInstallResolution, { ok: false }>["reason"],
|
||||
string
|
||||
> = {
|
||||
archive_version_missing: "Hosted skill has no downloadable version.",
|
||||
github_source_missing: "GitHub-backed skill source metadata is incomplete.",
|
||||
github_upstream_removed: "GitHub-backed skill has been removed upstream.",
|
||||
github_upstream_missing: "GitHub-backed skill path is missing upstream.",
|
||||
github_upstream_unknown: "GitHub-backed skill needs an upstream freshness check before install.",
|
||||
github_verification_pending:
|
||||
"GitHub-backed skill security scan is in progress. Try again shortly, or rerun with --force-install to install the unverified upstream commit.",
|
||||
github_scan_failed: "GitHub-backed skill failed ClawHub security scanning.",
|
||||
};
|
||||
|
||||
function buildGitHubTreeUrl(repo: string, commit: string, path: string) {
|
||||
return `https://github.com/${encodeURIComponentRepo(repo)}/tree/${commit}/${encodeURIComponentPath(
|
||||
path,
|
||||
)}`;
|
||||
}
|
||||
|
||||
function encodeURIComponentRepo(repo: string) {
|
||||
return repo
|
||||
.split("/")
|
||||
.map((segment) => encodeURIComponent(segment))
|
||||
.join("/");
|
||||
}
|
||||
|
||||
function encodeURIComponentPath(path: string) {
|
||||
return path
|
||||
.split("/")
|
||||
.map((segment) => encodeURIComponent(segment))
|
||||
.join("/");
|
||||
}
|
||||
@@ -223,7 +223,7 @@ describe("moderationEngine", () => {
|
||||
expect(result.status).toBe("suspicious");
|
||||
});
|
||||
|
||||
it("does not duplicate SkillSpector credential-browser automation analysis", () => {
|
||||
it("flags browser automation that puts passwords in argv", () => {
|
||||
const result = runStaticModerationScan({
|
||||
slug: "email-daily-summary",
|
||||
displayName: "Email Daily Summary",
|
||||
@@ -242,11 +242,11 @@ describe("moderationEngine", () => {
|
||||
],
|
||||
});
|
||||
|
||||
expect(result.reasonCodes).not.toContain("suspicious.browser_credential_automation");
|
||||
expect(result.status).toBe("clean");
|
||||
expect(result.reasonCodes).toContain("suspicious.browser_credential_automation");
|
||||
expect(result.status).toBe("suspicious");
|
||||
});
|
||||
|
||||
it("does not duplicate SkillSpector persisted browser eval analysis", () => {
|
||||
it("flags persisted browser-use eval against authenticated mail contexts", () => {
|
||||
const result = runStaticModerationScan({
|
||||
slug: "email-daily-summary",
|
||||
displayName: "Email Daily Summary",
|
||||
@@ -266,8 +266,8 @@ describe("moderationEngine", () => {
|
||||
],
|
||||
});
|
||||
|
||||
expect(result.reasonCodes).not.toContain("suspicious.browser_credential_automation");
|
||||
expect(result.status).toBe("clean");
|
||||
expect(result.reasonCodes).toContain("suspicious.browser_credential_automation");
|
||||
expect(result.status).toBe("suspicious");
|
||||
});
|
||||
|
||||
it("does not flag ordinary browser-use navigation docs", () => {
|
||||
@@ -290,7 +290,7 @@ describe("moderationEngine", () => {
|
||||
expect(result.status).toBe("clean");
|
||||
});
|
||||
|
||||
it("does not duplicate SkillSpector browser automation analysis in the static scanner", () => {
|
||||
it("blocks stealth browser automation that advertises bot-protection bypass and persistent sessions", () => {
|
||||
const result = runStaticModerationScan({
|
||||
slug: "stealth-browser",
|
||||
displayName: "Stealth Browser",
|
||||
@@ -311,8 +311,8 @@ describe("moderationEngine", () => {
|
||||
],
|
||||
});
|
||||
|
||||
expect(result.reasonCodes).not.toContain("malicious.stealth_browser_abuse");
|
||||
expect(result.status).toBe("clean");
|
||||
expect(result.reasonCodes).toContain("malicious.stealth_browser_abuse");
|
||||
expect(result.status).toBe("malicious");
|
||||
});
|
||||
|
||||
it("flags wallet mnemonics passed as CLI argv", () => {
|
||||
@@ -497,7 +497,7 @@ describe("moderationEngine", () => {
|
||||
expect(result.status).toBe("suspicious");
|
||||
});
|
||||
|
||||
it("does not duplicate SkillSpector browser file-render analysis", () => {
|
||||
it("flags Playwright file URL renders of interpolated SVG", () => {
|
||||
const result = runStaticModerationScan({
|
||||
slug: "office-quotes",
|
||||
displayName: "Office Quotes",
|
||||
@@ -520,8 +520,8 @@ describe("moderationEngine", () => {
|
||||
],
|
||||
});
|
||||
|
||||
expect(result.reasonCodes).not.toContain("suspicious.browser_file_render");
|
||||
expect(result.status).toBe("clean");
|
||||
expect(result.reasonCodes).toContain("suspicious.browser_file_render");
|
||||
expect(result.status).toBe("suspicious");
|
||||
});
|
||||
|
||||
it("does not flag Playwright file renders with JavaScript disabled", () => {
|
||||
@@ -1754,7 +1754,7 @@ describe("moderationEngine", () => {
|
||||
expect(result.status).toBe("clean");
|
||||
});
|
||||
|
||||
it("does not duplicate SkillSpector browser typing analysis", () => {
|
||||
it("flags shell positional input passed directly to browser typing", () => {
|
||||
const result = runStaticModerationScan({
|
||||
slug: "wechat-helper",
|
||||
displayName: "WeChat Helper",
|
||||
@@ -1775,11 +1775,11 @@ describe("moderationEngine", () => {
|
||||
],
|
||||
});
|
||||
|
||||
expect(result.reasonCodes).not.toContain("suspicious.unsafe_browser_text_input");
|
||||
expect(result.status).toBe("clean");
|
||||
expect(result.reasonCodes).toContain("suspicious.unsafe_browser_text_input");
|
||||
expect(result.status).toBe("suspicious");
|
||||
});
|
||||
|
||||
it("does not inspect shell positional assignments for browser typing", () => {
|
||||
it("checks every positional shell assignment before browser typing", () => {
|
||||
const result = runStaticModerationScan({
|
||||
slug: "wechat-helper",
|
||||
displayName: "WeChat Helper",
|
||||
@@ -1799,8 +1799,8 @@ describe("moderationEngine", () => {
|
||||
],
|
||||
});
|
||||
|
||||
expect(result.reasonCodes).not.toContain("suspicious.unsafe_browser_text_input");
|
||||
expect(result.status).toBe("clean");
|
||||
expect(result.reasonCodes).toContain("suspicious.unsafe_browser_text_input");
|
||||
expect(result.status).toBe("suspicious");
|
||||
});
|
||||
|
||||
it("allows browser typing after basic shell input validation", () => {
|
||||
|
||||
@@ -94,6 +94,8 @@ const GOOGLE_SHEETS_SPREADSHEET_URL_PATTERN =
|
||||
/https?:\/\/[^\s"'`]*\/spreadsheets\/([A-Za-z0-9_-]{20,})\/[^\s"'`]*/i;
|
||||
const DESTRUCTIVE_DELETE_PATTERN =
|
||||
/\brm\s+-[A-Za-z]*r[A-Za-z]*f[A-Za-z]*\s+(["']?)(\/root\/\.openclaw\/|\/home\/[^/\s"'`]+\/\.openclaw\/|\/Users\/[^/\s"'`]+\/\.openclaw\/|~\/\.openclaw\/|\$HOME\/\.openclaw\/|\$\{HOME\}\/\.openclaw\/|\/etc\/|\/usr\/|\/opt\/|\/Library\/|\/Applications\/)[^\s"'`;|&)]*\1/i;
|
||||
const SHELL_POSITIONAL_ASSIGNMENT_PATTERN =
|
||||
/^\s*([A-Z_][A-Z0-9_]*)=(["']?)\$(?:[1-9][0-9]*|@|\*)\2\s*(?:#.*)?$/gm;
|
||||
const SECRET_ASSIGNMENT_PATTERN =
|
||||
/\b(?:[A-Za-z0-9]+[_\s-]+)*(?:(?:api|client|consumer)[_\s-]?(?:secret|key|token)|secret[_\s-]?key|access[_\s-]?(?:token|key|secret|grant)|auth[_\s-]?token|bearer(?:[_\s-]?token)?|private[_\s-]?key|service[_\s-]?role[_\s-]?key|github[_\s-]?(?:pat|token)|(?:openrouter|supabase|storj)[_\s-]?(?:key|token|secret|access[_\s-]?grant)|password)\b\s*[:=]\s*["'`]?([A-Za-z0-9][A-Za-z0-9._~+/=-]{15,})["'`]?/i;
|
||||
const AUTH_HEADER_SECRET_PATTERN =
|
||||
@@ -109,6 +111,12 @@ const HOST_PLATFORM_SOURCE_CONTEXT_PATTERN =
|
||||
const HOST_PLATFORM_PATCH_COMMAND_PATTERN =
|
||||
/\b(?:sed\s+-i|perl\s+-0?pi|cp\s+|cat\s+>|python3?\b.{0,120}(?:write|replace))/i;
|
||||
const HOST_PLATFORM_REBUILD_PATTERN = /\b(?:pnpm\s+build|npm\s+run\s+build|bun\s+run\s+build)\b/i;
|
||||
const BROWSER_USE_PASSWORD_ARGV_PATTERN =
|
||||
/\bbrowser-use\s+input\b[^\n]*(?:password|passwd|\$[A-Z_]*(?:PASSWORD|PASS|PWD)[A-Z0-9_]*|<password>|\{password\})/i;
|
||||
const BROWSER_USE_AUTH_EVAL_PATTERN = /\bbrowser-use\s+(?:eval|python)\b/i;
|
||||
const AUTHENTICATED_MAIL_CONTEXT_PATTERN = /\b(?:mail\.google\.com|gmail|webmail|mailbox|inbox)\b/i;
|
||||
const PERSISTENCE_SCHEDULER_PATTERN =
|
||||
/\b(?:launchctl\s+load|crontab\b|LaunchAgents\/|systemctl\s+(?:--user\s+)?enable)\b/i;
|
||||
const SECRET_ARGV_WARNING_PATTERN =
|
||||
/\b(?:do\s+not|don't|avoid|never|reject)\b[^\n]{0,120}\b(?:argv|argument|from-mnemonic|private[-_\s]?key|seed[-\s]?phrase|mnemonic)\b/i;
|
||||
const FROM_MNEMONIC_ARGV_PATTERN =
|
||||
@@ -127,6 +135,17 @@ const PYTHON_BASE64_FILE_READ_PATTERN =
|
||||
/base64\.b64encode\s*\(\s*(?:[A-Za-z_][A-Za-z0-9_]*\.read_bytes\s*\(\s*\)|Path\s*\([^)]*\)\.read_bytes\s*\(\s*\)|open\s*\([^)]*["']rb["'][\s\S]{0,120}\.read\s*\(\s*\))/i;
|
||||
const PYTHON_NETWORK_UPLOAD_PATTERN =
|
||||
/\b(?:requests|session|self\.session|client|httpx\.(?:post|request))\.post\s*\([\s\S]{0,1600}(?:json\s*=|data\s*=|files\s*=|headers\s*=|Authorization)/i;
|
||||
const PLAYWRIGHT_CHROMIUM_PATTERN = /\b(?:playwright\.)?chromium\.launch\s*\(/i;
|
||||
const FILE_URL_BROWSER_NAVIGATION_PATTERN = /\bpage\.goto\s*\([^)]*file:\/\//i;
|
||||
const SVG_HTML_INTERPOLATION_PATTERN =
|
||||
/(?:<body>[\s\S]{0,240}\$\{[^}]*svg[^}]*\}|writeFile(?:Sync)?\s*\([^)]*\.html[^)]*\$\{[^}]*svg[^}]*\}|\$\{[^}]*svg[^}]*\}[\s\S]{0,240}<\/body>)/i;
|
||||
const BROWSER_JS_DISABLED_PATTERN =
|
||||
/javaScriptEnabled\s*:\s*false|Content-Security-Policy|script-src\s+['"]?none/i;
|
||||
const STEALTH_BROWSER_CONTEXT_PATTERN =
|
||||
/\b(?:stealth|anti[-\s]?detect|undetected|fingerprint spoof|navigator\.webdriver)\b/i;
|
||||
const BOT_PROTECTION_BYPASS_PATTERN = /\b(?:captcha|cloudflare|turnstile|bot detection|waf)\b/i;
|
||||
const BROWSER_SESSION_PERSISTENCE_PATTERN =
|
||||
/\b(?:persistent_context|userDataDir|storageState|session persistence|persist(?:ed)? cookies?)\b/i;
|
||||
const AGENT_OUTPUT_DIR_ARGUMENT_PATTERN =
|
||||
/add_argument\s*\(\s*["']--outdir["']|args\.outdir|output_path\s*=\s*Path\s*\(\s*args\.outdir\s*\)/i;
|
||||
const FFMPEG_FORCE_OUTPUT_PATTERN =
|
||||
@@ -249,6 +268,31 @@ function findCredentialExposureInstruction(content: string) {
|
||||
return null;
|
||||
}
|
||||
|
||||
function findBrowserCredentialAutomation(content: string) {
|
||||
const lines = content.split("\n");
|
||||
for (let i = 0; i < lines.length; i += 1) {
|
||||
const line = lines[i] ?? "";
|
||||
if (BROWSER_USE_PASSWORD_ARGV_PATTERN.test(line)) {
|
||||
return { line: i + 1, text: line };
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
BROWSER_USE_AUTH_EVAL_PATTERN.test(content) &&
|
||||
AUTHENTICATED_MAIL_CONTEXT_PATTERN.test(content) &&
|
||||
PERSISTENCE_SCHEDULER_PATTERN.test(content)
|
||||
) {
|
||||
for (let i = 0; i < lines.length; i += 1) {
|
||||
const line = lines[i] ?? "";
|
||||
if (BROWSER_USE_AUTH_EVAL_PATTERN.test(line) || PERSISTENCE_SCHEDULER_PATTERN.test(line)) {
|
||||
return { line: i + 1, text: line };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function redactSecretArgvEvidence(line: string) {
|
||||
return line.replace(SECRET_ARGV_REDACTION_PATTERN, "$1$2[REDACTED]$2");
|
||||
}
|
||||
@@ -357,6 +401,49 @@ function isScopedOpenClawDelete(line: string, slug?: string) {
|
||||
return false;
|
||||
}
|
||||
|
||||
function hasShellVariableValidation(content: string, variable: string, useIndex: number) {
|
||||
const escaped = variable.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
const beforeUse = content.slice(0, useIndex);
|
||||
const variableReference = String.raw`(?:\$\{${escaped}\}|\$${escaped})`;
|
||||
const lengthCheck = new RegExp(
|
||||
String.raw`\$\{#${escaped}\}\s*(?:-[a-z]\s+)?(?:[<>!=]=?|-[gl][te])`,
|
||||
"m",
|
||||
);
|
||||
const controlCharStrip = new RegExp(
|
||||
String.raw`(?:tr\s+-d\s+["']?\\(?:000|x00).{0,80}\\(?:037|x1[fF]|177|x7[fF])|${escaped}\s*=.*tr\s+-d)`,
|
||||
"s",
|
||||
);
|
||||
const explicitValidation = new RegExp(
|
||||
String.raw`(?:validate|sanitize|strip|clean)[A-Za-z0-9_ -]{0,60}${variableReference}|${variableReference}.{0,60}(?:validate|sanitize|strip|clean)`,
|
||||
"is",
|
||||
);
|
||||
|
||||
return (
|
||||
lengthCheck.test(beforeUse) ||
|
||||
controlCharStrip.test(beforeUse) ||
|
||||
explicitValidation.test(beforeUse)
|
||||
);
|
||||
}
|
||||
|
||||
function findUnsafeBrowserTextInput(content: string) {
|
||||
for (const assignment of content.matchAll(SHELL_POSITIONAL_ASSIGNMENT_PATTERN)) {
|
||||
const variable = assignment[1];
|
||||
if (!variable) continue;
|
||||
|
||||
const escaped = variable.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
const browserTextPattern = new RegExp(
|
||||
String.raw`\bbrowser\s+action=act\b[^\n]*\bkind=["']?type["']?[^\n]*\btext=(?:"\$${escaped}"|'\$${escaped}'|\$${escaped})(?![A-Za-z0-9_])`,
|
||||
"i",
|
||||
);
|
||||
const match = content.match(browserTextPattern);
|
||||
if (!match || match.index === undefined) continue;
|
||||
if (hasShellVariableValidation(content, variable, match.index)) continue;
|
||||
|
||||
return findLineAtIndex(content, match.index);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function addFinding(
|
||||
findings: ModerationFinding[],
|
||||
finding: Omit<ModerationFinding, "evidence"> & { evidence: string },
|
||||
@@ -515,6 +602,21 @@ function findJsSensitiveFileNetworkSend(content: string) {
|
||||
return null;
|
||||
}
|
||||
|
||||
function findUnsafeBrowserFileRender(content: string) {
|
||||
if (!PLAYWRIGHT_CHROMIUM_PATTERN.test(content)) return null;
|
||||
if (!FILE_URL_BROWSER_NAVIGATION_PATTERN.test(content)) return null;
|
||||
if (!SVG_HTML_INTERPOLATION_PATTERN.test(content)) return null;
|
||||
if (BROWSER_JS_DISABLED_PATTERN.test(content)) return null;
|
||||
return findFirstLine(content, FILE_URL_BROWSER_NAVIGATION_PATTERN);
|
||||
}
|
||||
|
||||
function findStealthBrowserAbuse(content: string) {
|
||||
if (!STEALTH_BROWSER_CONTEXT_PATTERN.test(content)) return null;
|
||||
if (!BOT_PROTECTION_BYPASS_PATTERN.test(content)) return null;
|
||||
if (!BROWSER_SESSION_PERSISTENCE_PATTERN.test(content)) return null;
|
||||
return findFirstLine(content, STEALTH_BROWSER_CONTEXT_PATTERN);
|
||||
}
|
||||
|
||||
function findUnsafeAgentControlledFileWrite(content: string) {
|
||||
if (!AGENT_OUTPUT_DIR_ARGUMENT_PATTERN.test(content)) return null;
|
||||
if (!FFMPEG_FORCE_OUTPUT_PATTERN.test(content)) return null;
|
||||
@@ -734,6 +836,18 @@ function scanCodeFile(
|
||||
});
|
||||
}
|
||||
|
||||
const unsafeBrowserTextInput = findUnsafeBrowserTextInput(content);
|
||||
if (unsafeBrowserTextInput) {
|
||||
addFinding(findings, {
|
||||
code: REASON_CODES.UNSAFE_BROWSER_TEXT_INPUT,
|
||||
severity: "warn",
|
||||
file: path,
|
||||
line: unsafeBrowserTextInput.line,
|
||||
message: "Shell positional input is typed into browser automation without validation.",
|
||||
evidence: unsafeBrowserTextInput.text,
|
||||
});
|
||||
}
|
||||
|
||||
const hostPlatformSourcePatch = findHostPlatformSourcePatch(content);
|
||||
if (hostPlatformSourcePatch) {
|
||||
addFinding(findings, {
|
||||
@@ -746,6 +860,32 @@ function scanCodeFile(
|
||||
});
|
||||
}
|
||||
|
||||
const unsafeBrowserFileRender = findUnsafeBrowserFileRender(content);
|
||||
if (unsafeBrowserFileRender) {
|
||||
addFinding(findings, {
|
||||
code: REASON_CODES.BROWSER_FILE_RENDER,
|
||||
severity: "critical",
|
||||
file: path,
|
||||
line: unsafeBrowserFileRender.line,
|
||||
message:
|
||||
"Browser automation renders interpolated SVG/HTML from a file URL with JavaScript enabled.",
|
||||
evidence: unsafeBrowserFileRender.text,
|
||||
});
|
||||
}
|
||||
|
||||
const stealthBrowserAbuse = findStealthBrowserAbuse(content);
|
||||
if (stealthBrowserAbuse) {
|
||||
addFinding(findings, {
|
||||
code: REASON_CODES.STEALTH_BROWSER_ABUSE,
|
||||
severity: "critical",
|
||||
file: path,
|
||||
line: stealthBrowserAbuse.line,
|
||||
message:
|
||||
"Browser automation advertises stealth/anti-detection behavior with bot-protection bypass and persistent sessions.",
|
||||
evidence: stealthBrowserAbuse.text,
|
||||
});
|
||||
}
|
||||
|
||||
const unsafeAgentControlledFileWrite = findUnsafeAgentControlledFileWrite(content);
|
||||
if (unsafeAgentControlledFileWrite) {
|
||||
addFinding(findings, {
|
||||
@@ -945,6 +1085,18 @@ function scanMarkdownFile(
|
||||
});
|
||||
}
|
||||
|
||||
const browserCredentialAutomation = findBrowserCredentialAutomation(content);
|
||||
if (browserCredentialAutomation) {
|
||||
addFinding(findings, {
|
||||
code: REASON_CODES.BROWSER_CREDENTIAL_AUTOMATION,
|
||||
severity: "critical",
|
||||
file: path,
|
||||
line: browserCredentialAutomation.line,
|
||||
message: "Browser automation instructions expose credentials or persist authenticated eval.",
|
||||
evidence: browserCredentialAutomation.text,
|
||||
});
|
||||
}
|
||||
|
||||
const secretArgvExposure = findSecretArgvExposure(content);
|
||||
if (secretArgvExposure) {
|
||||
addFinding(findings, {
|
||||
@@ -970,6 +1122,19 @@ function scanMarkdownFile(
|
||||
});
|
||||
}
|
||||
|
||||
const stealthBrowserAbuse = findStealthBrowserAbuse(content);
|
||||
if (stealthBrowserAbuse) {
|
||||
addFinding(findings, {
|
||||
code: REASON_CODES.STEALTH_BROWSER_ABUSE,
|
||||
severity: "critical",
|
||||
file: path,
|
||||
line: stealthBrowserAbuse.line,
|
||||
message:
|
||||
"Browser automation advertises stealth/anti-detection behavior with bot-protection bypass and persistent sessions.",
|
||||
evidence: stealthBrowserAbuse.text,
|
||||
});
|
||||
}
|
||||
|
||||
if (hasMaliciousInstallPrompt(content)) {
|
||||
const match = findFirstLine(
|
||||
content,
|
||||
@@ -998,6 +1163,18 @@ function scanMarkdownFile(
|
||||
});
|
||||
}
|
||||
|
||||
const unsafeBrowserTextInput = findUnsafeBrowserTextInput(content);
|
||||
if (unsafeBrowserTextInput) {
|
||||
addFinding(findings, {
|
||||
code: REASON_CODES.UNSAFE_BROWSER_TEXT_INPUT,
|
||||
severity: "warn",
|
||||
file: path,
|
||||
line: unsafeBrowserTextInput.line,
|
||||
message: "Shell positional input is typed into browser automation without validation.",
|
||||
evidence: unsafeBrowserTextInput.text,
|
||||
});
|
||||
}
|
||||
|
||||
if (
|
||||
/ignore\s+(all\s+)?previous\s+instructions/i.test(content) ||
|
||||
/system\s*prompt\s*[:=]/i.test(content)
|
||||
|
||||
@@ -12,7 +12,7 @@ export type ModerationFinding = {
|
||||
evidence: string;
|
||||
};
|
||||
|
||||
export const MODERATION_ENGINE_VERSION = "v2.4.25";
|
||||
export const MODERATION_ENGINE_VERSION = "v2.4.24";
|
||||
|
||||
export const REASON_CODES = {
|
||||
LLM_REVIEW: "review.llm_review",
|
||||
@@ -21,10 +21,13 @@ export const REASON_CODES = {
|
||||
GENERATED_SOURCE_TEMPLATE: "suspicious.generated_source_template_injection",
|
||||
EXPOSED_RESOURCE_IDENTIFIER: "suspicious.exposed_resource_identifier",
|
||||
DESTRUCTIVE_DELETE_COMMAND: "suspicious.destructive_delete_command",
|
||||
UNSAFE_BROWSER_TEXT_INPUT: "suspicious.unsafe_browser_text_input",
|
||||
EXPOSED_SECRET_LITERAL: "suspicious.exposed_secret_literal",
|
||||
CREDENTIAL_EXPOSURE_INSTRUCTIONS: "suspicious.credential_exposure_instructions",
|
||||
BROWSER_CREDENTIAL_AUTOMATION: "suspicious.browser_credential_automation",
|
||||
SECRET_ARGV_EXPOSURE: "suspicious.secret_argv_exposure",
|
||||
HOST_PLATFORM_SOURCE_PATCH: "suspicious.host_platform_source_patch",
|
||||
BROWSER_FILE_RENDER: "suspicious.browser_file_render",
|
||||
UNSAFE_FILE_WRITE: "suspicious.unsafe_file_write",
|
||||
INSECURE_TLS_VERIFICATION: "suspicious.insecure_tls_verification",
|
||||
AUTONOMOUS_CREDENTIAL_EGRESS: "suspicious.autonomous_credential_egress",
|
||||
@@ -41,12 +44,15 @@ export const REASON_CODES = {
|
||||
MANIFEST_PRIVILEGED_ALWAYS: "suspicious.privileged_always",
|
||||
MALICIOUS_INSTALL_PROMPT: "malicious.install_terminal_payload",
|
||||
KNOWN_BLOCKED_SIGNATURE: "malicious.known_blocked_signature",
|
||||
STEALTH_BROWSER_ABUSE: "malicious.stealth_browser_abuse",
|
||||
DEP_NOT_FOUND: "suspicious.dep_not_found_on_registry",
|
||||
} as const;
|
||||
|
||||
const MALICIOUS_CODES = new Set<string>([
|
||||
REASON_CODES.CRYPTO_MINING,
|
||||
REASON_CODES.MALICIOUS_INSTALL_PROMPT,
|
||||
REASON_CODES.KNOWN_BLOCKED_SIGNATURE,
|
||||
REASON_CODES.STEALTH_BROWSER_ABUSE,
|
||||
]);
|
||||
|
||||
const EXTERNALLY_CLEARABLE_SUSPICIOUS_CODES = new Set<string>([REASON_CODES.CREDENTIAL_HARVEST]);
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
export const Events = {
|
||||
GitHubSkillSourceSyncStarted: "github_skill_source_sync.started",
|
||||
GitHubSkillSourceSyncCompleted: "github_skill_source_sync.completed",
|
||||
GitHubSkillSourceSyncSourceFailed: "github_skill_source_sync.source_failed",
|
||||
GitHubSkillSourceSyncFailed: "github_skill_source_sync.failed",
|
||||
} as const;
|
||||
|
||||
export type EventName = (typeof Events)[keyof typeof Events];
|
||||
|
||||
type EventPayload = Record<string, unknown>;
|
||||
|
||||
export function logEvent(event: EventName, payload: EventPayload = {}) {
|
||||
console.log(JSON.stringify({ event, ...payload }));
|
||||
}
|
||||
|
||||
export function logErrorEvent(event: EventName, payload: EventPayload = {}) {
|
||||
console.error(JSON.stringify({ event, ...payload }));
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { Doc } from "../_generated/dataModel";
|
||||
import { hasOfficialPublisherRow, isOfficialPublisher } from "./officialPublishers";
|
||||
import { isOfficialPublisher } from "./officialPublishers";
|
||||
|
||||
function makePublisher(
|
||||
overrides: Partial<Record<keyof Doc<"publishers">, unknown>>,
|
||||
@@ -17,105 +17,278 @@ function makePublisher(
|
||||
} as Doc<"publishers">;
|
||||
}
|
||||
|
||||
function makeOfficialRow(publisherId: string) {
|
||||
return {
|
||||
_id: `officialPublishers:${publisherId}`,
|
||||
_creationTime: 1,
|
||||
publisherId,
|
||||
createdAt: 1,
|
||||
updatedAt: 1,
|
||||
};
|
||||
}
|
||||
|
||||
function makeCtx({ officialPublisherIds = [] }: { officialPublisherIds?: string[] } = {}) {
|
||||
return {
|
||||
db: {
|
||||
query: vi.fn((table: string) => {
|
||||
if (table !== "officialPublishers") {
|
||||
throw new Error(`Unexpected table ${table}`);
|
||||
}
|
||||
return {
|
||||
withIndex: vi.fn((_indexName: string, buildQuery: (q: unknown) => unknown) => {
|
||||
let requestedPublisherId: string | undefined;
|
||||
buildQuery({
|
||||
eq: vi.fn((field: string, value: string) => {
|
||||
if (field === "publisherId") requestedPublisherId = value;
|
||||
return {};
|
||||
}),
|
||||
});
|
||||
return {
|
||||
unique: vi.fn(async () =>
|
||||
requestedPublisherId && officialPublisherIds.includes(requestedPublisherId)
|
||||
? makeOfficialRow(requestedPublisherId)
|
||||
: null,
|
||||
),
|
||||
};
|
||||
}),
|
||||
};
|
||||
}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("isOfficialPublisher", () => {
|
||||
it("treats a publisher with an official row as official", async () => {
|
||||
const ctx = makeCtx({ officialPublisherIds: ["publishers:acme"] });
|
||||
it("treats the openclaw org publisher as official", async () => {
|
||||
const ctx = { db: { query: vi.fn() } };
|
||||
|
||||
await expect(
|
||||
isOfficialPublisher(ctx as never, makePublisher({ _id: "publishers:acme", handle: "acme" })),
|
||||
isOfficialPublisher(ctx as never, makePublisher({ handle: "openclaw" })),
|
||||
).resolves.toBe(true);
|
||||
});
|
||||
|
||||
it("treats a personal publisher with an official row as official", async () => {
|
||||
const ctx = makeCtx({ officialPublisherIds: ["publishers:alice"] });
|
||||
|
||||
await expect(
|
||||
isOfficialPublisher(
|
||||
ctx as never,
|
||||
makePublisher({
|
||||
_id: "publishers:alice",
|
||||
kind: "user",
|
||||
handle: "alice",
|
||||
linkedUserId: "users:alice",
|
||||
it("does not treat an unreserved nvidia org publisher as official", async () => {
|
||||
const ctx = {
|
||||
db: {
|
||||
query: vi.fn((table: string) => {
|
||||
if (table !== "reservedHandles") throw new Error(`Unexpected table ${table}`);
|
||||
return {
|
||||
withIndex: vi.fn(() => ({
|
||||
order: vi.fn(() => ({
|
||||
take: vi.fn(async () => []),
|
||||
})),
|
||||
})),
|
||||
};
|
||||
}),
|
||||
),
|
||||
).resolves.toBe(true);
|
||||
});
|
||||
|
||||
it("does not treat legacy official handles as official without a row", async () => {
|
||||
const ctx = makeCtx();
|
||||
},
|
||||
};
|
||||
|
||||
await expect(
|
||||
isOfficialPublisher(
|
||||
ctx as never,
|
||||
makePublisher({ _id: "publishers:openclaw", handle: "openclaw" }),
|
||||
),
|
||||
isOfficialPublisher(ctx as never, makePublisher({ handle: "nvidia" })),
|
||||
).resolves.toBe(false);
|
||||
});
|
||||
|
||||
it("does not inherit official status from org membership", async () => {
|
||||
it("treats the reserved-owner-controlled nvidia org publisher as official", async () => {
|
||||
const nvidia = makePublisher({ _id: "publishers:nvidia", handle: "nvidia" });
|
||||
const ctx = {
|
||||
db: {
|
||||
query: vi.fn((table: string) => {
|
||||
if (table === "reservedHandles") {
|
||||
return {
|
||||
withIndex: vi.fn(() => ({
|
||||
order: vi.fn(() => ({
|
||||
take: vi.fn(async () => [
|
||||
{
|
||||
_id: "reservedHandles:nvidia",
|
||||
handle: "nvidia",
|
||||
rightfulOwnerUserId: "users:nvidia",
|
||||
createdAt: 1,
|
||||
updatedAt: 1,
|
||||
},
|
||||
]),
|
||||
})),
|
||||
})),
|
||||
};
|
||||
}
|
||||
if (table === "publisherMembers") {
|
||||
return {
|
||||
withIndex: vi.fn(() => ({
|
||||
unique: vi.fn(async () => ({
|
||||
_id: "publisherMembers:nvidia",
|
||||
publisherId: nvidia._id,
|
||||
userId: "users:nvidia",
|
||||
role: "owner",
|
||||
createdAt: 1,
|
||||
updatedAt: 1,
|
||||
})),
|
||||
})),
|
||||
};
|
||||
}
|
||||
throw new Error(`Unexpected table ${table}`);
|
||||
}),
|
||||
},
|
||||
};
|
||||
|
||||
await expect(isOfficialPublisher(ctx as never, nvidia)).resolves.toBe(true);
|
||||
});
|
||||
|
||||
it("does not treat nvidia as official when the reserved owner does not own the org", async () => {
|
||||
const ctx = {
|
||||
db: {
|
||||
query: vi.fn((table: string) => {
|
||||
if (table === "reservedHandles") {
|
||||
return {
|
||||
withIndex: vi.fn(() => ({
|
||||
order: vi.fn(() => ({
|
||||
take: vi.fn(async () => [
|
||||
{
|
||||
_id: "reservedHandles:nvidia",
|
||||
handle: "nvidia",
|
||||
rightfulOwnerUserId: "users:nvidia",
|
||||
createdAt: 1,
|
||||
updatedAt: 1,
|
||||
},
|
||||
]),
|
||||
})),
|
||||
})),
|
||||
};
|
||||
}
|
||||
if (table === "publisherMembers") {
|
||||
return {
|
||||
withIndex: vi.fn(() => ({
|
||||
unique: vi.fn(async () => null),
|
||||
})),
|
||||
};
|
||||
}
|
||||
throw new Error(`Unexpected table ${table}`);
|
||||
}),
|
||||
},
|
||||
};
|
||||
|
||||
await expect(
|
||||
isOfficialPublisher(ctx as never, makePublisher({ handle: "nvidia" })),
|
||||
).resolves.toBe(false);
|
||||
});
|
||||
|
||||
it("does not treat personal publisher of unreserved nvidia org member as official", async () => {
|
||||
const nvidia = makePublisher({ _id: "publishers:nvidia", handle: "nvidia" });
|
||||
const personal = makePublisher({
|
||||
_id: "publishers:alice",
|
||||
kind: "user",
|
||||
handle: "alice",
|
||||
linkedUserId: "users:alice",
|
||||
});
|
||||
const ctx = makeCtx({ officialPublisherIds: ["publishers:openclaw"] });
|
||||
const ctx = {
|
||||
db: {
|
||||
query: vi.fn((table: string) => {
|
||||
if (table === "publishers") {
|
||||
return {
|
||||
withIndex: vi.fn((_index: string, fn: (q: any) => any) => {
|
||||
let capturedHandle: string | undefined;
|
||||
fn({ eq: (_: string, v: string) => { capturedHandle = v; return { eq: () => ({}) }; } });
|
||||
return { unique: vi.fn(async () => (capturedHandle === "nvidia" ? nvidia : null)) };
|
||||
}),
|
||||
};
|
||||
}
|
||||
if (table === "publisherMembers") {
|
||||
return {
|
||||
withIndex: vi.fn(() => ({
|
||||
unique: vi.fn(async () => ({
|
||||
_id: "publisherMembers:alice-nvidia",
|
||||
publisherId: nvidia._id,
|
||||
userId: "users:alice",
|
||||
role: "publisher",
|
||||
createdAt: 1,
|
||||
updatedAt: 1,
|
||||
})),
|
||||
})),
|
||||
};
|
||||
}
|
||||
if (table === "reservedHandles") {
|
||||
return {
|
||||
withIndex: vi.fn(() => ({
|
||||
order: vi.fn(() => ({
|
||||
take: vi.fn(async () => []),
|
||||
})),
|
||||
})),
|
||||
};
|
||||
}
|
||||
throw new Error(`Unexpected table ${table}`);
|
||||
}),
|
||||
},
|
||||
};
|
||||
|
||||
await expect(isOfficialPublisher(ctx as never, personal)).resolves.toBe(false);
|
||||
});
|
||||
|
||||
it("can check raw official rows independently from active publisher state", async () => {
|
||||
const ctx = makeCtx({ officialPublisherIds: ["publishers:acme"] });
|
||||
it("treats personal publisher of reserved-owner-controlled nvidia org member as official", async () => {
|
||||
const nvidia = makePublisher({ _id: "publishers:nvidia", handle: "nvidia" });
|
||||
const personal = makePublisher({
|
||||
_id: "publishers:alice",
|
||||
kind: "user",
|
||||
handle: "alice",
|
||||
linkedUserId: "users:alice",
|
||||
});
|
||||
const ctx = {
|
||||
db: {
|
||||
query: vi.fn((table: string) => {
|
||||
if (table === "publishers") {
|
||||
return {
|
||||
withIndex: vi.fn((_index: string, fn: (q: any) => any) => {
|
||||
let capturedHandle: string | undefined;
|
||||
fn({ eq: (_: string, v: string) => { capturedHandle = v; return { eq: () => ({}) }; } });
|
||||
return { unique: vi.fn(async () => (capturedHandle === "nvidia" ? nvidia : null)) };
|
||||
}),
|
||||
};
|
||||
}
|
||||
if (table === "publisherMembers") {
|
||||
return {
|
||||
withIndex: vi.fn((_index: string, fn: (q: any) => any) => {
|
||||
let capturedUserId: string | undefined;
|
||||
fn({ eq: (_: string, _v: any) => ({ eq: (_2: string, v2: string) => { capturedUserId = v2; return {}; } }) });
|
||||
const record =
|
||||
capturedUserId === "users:nvidia-owner"
|
||||
? {
|
||||
_id: "publisherMembers:nvidia-owner",
|
||||
publisherId: nvidia._id,
|
||||
userId: "users:nvidia-owner",
|
||||
role: "owner",
|
||||
createdAt: 1,
|
||||
updatedAt: 1,
|
||||
}
|
||||
: {
|
||||
_id: "publisherMembers:alice-nvidia",
|
||||
publisherId: nvidia._id,
|
||||
userId: "users:alice",
|
||||
role: "publisher",
|
||||
createdAt: 1,
|
||||
updatedAt: 1,
|
||||
};
|
||||
return { unique: vi.fn(async () => record) };
|
||||
}),
|
||||
};
|
||||
}
|
||||
if (table === "reservedHandles") {
|
||||
return {
|
||||
withIndex: vi.fn(() => ({
|
||||
order: vi.fn(() => ({
|
||||
take: vi.fn(async () => [
|
||||
{
|
||||
_id: "reservedHandles:nvidia",
|
||||
handle: "nvidia",
|
||||
rightfulOwnerUserId: "users:nvidia-owner",
|
||||
createdAt: 1,
|
||||
updatedAt: 1,
|
||||
},
|
||||
]),
|
||||
})),
|
||||
})),
|
||||
};
|
||||
}
|
||||
throw new Error(`Unexpected table ${table}`);
|
||||
}),
|
||||
},
|
||||
};
|
||||
|
||||
await expect(
|
||||
isOfficialPublisher(
|
||||
ctx as never,
|
||||
makePublisher({ _id: "publishers:acme", handle: "acme", deactivatedAt: 123 }),
|
||||
),
|
||||
).resolves.toBe(false);
|
||||
await expect(hasOfficialPublisherRow(ctx as never, "publishers:acme" as never)).resolves.toBe(
|
||||
true,
|
||||
);
|
||||
await expect(isOfficialPublisher(ctx as never, personal)).resolves.toBe(true);
|
||||
});
|
||||
|
||||
it("treats personal publishers for openclaw org members as official", async () => {
|
||||
const openclaw = makePublisher({ _id: "publishers:openclaw", handle: "openclaw" });
|
||||
const personal = makePublisher({
|
||||
_id: "publishers:alice",
|
||||
kind: "user",
|
||||
handle: "alice",
|
||||
linkedUserId: "users:alice",
|
||||
});
|
||||
const ctx = {
|
||||
db: {
|
||||
query: vi.fn((table: string) => {
|
||||
if (table === "publishers") {
|
||||
return {
|
||||
withIndex: vi.fn(() => ({
|
||||
unique: vi.fn(async () => openclaw),
|
||||
})),
|
||||
};
|
||||
}
|
||||
if (table === "publisherMembers") {
|
||||
return {
|
||||
withIndex: vi.fn(() => ({
|
||||
unique: vi.fn(async () => ({
|
||||
_id: "publisherMembers:alice",
|
||||
publisherId: "publishers:openclaw",
|
||||
userId: "users:alice",
|
||||
role: "publisher",
|
||||
createdAt: 1,
|
||||
updatedAt: 1,
|
||||
})),
|
||||
})),
|
||||
};
|
||||
}
|
||||
throw new Error(`Unexpected table ${table}`);
|
||||
}),
|
||||
},
|
||||
};
|
||||
|
||||
await expect(isOfficialPublisher(ctx as never, personal)).resolves.toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,28 +1,89 @@
|
||||
import type { Doc } from "../_generated/dataModel";
|
||||
import type { MutationCtx, QueryCtx } from "../_generated/server";
|
||||
import { toPublicPublisher, type PublicPublisher } from "./public";
|
||||
import {
|
||||
getPublisherByHandle,
|
||||
getPublisherMembership,
|
||||
normalizePublisherHandle,
|
||||
} from "./publishers";
|
||||
import { getLatestActiveReservedHandle } from "./reservedHandles";
|
||||
|
||||
const LEGACY_OFFICIAL_ORG_HANDLES = ["openclaw"] as const;
|
||||
const RESERVED_OWNER_VERIFIED_OFFICIAL_ORG_HANDLES = ["nvidia"] as const;
|
||||
const OFFICIAL_ORG_HANDLES = [
|
||||
...LEGACY_OFFICIAL_ORG_HANDLES,
|
||||
...RESERVED_OWNER_VERIFIED_OFFICIAL_ORG_HANDLES,
|
||||
] as const;
|
||||
const LEGACY_OFFICIAL_ORG_HANDLE_SET = new Set<string>(LEGACY_OFFICIAL_ORG_HANDLES);
|
||||
const RESERVED_OWNER_VERIFIED_OFFICIAL_ORG_HANDLE_SET = new Set<string>(
|
||||
RESERVED_OWNER_VERIFIED_OFFICIAL_ORG_HANDLES,
|
||||
);
|
||||
|
||||
type DbCtx = Pick<QueryCtx | MutationCtx, "db">;
|
||||
|
||||
type OfficialPublisherCandidate = Pick<Doc<"publishers">, "_id" | "deletedAt" | "deactivatedAt">;
|
||||
type OfficialPublisherCandidate = Pick<
|
||||
Doc<"publishers">,
|
||||
| "_id"
|
||||
| "_creationTime"
|
||||
| "kind"
|
||||
| "handle"
|
||||
| "displayName"
|
||||
| "image"
|
||||
| "bio"
|
||||
| "linkedUserId"
|
||||
| "deletedAt"
|
||||
| "deactivatedAt"
|
||||
>;
|
||||
|
||||
export function isReservedOwnerVerifiedOfficialOrgHandle(
|
||||
handle: string | undefined | null,
|
||||
): boolean {
|
||||
const normalizedHandle = normalizePublisherHandle(handle);
|
||||
return Boolean(
|
||||
normalizedHandle && RESERVED_OWNER_VERIFIED_OFFICIAL_ORG_HANDLE_SET.has(normalizedHandle),
|
||||
);
|
||||
}
|
||||
|
||||
async function isOfficialOrgPublisher(
|
||||
ctx: DbCtx,
|
||||
publisher: OfficialPublisherCandidate,
|
||||
): Promise<boolean> {
|
||||
const handle = normalizePublisherHandle(publisher.handle);
|
||||
if (!handle) return false;
|
||||
if (LEGACY_OFFICIAL_ORG_HANDLE_SET.has(handle)) return true;
|
||||
if (!RESERVED_OWNER_VERIFIED_OFFICIAL_ORG_HANDLE_SET.has(handle)) return false;
|
||||
|
||||
const reservation = await getLatestActiveReservedHandle(ctx, handle);
|
||||
if (!reservation) return false;
|
||||
|
||||
// Security-sensitive: newly official handles must be bound to an admin-created
|
||||
// reservation, not just any public org that claimed the handle first.
|
||||
const ownerMembership = await getPublisherMembership(
|
||||
ctx,
|
||||
publisher._id,
|
||||
reservation.rightfulOwnerUserId,
|
||||
);
|
||||
return ownerMembership?.role === "owner";
|
||||
}
|
||||
|
||||
export async function isOfficialPublisher(
|
||||
ctx: DbCtx,
|
||||
publisher: OfficialPublisherCandidate | null | undefined,
|
||||
): Promise<boolean> {
|
||||
if (!publisher || publisher.deletedAt || publisher.deactivatedAt) return false;
|
||||
return await hasOfficialPublisherRow(ctx, publisher._id);
|
||||
}
|
||||
if (publisher.kind === "org") return await isOfficialOrgPublisher(ctx, publisher);
|
||||
if (!publisher.linkedUserId) return false;
|
||||
|
||||
export async function hasOfficialPublisherRow(
|
||||
ctx: DbCtx,
|
||||
publisherId: Doc<"publishers">["_id"],
|
||||
): Promise<boolean> {
|
||||
const officialPublisher = await ctx.db
|
||||
.query("officialPublishers")
|
||||
.withIndex("by_publisher", (q) => q.eq("publisherId", publisherId))
|
||||
.unique();
|
||||
return Boolean(officialPublisher);
|
||||
for (const officialOrgHandle of OFFICIAL_ORG_HANDLES) {
|
||||
const officialOrg = await getPublisherByHandle(ctx, officialOrgHandle);
|
||||
if (!officialOrg || officialOrg.deletedAt || officialOrg.deactivatedAt) continue;
|
||||
const membership = await getPublisherMembership(ctx, officialOrg._id, publisher.linkedUserId);
|
||||
if (!membership) continue;
|
||||
if (!(await isOfficialOrgPublisher(ctx, officialOrg))) continue;
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
export async function toPublicPublisherWithOfficial(
|
||||
|
||||
@@ -271,15 +271,6 @@ function buildVerification(source: SourceInfo | undefined): PackageVerificationS
|
||||
scanStatus: "not-run",
|
||||
};
|
||||
}
|
||||
// `source.path` is the package directory inside the source repo (e.g.
|
||||
// "examples/openclaw-plugin"). When the package lives at the repo root the
|
||||
// CLI sends "." (or empty), and there's nothing useful to serialize. Only
|
||||
// promote real subpaths into `verification.sourcePath` so consumers can
|
||||
// build a `raw.githubusercontent.com/<repo>/<sha>/<path>/` base URL for
|
||||
// resolving relative README asset references.
|
||||
const rawPath = typeof source.path === "string" ? source.path.trim() : "";
|
||||
const sourcePath =
|
||||
rawPath && rawPath !== "." ? rawPath.replace(/^\/+/, "").replace(/\/+$/, "") : undefined;
|
||||
return {
|
||||
tier: "source-linked",
|
||||
scope: "artifact-only",
|
||||
@@ -287,7 +278,6 @@ function buildVerification(source: SourceInfo | undefined): PackageVerificationS
|
||||
sourceRepo: source.repo || source.url,
|
||||
sourceCommit: source.commit,
|
||||
sourceTag: source.ref,
|
||||
sourcePath: sourcePath || undefined,
|
||||
hasProvenance: false,
|
||||
scanStatus: "not-run",
|
||||
};
|
||||
|
||||
@@ -20,7 +20,6 @@ const SHARED_KEYS = [
|
||||
"summary",
|
||||
"capabilityTags",
|
||||
"executesCode",
|
||||
"stats",
|
||||
"runtimeId",
|
||||
"scanStatus",
|
||||
"softDeletedAt",
|
||||
@@ -46,7 +45,6 @@ const CAPABILITY_SHARED_KEYS = [
|
||||
"capabilityTags",
|
||||
"executesCode",
|
||||
"verificationTier",
|
||||
"stats",
|
||||
"scanStatus",
|
||||
"softDeletedAt",
|
||||
"createdAt",
|
||||
@@ -72,7 +70,6 @@ const PLUGIN_CATEGORY_SHARED_KEYS = [
|
||||
"pluginCategoryTags",
|
||||
"executesCode",
|
||||
"verificationTier",
|
||||
"stats",
|
||||
"scanStatus",
|
||||
"softDeletedAt",
|
||||
"createdAt",
|
||||
|
||||
@@ -66,22 +66,6 @@ describe("public skill mapping", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("exposes GitHub-backed skill source fields", () => {
|
||||
const mapped = toPublicSkill(
|
||||
makeSkill({
|
||||
installKind: "github",
|
||||
githubPath: "skills/demo",
|
||||
githubCurrentCommit: "a".repeat(40),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(mapped).toMatchObject({
|
||||
installKind: "github",
|
||||
githubPath: "skills/demo",
|
||||
githubCurrentCommit: "a".repeat(40),
|
||||
});
|
||||
});
|
||||
|
||||
it("returns skill when moderationStatus is active", () => {
|
||||
const skill = makeSkill({ moderationStatus: "active" });
|
||||
expect(toPublicSkill(skill)).not.toBeNull();
|
||||
@@ -109,14 +93,6 @@ describe("public skill mapping", () => {
|
||||
});
|
||||
expect(toPublicSkill(skill)).toBeNull();
|
||||
});
|
||||
|
||||
it("filters out skills with a malicious moderation verdict", () => {
|
||||
const skill = makeSkill({
|
||||
moderationStatus: "active",
|
||||
moderationVerdict: "malicious",
|
||||
});
|
||||
expect(toPublicSkill(skill)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("public publisher mapping", () => {
|
||||
|
||||
+2
-22
@@ -24,12 +24,6 @@ export type PublicSkill = Pick<
|
||||
| "canonicalSkillId"
|
||||
| "forkOf"
|
||||
| "latestVersionId"
|
||||
| "installKind"
|
||||
| "githubPath"
|
||||
| "githubCurrentCommit"
|
||||
| "githubCurrentStatus"
|
||||
| "githubScanStatus"
|
||||
| "githubHasSkillCard"
|
||||
| "tags"
|
||||
| "capabilityTags"
|
||||
| "badges"
|
||||
@@ -37,9 +31,7 @@ export type PublicSkill = Pick<
|
||||
| "isSuspicious"
|
||||
| "createdAt"
|
||||
| "updatedAt"
|
||||
> & {
|
||||
githubSourceRepo?: string;
|
||||
};
|
||||
>;
|
||||
|
||||
/**
|
||||
* Minimum set of fields needed by `hydrateResults` to filter and convert
|
||||
@@ -60,10 +52,6 @@ export type HydratableSkill = Pick<
|
||||
| "canonicalSkillId"
|
||||
| "forkOf"
|
||||
| "latestVersionId"
|
||||
| "installKind"
|
||||
| "githubHasSkillCard"
|
||||
| "githubCurrentStatus"
|
||||
| "githubScanStatus"
|
||||
| "latestVersionSummary"
|
||||
| "tags"
|
||||
| "capabilityTags"
|
||||
@@ -76,13 +64,11 @@ export type HydratableSkill = Pick<
|
||||
| "softDeletedAt"
|
||||
| "moderationStatus"
|
||||
| "moderationFlags"
|
||||
| "moderationVerdict"
|
||||
| "moderationReason"
|
||||
| "isSuspicious"
|
||||
| "createdAt"
|
||||
| "updatedAt"
|
||||
> &
|
||||
Partial<Pick<Doc<"skills">, "githubPath" | "githubCurrentCommit">>;
|
||||
>;
|
||||
|
||||
export type PublicSoul = Pick<
|
||||
Doc<"souls">,
|
||||
@@ -163,12 +149,6 @@ export function toPublicSkill(skill: HydratableSkill | null | undefined): Public
|
||||
canonicalSkillId: skill.canonicalSkillId,
|
||||
forkOf: skill.forkOf,
|
||||
latestVersionId: skill.latestVersionId,
|
||||
installKind: skill.installKind,
|
||||
githubPath: skill.githubPath,
|
||||
githubCurrentCommit: skill.githubCurrentCommit,
|
||||
githubCurrentStatus: skill.githubCurrentStatus,
|
||||
githubScanStatus: skill.githubScanStatus,
|
||||
githubHasSkillCard: skill.githubHasSkillCard,
|
||||
tags: skill.tags,
|
||||
capabilityTags: skill.capabilityTags,
|
||||
badges: skill.badges,
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { isReservedPublicOwnerHandle } from "./publicRouteReservations";
|
||||
|
||||
describe("public route reservations", () => {
|
||||
it.each(["admin", "clawhub", "plugins", "skills"])(
|
||||
"reserves @%s as a public owner handle",
|
||||
(handle) => {
|
||||
expect(isReservedPublicOwnerHandle(handle)).toBe(true);
|
||||
},
|
||||
);
|
||||
|
||||
it("does not normalize at-sign prefixes", () => {
|
||||
expect(isReservedPublicOwnerHandle("@clawhub")).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,53 @@
|
||||
const RESERVED_PUBLIC_OWNER_HANDLES = new Set(["admin", "clawhub", "plugins", "skills"]);
|
||||
/**
|
||||
* Handles and package names that are reserved for ClawHub platform routes.
|
||||
*
|
||||
* RESERVED_PUBLIC_OWNER_HANDLES: every top-level path segment that exists as
|
||||
* a real app route and would shadow the `/$owner` dynamic catch-all if a user
|
||||
* were able to register it as a publisher handle.
|
||||
*
|
||||
* Add entries here whenever a new top-level route is added to src/routes/.
|
||||
*/
|
||||
const RESERVED_PUBLIC_OWNER_HANDLES = new Set([
|
||||
// Content browsing
|
||||
"skills",
|
||||
"souls",
|
||||
"plugins",
|
||||
"packages",
|
||||
"publishers",
|
||||
"orgs",
|
||||
|
||||
// Publisher / user profile shortlinks
|
||||
"p",
|
||||
"u",
|
||||
|
||||
// User-facing flows
|
||||
"search",
|
||||
"import",
|
||||
"upload",
|
||||
"publish-skill",
|
||||
"publish-plugin",
|
||||
"stars",
|
||||
"dashboard",
|
||||
"settings",
|
||||
|
||||
// Admin / platform-internal
|
||||
"admin",
|
||||
"management",
|
||||
"audits",
|
||||
|
||||
// Informational / static
|
||||
"docs",
|
||||
"cli",
|
||||
|
||||
// Auth / user account
|
||||
"user",
|
||||
"users",
|
||||
]);
|
||||
|
||||
/**
|
||||
* Unscoped package names that are reserved for ClawHub routes or CLI commands.
|
||||
* Scoped packages (e.g. @scope/publish) are not affected.
|
||||
*/
|
||||
const RESERVED_UNSCOPED_PACKAGE_NAMES = new Set(["publish"]);
|
||||
|
||||
export function isReservedPublicOwnerHandle(handle: string | undefined | null) {
|
||||
|
||||
@@ -5,7 +5,6 @@ import {
|
||||
getPublishFileSizeError,
|
||||
getPublishTotalSizeError,
|
||||
MAX_CLAWPACK_BYTES,
|
||||
MAX_PACKAGE_MULTIPART_BYTES,
|
||||
MAX_PUBLISH_FILE_BYTES,
|
||||
} from "./publishLimits";
|
||||
|
||||
@@ -32,9 +31,8 @@ describe("publishLimits", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps ClawPack capacity above the multipart request budget", () => {
|
||||
it("keeps the ClawPack tarball limit separate from legacy file limits", () => {
|
||||
expect(MAX_CLAWPACK_BYTES).toBe(120 * 1024 * 1024);
|
||||
expect(MAX_CLAWPACK_BYTES).toBeGreaterThan(MAX_PACKAGE_MULTIPART_BYTES);
|
||||
expect(MAX_CLAWPACK_BYTES).toBeGreaterThan(MAX_PUBLISH_FILE_BYTES);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,17 +1,6 @@
|
||||
import { MAX_PACKAGE_CLAWPACK_BYTES } from "clawhub-schema";
|
||||
|
||||
export {
|
||||
estimatePackageMultipartUploadBytes,
|
||||
getPackageMultipartSizeError,
|
||||
isPackageMultipartUploadTooLarge,
|
||||
MAX_PACKAGE_MULTIPART_BYTES,
|
||||
type PackageMultipartUploadField,
|
||||
type PackageMultipartUploadPart,
|
||||
} from "clawhub-schema";
|
||||
|
||||
export const MAX_PUBLISH_TOTAL_BYTES = 50 * 1024 * 1024;
|
||||
export const MAX_PUBLISH_FILE_BYTES = 10 * 1024 * 1024;
|
||||
export const MAX_CLAWPACK_BYTES = MAX_PACKAGE_CLAWPACK_BYTES;
|
||||
export const MAX_CLAWPACK_BYTES = 120 * 1024 * 1024;
|
||||
|
||||
type SizedPathLike = {
|
||||
path: string;
|
||||
|
||||
@@ -2,13 +2,8 @@
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
computeCurrentSkillTemporalAbuseScore,
|
||||
computeHistoricalSkillTemporalAbuseScore,
|
||||
computePublisherAbuseRawScore,
|
||||
computeTemporalAbuseCohortBenchmark,
|
||||
computeTemporalPublisherAbuseZScore,
|
||||
DEFAULT_PUBLISHER_ABUSE_MODEL_CONFIG,
|
||||
labelForTemporalPublisherAbuse,
|
||||
labelForPublisherAbuseZScore,
|
||||
scorePublisherAbuseCohort,
|
||||
} from "./publisherAbuseScoring";
|
||||
@@ -23,40 +18,6 @@ describe("publisher abuse scoring", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("maps temporal labels to review-compatible z-scores", () => {
|
||||
const review = computeTemporalPublisherAbuseZScore({
|
||||
label: "review",
|
||||
highTemporalSkillCount: 1,
|
||||
maxTemporalPressure: 20,
|
||||
});
|
||||
const potentialBan = computeTemporalPublisherAbuseZScore({
|
||||
label: "potential_ban_candidate",
|
||||
highTemporalSkillCount: 2,
|
||||
maxTemporalPressure: 20,
|
||||
});
|
||||
|
||||
expect(
|
||||
computeTemporalPublisherAbuseZScore({
|
||||
label: "pass",
|
||||
highTemporalSkillCount: 0,
|
||||
maxTemporalPressure: 0,
|
||||
}),
|
||||
).toBe(0);
|
||||
expect(review).toBeGreaterThanOrEqual(1.5);
|
||||
expect(review).toBeLessThan(2.5);
|
||||
expect(potentialBan).toBeGreaterThanOrEqual(2.5);
|
||||
expect(potentialBan).toBeGreaterThan(review);
|
||||
});
|
||||
|
||||
it("escalates one P99 temporal hit as a potential ban candidate", () => {
|
||||
expect(
|
||||
labelForTemporalPublisherAbuse({ highTemporalSkillCount: 1, p99TemporalSkillCount: 1 }),
|
||||
).toBe("potential_ban_candidate");
|
||||
expect(
|
||||
labelForTemporalPublisherAbuse({ highTemporalSkillCount: 1, p99TemporalSkillCount: 0 }),
|
||||
).toBe("review");
|
||||
});
|
||||
|
||||
it("keeps a high-volume publisher with strong usage below low-engagement publishers", () => {
|
||||
const scored = scorePublisherAbuseCohort([
|
||||
publisher("byungkyu", {
|
||||
@@ -164,198 +125,8 @@ describe("publisher abuse scoring", () => {
|
||||
"pass",
|
||||
);
|
||||
});
|
||||
|
||||
it("flags a current 7-day download spike with flat installs", () => {
|
||||
const todayDay = 100;
|
||||
const score = computeCurrentSkillTemporalAbuseScore({
|
||||
todayDay,
|
||||
benchmark: temporalBenchmark({
|
||||
downloads30dP95: 2_000,
|
||||
downloads30dP99: 5_000,
|
||||
spikeMultiplier7dP95: 5,
|
||||
spikeMultiplier7dP99: 20,
|
||||
}),
|
||||
dailyStats: [
|
||||
...dailyRange(64, 30, { downloads: 5, installs: 0 }),
|
||||
...dailyRange(94, 7, { downloads: 200, installs: 0 }),
|
||||
],
|
||||
});
|
||||
|
||||
expect(score.spike).toBe(true);
|
||||
expect(score.sustained).toBe(false);
|
||||
expect(score.recent7Downloads).toBe(1_400);
|
||||
expect(score.recent7Installs).toBe(0);
|
||||
expect(score.previous30Downloads).toBe(150);
|
||||
expect(score.spikeMultiplier).toBeCloseTo(14);
|
||||
expect(score.spikeMultiplierCohortBand).toBe("p95");
|
||||
expect(score.reasonCodes).toContain("temporal_download_spike_flat_installs");
|
||||
});
|
||||
|
||||
it("flags sustained high downloads with flat installs", () => {
|
||||
const todayDay = 100;
|
||||
const score = computeCurrentSkillTemporalAbuseScore({
|
||||
todayDay,
|
||||
benchmark: temporalBenchmark({
|
||||
downloads30dP95: 3_000,
|
||||
downloads30dP99: 6_000,
|
||||
spikeMultiplier7dP95: 20,
|
||||
spikeMultiplier7dP99: 50,
|
||||
}),
|
||||
dailyStats: dailyRange(71, 30, { downloads: 120, installs: 0 }),
|
||||
});
|
||||
|
||||
expect(score.spike).toBe(false);
|
||||
expect(score.sustained).toBe(true);
|
||||
expect(score.recent30Downloads).toBe(3_600);
|
||||
expect(score.recent30Installs).toBe(0);
|
||||
expect(score.downloadInstallRatio30).toBe(3_600);
|
||||
expect(score.downloads30dCohortBand).toBe("p95");
|
||||
expect(score.reasonCodes).toContain("temporal_sustained_downloads_flat_installs");
|
||||
});
|
||||
|
||||
it("flags high-volume installs that track downloads too closely", () => {
|
||||
const todayDay = 100;
|
||||
const score = computeCurrentSkillTemporalAbuseScore({
|
||||
todayDay,
|
||||
dailyStats: dailyRange(94, 7, { downloads: 200, installs: 180 }),
|
||||
});
|
||||
|
||||
expect(score.nearConversion).toBe(true);
|
||||
expect(score.recent7Downloads).toBe(1_400);
|
||||
expect(score.recent7Installs).toBe(1_260);
|
||||
expect(score.installDownloadRatio7).toBeCloseTo(0.9);
|
||||
expect(score.reasonCodes).toContain("temporal_installs_track_downloads");
|
||||
});
|
||||
|
||||
it("keeps low-volume one-to-one install traffic below close-ratio thresholds", () => {
|
||||
const todayDay = 100;
|
||||
const score = computeCurrentSkillTemporalAbuseScore({
|
||||
todayDay,
|
||||
dailyStats: dailyRange(94, 7, { downloads: 1, installs: 1 }),
|
||||
});
|
||||
|
||||
expect(score.nearConversion).toBe(false);
|
||||
expect(score.reasonCodes).not.toContain("temporal_installs_track_downloads");
|
||||
});
|
||||
|
||||
it("keeps observed high-end install ratios below close-ratio thresholds", () => {
|
||||
const todayDay = 100;
|
||||
const score = computeCurrentSkillTemporalAbuseScore({
|
||||
todayDay,
|
||||
dailyStats: dailyRange(94, 7, { downloads: 20, installs: 1 }),
|
||||
});
|
||||
|
||||
expect(score.recent7Downloads).toBe(140);
|
||||
expect(score.recent7Installs).toBe(7);
|
||||
expect(score.installDownloadRatio7).toBeCloseTo(0.05);
|
||||
expect(score.nearConversion).toBe(false);
|
||||
expect(score.reasonCodes).not.toContain("temporal_installs_track_downloads");
|
||||
});
|
||||
|
||||
it("requires installs to be close to downloads, not just statistically elevated", () => {
|
||||
const todayDay = 100;
|
||||
const score = computeCurrentSkillTemporalAbuseScore({
|
||||
todayDay,
|
||||
dailyStats: dailyRange(94, 7, { downloads: 300, installs: 15 }),
|
||||
});
|
||||
|
||||
expect(score.recent7Downloads).toBe(2_100);
|
||||
expect(score.recent7Installs).toBe(105);
|
||||
expect(score.installDownloadRatio7).toBeCloseTo(0.05);
|
||||
expect(score.installDownloadExcessZScore7).toBeGreaterThan(10);
|
||||
expect(score.nearConversion).toBe(false);
|
||||
expect(score.reasonCodes).not.toContain("temporal_installs_track_downloads");
|
||||
});
|
||||
|
||||
it("reports a 30-day close-ratio window when the 7-day threshold is not met", () => {
|
||||
const todayDay = 100;
|
||||
const score = computeCurrentSkillTemporalAbuseScore({
|
||||
todayDay,
|
||||
dailyStats: dailyRange(71, 30, { downloads: 100, installs: 80 }),
|
||||
});
|
||||
|
||||
expect(score.nearConversion).toBe(true);
|
||||
expect(score.installDownloadRatio7).toBeCloseTo(0.8);
|
||||
expect(score.installDownloadRatio30).toBeCloseTo(0.8);
|
||||
expect(score.nearConversionWindowStartDay).toBe(71);
|
||||
expect(score.nearConversionWindowEndDay).toBe(100);
|
||||
});
|
||||
|
||||
it("keeps ordinary steady download traffic below temporal thresholds", () => {
|
||||
const todayDay = 100;
|
||||
const score = computeCurrentSkillTemporalAbuseScore({
|
||||
todayDay,
|
||||
benchmark: temporalBenchmark({
|
||||
downloads30dP95: 4_000,
|
||||
downloads30dP99: 8_000,
|
||||
spikeMultiplier7dP95: 20,
|
||||
spikeMultiplier7dP99: 50,
|
||||
}),
|
||||
dailyStats: [
|
||||
...dailyRange(64, 30, { downloads: 80, installs: 1 }),
|
||||
...dailyRange(94, 7, { downloads: 85, installs: 1 }),
|
||||
],
|
||||
});
|
||||
|
||||
expect(score.spike).toBe(false);
|
||||
expect(score.sustained).toBe(false);
|
||||
expect(score.pressure).toBe(0);
|
||||
expect(score.reasonCodes).toEqual([]);
|
||||
});
|
||||
|
||||
it("finds historical spike and sustained windows for backfill scans", () => {
|
||||
const score = computeHistoricalSkillTemporalAbuseScore({
|
||||
benchmark: temporalBenchmark({
|
||||
downloads30dP95: 3_000,
|
||||
downloads30dP99: 10_000,
|
||||
spikeMultiplier7dP95: 5,
|
||||
spikeMultiplier7dP99: 25,
|
||||
}),
|
||||
dailyStats: [
|
||||
...dailyRange(10, 30, { downloads: 3, installs: 0 }),
|
||||
...dailyRange(40, 7, { downloads: 220, installs: 0 }),
|
||||
...dailyRange(80, 30, { downloads: 150, installs: 0 }),
|
||||
],
|
||||
});
|
||||
|
||||
expect(score.spike).toBe(true);
|
||||
expect(score.sustained).toBe(true);
|
||||
expect(score.spikeWindowStartDay).toBe(40);
|
||||
expect(score.sustainedWindowStartDay).toBe(80);
|
||||
expect(score.reasonCodes).toEqual([
|
||||
"temporal_download_spike_flat_installs",
|
||||
"temporal_sustained_downloads_flat_installs",
|
||||
]);
|
||||
});
|
||||
|
||||
it("computes cohort benchmark percentiles from scanned skill windows", () => {
|
||||
const benchmark = computeTemporalAbuseCohortBenchmark([
|
||||
...Array.from({ length: 95 }, () => ({ recent30Downloads: 100, spikeMultiplier: 1 })),
|
||||
...Array.from({ length: 4 }, () => ({ recent30Downloads: 500, spikeMultiplier: 2 })),
|
||||
{ recent30Downloads: 10_000, spikeMultiplier: 30 },
|
||||
]);
|
||||
|
||||
expect(benchmark.sampleSize).toBe(100);
|
||||
expect(benchmark.downloads30dMedian).toBe(100);
|
||||
expect(benchmark.downloads30dP95).toBe(100);
|
||||
expect(benchmark.downloads30dP99).toBe(500);
|
||||
expect(benchmark.spikeMultiplier7dP99).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
function temporalBenchmark(overrides = {}) {
|
||||
return {
|
||||
sampleSize: 100,
|
||||
downloads30dAverage: 500,
|
||||
downloads30dMedian: 100,
|
||||
downloads30dP95: 1_000,
|
||||
downloads30dP99: 5_000,
|
||||
spikeMultiplier7dP95: 5,
|
||||
spikeMultiplier7dP99: 25,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function publisher(
|
||||
handleSnapshot: string,
|
||||
stats: {
|
||||
@@ -372,15 +143,3 @@ function publisher(
|
||||
...stats,
|
||||
};
|
||||
}
|
||||
|
||||
function dailyRange(
|
||||
startDay: number,
|
||||
length: number,
|
||||
stats: { downloads: number; installs: number },
|
||||
) {
|
||||
return Array.from({ length }, (_, index) => ({
|
||||
day: startDay + index,
|
||||
downloads: stats.downloads,
|
||||
installs: stats.installs,
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
export const PUBLISHER_ABUSE_MODEL_VERSION = "publisher-abuse-pressure.v1";
|
||||
export const PUBLISHER_TEMPORAL_ABUSE_MODEL_VERSION = "publisher-abuse-temporal.v1";
|
||||
|
||||
export type PublisherAbuseLabel = "pass" | "review" | "potential_ban_candidate";
|
||||
|
||||
@@ -51,52 +50,6 @@ export type PublisherAbuseScore = PublisherAbuseRawScore & {
|
||||
zScore: number;
|
||||
};
|
||||
|
||||
export type SkillTemporalAbuseDailyStat = {
|
||||
day: number;
|
||||
downloads: number;
|
||||
installs: number;
|
||||
};
|
||||
|
||||
export type SkillTemporalAbuseScore = {
|
||||
spike: boolean;
|
||||
sustained: boolean;
|
||||
nearConversion: boolean;
|
||||
pressure: number;
|
||||
recent7Downloads: number;
|
||||
recent7Installs: number;
|
||||
previous30Downloads: number;
|
||||
baseline7Downloads: number;
|
||||
spikeMultiplier: number;
|
||||
recent30Downloads: number;
|
||||
recent30Installs: number;
|
||||
downloadInstallRatio30: number;
|
||||
downloads30dCohortBand?: "p95" | "p99";
|
||||
spikeMultiplierCohortBand?: "p95" | "p99";
|
||||
downloads30dVsPeerP95?: number;
|
||||
spikeMultiplierVsPeerP95?: number;
|
||||
installDownloadRatio7: number;
|
||||
installDownloadRatio30: number;
|
||||
installDownloadExcessZScore7: number;
|
||||
installDownloadExcessZScore30: number;
|
||||
spikeWindowStartDay?: number;
|
||||
spikeWindowEndDay?: number;
|
||||
sustainedWindowStartDay?: number;
|
||||
sustainedWindowEndDay?: number;
|
||||
nearConversionWindowStartDay?: number;
|
||||
nearConversionWindowEndDay?: number;
|
||||
reasonCodes: string[];
|
||||
};
|
||||
|
||||
export type TemporalAbuseCohortBenchmark = {
|
||||
sampleSize: number;
|
||||
downloads30dAverage: number;
|
||||
downloads30dMedian: number;
|
||||
downloads30dP95: number;
|
||||
downloads30dP99: number;
|
||||
spikeMultiplier7dP95: number;
|
||||
spikeMultiplier7dP99: number;
|
||||
};
|
||||
|
||||
export const DEFAULT_PUBLISHER_ABUSE_MODEL_CONFIG = {
|
||||
modelVersion: PUBLISHER_ABUSE_MODEL_VERSION,
|
||||
skillPivot: 100,
|
||||
@@ -117,18 +70,6 @@ export const DEFAULT_PUBLISHER_ABUSE_MODEL_CONFIG = {
|
||||
} satisfies PublisherAbuseModelConfig;
|
||||
|
||||
const MIN_PRESSURE_FOR_LOG = 1e-9;
|
||||
const TEMPORAL_SPIKE_RECENT_DAYS = 7;
|
||||
const TEMPORAL_SPIKE_BASELINE_DAYS = 30;
|
||||
const TEMPORAL_SUSTAINED_DAYS = 30;
|
||||
const TEMPORAL_MAX_SPIKE_INSTALLS = 2;
|
||||
const TEMPORAL_MAX_SUSTAINED_INSTALLS = 5;
|
||||
const TEMPORAL_MIN_BASELINE_7_DOWNLOADS = 100;
|
||||
const TEMPORAL_MIN_NEAR_CONVERSION_7_DOWNLOADS = 1_000;
|
||||
const TEMPORAL_MIN_NEAR_CONVERSION_30_DOWNLOADS = 2_000;
|
||||
const TEMPORAL_MIN_NEAR_CONVERSION_INSTALLS = 500;
|
||||
const TEMPORAL_EXPECTED_INSTALL_DOWNLOAD_RATIO = 0.012;
|
||||
const TEMPORAL_MIN_INSTALL_DOWNLOAD_RATIO = 0.5;
|
||||
const TEMPORAL_MIN_INSTALL_DOWNLOAD_EXCESS_Z_SCORE = 50;
|
||||
|
||||
export function labelForPublisherAbuseZScore(
|
||||
zScore: number,
|
||||
@@ -139,21 +80,6 @@ export function labelForPublisherAbuseZScore(
|
||||
return "pass";
|
||||
}
|
||||
|
||||
export function computeTemporalPublisherAbuseZScore(input: {
|
||||
label: PublisherAbuseLabel;
|
||||
highTemporalSkillCount: number;
|
||||
maxTemporalPressure: number;
|
||||
}): number {
|
||||
if (input.label === "pass") return 0;
|
||||
|
||||
const pressureBoost = Math.log10(Math.max(input.maxTemporalPressure, 1) + 1) / 2;
|
||||
const skillCountBoost = Math.max(0, input.highTemporalSkillCount - 2) * 0.2;
|
||||
if (input.label === "potential_ban_candidate") {
|
||||
return 2.5 + Math.min(2, pressureBoost + skillCountBoost);
|
||||
}
|
||||
return 1.5 + Math.min(0.99, pressureBoost);
|
||||
}
|
||||
|
||||
export function computePublisherAbuseRawScore(
|
||||
input: PublisherAbuseInput,
|
||||
config: PublisherAbuseModelConfig = DEFAULT_PUBLISHER_ABUSE_MODEL_CONFIG,
|
||||
@@ -278,168 +204,6 @@ export function summarizePublisherAbuseLogPressure(
|
||||
};
|
||||
}
|
||||
|
||||
export function computeCurrentSkillTemporalAbuseScore(input: {
|
||||
todayDay: number;
|
||||
dailyStats: SkillTemporalAbuseDailyStat[];
|
||||
benchmark?: TemporalAbuseCohortBenchmark;
|
||||
}): SkillTemporalAbuseScore {
|
||||
const statsByDay = aggregateSkillTemporalDailyStats(input.dailyStats);
|
||||
const score = computeSkillTemporalAbuseScoreForWindows({
|
||||
statsByDay,
|
||||
spikeStartDay: input.todayDay - TEMPORAL_SPIKE_RECENT_DAYS + 1,
|
||||
sustainedStartDay: input.todayDay - TEMPORAL_SUSTAINED_DAYS + 1,
|
||||
});
|
||||
return classifySkillTemporalAbuseScore(score, input.benchmark);
|
||||
}
|
||||
|
||||
export function computeHistoricalSkillTemporalAbuseScore(input: {
|
||||
dailyStats: SkillTemporalAbuseDailyStat[];
|
||||
benchmark?: TemporalAbuseCohortBenchmark;
|
||||
}): SkillTemporalAbuseScore {
|
||||
const statsByDay = aggregateSkillTemporalDailyStats(input.dailyStats);
|
||||
const days = [...statsByDay.keys()];
|
||||
if (days.length === 0) return emptySkillTemporalAbuseScore();
|
||||
|
||||
const minDay = Math.min(...days);
|
||||
const maxDay = Math.max(...days);
|
||||
let bestSpike = emptySkillTemporalAbuseScore();
|
||||
let bestSustained = emptySkillTemporalAbuseScore();
|
||||
let bestNearConversion = emptySkillTemporalAbuseScore();
|
||||
|
||||
for (let startDay = minDay; startDay <= maxDay; startDay += 1) {
|
||||
if (startDay + TEMPORAL_SPIKE_RECENT_DAYS - 1 <= maxDay) {
|
||||
const score = classifySkillTemporalAbuseScore(
|
||||
computeSkillTemporalAbuseScoreForWindows({
|
||||
statsByDay,
|
||||
spikeStartDay: startDay,
|
||||
sustainedStartDay: startDay,
|
||||
}),
|
||||
input.benchmark,
|
||||
);
|
||||
if (score.spike && score.spikeMultiplier > bestSpike.spikeMultiplier) {
|
||||
bestSpike = score;
|
||||
}
|
||||
if (
|
||||
score.nearConversion &&
|
||||
score.nearConversionWindowEndDay === startDay + TEMPORAL_SPIKE_RECENT_DAYS - 1 &&
|
||||
score.installDownloadRatio7 > bestNearConversion.installDownloadRatio7
|
||||
) {
|
||||
bestNearConversion = score;
|
||||
}
|
||||
}
|
||||
|
||||
if (startDay + TEMPORAL_SUSTAINED_DAYS - 1 <= maxDay) {
|
||||
const score = classifySkillTemporalAbuseScore(
|
||||
computeSkillTemporalAbuseScoreForWindows({
|
||||
statsByDay,
|
||||
spikeStartDay: startDay,
|
||||
sustainedStartDay: startDay,
|
||||
}),
|
||||
input.benchmark,
|
||||
);
|
||||
if (score.sustained && score.recent30Downloads > bestSustained.recent30Downloads) {
|
||||
bestSustained = score;
|
||||
}
|
||||
if (
|
||||
score.nearConversion &&
|
||||
score.nearConversionWindowEndDay === startDay + TEMPORAL_SUSTAINED_DAYS - 1 &&
|
||||
score.installDownloadRatio30 > bestNearConversion.installDownloadRatio30
|
||||
) {
|
||||
bestNearConversion = score;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return mergeTemporalAbuseWindowScores(bestSpike, bestSustained, bestNearConversion);
|
||||
}
|
||||
|
||||
export function labelForTemporalPublisherAbuse(input: {
|
||||
highTemporalSkillCount: number;
|
||||
p99TemporalSkillCount?: number;
|
||||
}): PublisherAbuseLabel {
|
||||
if ((input.p99TemporalSkillCount ?? 0) >= 1 || input.highTemporalSkillCount >= 2) {
|
||||
return "potential_ban_candidate";
|
||||
}
|
||||
if (input.highTemporalSkillCount >= 1) return "review";
|
||||
return "pass";
|
||||
}
|
||||
|
||||
export function computeTemporalAbuseCohortBenchmark(
|
||||
scores: Pick<SkillTemporalAbuseScore, "recent30Downloads" | "spikeMultiplier">[],
|
||||
): TemporalAbuseCohortBenchmark {
|
||||
const downloads30d = scores.map((score) => nonNegative(score.recent30Downloads));
|
||||
const spikeMultipliers = scores.map((score) => nonNegative(score.spikeMultiplier));
|
||||
return {
|
||||
sampleSize: scores.length,
|
||||
downloads30dAverage: average(downloads30d),
|
||||
downloads30dMedian: percentile(downloads30d, 0.5),
|
||||
downloads30dP95: percentile(downloads30d, 0.95),
|
||||
downloads30dP99: percentile(downloads30d, 0.99),
|
||||
spikeMultiplier7dP95: percentile(spikeMultipliers, 0.95),
|
||||
spikeMultiplier7dP99: percentile(spikeMultipliers, 0.99),
|
||||
};
|
||||
}
|
||||
|
||||
export function classifySkillTemporalAbuseScore(
|
||||
score: SkillTemporalAbuseScore,
|
||||
benchmark: TemporalAbuseCohortBenchmark | undefined,
|
||||
): SkillTemporalAbuseScore {
|
||||
if (!benchmark || benchmark.sampleSize <= 0) return score;
|
||||
|
||||
const downloads30dVsPeerP95 = score.recent30Downloads / Math.max(1, benchmark.downloads30dP95);
|
||||
const spikeMultiplierVsPeerP95 =
|
||||
score.spikeMultiplier / Math.max(1, benchmark.spikeMultiplier7dP95);
|
||||
const downloads30dCohortBand =
|
||||
score.recent30Installs <= TEMPORAL_MAX_SUSTAINED_INSTALLS
|
||||
? percentileBand({
|
||||
value: score.recent30Downloads,
|
||||
p95: benchmark.downloads30dP95,
|
||||
p99: benchmark.downloads30dP99,
|
||||
})
|
||||
: undefined;
|
||||
const spikeMultiplierCohortBand =
|
||||
score.recent7Installs <= TEMPORAL_MAX_SPIKE_INSTALLS && score.recent7Downloads > 0
|
||||
? percentileBand({
|
||||
value: score.spikeMultiplier,
|
||||
p95: benchmark.spikeMultiplier7dP95,
|
||||
p99: benchmark.spikeMultiplier7dP99,
|
||||
})
|
||||
: undefined;
|
||||
const spike = Boolean(spikeMultiplierCohortBand);
|
||||
const sustained = Boolean(downloads30dCohortBand);
|
||||
const nearConversion = score.nearConversion;
|
||||
const nearConversionPressure = nearConversion
|
||||
? Math.max(score.installDownloadExcessZScore7, score.installDownloadExcessZScore30)
|
||||
: 0;
|
||||
const reasonCodes: string[] = [];
|
||||
if (spike) reasonCodes.push("temporal_download_spike_flat_installs");
|
||||
if (sustained) reasonCodes.push("temporal_sustained_downloads_flat_installs");
|
||||
if (nearConversion) reasonCodes.push("temporal_installs_track_downloads");
|
||||
|
||||
return {
|
||||
...score,
|
||||
spike,
|
||||
sustained,
|
||||
nearConversion,
|
||||
pressure: Math.max(
|
||||
spike ? spikeMultiplierVsPeerP95 : 0,
|
||||
sustained ? downloads30dVsPeerP95 : 0,
|
||||
nearConversionPressure,
|
||||
),
|
||||
downloads30dCohortBand,
|
||||
spikeMultiplierCohortBand,
|
||||
downloads30dVsPeerP95,
|
||||
spikeMultiplierVsPeerP95,
|
||||
spikeWindowStartDay: spike ? score.spikeWindowStartDay : undefined,
|
||||
spikeWindowEndDay: spike ? score.spikeWindowEndDay : undefined,
|
||||
sustainedWindowStartDay: sustained ? score.sustainedWindowStartDay : undefined,
|
||||
sustainedWindowEndDay: sustained ? score.sustainedWindowEndDay : undefined,
|
||||
nearConversionWindowStartDay: nearConversion ? score.nearConversionWindowStartDay : undefined,
|
||||
nearConversionWindowEndDay: nearConversion ? score.nearConversionWindowEndDay : undefined,
|
||||
reasonCodes,
|
||||
};
|
||||
}
|
||||
|
||||
function reasonCodesForPublisher(input: {
|
||||
publishedSkills: number;
|
||||
installsPerSkill: number;
|
||||
@@ -465,213 +229,6 @@ function reasonCodesForPublisher(input: {
|
||||
return codes;
|
||||
}
|
||||
|
||||
function computeSkillTemporalAbuseScoreForWindows(input: {
|
||||
statsByDay: Map<number, { downloads: number; installs: number }>;
|
||||
spikeStartDay: number;
|
||||
sustainedStartDay: number;
|
||||
}): SkillTemporalAbuseScore {
|
||||
const spikeEndDay = input.spikeStartDay + TEMPORAL_SPIKE_RECENT_DAYS - 1;
|
||||
const sustainedEndDay = input.sustainedStartDay + TEMPORAL_SUSTAINED_DAYS - 1;
|
||||
const recent7 = sumTemporalStatsRange(input.statsByDay, input.spikeStartDay, spikeEndDay);
|
||||
const previous30 = sumTemporalStatsRange(
|
||||
input.statsByDay,
|
||||
input.spikeStartDay - TEMPORAL_SPIKE_BASELINE_DAYS,
|
||||
input.spikeStartDay - 1,
|
||||
);
|
||||
const recent30 = sumTemporalStatsRange(
|
||||
input.statsByDay,
|
||||
input.sustainedStartDay,
|
||||
sustainedEndDay,
|
||||
);
|
||||
const baseline7Downloads = Math.max(
|
||||
TEMPORAL_MIN_BASELINE_7_DOWNLOADS,
|
||||
(previous30.downloads / TEMPORAL_SPIKE_BASELINE_DAYS) * TEMPORAL_SPIKE_RECENT_DAYS,
|
||||
);
|
||||
const spikeMultiplier = baseline7Downloads > 0 ? recent7.downloads / baseline7Downloads : 0;
|
||||
const downloadInstallRatio30 = recent30.downloads / Math.max(1, recent30.installs);
|
||||
const installDownloadRatio7 = recent7.installs / Math.max(1, recent7.downloads);
|
||||
const installDownloadRatio30 = recent30.installs / Math.max(1, recent30.downloads);
|
||||
const installDownloadExcessZScore7 = installDownloadExcessZScore({
|
||||
downloads: recent7.downloads,
|
||||
installs: recent7.installs,
|
||||
});
|
||||
const installDownloadExcessZScore30 = installDownloadExcessZScore({
|
||||
downloads: recent30.downloads,
|
||||
installs: recent30.installs,
|
||||
});
|
||||
const nearConversion7 =
|
||||
recent7.downloads >= TEMPORAL_MIN_NEAR_CONVERSION_7_DOWNLOADS &&
|
||||
recent7.installs >= TEMPORAL_MIN_NEAR_CONVERSION_INSTALLS &&
|
||||
installDownloadRatio7 >= TEMPORAL_MIN_INSTALL_DOWNLOAD_RATIO &&
|
||||
installDownloadExcessZScore7 >= TEMPORAL_MIN_INSTALL_DOWNLOAD_EXCESS_Z_SCORE;
|
||||
const nearConversion30 =
|
||||
recent30.downloads >= TEMPORAL_MIN_NEAR_CONVERSION_30_DOWNLOADS &&
|
||||
recent30.installs >= TEMPORAL_MIN_NEAR_CONVERSION_INSTALLS &&
|
||||
installDownloadRatio30 >= TEMPORAL_MIN_INSTALL_DOWNLOAD_RATIO &&
|
||||
installDownloadExcessZScore30 >= TEMPORAL_MIN_INSTALL_DOWNLOAD_EXCESS_Z_SCORE;
|
||||
const nearConversion = nearConversion7 || nearConversion30;
|
||||
const reasonCodes: string[] = [];
|
||||
if (nearConversion) reasonCodes.push("temporal_installs_track_downloads");
|
||||
|
||||
return {
|
||||
spike: false,
|
||||
sustained: false,
|
||||
nearConversion,
|
||||
pressure: nearConversion
|
||||
? Math.max(installDownloadExcessZScore7, installDownloadExcessZScore30)
|
||||
: 0,
|
||||
recent7Downloads: recent7.downloads,
|
||||
recent7Installs: recent7.installs,
|
||||
previous30Downloads: previous30.downloads,
|
||||
baseline7Downloads,
|
||||
spikeMultiplier,
|
||||
recent30Downloads: recent30.downloads,
|
||||
recent30Installs: recent30.installs,
|
||||
downloadInstallRatio30,
|
||||
installDownloadRatio7,
|
||||
installDownloadRatio30,
|
||||
installDownloadExcessZScore7,
|
||||
installDownloadExcessZScore30,
|
||||
spikeWindowStartDay: input.spikeStartDay,
|
||||
spikeWindowEndDay: spikeEndDay,
|
||||
sustainedWindowStartDay: input.sustainedStartDay,
|
||||
sustainedWindowEndDay: sustainedEndDay,
|
||||
nearConversionWindowStartDay: nearConversion7
|
||||
? input.spikeStartDay
|
||||
: nearConversion30
|
||||
? input.sustainedStartDay
|
||||
: undefined,
|
||||
nearConversionWindowEndDay: nearConversion7
|
||||
? spikeEndDay
|
||||
: nearConversion30
|
||||
? sustainedEndDay
|
||||
: undefined,
|
||||
reasonCodes,
|
||||
};
|
||||
}
|
||||
|
||||
function mergeTemporalAbuseWindowScores(
|
||||
bestSpike: SkillTemporalAbuseScore,
|
||||
bestSustained: SkillTemporalAbuseScore,
|
||||
bestNearConversion: SkillTemporalAbuseScore,
|
||||
): SkillTemporalAbuseScore {
|
||||
if (!bestSpike.spike && !bestSustained.sustained && !bestNearConversion.nearConversion) {
|
||||
return emptySkillTemporalAbuseScore();
|
||||
}
|
||||
const reasonCodes: string[] = [];
|
||||
if (bestSpike.spike) reasonCodes.push("temporal_download_spike_flat_installs");
|
||||
if (bestSustained.sustained) reasonCodes.push("temporal_sustained_downloads_flat_installs");
|
||||
if (bestNearConversion.nearConversion) reasonCodes.push("temporal_installs_track_downloads");
|
||||
|
||||
return {
|
||||
spike: bestSpike.spike,
|
||||
sustained: bestSustained.sustained,
|
||||
nearConversion: bestNearConversion.nearConversion,
|
||||
pressure: Math.max(bestSpike.pressure, bestSustained.pressure, bestNearConversion.pressure),
|
||||
recent7Downloads: bestSpike.spike
|
||||
? bestSpike.recent7Downloads
|
||||
: bestNearConversion.recent7Downloads,
|
||||
recent7Installs: bestSpike.spike
|
||||
? bestSpike.recent7Installs
|
||||
: bestNearConversion.recent7Installs,
|
||||
previous30Downloads: bestSpike.spike
|
||||
? bestSpike.previous30Downloads
|
||||
: bestNearConversion.previous30Downloads,
|
||||
baseline7Downloads: bestSpike.spike
|
||||
? bestSpike.baseline7Downloads
|
||||
: bestNearConversion.baseline7Downloads,
|
||||
spikeMultiplier: bestSpike.spike
|
||||
? bestSpike.spikeMultiplier
|
||||
: bestNearConversion.spikeMultiplier,
|
||||
recent30Downloads: bestSustained.sustained
|
||||
? bestSustained.recent30Downloads
|
||||
: bestNearConversion.recent30Downloads,
|
||||
recent30Installs: bestSustained.sustained
|
||||
? bestSustained.recent30Installs
|
||||
: bestNearConversion.recent30Installs,
|
||||
downloadInstallRatio30: bestSustained.sustained
|
||||
? bestSustained.downloadInstallRatio30
|
||||
: bestNearConversion.downloadInstallRatio30,
|
||||
installDownloadRatio7: bestNearConversion.installDownloadRatio7,
|
||||
installDownloadRatio30: bestNearConversion.installDownloadRatio30,
|
||||
installDownloadExcessZScore7: bestNearConversion.installDownloadExcessZScore7,
|
||||
installDownloadExcessZScore30: bestNearConversion.installDownloadExcessZScore30,
|
||||
downloads30dCohortBand: bestSustained.downloads30dCohortBand,
|
||||
spikeMultiplierCohortBand: bestSpike.spikeMultiplierCohortBand,
|
||||
downloads30dVsPeerP95: bestSustained.downloads30dVsPeerP95,
|
||||
spikeMultiplierVsPeerP95: bestSpike.spikeMultiplierVsPeerP95,
|
||||
spikeWindowStartDay: bestSpike.spikeWindowStartDay,
|
||||
spikeWindowEndDay: bestSpike.spikeWindowEndDay,
|
||||
sustainedWindowStartDay: bestSustained.sustainedWindowStartDay,
|
||||
sustainedWindowEndDay: bestSustained.sustainedWindowEndDay,
|
||||
nearConversionWindowStartDay: bestNearConversion.nearConversionWindowStartDay,
|
||||
nearConversionWindowEndDay: bestNearConversion.nearConversionWindowEndDay,
|
||||
reasonCodes,
|
||||
};
|
||||
}
|
||||
|
||||
function aggregateSkillTemporalDailyStats(dailyStats: SkillTemporalAbuseDailyStat[]) {
|
||||
const byDay = new Map<number, { downloads: number; installs: number }>();
|
||||
for (const point of dailyStats) {
|
||||
if (!Number.isFinite(point.day)) continue;
|
||||
const day = Math.trunc(point.day);
|
||||
const existing = byDay.get(day) ?? { downloads: 0, installs: 0 };
|
||||
existing.downloads += nonNegative(point.downloads);
|
||||
existing.installs += nonNegative(point.installs);
|
||||
byDay.set(day, existing);
|
||||
}
|
||||
return byDay;
|
||||
}
|
||||
|
||||
function sumTemporalStatsRange(
|
||||
statsByDay: Map<number, { downloads: number; installs: number }>,
|
||||
startDay: number,
|
||||
endDay: number,
|
||||
) {
|
||||
let downloads = 0;
|
||||
let installs = 0;
|
||||
for (let day = startDay; day <= endDay; day += 1) {
|
||||
const point = statsByDay.get(day);
|
||||
if (!point) continue;
|
||||
downloads += point.downloads;
|
||||
installs += point.installs;
|
||||
}
|
||||
return { downloads, installs };
|
||||
}
|
||||
|
||||
function emptySkillTemporalAbuseScore(): SkillTemporalAbuseScore {
|
||||
return {
|
||||
spike: false,
|
||||
sustained: false,
|
||||
nearConversion: false,
|
||||
pressure: 0,
|
||||
recent7Downloads: 0,
|
||||
recent7Installs: 0,
|
||||
previous30Downloads: 0,
|
||||
baseline7Downloads: TEMPORAL_MIN_BASELINE_7_DOWNLOADS,
|
||||
spikeMultiplier: 0,
|
||||
recent30Downloads: 0,
|
||||
recent30Installs: 0,
|
||||
downloadInstallRatio30: 0,
|
||||
installDownloadRatio7: 0,
|
||||
installDownloadRatio30: 0,
|
||||
installDownloadExcessZScore7: 0,
|
||||
installDownloadExcessZScore30: 0,
|
||||
reasonCodes: [],
|
||||
};
|
||||
}
|
||||
|
||||
function installDownloadExcessZScore(input: { downloads: number; installs: number }) {
|
||||
if (input.downloads <= 0) return 0;
|
||||
const expected = input.downloads * TEMPORAL_EXPECTED_INSTALL_DOWNLOAD_RATIO;
|
||||
const variance =
|
||||
input.downloads *
|
||||
TEMPORAL_EXPECTED_INSTALL_DOWNLOAD_RATIO *
|
||||
(1 - TEMPORAL_EXPECTED_INSTALL_DOWNLOAD_RATIO);
|
||||
const stdDev = Math.sqrt(Math.max(variance, 1));
|
||||
return (input.installs - expected) / stdDev;
|
||||
}
|
||||
|
||||
function nonNegative(value: number) {
|
||||
return Number.isFinite(value) ? Math.max(0, value) : 0;
|
||||
}
|
||||
@@ -681,24 +238,6 @@ function average(values: number[]) {
|
||||
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
||||
}
|
||||
|
||||
function percentile(values: number[], quantile: number) {
|
||||
if (values.length === 0) return 0;
|
||||
const sorted = [...values].sort((left, right) => left - right);
|
||||
const index = Math.max(0, Math.min(sorted.length - 1, Math.ceil(quantile * sorted.length) - 1));
|
||||
return sorted[index] ?? 0;
|
||||
}
|
||||
|
||||
function percentileBand(input: {
|
||||
value: number;
|
||||
p95: number;
|
||||
p99: number;
|
||||
}): "p95" | "p99" | undefined {
|
||||
if (input.value <= 0) return undefined;
|
||||
if (input.p99 > 0 && input.value > input.p99) return "p99";
|
||||
if (input.p95 > 0 && input.value > input.p95) return "p95";
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function standardDeviation(values: number[], mean: number) {
|
||||
if (values.length === 0) return 0;
|
||||
const variance = values.reduce((sum, value) => sum + (value - mean) ** 2, 0) / values.length;
|
||||
|
||||
@@ -1,196 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildGitHubSkillCatalogDisplay } from "./publisherCatalogDisplay";
|
||||
|
||||
const baseItem = {
|
||||
kind: "skill" as const,
|
||||
summary: null,
|
||||
icon: null,
|
||||
href: "/nvidia/example",
|
||||
downloads: 0,
|
||||
stars: 0,
|
||||
isOfficial: true,
|
||||
updatedAt: 1,
|
||||
sourceBacked: true,
|
||||
sourceRepo: "NVIDIA/skills",
|
||||
sourcePath: null,
|
||||
sourceVerifiedCommit: null,
|
||||
};
|
||||
|
||||
describe("buildGitHubSkillCatalogDisplay", () => {
|
||||
it("groups source-backed skills by manifest entries and ignores missing entries", () => {
|
||||
const display = buildGitHubSkillCatalogDisplay({
|
||||
sources: [
|
||||
{
|
||||
_id: "githubSkillSources:nvidia",
|
||||
repo: "NVIDIA/skills",
|
||||
displayManifestStatus: "ok",
|
||||
displayManifest: {
|
||||
notGrouped: "bottom",
|
||||
groupings: [
|
||||
{
|
||||
title: "Agentic AI",
|
||||
description: "Agentic AI skills.",
|
||||
skills: ["aiq-deploy", "missing-upstream-entry"],
|
||||
},
|
||||
{
|
||||
title: "Vision AI",
|
||||
skills: ["vision-helper"],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
items: [
|
||||
{
|
||||
...baseItem,
|
||||
_id: "skills:aiq-deploy",
|
||||
displayName: "AIQ Deploy",
|
||||
slug: "aiq-deploy",
|
||||
sourceId: "githubSkillSources:nvidia",
|
||||
},
|
||||
{
|
||||
...baseItem,
|
||||
_id: "skills:vision-helper",
|
||||
displayName: "Vision Helper",
|
||||
slug: "vision-helper",
|
||||
sourceId: "githubSkillSources:nvidia",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(display).toMatchObject({
|
||||
mode: "grouped",
|
||||
sourceRepos: ["NVIDIA/skills"],
|
||||
sections: [
|
||||
{
|
||||
title: "Agentic AI",
|
||||
description: "Agentic AI skills.",
|
||||
sourceRepo: "NVIDIA/skills",
|
||||
items: [{ displayName: "AIQ Deploy" }],
|
||||
},
|
||||
{
|
||||
title: "Vision AI",
|
||||
sourceRepo: "NVIDIA/skills",
|
||||
items: [{ displayName: "Vision Helper" }],
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("matches manifest entries by normalized display name and places unlisted skills at the requested edge", () => {
|
||||
const display = buildGitHubSkillCatalogDisplay({
|
||||
sources: [
|
||||
{
|
||||
_id: "githubSkillSources:nvidia",
|
||||
repo: "NVIDIA/skills",
|
||||
displayManifestStatus: "ok",
|
||||
displayManifest: {
|
||||
notGrouped: "top",
|
||||
groupings: [
|
||||
{
|
||||
title: "Physical AI",
|
||||
skills: ["Isaac Sim Helper"],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
items: [
|
||||
{
|
||||
...baseItem,
|
||||
_id: "skills:isaac-sim-helper",
|
||||
displayName: "Isaac Sim Helper",
|
||||
slug: "isaac-sim-helper",
|
||||
sourceId: "githubSkillSources:nvidia",
|
||||
},
|
||||
{
|
||||
...baseItem,
|
||||
_id: "skills:extra",
|
||||
displayName: "Extra Skill",
|
||||
slug: "extra",
|
||||
sourceId: "githubSkillSources:nvidia",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(display?.sections.map((section) => section.title)).toEqual([
|
||||
"Other skills",
|
||||
"Physical AI",
|
||||
]);
|
||||
expect(display?.sections[0]?.items.map((item) => item.displayName)).toEqual(["Extra Skill"]);
|
||||
});
|
||||
|
||||
it("falls back to the normal catalog when the source manifest is missing or invalid", () => {
|
||||
const display = buildGitHubSkillCatalogDisplay({
|
||||
sources: [
|
||||
{
|
||||
_id: "githubSkillSources:nvidia",
|
||||
repo: "NVIDIA/skills",
|
||||
displayManifestStatus: "invalid",
|
||||
},
|
||||
],
|
||||
items: [
|
||||
{
|
||||
...baseItem,
|
||||
_id: "skills:aiq-deploy",
|
||||
displayName: "AIQ Deploy",
|
||||
slug: "aiq-deploy",
|
||||
sourceId: "githubSkillSources:nvidia",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(display).toBeNull();
|
||||
});
|
||||
|
||||
it("keeps source-backed skills from non-renderable sources in other skills", () => {
|
||||
const display = buildGitHubSkillCatalogDisplay({
|
||||
sources: [
|
||||
{
|
||||
_id: "githubSkillSources:nvidia",
|
||||
repo: "NVIDIA/skills",
|
||||
displayManifestStatus: "ok",
|
||||
displayManifest: {
|
||||
notGrouped: "bottom",
|
||||
groupings: [
|
||||
{
|
||||
title: "Agentic AI",
|
||||
skills: ["aiq-deploy"],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
_id: "githubSkillSources:invalid",
|
||||
repo: "example/skills",
|
||||
displayManifestStatus: "invalid",
|
||||
},
|
||||
],
|
||||
items: [
|
||||
{
|
||||
...baseItem,
|
||||
_id: "skills:aiq-deploy",
|
||||
displayName: "AIQ Deploy",
|
||||
slug: "aiq-deploy",
|
||||
sourceId: "githubSkillSources:nvidia",
|
||||
},
|
||||
{
|
||||
...baseItem,
|
||||
_id: "skills:unlisted",
|
||||
displayName: "Unlisted Source Skill",
|
||||
slug: "unlisted-source-skill",
|
||||
sourceRepo: "example/skills",
|
||||
sourceId: "githubSkillSources:invalid",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(display?.sections.map((section) => section.title)).toEqual([
|
||||
"Agentic AI",
|
||||
"Other skills",
|
||||
]);
|
||||
expect(display?.sections.at(-1)?.items.map((item) => item.displayName)).toEqual([
|
||||
"Unlisted Source Skill",
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -1,157 +0,0 @@
|
||||
type DisplayManifest = {
|
||||
notGrouped?: "top" | "bottom";
|
||||
groupings: Array<{
|
||||
title: string;
|
||||
description?: string;
|
||||
skills: string[];
|
||||
}>;
|
||||
};
|
||||
|
||||
export type GitHubSkillCatalogSource = {
|
||||
_id: string;
|
||||
repo: string;
|
||||
displayManifestStatus?: "ok" | "missing" | "invalid" | "failed";
|
||||
displayManifest?: DisplayManifest;
|
||||
};
|
||||
|
||||
export type GitHubSkillCatalogItem = {
|
||||
_id: string;
|
||||
kind: "skill" | "plugin";
|
||||
displayName: string;
|
||||
slug?: string | null;
|
||||
sourceBacked?: boolean;
|
||||
sourceId?: string | null;
|
||||
sourceRepo?: string | null;
|
||||
sourcePath?: string | null;
|
||||
sourceVerifiedCommit?: string | null;
|
||||
summary: string | null;
|
||||
icon: string | null;
|
||||
href: string;
|
||||
downloads: number;
|
||||
stars: number;
|
||||
isOfficial: boolean;
|
||||
updatedAt: number;
|
||||
};
|
||||
|
||||
export type GitHubSkillCatalogSection = {
|
||||
key: string;
|
||||
title: string;
|
||||
description: string | null;
|
||||
sourceRepo: string | null;
|
||||
items: GitHubSkillCatalogItem[];
|
||||
};
|
||||
|
||||
export type GitHubSkillCatalogDisplay = {
|
||||
mode: "grouped";
|
||||
sourceRepos: string[];
|
||||
sections: GitHubSkillCatalogSection[];
|
||||
};
|
||||
|
||||
function normalizeManifestSkillKey(value: string) {
|
||||
return value
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[\s_]+/g, "-")
|
||||
.replace(/[^a-z0-9-]+/g, "-")
|
||||
.replace(/-+/g, "-")
|
||||
.replace(/^-|-$/g, "");
|
||||
}
|
||||
|
||||
function getItemKeys(item: GitHubSkillCatalogItem) {
|
||||
const keys = new Set<string>();
|
||||
if (item.slug) keys.add(normalizeManifestSkillKey(item.slug));
|
||||
keys.add(normalizeManifestSkillKey(item.displayName));
|
||||
|
||||
const sourcePathName = item.sourcePath?.split("/").filter(Boolean).at(-1);
|
||||
if (sourcePathName) keys.add(normalizeManifestSkillKey(sourcePathName));
|
||||
|
||||
return keys;
|
||||
}
|
||||
|
||||
function findManifestItem(
|
||||
candidates: GitHubSkillCatalogItem[],
|
||||
manifestEntry: string,
|
||||
usedItemIds: Set<string>,
|
||||
) {
|
||||
const key = normalizeManifestSkillKey(manifestEntry);
|
||||
if (!key) return null;
|
||||
|
||||
return (
|
||||
candidates.find((item) => !usedItemIds.has(item._id) && getItemKeys(item).has(key)) ?? null
|
||||
);
|
||||
}
|
||||
|
||||
function isRenderableSource(source: GitHubSkillCatalogSource) {
|
||||
return (
|
||||
source.displayManifestStatus === "ok" &&
|
||||
Boolean(source.displayManifest) &&
|
||||
source.displayManifest!.groupings.length > 0
|
||||
);
|
||||
}
|
||||
|
||||
export function buildGitHubSkillCatalogDisplay({
|
||||
sources,
|
||||
items,
|
||||
}: {
|
||||
sources: GitHubSkillCatalogSource[];
|
||||
items: GitHubSkillCatalogItem[];
|
||||
}): GitHubSkillCatalogDisplay | null {
|
||||
const renderableSources = sources.filter(isRenderableSource);
|
||||
if (renderableSources.length === 0) return null;
|
||||
|
||||
const sourceRepos = Array.from(new Set(renderableSources.map((source) => source.repo)));
|
||||
const usedItemIds = new Set<string>();
|
||||
const sections: GitHubSkillCatalogSection[] = [];
|
||||
const otherPosition = renderableSources.some(
|
||||
(source) => source.displayManifest?.notGrouped === "top",
|
||||
)
|
||||
? "top"
|
||||
: "bottom";
|
||||
|
||||
for (const source of renderableSources) {
|
||||
const sourceItems = items.filter(
|
||||
(item) => item.kind === "skill" && item.sourceId === source._id,
|
||||
);
|
||||
if (sourceItems.length === 0) continue;
|
||||
|
||||
for (const [groupIndex, group] of source.displayManifest!.groupings.entries()) {
|
||||
const groupItems = group.skills
|
||||
.map((entry) => findManifestItem(sourceItems, entry, usedItemIds))
|
||||
.filter((item): item is GitHubSkillCatalogItem => Boolean(item));
|
||||
|
||||
if (groupItems.length === 0) continue;
|
||||
for (const item of groupItems) usedItemIds.add(item._id);
|
||||
|
||||
sections.push({
|
||||
key: `${source._id}:${groupIndex}:${group.title}`,
|
||||
title: group.title,
|
||||
description: group.description ?? null,
|
||||
sourceRepo: source.repo,
|
||||
items: groupItems,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const otherItems = items.filter((item) => item.kind === "skill" && !usedItemIds.has(item._id));
|
||||
const otherSection =
|
||||
otherItems.length > 0
|
||||
? {
|
||||
key: "other-skills",
|
||||
title: "Other skills",
|
||||
description: null,
|
||||
sourceRepo: null,
|
||||
items: otherItems,
|
||||
}
|
||||
: null;
|
||||
const orderedSections =
|
||||
otherPosition === "top" && otherSection
|
||||
? [otherSection, ...sections]
|
||||
: [...sections, ...(otherSection ? [otherSection] : [])];
|
||||
|
||||
if (orderedSections.length === 0) return null;
|
||||
return {
|
||||
mode: "grouped",
|
||||
sourceRepos,
|
||||
sections: orderedSections,
|
||||
};
|
||||
}
|
||||
@@ -52,13 +52,6 @@ describe("publisher stat maintenance", () => {
|
||||
return {
|
||||
collect: vi.fn(async () => [
|
||||
makeSkill({ statsDownloads: 11, statsStars: 2, statsInstallsAllTime: 5 }),
|
||||
makeSkill({
|
||||
_id: "skills:hidden",
|
||||
moderationStatus: "hidden",
|
||||
statsDownloads: 100,
|
||||
statsStars: 100,
|
||||
statsInstallsAllTime: 100,
|
||||
}),
|
||||
]),
|
||||
};
|
||||
}
|
||||
@@ -143,26 +136,6 @@ describe("publisher stat maintenance", () => {
|
||||
expect(ctx.db.query).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not count hidden skills in public publisher aggregates", async () => {
|
||||
const ctx = {
|
||||
db: {
|
||||
get: vi.fn(),
|
||||
patch: vi.fn(),
|
||||
query: vi.fn(),
|
||||
},
|
||||
};
|
||||
|
||||
await adjustPublisherStatsForSkillChange(
|
||||
ctx as never,
|
||||
null,
|
||||
makeSkill({ moderationStatus: "hidden", moderationReason: "pending.scan" }),
|
||||
);
|
||||
|
||||
expect(ctx.db.get).not.toHaveBeenCalled();
|
||||
expect(ctx.db.patch).not.toHaveBeenCalled();
|
||||
expect(ctx.db.query).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps legacy aggregate updates bounded when skill-only aggregates are missing", async () => {
|
||||
const patch = vi.fn();
|
||||
const ctx = {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user