From 0f84533e9c93e7b4f712c4d4cdb071e1f435baea Mon Sep 17 00:00:00 2001 From: Patrick Erichsen Date: Fri, 24 Jul 2026 14:32:00 -0500 Subject: [PATCH] feat: add permanent skills.sh mirror storage (#3227) * feat: add staged skills.sh mirror storage * ci: allow guarded CLAW-563 Test deploy * ci: expose guarded Test deploy diagnostics * ci: defer branch guard to deploy step * ci: deploy CLAW-563 PR head to Test * ci: admit CLAW-563 PR Test job * fix: make mirror source recovery durable * ci: trigger labeled mirror load * feat: activate mirror search queries * fix: tighten mirror source typing * fix: bypass protected Test mirror proof * feat: attribute skill metrics by source * feat: present stars as bookmarks * style: format mirror proof changes * fix: bypass protected mirror readback * fix: resume mirror past missing scanner pages * fix: fetch skills.sh mirror audits from api * fix: validate structural skills.sh identities * fix: resolve ambiguous skills.sh mirror identities * feat: stabilize skills.sh mirror ingestion * fix: account mirror identity conflicts in proof * fix: quarantine invalid skills.sh detail ids * fix: resume skills.sh mirror proof * fix: preserve skills.sh mirror provenance * fix: recover exact skills.sh mirror runs * fix: recover stale skills.sh mirror runs * fix: normalize skills.sh mirror topic facets * feat: prove complete skills.sh leaderboard mirror * fix: canonicalize skills.sh source page hashes * test: enable skills.sh rollout in mirror tests * ci: skip unrelated Test deploy pull requests * fix: preserve Vercel preview marker in Test deploy * fix: tighten Test deploy and metric reconciliation * fix: bound mirror detail proof pages * fix: delegate controlled mirror rate limits * fix: preserve mirror reconciliation progress * fix: release mirror retry responses * fix: preserve stale mirror replay state * fix: authenticate mirror source starts * fix: delegate mirror identity rate limits * ci: trigger mirror proof when labeled * ci: couple mirror deploy and proof opt-in * fix: admit permanent Vercel Test runtime * fix: pass Test target to Vercel runtime * test: align bookmark sync browser labels * fix: preserve skills.sh source accounting * fix: preflight active mirror runs * fix: bind mirror snapshot accounting * fix: reject truncated replay hashes * fix: preserve live mirror overlay metadata --- .github/workflows/deploy-test.yml | 231 +- bun.lock | 1 + convex/_generated/api.d.ts | 2 + convex/dashboard.ts | 4 +- convex/githubSkillSync.test.ts | 12 + convex/httpApiV1.handlers.test.ts | 5 +- convex/httpApiV1/skillsShCatalogV1.test.ts | 399 +++ convex/httpApiV1/skillsShCatalogV1.ts | 280 ++ convex/httpApiV1/skillsV1.ts | 6 +- convex/lib/public.test.ts | 18 +- convex/lib/public.ts | 6 +- convex/lib/retentionPolicy.ts | 9 + convex/lib/skillSearchDigest.test.ts | 4 + convex/lib/skillSearchDigest.ts | 2 + convex/lib/skillStats.test.ts | 49 + convex/lib/skillStats.ts | 38 + convex/lib/skillsShCatalogFixtures.ts | 4 +- convex/publishers.test.ts | 14 +- convex/publishers.ts | 10 +- convex/schema.ts | 260 ++ convex/search.test.ts | 48 + convex/search.ts | 14 +- convex/skills.dashboard.test.ts | 34 +- convex/skills.packageCatalog.test.ts | 29 + convex/skills.ts | 10 +- convex/skillsShCatalog.ts | 136 +- convex/skillsShCatalogCanary.test.ts | 165 +- convex/skillsShMirror.test.ts | 1460 ++++++++++ convex/skillsShMirror.ts | 1803 ++++++++++++ docs/cli.md | 3 +- docs/http-api.md | 3 +- e2e/local-auth/skill-star-sync.pw.test.ts | 6 +- package.json | 2 + .../prove-mirror-request.test.ts | 352 +++ .../skills-sh-catalog/prove-mirror-request.ts | 268 ++ .../skills-sh-catalog/prove-mirror-test.ts | 776 +++++ .../routes/ops/skills-sh/mirror-test.post.ts | 638 ++++ server/skillsShCatalogSource.test.ts | 2575 ++++++++++++++++- server/skillsShCatalogSource.ts | 2127 +++++++++++++- server/skillsShMirrorClassification.test.ts | 353 +++ server/skillsShMirrorClassification.ts | 264 ++ server/skillsShMirrorTestRoute.test.ts | 1258 ++++++++ specs/download-metering.md | 28 +- specs/github-backed-skills.md | 29 + specs/spec.md | 17 +- src/__tests__/deploy-test-workflow.test.ts | 53 +- src/__tests__/header.test.tsx | 4 +- src/__tests__/skill-detail-page.test.tsx | 10 +- src/__tests__/skills-index.test.tsx | 4 +- src/components/Header.tsx | 2 +- src/components/SkillDetailPage.tsx | 4 +- src/components/SkillHeader.test.tsx | 12 +- src/components/SkillHeader.tsx | 6 +- src/components/UserBadge.tsx | 2 +- .../dashboard/DashboardCatalogView.test.tsx | 60 + .../dashboard/DashboardCatalogView.tsx | 20 +- src/components/dashboard/types.ts | 8 + .../skeletons/ProtectedPageSkeletons.tsx | 2 +- src/routes/-stars.test.tsx | 14 +- src/routes/skills/index.tsx | 2 +- src/routes/skills/publish.tsx | 4 +- src/routes/stars.tsx | 26 +- src/routes/user/$handle.tsx | 8 +- 63 files changed, 13855 insertions(+), 138 deletions(-) create mode 100644 convex/lib/skillStats.test.ts create mode 100644 convex/skillsShMirror.test.ts create mode 100644 convex/skillsShMirror.ts create mode 100644 scripts/skills-sh-catalog/prove-mirror-request.test.ts create mode 100644 scripts/skills-sh-catalog/prove-mirror-request.ts create mode 100644 scripts/skills-sh-catalog/prove-mirror-test.ts create mode 100644 server/routes/ops/skills-sh/mirror-test.post.ts create mode 100644 server/skillsShMirrorClassification.test.ts create mode 100644 server/skillsShMirrorClassification.ts create mode 100644 server/skillsShMirrorTestRoute.test.ts create mode 100644 src/components/dashboard/DashboardCatalogView.test.tsx diff --git a/.github/workflows/deploy-test.yml b/.github/workflows/deploy-test.yml index 668c1adb..28630bab 100644 --- a/.github/workflows/deploy-test.yml +++ b/.github/workflows/deploy-test.yml @@ -5,7 +5,18 @@ on: workflows: ["CI"] types: [completed] branches: [main] + pull_request: + types: [synchronize, labeled] workflow_dispatch: + inputs: + expected_sha: + description: Exact branch SHA permitted for a temporary protected Test deploy + required: false + type: string + branch_test_confirm: + description: Confirmation phrase for the CLAW-563 branch-only Test deploy + required: false + type: string concurrency: group: deploy-test @@ -18,7 +29,16 @@ jobs: deploy-test: if: >- (github.event_name == 'workflow_dispatch' && - github.ref == 'refs/heads/main') || + (github.ref == 'refs/heads/main' || + (github.ref == 'refs/heads/pe/claw-563-skills-sh-mirror-10k' && + github.actor == 'Patrick-Erichsen' && + inputs.branch_test_confirm == 'deploy-claw-563-to-permanent-test' && + inputs.expected_sha != ''))) || + (github.event_name == 'pull_request' && + github.event.pull_request.head.ref == 'pe/claw-563-skills-sh-mirror-10k' && + github.event.pull_request.head.repo.full_name == github.repository && + github.actor == 'Patrick-Erichsen' && + contains(github.event.pull_request.labels.*.name, 'test-mirror-load')) || (github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push') runs-on: ubuntu-latest @@ -30,11 +50,17 @@ jobs: deployment_url: ${{ steps.vercel.outputs.deployment_url }} deploy_sha: ${{ steps.revision.outputs.deploy_sha }} fixture_version: ${{ steps.revision.outputs.fixture_version }} + branch_test: ${{ steps.revision.outputs.branch_test }} steps: - uses: actions/checkout@v7.0.1 with: fetch-depth: 0 - ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.ref }} + ref: >- + ${{ + github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || + github.event_name == 'pull_request' && github.event.pull_request.head.sha || + github.ref + }} - name: Resolve deployment revision id: revision @@ -44,8 +70,30 @@ jobs: git fetch --no-tags origin main main_sha="$(git rev-parse origin/main)" if [[ "$deploy_sha" != "$main_sha" ]]; then - echo "::error::Refusing stale Test deploy: $deploy_sha is not current main $main_sha" - exit 1 + branch_test_allowed=false + if [[ "$GITHUB_EVENT_NAME" == workflow_dispatch ]] && + [[ "$GITHUB_REF" == refs/heads/pe/claw-563-skills-sh-mirror-10k ]] && + [[ "$GITHUB_ACTOR" == Patrick-Erichsen ]] && + [[ "${{ inputs.branch_test_confirm }}" == deploy-claw-563-to-permanent-test ]] && + [[ "${{ inputs.expected_sha }}" == "$deploy_sha" ]] + then + branch_test_allowed=true + fi + if [[ "$GITHUB_EVENT_NAME" == pull_request ]] && + [[ "$GITHUB_HEAD_REF" == pe/claw-563-skills-sh-mirror-10k ]] && + [[ "$GITHUB_ACTOR" == Patrick-Erichsen ]] && + [[ "${{ github.event.pull_request.head.repo.full_name }}" == "$GITHUB_REPOSITORY" ]] && + [[ "${{ github.event.pull_request.head.sha }}" == "$deploy_sha" ]] + then + branch_test_allowed=true + fi + if [[ "$branch_test_allowed" != true ]]; then + echo "::error::Refusing non-main Test deploy without the exact CLAW-563 branch guard" + exit 1 + fi + echo "branch_test=true" >> "$GITHUB_OUTPUT" + else + echo "branch_test=false" >> "$GITHUB_OUTPUT" fi fixture_version="$( git hash-object convex/devSeed.ts convex/lib/testSeed.ts | @@ -138,6 +186,7 @@ jobs: ' <<< "$capabilities" - name: Apply additive Test fixtures + if: steps.revision.outputs.branch_test != 'true' env: CONVEX_DEPLOY_KEY: ${{ secrets.CONVEX_DEPLOY_KEY }} CONVEX_DEPLOYMENT: ${{ vars.CONVEX_DEPLOYMENT }} @@ -164,20 +213,22 @@ jobs: --meta githubSha="${{ steps.revision.outputs.deploy_sha }}" \ --meta clawhubEnvironment=test \ --build-env CONVEX_DEPLOY_KEY= \ - --build-env VERCEL_ENV=test \ --build-env VERCEL_TARGET_ENV=test \ --build-env CLAWHUB_ENV=test \ --build-env CLAWHUB_SKILLS_SH_ROLLOUT_MODE=test \ --build-env CLAWHUB_GITHUB_SKILL_SYNC_ROLLOUT_MODE=test \ + --build-env CLAWHUB_SKILLS_SH_TEST_LIVE_FETCH_ENABLED=1 \ --build-env SITE_URL="$TEST_SITE_URL" \ --build-env VITE_CLAWHUB_DEPLOY_ENV=test \ --build-env VITE_CONVEX_URL="$VITE_CONVEX_URL" \ --build-env VITE_CONVEX_SITE_URL="$VITE_CONVEX_SITE_URL" \ --build-env VITE_SITE_URL="$TEST_SITE_URL" \ --env CONVEX_DEPLOY_KEY= \ + --env VERCEL_TARGET_ENV=test \ --env CLAWHUB_ENV=test \ --env CLAWHUB_SKILLS_SH_ROLLOUT_MODE=test \ --env CLAWHUB_GITHUB_SKILL_SYNC_ROLLOUT_MODE=test \ + --env CLAWHUB_SKILLS_SH_TEST_LIVE_FETCH_ENABLED=1 \ --env SITE_URL="$TEST_SITE_URL" \ --env VITE_CLAWHUB_DEPLOY_ENV=test \ --env VITE_CONVEX_URL="$VITE_CONVEX_URL" \ @@ -256,3 +307,173 @@ jobs: echo "- Fixture version: \`${FIXTURE_VERSION:-unresolved}\`" echo "- Result: \`${{ job.status }}\`" } >> "$GITHUB_STEP_SUMMARY" + + claw563-mirror-load: + needs: deploy-test + if: >- + github.event_name == 'pull_request' && + github.event.pull_request.head.ref == 'pe/claw-563-skills-sh-mirror-10k' && + contains(github.event.pull_request.labels.*.name, 'test-mirror-load') + runs-on: ubuntu-latest + timeout-minutes: 180 + environment: + name: Test + steps: + - uses: actions/checkout@v7.0.1 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - uses: ./.github/actions/setup-bun + + - name: Load and prove the authenticated leaderboard mirror foundation + env: + CONVEX_DEPLOY_KEY: ${{ secrets.CONVEX_DEPLOY_KEY }} + DEPLOY_SHA: ${{ needs.deploy-test.outputs.deploy_sha }} + TEST_SITE_URL: ${{ vars.SITE_URL }} + VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} + run: | + set -euo pipefail + operator_token="" + cleanup() { + set +e + bunx convex run --no-push skillsShMirror:configureInternal \ + '{ + "actor":"CLAW-563 Test workflow", + "reason":"CLAW-563 proof cleanup", + "confirm":"enable-skills-sh-mirror-test", + "enabled":false, + "maxRowsPerRun":50000, + "maxRowsPerBatch":50, + "maxDetailBytes":65536 + }' > claw563-control-disabled.json 2>&1 + disable_exit=$? + bunx convex run --no-push devSeed:seedCliRoleHelpFixtures '{}' >/dev/null + token_rotation_exit=$? + if [[ -n "$operator_token" ]]; then + revoked_status="$( + curl \ + --silent \ + --output /dev/null \ + --write-out '%{http_code}' \ + --header "Authorization: Bearer $operator_token" \ + https://academic-chihuahua-392.convex.site/api/v1/operator/skills-sh/catalog-test + )" + else + revoked_status="not-created" + fi + jq -n \ + --argjson disableExit "$disable_exit" \ + --argjson tokenRotationExit "$token_rotation_exit" \ + --arg revokedStatus "$revoked_status" \ + '{ + disableExit:$disableExit, + tokenRotationExit:$tokenRotationExit, + revokedStatus:$revokedStatus + }' > claw563-cleanup.json + set -e + [[ + "$disable_exit" -eq 0 && + "$token_rotation_exit" -eq 0 && + "$revoked_status" == "401" + ]] + } + trap cleanup EXIT + + deployment_json="$(bunx convex run --no-push appMeta:getDeploymentInfo)" + build_sha="$(jq -r '.appBuildSha // empty' <<<"$deployment_json")" + if [[ "$build_sha" != "$DEPLOY_SHA" ]]; then + echo "::error::Unexpected Convex Test build SHA: $build_sha" + exit 1 + fi + + token_seed="$(bunx convex run --no-push devSeed:seedCliRoleHelpFixtures '{}')" + operator_token="$(jq -r '.admin.token // empty' <<<"$token_seed")" + token_seed="" + if [[ -z "$operator_token" ]]; then + echo "::error::Test admin fixture token is missing" + exit 1 + fi + echo "::add-mask::$operator_token" + + export CLAWHUB_TEST_MIRROR_GATE_URL="$TEST_SITE_URL/ops/skills-sh/mirror-test" + export CLAWHUB_TEST_OPERATOR_TOKEN="$operator_token" + bun run skills-sh:prove-mirror > claw563-proof-output.json + + curl \ + --fail-with-body \ + --silent \ + --show-error \ + --request POST \ + --header "Authorization: Bearer $operator_token" \ + --header "Content-Type: application/json" \ + --header "x-vercel-protection-bypass: $VERCEL_AUTOMATION_BYPASS_SECRET" \ + --data '{"operation":"read","externalId":"patrick-erichsen/skills/html"}' \ + "$CLAWHUB_TEST_MIRROR_GATE_URL" > claw563-controlled-entry.json + jq -e ' + .digest.externalId == "patrick-erichsen/skills/html" and + .digest.sourceType == "github" and + .digest.owner == "patrick-erichsen" and + .digest.repo == "skills" and + .digest.githubPath == "skills/html" and + .digest.githubCommit == "050daba89f6b6636470add5cb300aac46a412cf8" and + .digest.sourceContentHash == "42d2e89358ea927441dfede45c3b0cf89a21603bc7c32246f098d24a9cbea1ff" and + .digest.active == true and + .digest.publicVisible == false and + .digest.installable == false + ' claw563-controlled-entry.json >/dev/null + + curl \ + --fail-with-body \ + --silent \ + --show-error \ + --request POST \ + --header "Authorization: Bearer $operator_token" \ + --header "Content-Type: application/json" \ + --header "x-vercel-protection-bypass: $VERCEL_AUTOMATION_BYPASS_SECRET" \ + --data '{"operation":"read","externalId":"steipete/clawdis/discrawl"}' \ + "$CLAWHUB_TEST_MIRROR_GATE_URL" > claw563-discrawl-entry.json + jq -e ' + .digest.externalId == "steipete/clawdis/discrawl" and + .digest.sourceType == "github" and + .digest.owner == "steipete" and + .digest.repo == "clawdis" and + .digest.githubPath == ".agents/skills/discrawl" and + .digest.githubCommit == "690ed564419291ca6e832dc69b53061300075b62" and + .digest.sourceContentHash == "889dc43180b210dbca12f8291e007feb231250ecfdba90c4d3938a18125efb6d" and + .digest.active == true and + .digest.publicVisible == false and + .digest.installable == false + ' claw563-discrawl-entry.json >/dev/null + + jq -n \ + --arg sourceSha "$DEPLOY_SHA" \ + --arg deploymentUrl "$TEST_SITE_URL" \ + '{ + sourceSha:$sourceSha, + deploymentUrl:$deploymentUrl, + convexDeployment:"academic-chihuahua-392" + }' > claw563-deployment.json + + cleanup + trap - EXIT + jq -e ' + .disableExit == 0 and + .tokenRotationExit == 0 and + .revokedStatus == "401" + ' claw563-cleanup.json >/dev/null + + - name: Upload permanent Test mirror proof + if: always() + uses: actions/upload-artifact@v7 + with: + name: claw563-mirror-proof + if-no-files-found: warn + path: | + proof/claw-563/skills-sh-mirror-test-proof.json + claw563-proof-output.json + claw563-controlled-entry.json + claw563-discrawl-entry.json + claw563-control-disabled.json + claw563-cleanup.json + claw563-deployment.json diff --git a/bun.lock b/bun.lock index c87607e2..82e05c65 100644 --- a/bun.lock +++ b/bun.lock @@ -46,6 +46,7 @@ "mermaid": "^11.16.0", "mime": "4.1.0", "monaco-editor": "0.56.0", + "parse5": "8.0.1", "pino": "10.3.1", "react": "19.2.7", "react-dom": "19.2.7", diff --git a/convex/_generated/api.d.ts b/convex/_generated/api.d.ts index 29d9162d..c2fdc7e2 100644 --- a/convex/_generated/api.d.ts +++ b/convex/_generated/api.d.ts @@ -173,6 +173,7 @@ import type * as skillStatEvents from "../skillStatEvents.js"; import type * as skillTransfers from "../skillTransfers.js"; import type * as skills from "../skills.js"; import type * as skillsShCatalog from "../skillsShCatalog.js"; +import type * as skillsShMirror from "../skillsShMirror.js"; import type * as stars from "../stars.js"; import type * as statsMaintenance from "../statsMaintenance.js"; import type * as telemetry from "../telemetry.js"; @@ -354,6 +355,7 @@ declare const fullApi: ApiFromModules<{ skillTransfers: typeof skillTransfers; skills: typeof skills; skillsShCatalog: typeof skillsShCatalog; + skillsShMirror: typeof skillsShMirror; stars: typeof stars; statsMaintenance: typeof statsMaintenance; telemetry: typeof telemetry; diff --git a/convex/dashboard.ts b/convex/dashboard.ts index dfd0ad44..02449e0c 100644 --- a/convex/dashboard.ts +++ b/convex/dashboard.ts @@ -11,7 +11,7 @@ import { } from "./lib/downloadTrend"; import { normalizePackageName } from "./lib/packageRegistry"; import { canAccessPublisherOwnerScope } from "./lib/publishers"; -import { readCanonicalStat } from "./lib/skillStats"; +import { readPublicDownloads } from "./lib/skillStats"; const dashboardMetricSelectionValidator = v.union( v.object({ kind: v.literal("skill"), slug: v.string() }), @@ -46,7 +46,7 @@ async function aggregateSkillDownloads(ctx: QueryCtx, skills: Doc<"skills">[], e ); for (const trend of trends) addPoints(points, trend); return { - allTimeDownloads: skills.reduce((sum, skill) => sum + readCanonicalStat(skill, "downloads"), 0), + allTimeDownloads: skills.reduce((sum, skill) => sum + readPublicDownloads(skill), 0), points, }; } diff --git a/convex/githubSkillSync.test.ts b/convex/githubSkillSync.test.ts index 077b9de7..8a849098 100644 --- a/convex/githubSkillSync.test.ts +++ b/convex/githubSkillSync.test.ts @@ -2400,6 +2400,8 @@ description: Build HTML artifacts. statsStars: 3, statsInstallsCurrent: 2, statsInstallsAllTime: 5, + statsSkillsShInstalls: 17, + statsGithubStars: 321, stats: { downloads: 7, stars: 3, installsCurrent: 2, installsAllTime: 5, versions: 0 }, createdAt: 1, updatedAt: 60, @@ -2436,6 +2438,16 @@ description: Build HTML artifacts. moderationReason: "pending.scan", statsDownloads: 7, statsStars: 3, + statsInstallsCurrent: 2, + statsInstallsAllTime: 5, + statsSkillsShInstalls: 17, + statsGithubStars: 321, + stats: { + downloads: 7, + stars: 3, + installsCurrent: 2, + installsAllTime: 5, + }, }); expect(tables.skills[0]).not.toHaveProperty("githubRemovedAt"); expect(tables.skills[0]).not.toHaveProperty("softDeletedAt"); diff --git a/convex/httpApiV1.handlers.test.ts b/convex/httpApiV1.handlers.test.ts index 7790bc32..65848cfb 100644 --- a/convex/httpApiV1.handlers.test.ts +++ b/convex/httpApiV1.handlers.test.ts @@ -1922,8 +1922,7 @@ describe("httpApiV1 handlers", () => { displayName: "Demo", summary: "Summary", updatedAt: 1, - statsDownloads: 42, - stats: { downloads: 1 }, + stats: { downloads: 50 }, }, version: { version: "1.0.0" }, ownerHandle: "openclaw", @@ -1951,7 +1950,7 @@ describe("httpApiV1 handlers", () => { displayName: "Demo", summary: "Summary", version: "1.0.0", - downloads: 42, + downloads: 50, updatedAt: 1, ownerHandle: "openclaw", owner: { diff --git a/convex/httpApiV1/skillsShCatalogV1.test.ts b/convex/httpApiV1/skillsShCatalogV1.test.ts index 21704148..5e00c02d 100644 --- a/convex/httpApiV1/skillsShCatalogV1.test.ts +++ b/convex/httpApiV1/skillsShCatalogV1.test.ts @@ -235,6 +235,405 @@ describe("skills.sh catalog Test HTTP API", () => { expect(buildGitHubApiHeaders).not.toHaveBeenCalled(); }); + it("routes mirror batches through the scanless mirror mutation", async () => { + const runMutation = vi.fn(async (_ref, args: Record) => ({ + status: "running", + page: args.page, + offset: 1, + })); + const ctx = { + runQuery: vi.fn(async () => ({ + environment: "test", + deploymentName: "academic-chihuahua-392", + buildSha: "test-sha", + control: {}, + })), + runMutation, + } as never; + const request = new Request("https://academic-chihuahua-392.convex.site/api/v1/ops", { + method: "POST", + body: JSON.stringify({ + operation: "mirror-batch", + runId: "skillsShMirrorRuns:test", + leaseToken: "lease:test", + page: 0, + offset: 0, + pageLength: 500, + hasMore: true, + sourceTotal: 9_571, + sourceRequests: 2, + sourceBytes: 1_024, + rows: [{ externalId: "vercel-labs/skills/find-skills" }], + }), + }); + + const response = await skillsShCatalogTestV1Handler(ctx, request); + + expect(response.status).toBe(200); + expect(runMutation).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + runId: "skillsShMirrorRuns:test", + leaseToken: "lease:test", + page: 0, + offset: 0, + sourceTotal: 9_571, + }), + ); + }); + + it("stores and summarizes immutable mirror source pages", async () => { + const staging = { + environment: "test", + deploymentName: "academic-chihuahua-392", + buildSha: "test-sha", + control: {}, + }; + const rows = [ + { + id: "vercel-labs/skills/find-skills", + installUrl: "https://github.com/vercel-labs/skills", + installs: 42, + name: "Find Skills", + slug: "find-skills", + source: "vercel-labs/skills", + sourceType: "github", + url: "https://skills.sh/vercel-labs/skills/find-skills", + }, + ]; + const runMutation = vi.fn(async () => ({ stored: true, page: 0, rows: 1 })); + const storeCtx = { runQuery: vi.fn(async () => staging), runMutation } as never; + const storeResponse = await skillsShCatalogTestV1Handler( + storeCtx, + new Request("https://academic-chihuahua-392.convex.site/api/v1/ops", { + method: "POST", + body: JSON.stringify({ + operation: "mirror-source-page-store", + snapshotHash: "a".repeat(64), + page: 0, + sourceTotal: 1, + pageLength: 1, + hasMore: false, + identityHash: "b".repeat(64), + contentHash: "c".repeat(64), + sourceBytes: 512, + serializedBytes: 768, + rows, + }), + }), + ); + + expect(storeResponse.status).toBe(200); + expect(runMutation).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + snapshotHash: "a".repeat(64), + page: 0, + rows, + }), + ); + + const summary = { + snapshotHash: "a".repeat(64), + pageDocuments: 1, + rows: 1, + sourceBytes: 512, + serializedBytes: 768, + }; + const runQuery = vi.fn().mockResolvedValueOnce(staging).mockResolvedValueOnce(summary); + const summaryResponse = await skillsShCatalogTestV1Handler( + { runQuery } as never, + new Request("https://academic-chihuahua-392.convex.site/api/v1/ops", { + method: "POST", + body: JSON.stringify({ + operation: "mirror-source-summary", + snapshotHash: "a".repeat(64), + }), + }), + ); + + expect(summaryResponse.status).toBe(200); + expect(await summaryResponse.json()).toEqual(summary); + expect(runQuery).toHaveBeenCalledTimes(2); + }); + + it("routes guarded mirror batch lease claims and releases", async () => { + const runMutation = vi + .fn() + .mockResolvedValueOnce({ + runId: "skillsShMirrorRuns:test", + page: 3, + offset: 50, + leaseToken: "lease:test", + leaseExpiresAt: Date.now() + 300_000, + }) + .mockResolvedValueOnce({ released: true }); + const ctx = { + runQuery: vi.fn(async () => ({ + environment: "test", + deploymentName: "academic-chihuahua-392", + buildSha: "test-sha", + control: {}, + })), + runMutation, + } as never; + + for (const operation of ["mirror-batch-claim", "mirror-batch-release"] as const) { + const response = await skillsShCatalogTestV1Handler( + ctx, + new Request("https://academic-chihuahua-392.convex.site/api/v1/ops", { + method: "POST", + body: JSON.stringify({ + operation, + runId: "skillsShMirrorRuns:test", + page: 3, + offset: 50, + leaseToken: "lease:test", + }), + }), + ); + expect(response.status).toBe(200); + } + + expect(runMutation).toHaveBeenNthCalledWith( + 1, + expect.anything(), + expect.objectContaining({ + runId: "skillsShMirrorRuns:test", + page: 3, + offset: 50, + leaseToken: "lease:test", + }), + ); + expect(runMutation).toHaveBeenNthCalledWith( + 2, + expect.anything(), + expect.objectContaining({ + runId: "skillsShMirrorRuns:test", + page: 3, + offset: 50, + leaseToken: "lease:test", + }), + ); + }); + + it("reads one mirror run for source-fetch preflight", async () => { + const runQuery = vi + .fn() + .mockResolvedValueOnce({ + environment: "test", + deploymentName: "academic-chihuahua-392", + buildSha: "test-sha", + control: {}, + }) + .mockResolvedValueOnce({ + runId: "skillsShMirrorRuns:test", + status: "paused", + page: 3, + offset: 50, + }); + const ctx = { runQuery } as never; + const request = new Request("https://academic-chihuahua-392.convex.site/api/v1/ops", { + method: "POST", + body: JSON.stringify({ + operation: "mirror-run", + runId: "skillsShMirrorRuns:test", + }), + }); + + const response = await skillsShCatalogTestV1Handler(ctx, request); + + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ + status: "paused", + page: 3, + offset: 50, + }); + expect(runQuery).toHaveBeenCalledTimes(2); + }); + + it("cancels one stale active mirror run with explicit confirmation", async () => { + const runQuery = vi.fn().mockResolvedValue({ + environment: "test", + deploymentName: "academic-chihuahua-392", + buildSha: "test-sha", + control: {}, + }); + const runMutation = vi.fn().mockResolvedValue({ + runId: "skillsShMirrorRuns:stale", + status: "canceled", + }); + const ctx = { runQuery, runMutation } as never; + const request = new Request("https://academic-chihuahua-392.convex.site/api/v1/ops", { + method: "POST", + body: JSON.stringify({ + operation: "mirror-cancel", + runId: "skillsShMirrorRuns:stale", + reason: "discard stale captured recovery", + confirm: "cancel-skills-sh-mirror-test-run", + }), + }); + + const response = await skillsShCatalogTestV1Handler(ctx, request); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ + runId: "skillsShMirrorRuns:stale", + status: "canceled", + }); + expect(runMutation).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + runId: "skillsShMirrorRuns:stale", + reason: "discard stale captured recovery", + confirm: "cancel-skills-sh-mirror-test-run", + }), + ); + }); + + it("reads bounded mirror conflicts for an exact completed run", async () => { + const runQuery = vi + .fn() + .mockResolvedValueOnce({ + environment: "test", + deploymentName: "academic-chihuahua-392", + buildSha: "test-sha", + control: {}, + }) + .mockResolvedValueOnce([ + { + runId: "skillsShMirrorRuns:live", + externalId: "larksuite/cli/lark-doc", + kind: "source-quarantine", + }, + ]); + const ctx = { runQuery } as never; + const response = await skillsShCatalogTestV1Handler( + ctx, + new Request("https://academic-chihuahua-392.convex.site/api/v1/ops", { + method: "POST", + body: JSON.stringify({ + operation: "mirror-conflicts", + runId: "skillsShMirrorRuns:live", + limit: 50, + }), + }), + ); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ + conflicts: [ + { + runId: "skillsShMirrorRuns:live", + externalId: "larksuite/cli/lark-doc", + kind: "source-quarantine", + }, + ], + }); + expect(runQuery).toHaveBeenCalledTimes(2); + }); + + it("reads bounded mirror classification reuse state", async () => { + const runQuery = vi + .fn() + .mockResolvedValueOnce({ + environment: "test", + deploymentName: "academic-chihuahua-392", + buildSha: "test-sha", + control: {}, + }) + .mockResolvedValueOnce([ + { + externalId: "patrick-erichsen/skills/html", + inferredClassifierVersion: "taxonomy-prototype-v9", + }, + ]); + const ctx = { runQuery } as never; + const response = await skillsShCatalogTestV1Handler( + ctx, + new Request("https://academic-chihuahua-392.convex.site/api/v1/ops", { + method: "POST", + body: JSON.stringify({ + operation: "mirror-classification-states", + externalIds: ["patrick-erichsen/skills/html"], + }), + }), + ); + + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ + states: [{ externalId: "patrick-erichsen/skills/html" }], + }); + expect(runQuery).toHaveBeenCalledTimes(2); + }); + + it("reads a bounded mirror facet proof page", async () => { + const runQuery = vi + .fn() + .mockResolvedValueOnce({ + environment: "test", + deploymentName: "academic-chihuahua-392", + buildSha: "test-sha", + control: {}, + }) + .mockResolvedValueOnce({ + page: [{ kind: "category", term: "development" }], + isDone: true, + continueCursor: "", + }); + const ctx = { runQuery } as never; + const response = await skillsShCatalogTestV1Handler( + ctx, + new Request("https://academic-chihuahua-392.convex.site/api/v1/ops", { + method: "POST", + body: JSON.stringify({ + operation: "mirror-facet-page", + cursor: null, + limit: 500, + }), + }), + ); + + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ + page: [{ kind: "category", term: "development" }], + isDone: true, + }); + }); + + it("reads bounded captured mirror rows for replay", async () => { + const runQuery = vi + .fn() + .mockResolvedValueOnce({ + environment: "test", + deploymentName: "academic-chihuahua-392", + buildSha: "test-sha", + control: {}, + }) + .mockResolvedValueOnce([ + { + digest: { externalId: "patrick-erichsen/skills/html", active: true }, + detail: null, + }, + ]); + const ctx = { runQuery } as never; + const response = await skillsShCatalogTestV1Handler( + ctx, + new Request("https://academic-chihuahua-392.convex.site/api/v1/ops", { + method: "POST", + body: JSON.stringify({ + operation: "mirror-replay-rows", + externalIds: ["patrick-erichsen/skills/html"], + }), + }), + ); + + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ + rows: [{ digest: { externalId: "patrick-erichsen/skills/html" }, detail: null }], + }); + }); + it("fetches only owners missing from authenticated staging-live state", async () => { const githubFetch = vi.fn(async (url: string, init?: RequestInit) => { expect(init?.headers).toMatchObject({ Authorization: "Bearer placeholder" }); diff --git a/convex/httpApiV1/skillsShCatalogV1.ts b/convex/httpApiV1/skillsShCatalogV1.ts index 5f93c1a2..c830909b 100644 --- a/convex/httpApiV1/skillsShCatalogV1.ts +++ b/convex/httpApiV1/skillsShCatalogV1.ts @@ -28,6 +28,29 @@ const internalRefs = internal as unknown as { startFixtureRunInternal: unknown; startStagingLiveRunInternal: unknown; }; + skillsShMirror: { + cancelRunInternal: unknown; + claimBatchLeaseInternal: unknown; + configureInternal: unknown; + getByExternalIdInternal: unknown; + getClassificationStatesInternal: unknown; + getDetailByExternalIdInternal: unknown; + getIsolationInternal: unknown; + getReplayRowsInternal: unknown; + getRunInternal: unknown; + getSourceCaptureSummaryInternal: unknown; + getStatusInternal: unknown; + listDetailsPageInternal: unknown; + listDigestsPageInternal: unknown; + listConflictsByRunInternal: unknown; + listFacetsPageInternal: unknown; + processBatchInternal: unknown; + reconcileBatchInternal: unknown; + releaseBatchLeaseInternal: unknown; + setPausedInternal: unknown; + startRunInternal: unknown; + storeSourcePageInternal: unknown; + }; }; const MAX_GITHUB_OWNER_RESOLUTIONS = 500; const GITHUB_OWNER_RESOLUTION_CONCURRENCY = 8; @@ -91,6 +114,19 @@ function requireBoolean(record: Record, key: string) { return value; } +function requireStringArray(record: Record, key: string, maxItems: number) { + const value = record[key]; + if ( + !Array.isArray(value) || + value.length < 1 || + value.length > maxItems || + value.some((item) => typeof item !== "string" || !item.trim()) + ) { + throw new Error(`${key} must contain between 1 and ${maxItems} strings`); + } + return value as string[]; +} + async function sha256Hex(bytes: Uint8Array) { const digest = await crypto.subtle.digest("SHA-256", new Uint8Array(bytes).buffer); return Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join(""); @@ -426,6 +462,250 @@ export async function skillsShCatalogTestV1Handler(ctx: ActionCtx, request: Requ const body = asRecord(await request.json()); if (!body) return text("Invalid JSON", 400, rate.headers); const operation = requireString(body, "operation"); + if (operation === "mirror-status") { + return json( + await runQueryRef(ctx, internalRefs.skillsShMirror.getStatusInternal, {}), + 200, + rate.headers, + ); + } + if (operation === "mirror-isolation") { + return json( + await runQueryRef(ctx, internalRefs.skillsShMirror.getIsolationInternal, {}), + 200, + rate.headers, + ); + } + if (operation === "mirror-run") { + return json( + await runQueryRef(ctx, internalRefs.skillsShMirror.getRunInternal, { + runId: requireString(body, "runId"), + }), + 200, + rate.headers, + ); + } + if (operation === "mirror-conflicts") { + const conflicts = await runQueryRef( + ctx, + internalRefs.skillsShMirror.listConflictsByRunInternal, + { + runId: requireString(body, "runId"), + limit: requireNumber(body, "limit"), + }, + ); + return json({ conflicts }, 200, rate.headers); + } + if (operation === "mirror-read") { + const externalId = requireString(body, "externalId"); + const [digest, detail] = await Promise.all([ + runQueryRef(ctx, internalRefs.skillsShMirror.getByExternalIdInternal, { externalId }), + runQueryRef(ctx, internalRefs.skillsShMirror.getDetailByExternalIdInternal, { + externalId, + }), + ]); + return json({ digest, detail }, 200, rate.headers); + } + if (operation === "mirror-classification-states") { + const states = await runQueryRef( + ctx, + internalRefs.skillsShMirror.getClassificationStatesInternal, + { + externalIds: requireStringArray(body, "externalIds", 50), + }, + ); + return json({ states }, 200, rate.headers); + } + if (operation === "mirror-replay-rows") { + const rows = await runQueryRef(ctx, internalRefs.skillsShMirror.getReplayRowsInternal, { + externalIds: requireStringArray(body, "externalIds", 50), + }); + return json({ rows }, 200, rate.headers); + } + if (operation === "mirror-source-summary") { + return json( + await runQueryRef(ctx, internalRefs.skillsShMirror.getSourceCaptureSummaryInternal, { + snapshotHash: requireString(body, "snapshotHash"), + }), + 200, + rate.headers, + ); + } + if (operation === "mirror-page") { + const cursor = + body.cursor === null || typeof body.cursor === "string" + ? body.cursor + : (() => { + throw new Error("cursor is required"); + })(); + return json( + await runQueryRef(ctx, internalRefs.skillsShMirror.listDigestsPageInternal, { + cursor, + limit: requireNumber(body, "limit"), + }), + 200, + rate.headers, + ); + } + if (operation === "mirror-detail-page") { + const cursor = + body.cursor === null || typeof body.cursor === "string" + ? body.cursor + : (() => { + throw new Error("cursor is required"); + })(); + return json( + await runQueryRef(ctx, internalRefs.skillsShMirror.listDetailsPageInternal, { + cursor, + limit: requireNumber(body, "limit"), + }), + 200, + rate.headers, + ); + } + if (operation === "mirror-facet-page") { + const cursor = + body.cursor === null || typeof body.cursor === "string" + ? body.cursor + : (() => { + throw new Error("cursor is required"); + })(); + return json( + await runQueryRef(ctx, internalRefs.skillsShMirror.listFacetsPageInternal, { + cursor, + limit: requireNumber(body, "limit"), + }), + 200, + rate.headers, + ); + } + if (operation === "mirror-configure") { + return json( + await runMutationRef(ctx, internalRefs.skillsShMirror.configureInternal, { + actor: auth.user.handle, + reason: requireString(body, "reason"), + confirm: requireString(body, "confirm"), + enabled: requireBoolean(body, "enabled"), + maxRowsPerRun: requireNumber(body, "maxRowsPerRun"), + maxRowsPerBatch: requireNumber(body, "maxRowsPerBatch"), + maxDetailBytes: requireNumber(body, "maxDetailBytes"), + }), + 200, + rate.headers, + ); + } + if (operation === "mirror-start") { + return json( + await runMutationRef(ctx, internalRefs.skillsShMirror.startRunInternal, { + actor: auth.user.handle, + reason: requireString(body, "reason"), + snapshotId: requireString(body, "snapshotId"), + ...(typeof body.sourceSnapshotHash === "string" + ? { sourceSnapshotHash: requireString(body, "sourceSnapshotHash") } + : {}), + ...(typeof body.sourceCaptureWrites === "number" + ? { sourceCaptureWrites: requireNumber(body, "sourceCaptureWrites") } + : {}), + sourceTotal: requireNumber(body, "sourceTotal"), + sourcePageSize: requireNumber(body, "sourcePageSize"), + sourceMeasuredAt: requireString(body, "sourceMeasuredAt"), + }), + 200, + rate.headers, + ); + } + if (operation === "mirror-source-page-store") { + if (!Array.isArray(body.rows)) throw new Error("rows is required"); + return json( + await runMutationRef(ctx, internalRefs.skillsShMirror.storeSourcePageInternal, { + snapshotHash: requireString(body, "snapshotHash"), + page: requireNumber(body, "page"), + sourceTotal: requireNumber(body, "sourceTotal"), + pageLength: requireNumber(body, "pageLength"), + hasMore: requireBoolean(body, "hasMore"), + identityHash: requireString(body, "identityHash"), + contentHash: requireString(body, "contentHash"), + sourceBytes: requireNumber(body, "sourceBytes"), + serializedBytes: requireNumber(body, "serializedBytes"), + rows: body.rows, + }), + 200, + rate.headers, + ); + } + if (operation === "mirror-batch") { + if (!Array.isArray(body.rows)) throw new Error("rows is required"); + return json( + await runMutationRef(ctx, internalRefs.skillsShMirror.processBatchInternal, { + runId: requireString(body, "runId"), + leaseToken: requireString(body, "leaseToken"), + page: requireNumber(body, "page"), + offset: requireNumber(body, "offset"), + pageLength: requireNumber(body, "pageLength"), + hasMore: requireBoolean(body, "hasMore"), + sourceTotal: requireNumber(body, "sourceTotal"), + sourceRequests: requireNumber(body, "sourceRequests"), + sourceBytes: requireNumber(body, "sourceBytes"), + rows: body.rows, + }), + 200, + rate.headers, + ); + } + if (operation === "mirror-batch-claim" || operation === "mirror-batch-release") { + const args = { + runId: requireString(body, "runId"), + page: requireNumber(body, "page"), + offset: requireNumber(body, "offset"), + leaseToken: requireString(body, "leaseToken"), + }; + return json( + await runMutationRef( + ctx, + operation === "mirror-batch-claim" + ? internalRefs.skillsShMirror.claimBatchLeaseInternal + : internalRefs.skillsShMirror.releaseBatchLeaseInternal, + args, + ), + 200, + rate.headers, + ); + } + if (operation === "mirror-pause") { + return json( + await runMutationRef(ctx, internalRefs.skillsShMirror.setPausedInternal, { + runId: requireString(body, "runId"), + paused: requireBoolean(body, "paused"), + actor: auth.user.handle, + reason: requireString(body, "reason"), + confirm: requireString(body, "confirm"), + }), + 200, + rate.headers, + ); + } + if (operation === "mirror-cancel") { + return json( + await runMutationRef(ctx, internalRefs.skillsShMirror.cancelRunInternal, { + runId: requireString(body, "runId"), + actor: auth.user.handle, + reason: requireString(body, "reason"), + confirm: requireString(body, "confirm"), + }), + 200, + rate.headers, + ); + } + if (operation === "mirror-reconcile") { + return json( + await runMutationRef(ctx, internalRefs.skillsShMirror.reconcileBatchInternal, { + runId: requireString(body, "runId"), + limit: requireNumber(body, "limit"), + }), + 200, + rate.headers, + ); + } if (operation === "verify-canary") { return json(await verifyControlledCanaryGitHubSource({}), 200, rate.headers); } diff --git a/convex/httpApiV1/skillsV1.ts b/convex/httpApiV1/skillsV1.ts index b91b4611..333f3471 100644 --- a/convex/httpApiV1/skillsV1.ts +++ b/convex/httpApiV1/skillsV1.ts @@ -50,7 +50,6 @@ import { getSkillFileModerationInfoFromSkill, isSkillVersionForSkill, } from "../lib/skillFileAccess"; -import { readCanonicalStat } from "../lib/skillStats"; import { buildDeterministicZip, buildMergedExportZip, @@ -101,7 +100,6 @@ type SearchSkillEntry = { stars?: number; installs?: number; }; - statsDownloads?: number; } | null; version: { version?: string; createdAt?: number } | null; ownerHandle?: string | null; @@ -1394,7 +1392,9 @@ export async function searchSkillsV1Handler(ctx: ActionCtx, request: Request) { displayName: result.skill?.displayName, summary: result.skill?.summary ?? null, version: result.version?.version ?? null, - downloads: result.skill ? readCanonicalStat(result.skill, "downloads") : 0, + // searchSkills already returns the ordinary public skill shape, including + // the combined presentation value and no source-attribution fields. + downloads: result.skill?.stats.downloads ?? 0, updatedAt: result.skill?.updatedAt, ownerHandle: result.ownerHandle ?? owner?.handle ?? null, owner, diff --git a/convex/lib/public.test.ts b/convex/lib/public.test.ts index 35070e16..e0e5d8c4 100644 --- a/convex/lib/public.test.ts +++ b/convex/lib/public.test.ts @@ -51,13 +51,15 @@ describe("public skill mapping", () => { statsStars: 3, statsInstallsCurrent: 5, statsInstallsAllTime: 7, + statsSkillsShInstalls: 8, + statsGithubStars: 99, }); const mapped = toPublicSkill(legacySkill); expect(mapped).not.toBeNull(); expect(mapped?.stats).toEqual({ - downloads: 12, + downloads: 20, stars: 3, installs: 7, versions: 0, @@ -65,6 +67,20 @@ describe("public skill mapping", () => { }); }); + it("does not expose source breakdowns on the ordinary public skill shape", () => { + const mapped = toPublicSkill( + makeSkill({ + statsDownloads: 12, + statsSkillsShInstalls: 8, + statsGithubStars: 99, + }), + ); + + expect(mapped?.stats.downloads).toBe(20); + expect(mapped).not.toHaveProperty("statsSkillsShInstalls"); + expect(mapped).not.toHaveProperty("statsGithubStars"); + }); + it("exposes GitHub-backed skill source fields", () => { const mapped = toPublicSkill( makeSkill({ diff --git a/convex/lib/public.ts b/convex/lib/public.ts index 53bd7429..b8a56859 100644 --- a/convex/lib/public.ts +++ b/convex/lib/public.ts @@ -1,6 +1,6 @@ import type { Doc } from "../_generated/dataModel"; import { isPublicSkillDoc } from "./globalStats"; -import { readCanonicalStat } from "./skillStats"; +import { readCanonicalStat, readPublicDownloads } from "./skillStats"; export type PublicUser = Pick< Doc<"users">, @@ -101,6 +101,8 @@ export type HydratableSkill = Pick< | "statsStars" | "statsInstallsCurrent" | "statsInstallsAllTime" + | "statsSkillsShInstalls" + | "statsGithubStars" | "softDeletedAt" | "moderationStatus" | "moderationFlags" @@ -149,7 +151,7 @@ export function toPublicSkill(skill: HydratableSkill | null | undefined): Public if (!skill) return null; if (!isPublicSkillDoc(skill)) return null; const stats = { - downloads: readCanonicalStat(skill, "downloads"), + downloads: readPublicDownloads(skill), stars: readCanonicalStat(skill, "stars"), installs: readCanonicalStat(skill, "installsAllTime"), versions: skill.stats?.versions ?? 0, diff --git a/convex/lib/retentionPolicy.ts b/convex/lib/retentionPolicy.ts index c5b96a02..7acdc4e5 100644 --- a/convex/lib/retentionPolicy.ts +++ b/convex/lib/retentionPolicy.ts @@ -222,6 +222,15 @@ export const RETENTION_POLICIES = { skillsShCatalogScanAttempts: permanent( "Exact-hash skills.sh scan attempts are durable audit and deduplication history.", ), + skillsShMirrorControls: permanent("Durable skills.sh external mirror operator controls."), + skillsShMirrorRuns: permanent("Skills.sh external mirror cursor and reconciliation history."), + skillsShMirrorSourcePages: permanent( + "Immutable authenticated leaderboard source pages retained for mirror provenance.", + ), + skillsShMirrorDigests: permanent("Normalized skills.sh external search digests."), + skillsShMirrorDetails: permanent("Bounded skills.sh external detail content."), + skillsShMirrorFacets: permanent("Indexed skills.sh external category and topic metadata."), + skillsShMirrorConflicts: permanent("Skills.sh external observation conflict audit history."), publisherAbuseScoreRuns: permanent("Abuse scoring run history."), publisherAbuseTemporalScanSamples: ephemeral( "Exact temporal percentile samples are temporary scan working state.", diff --git a/convex/lib/skillSearchDigest.test.ts b/convex/lib/skillSearchDigest.test.ts index 9597cddd..69f619da 100644 --- a/convex/lib/skillSearchDigest.test.ts +++ b/convex/lib/skillSearchDigest.test.ts @@ -58,6 +58,8 @@ function makeSkillDoc(overrides: Record = {}) { statsStars: 5, statsInstallsCurrent: 10, statsInstallsAllTime: 100, + statsSkillsShInstalls: 8, + statsGithubStars: 250, stats: { downloads: 42, installsCurrent: 10, @@ -88,6 +90,8 @@ describe("extractDigestFields", () => { expect(digest.statsStars).toBe(5); expect(digest.statsInstallsCurrent).toBe(10); expect(digest.statsInstallsAllTime).toBe(100); + expect(digest.statsSkillsShInstalls).toBe(8); + expect(digest.statsGithubStars).toBe(250); expect(digest.recommendedScore).toBe( computeRecommendationScore({ downloads: 42, installs: 100, stars: 5 }), ); diff --git a/convex/lib/skillSearchDigest.ts b/convex/lib/skillSearchDigest.ts index 312aaff6..eba87c0a 100644 --- a/convex/lib/skillSearchDigest.ts +++ b/convex/lib/skillSearchDigest.ts @@ -47,6 +47,8 @@ const SHARED_KEYS = [ "statsStars", "statsInstallsCurrent", "statsInstallsAllTime", + "statsSkillsShInstalls", + "statsGithubStars", "softDeletedAt", "moderationStatus", "moderationFlags", diff --git a/convex/lib/skillStats.test.ts b/convex/lib/skillStats.test.ts new file mode 100644 index 00000000..459d76a6 --- /dev/null +++ b/convex/lib/skillStats.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "vitest"; +import { + buildExternalSkillMetricPatch, + readPublicDownloads, + readSkillMetricSources, +} from "./skillStats"; + +const skill = { + statsDownloads: 40, + statsStars: 7, + statsInstallsCurrent: 3, + statsInstallsAllTime: 12, + statsSkillsShInstalls: 9, + statsGithubStars: 250, + stats: { + downloads: 40, + stars: 7, + installsCurrent: 3, + installsAllTime: 12, + }, +}; + +describe("source-attributed skill metrics", () => { + it("keeps every source independently attributable", () => { + expect(readSkillMetricSources(skill)).toEqual({ + clawHubDownloads: 40, + skillsShInstalls: 9, + openClawInstallsCurrent: 3, + openClawInstallsAllTime: 12, + githubStars: 250, + bookmarks: 7, + }); + }); + + it("adds skills.sh installs to public downloads without double-counting OpenClaw installs", () => { + expect(readPublicDownloads(skill)).toBe(49); + }); + + it("builds a source-only refresh patch and preserves unknown GitHub popularity", () => { + expect( + buildExternalSkillMetricPatch({ + skillsShInstalls: 11, + githubStars: undefined, + }), + ).toEqual({ + statsSkillsShInstalls: 11, + }); + }); +}); diff --git a/convex/lib/skillStats.ts b/convex/lib/skillStats.ts index a48405d1..bcabfe8b 100644 --- a/convex/lib/skillStats.ts +++ b/convex/lib/skillStats.ts @@ -17,8 +17,20 @@ export type SkillStatReadable = { statsStars?: number; statsInstallsCurrent?: number; statsInstallsAllTime?: number; + statsSkillsShInstalls?: number; + statsGithubStars?: number; }; +type ExternalSkillMetricSnapshot = { + skillsShInstalls?: number; + githubStars?: number; +}; + +function nonNegativeCount(value: number | undefined): number { + if (typeof value !== "number" || !Number.isFinite(value)) return 0; + return Math.max(0, Math.trunc(value)); +} + /** * Read the canonical value of a migrated stat field from a skill document. * @@ -42,6 +54,32 @@ export function readCanonicalStat( return typeof skill[topLevelKey] === "number" ? skill[topLevelKey]! : (skill.stats[field] ?? 0); } +export function readSkillMetricSources(skill: SkillStatReadable) { + return { + clawHubDownloads: readCanonicalStat(skill, "downloads"), + skillsShInstalls: nonNegativeCount(skill.statsSkillsShInstalls), + openClawInstallsCurrent: readCanonicalStat(skill, "installsCurrent"), + openClawInstallsAllTime: readCanonicalStat(skill, "installsAllTime"), + githubStars: nonNegativeCount(skill.statsGithubStars), + bookmarks: readCanonicalStat(skill, "stars"), + }; +} + +export function readPublicDownloads(skill: SkillStatReadable): number { + return readCanonicalStat(skill, "downloads") + nonNegativeCount(skill.statsSkillsShInstalls); +} + +export function buildExternalSkillMetricPatch(snapshot: ExternalSkillMetricSnapshot) { + return { + ...(snapshot.skillsShInstalls === undefined + ? {} + : { statsSkillsShInstalls: nonNegativeCount(snapshot.skillsShInstalls) }), + ...(snapshot.githubStars === undefined + ? {} + : { statsGithubStars: nonNegativeCount(snapshot.githubStars) }), + }; +} + export function applySkillStatDeltas(skill: Doc<"skills">, deltas: SkillStatDeltas) { const currentDownloads = readCanonicalStat(skill, "downloads"); const currentStars = readCanonicalStat(skill, "stars"); diff --git a/convex/lib/skillsShCatalogFixtures.ts b/convex/lib/skillsShCatalogFixtures.ts index ffe20269..487f8383 100644 --- a/convex/lib/skillsShCatalogFixtures.ts +++ b/convex/lib/skillsShCatalogFixtures.ts @@ -15,6 +15,7 @@ export type SkillsShCatalogFixtureRow = { claimPublisherHandle?: string; sourceContentHash: string; installs: number; + githubStars?: number; }; type SkillsShCatalogFixture = { @@ -61,7 +62,8 @@ const PATRICK_HTML_CANARY: SkillsShCatalogFixtureRow = { githubContentHash: "a47adb2c1ac33c088f664b5187971b63d2b958a7b9f01516d26005ca941a108f", claimPublisherHandle: "patrick-erichsen", sourceContentHash: "a47adb2c1ac33c088f664b5187971b63d2b958a7b9f01516d26005ca941a108f", - installs: 0, + installs: 17, + githubStars: 321, }; const FROZEN_ROWS = frozenSnapshot.rows satisfies SkillsShCatalogFixtureRow[]; diff --git a/convex/publishers.test.ts b/convex/publishers.test.ts index 908a6bbf..479e914c 100644 --- a/convex/publishers.test.ts +++ b/convex/publishers.test.ts @@ -283,7 +283,12 @@ const listPublishedPageHandler = ( paginationOpts: { cursor: string | null; numItems: number }; }, { - page: Array<{ displayName: string; href: string; kind: "skill" | "plugin" }>; + page: Array<{ + displayName: string; + href: string; + kind: "skill" | "plugin"; + downloads: number; + }>; continueCursor: string; isDone: boolean; } @@ -3139,6 +3144,7 @@ describe("publishers membership controls", () => { slug: "demo", displayName: "Demo Skill", moderationStatus: "active", + statsSkillsShInstalls: 6, stats: { downloads: 10, stars: 1, installsCurrent: 1, installsAllTime: 2 }, updatedAt: 5, }; @@ -3188,6 +3194,7 @@ describe("publishers membership controls", () => { }); expect(firstPage.page.map((item) => item.kind)).toEqual(["skill"]); + expect(firstPage.page[0]?.downloads).toBe(16); expect(firstPage.continueCursor).toBe("1"); expect(firstPage.isDone).toBe(false); expect(secondPage.page.map((item) => item.kind)).toEqual(["plugin"]); @@ -3231,6 +3238,7 @@ describe("publishers membership controls", () => { displayName: "Modern Low", moderationStatus: "active", statsDownloads: 7, + statsSkillsShInstalls: 100, stats: { downloads: 7, stars: 1, installsCurrent: 1, installsAllTime: 2 }, updatedAt: 6, }; @@ -3283,6 +3291,7 @@ describe("publishers membership controls", () => { paginationOpts: { cursor: null, numItems: 1 }, }); legacyLookup.first.mockResolvedValue(null); + expect(firstPage.page[0]?.displayName).toBe("Legacy High"); const secondPage = await listPublishedPageHandler(ctx as never, { handle: "legacy", kind: "skill", @@ -3463,6 +3472,7 @@ describe("publishers membership controls", () => { ownerPublisherId: "publishers:openclaw", softDeletedAt: undefined, moderationStatus: "active", + statsSkillsShInstalls: 8, stats: { downloads: 42, stars: 2, installsCurrent: 4, installsAllTime: 7 }, }; const pkg = { @@ -3509,7 +3519,7 @@ describe("publishers membership controls", () => { skills: 1, packages: 1, installs: 12, - downloads: 50, + downloads: 58, stars: 3, }); }); diff --git a/convex/publishers.ts b/convex/publishers.ts index 0bbfb099..1c8430fd 100644 --- a/convex/publishers.ts +++ b/convex/publishers.ts @@ -44,7 +44,7 @@ import { isHandleReservedForAnotherUser, } from "./lib/reservedHandles"; import { syncSkillSearchDigestForSkill } from "./lib/skillSearchDigest"; -import { readCanonicalStat } from "./lib/skillStats"; +import { readCanonicalStat, readPublicDownloads } from "./lib/skillStats"; import { adjustUserSkillStatsForSkillChange } from "./lib/userSkillStats"; const MAX_PUBLIC_PUBLISHER_LIST_LIMIT = 500; @@ -388,7 +388,7 @@ function getIndexedPublisherStatsFromRows(rows: PublisherPublishedRows): Publish for (const skill of rows.skills) { stats.skills += 1; stats.installs += readCanonicalStat(skill, "installsAllTime"); - stats.downloads += readCanonicalStat(skill, "downloads"); + stats.downloads += readPublicDownloads(skill); stats.stars += readCanonicalStat(skill, "stars"); } @@ -416,7 +416,7 @@ function getPublisherPublishedItems( inferredCategories: skill.inferredCategories, latestVersionId: skill.latestVersionId, inferredFromVersionId: skill.inferredFromVersionId, - downloads: readCanonicalStat(skill, "downloads"), + downloads: readPublicDownloads(skill), installs: readCanonicalStat(skill, "installsAllTime"), })), ...rows.packages.map((pkg) => ({ @@ -551,7 +551,7 @@ function toPublisherSkillCatalogItem( icon: skill.icon ?? null, href: `/${encodeURIComponent(publisher.handle)}/${encodeURIComponent(skill.slug)}`, installs: readCanonicalStat(skill, "installsAllTime"), - downloads: readCanonicalStat(skill, "downloads"), + downloads: readPublicDownloads(skill), stars: readCanonicalStat(skill, "stars"), isOfficial: publisherOfficial || Boolean(skill.badges?.official), updatedAt: skill.updatedAt, @@ -2578,7 +2578,7 @@ export const listStarredPage = query({ icon: skill.icon ?? null, href: `/${encodeURIComponent(ownerHandle)}/${encodeURIComponent(skill.slug)}`, installs: readCanonicalStat(skill, "installsAllTime"), - downloads: readCanonicalStat(skill, "downloads"), + downloads: readPublicDownloads(skill), stars: readCanonicalStat(skill, "stars"), isOfficial: official || Boolean(skill.badges?.official), updatedAt: skill.updatedAt, diff --git a/convex/schema.ts b/convex/schema.ts index c73cb447..c04b492f 100644 --- a/convex/schema.ts +++ b/convex/schema.ts @@ -921,6 +921,8 @@ const skills = defineTable({ statsStars: v.optional(v.number()), statsInstallsCurrent: v.optional(v.number()), statsInstallsAllTime: v.optional(v.number()), + statsSkillsShInstalls: v.optional(v.number()), + statsGithubStars: v.optional(v.number()), installBackfill: v.optional( v.object({ modelVersion: v.string(), @@ -1347,6 +1349,8 @@ const skillSearchDigest = defineTable({ statsStars: v.optional(v.number()), statsInstallsCurrent: v.optional(v.number()), statsInstallsAllTime: v.optional(v.number()), + statsSkillsShInstalls: v.optional(v.number()), + statsGithubStars: v.optional(v.number()), recommendedScore: v.optional(v.number()), recommendedScoreVersion: v.optional(v.number()), softDeletedAt: v.optional(v.number()), @@ -2916,6 +2920,7 @@ const skillsShCatalogEntries = defineTable({ githubContentHash: v.optional(v.string()), sourceContentHash: v.string(), installs: v.number(), + githubStars: v.optional(v.number()), sourceSnapshotId: v.string(), reconciliation: v.optional( v.object({ @@ -2996,6 +3001,254 @@ const skillsShCatalogScanAttempts = defineTable({ .index("by_status_and_created_at", ["status", "createdAt"]) .index("by_dispatch_kind_and_status_and_created_at", ["dispatchKind", "status", "createdAt"]); +const skillsShMirrorControls = defineTable({ + key: v.literal("global"), + enabled: v.boolean(), + paused: v.boolean(), + maxRowsPerRun: v.number(), + maxRowsPerBatch: v.number(), + maxDetailBytes: v.number(), + updatedBy: v.string(), + reason: v.string(), + updatedAt: v.number(), +}).index("by_key", ["key"]); + +const skillsShMirrorRunCountsValidator = v.object({ + observed: v.number(), + inserted: v.number(), + updated: v.number(), + unchanged: v.number(), + rejected: v.number(), + quarantined: v.optional(v.number()), + quarantinedPreserved: v.optional(v.number()), + conflicts: v.number(), + detailsInserted: v.number(), + detailsUpdated: v.number(), + detailsUnchanged: v.number(), + detailsMissing: v.number(), + detailsTruncated: v.number(), + tombstoned: v.number(), + reactivated: v.number(), + scansPlanned: v.literal(0), + scansAdmitted: v.literal(0), +}); + +const skillsShMirrorRuns = defineTable({ + snapshotId: v.string(), + sourceSnapshotHash: v.optional(v.string()), + sourceCaptureWrites: v.optional(v.number()), + status: v.union( + v.literal("running"), + v.literal("paused"), + v.literal("reconciling"), + v.literal("completed"), + v.literal("failed"), + v.literal("canceled"), + ), + sourceTotal: v.number(), + sourcePageSize: v.number(), + sourceMeasuredAt: v.string(), + page: v.number(), + offset: v.number(), + batchLeaseToken: v.optional(v.string()), + batchLeaseExpiresAt: v.optional(v.number()), + reconcileCursor: v.optional(v.string()), + counts: skillsShMirrorRunCountsValidator, + operations: v.object({ + functionCalls: v.number(), + dbReads: v.number(), + dbWrites: v.number(), + sourceRequests: v.number(), + sourceBytes: v.number(), + }), + actor: v.string(), + reason: v.string(), + startedAt: v.number(), + completedAt: v.optional(v.number()), + updatedAt: v.number(), +}) + .index("by_started_at", ["startedAt"]) + .index("by_status_and_updated_at", { + fields: ["status", "updatedAt"], + }); + +const skillsShMirrorSourcePages = defineTable({ + snapshotHash: v.string(), + page: v.number(), + sourceTotal: v.number(), + pageLength: v.number(), + hasMore: v.boolean(), + identityHash: v.string(), + contentHash: v.string(), + sourceBytes: v.number(), + serializedBytes: v.number(), + rows: v.array( + v.object({ + id: v.string(), + installUrl: v.union(v.string(), v.null()), + installs: v.number(), + name: v.string(), + slug: v.string(), + source: v.string(), + sourceType: v.string(), + url: v.string(), + }), + ), + createdAt: v.number(), +}).index("by_snapshot_hash_and_page", ["snapshotHash", "page"]); + +const skillsShMirrorUpstreamScannerValidator = v.object({ + status: v.string(), + sourceCheckedAt: v.optional(v.string()), + sourceUrl: v.optional(v.string()), +}); + +const skillsShMirrorClassificationConfidenceValidator = v.union( + v.literal("high"), + v.literal("medium"), + v.literal("low"), +); + +const skillsShMirrorDigests = defineTable({ + externalId: v.string(), + sourceType: v.union(v.literal("github"), v.literal("well-known")), + upstreamSourceType: v.optional(v.string()), + owner: v.optional(v.string()), + repo: v.optional(v.string()), + sourceHost: v.optional(v.string()), + slug: v.string(), + normalizedSlug: v.string(), + normalizedSlugFirstToken: v.string(), + displayName: v.string(), + normalizedDisplayName: v.string(), + normalizedDisplayNameFirstToken: v.string(), + searchText: v.string(), + sourceUrl: v.string(), + canonicalRepoUrl: v.optional(v.string()), + githubPath: v.optional(v.string()), + githubCommit: v.optional(v.string()), + sourceContentHash: v.optional(v.string()), + upstreamInstalls: v.number(), + upstreamScanners: v.object({ + genAgentTrustHub: skillsShMirrorUpstreamScannerValidator, + socket: skillsShMirrorUpstreamScannerValidator, + snyk: skillsShMirrorUpstreamScannerValidator, + }), + inferredCategories: v.optional(v.array(v.string())), + inferredTopics: v.optional(v.array(v.string())), + inferredCategoryConfidence: v.optional(skillsShMirrorClassificationConfidenceValidator), + inferredTopicConfidence: v.optional(skillsShMirrorClassificationConfidenceValidator), + inferredClassifierVersion: v.optional(v.string()), + inferredTopicClassifierVersion: v.optional(v.string()), + inferredInputHash: v.optional(v.string()), + inferredTopicInputHash: v.optional(v.string()), + inferredAt: v.optional(v.number()), + sourceFreshnessStatus: v.union(v.literal("observed-only"), v.literal("stale")), + staleQuarantineReason: v.optional(v.string()), + detailStatus: v.union(v.literal("available"), v.literal("missing")), + observationFingerprint: v.string(), + sourceSnapshotId: v.string(), + lastObservedRunId: v.id("skillsShMirrorRuns"), + active: v.boolean(), + publicVisible: v.literal(false), + installable: v.literal(false), + tombstonedAt: v.optional(v.number()), + firstObservedAt: v.number(), + lastObservedAt: v.number(), + createdAt: v.number(), + updatedAt: v.number(), +}) + .index("by_external_id", ["externalId"]) + .index("by_active_and_normalized_slug", { + fields: ["active", "normalizedSlug"], + }) + .index("by_active_and_normalized_display_name", { + fields: ["active", "normalizedDisplayName"], + }) + .index("by_active_and_normalized_slug_first_token", { + fields: ["active", "normalizedSlugFirstToken"], + }) + .index("by_active_and_normalized_display_name_first_token", { + fields: ["active", "normalizedDisplayNameFirstToken"], + }) + .index("by_active_and_source_type_and_owner_and_repo_and_external_id", { + fields: ["active", "sourceType", "owner", "repo", "externalId"], + }) + .index("by_active_and_upstream_installs", { + fields: ["active", "upstreamInstalls"], + }) + .index("by_source_type_and_external_id", { + fields: ["sourceType", "externalId"], + }) + .index("by_last_observed_at", { + fields: ["lastObservedAt"], + }) + .searchIndex("search_by_search_text", { + searchField: "searchText", + filterFields: ["active"], + }); + +const skillsShMirrorDetails = defineTable({ + externalId: v.string(), + digestId: v.id("skillsShMirrorDigests"), + contentKind: v.union(v.literal("skill-md"), v.literal("readme")), + path: v.string(), + content: v.string(), + contentBytes: v.number(), + sourceBytes: v.number(), + sourceFileCount: v.number(), + truncated: v.boolean(), + sourceContentHash: v.optional(v.string()), + sourceSnapshotId: v.string(), + lastObservedRunId: v.id("skillsShMirrorRuns"), + createdAt: v.number(), + updatedAt: v.number(), +}) + .index("by_external_id", ["externalId"]) + .index("by_digest_id", { + fields: ["digestId"], + }); + +const skillsShMirrorFacets = defineTable({ + digestId: v.id("skillsShMirrorDigests"), + externalId: v.string(), + kind: v.union(v.literal("category"), v.literal("topic")), + term: v.string(), + active: v.boolean(), + installs: v.number(), + createdAt: v.number(), + updatedAt: v.number(), +}) + .index("by_digest_id_and_kind_and_term", ["digestId", "kind", "term"]) + .index("by_active_and_kind_and_term_and_installs_and_external_id", [ + "active", + "kind", + "term", + "installs", + "externalId", + ]); + +const skillsShMirrorConflicts = defineTable({ + runId: v.id("skillsShMirrorRuns"), + externalId: v.string(), + kind: v.union( + v.literal("same-run-drift"), + v.literal("identity-mismatch"), + v.literal("source-quarantine"), + ), + reason: v.optional(v.string()), + upstreamSourceType: v.optional(v.string()), + previousFingerprint: v.optional(v.string()), + observedFingerprint: v.string(), + page: v.number(), + offset: v.number(), + createdAt: v.number(), +}) + .index("by_run_id", ["runId"]) + .index("by_external_id_and_created_at", { + fields: ["externalId", "createdAt"], + }); + const publisherAbuseScoreRuns = defineTable({ modelVersion: v.string(), modelConfig: publisherAbuseModelConfigValidator, @@ -3692,6 +3945,13 @@ export default defineSchema({ skillsShCatalogRuns, skillsShCatalogEntries, skillsShCatalogScanAttempts, + skillsShMirrorControls, + skillsShMirrorRuns, + skillsShMirrorSourcePages, + skillsShMirrorDigests, + skillsShMirrorDetails, + skillsShMirrorFacets, + skillsShMirrorConflicts, publisherAbuseScoreRuns, publisherAbuseTemporalScanSamples, publisherAbuseTemporalScanCandidates, diff --git a/convex/search.test.ts b/convex/search.test.ts index 78e923d2..9d87afe0 100644 --- a/convex/search.test.ts +++ b/convex/search.test.ts @@ -2014,6 +2014,54 @@ describe("search helpers", () => { expect(result.map((entry) => entry.skill.slug)).toEqual(["tool-installed", "tool-downloaded"]); }); + it("keeps skills.sh installs out of the native download ranking tie-breaker", async () => { + generateEmbeddingMock.mockResolvedValueOnce([0, 1, 2]); + const indexed = { + nativeDownloads: 10, + skill: makePublicSkill({ + id: "skills:indexed", + slug: "tool-indexed", + displayName: "Tool", + downloads: 10_010, + installs: 0, + stars: 0, + }), + version: null, + ownerHandle: "owner", + owner: null, + }; + const native = { + nativeDownloads: 20, + skill: makePublicSkill({ + id: "skills:native", + slug: "tool-native", + displayName: "Tool", + downloads: 20, + installs: 0, + stars: 0, + }), + version: null, + ownerHandle: "owner", + owner: null, + }; + const runQuery = vi + .fn() + .mockResolvedValueOnce(null) // getExactSkillSlugMatch + .mockResolvedValueOnce([]) // directPrefixSkillMatches + .mockResolvedValueOnce([indexed, native]); // lexicalFallbackSkills + + const result = await searchSkillsHandler( + { + vectorSearch: vi.fn().mockResolvedValue([]), + runQuery, + }, + { query: "tool", limit: 2 }, + ); + + expect(result.map((entry) => entry.skill.slug)).toEqual(["tool-native", "tool-indexed"]); + expect(result[0]).not.toHaveProperty("nativeDownloads"); + }); + it("uses digest doc instead of full skill doc in hydrateResults but revalidates the owner", async () => { // Derive digest from makeSkillDoc so it stays in sync with schema changes. const skillDoc = makeSkillDoc({ diff --git a/convex/search.ts b/convex/search.ts index bc6b33c5..6d6a6de1 100644 --- a/convex/search.ts +++ b/convex/search.ts @@ -35,6 +35,7 @@ import { normalizeSkillSearchText, } from "./lib/skillSearchDigest"; import { isSearchableSkillSlugShape, normalizeSkillSlug } from "./lib/skillSlugValidator"; +import { readCanonicalStat } from "./lib/skillStats"; type OwnerInfo = { ownerHandle: string | null; owner: PublicPublisher | null }; @@ -75,6 +76,7 @@ async function withOfficialOwnerInfo(ctx: Pick, ownerInfo: Owner type SkillSearchEntry = { embeddingId?: Id<"skillEmbeddings">; + nativeDownloads: number; skill: NonNullable>; version: Doc<"skillVersions"> | null; ownerHandle: string | null; @@ -89,7 +91,7 @@ type SearchResult = SkillSearchEntry & SearchMatch & { score: number; }; -type PublicSearchResult = SkillSearchEntry & { +type PublicSearchResult = Omit & { score: number; }; @@ -239,7 +241,7 @@ function compareSkillTrustAndUsage(a: SkillSearchEntry, b: SkillSearchEntry) { { stars: a.skill.stats.stars, installsAllTime: a.skill.stats.installs }, { stars: b.skill.stats.stars, installsAllTime: b.skill.stats.installs }, ) || - (b.skill.stats.downloads ?? 0) - (a.skill.stats.downloads ?? 0) + b.nativeDownloads - a.nativeDownloads ); } @@ -508,7 +510,9 @@ export const searchSkills: ReturnType = action({ b.skill.updatedAt - a.skill.updatedAt, ) .slice(0, limit); - return rankedMatches.map(({ rankTier: _rankTier, ...entry }) => entry); + return rankedMatches.map( + ({ nativeDownloads: _nativeDownloads, rankTier: _rankTier, ...entry }) => entry, + ); }, }); @@ -541,6 +545,7 @@ export const getExactSkillSlugMatch = internalQuery({ if (!publicSkill || !resolved.owner) return null; const entry: SkillSearchEntry = { + nativeDownloads: readCanonicalStat(skill, "downloads"), skill: publicSkill, version: null as Doc<"skillVersions"> | null, ownerHandle: resolved.ownerHandle, @@ -851,6 +856,7 @@ export const directPrefixSkillMatches = internalQuery({ const publicSkill = toPublicSearchSkill(skill); if (!publicSkill || !resolved.owner) return null; return { + nativeDownloads: readCanonicalStat(skill, "downloads"), skill: publicSkill, version: null as Doc<"skillVersions"> | null, ownerHandle: resolved.ownerHandle, @@ -920,6 +926,7 @@ export const hydrateResults = internalQuery({ if (!publicSkill) return null; return { embeddingId, + nativeDownloads: readCanonicalStat(skill, "downloads"), skill: publicSkill, version: null as Doc<"skillVersions"> | null, ownerHandle: resolved.ownerHandle, @@ -1093,6 +1100,7 @@ export const lexicalFallbackSkills = internalQuery({ const publicSkill = toPublicSearchSkill(skill); if (!publicSkill) return null; return { + nativeDownloads: readCanonicalStat(skill, "downloads"), skill: publicSkill, version: null as Doc<"skillVersions"> | null, ownerHandle: resolved.ownerHandle, diff --git a/convex/skills.dashboard.test.ts b/convex/skills.dashboard.test.ts index 27eedd12..400ee00c 100644 --- a/convex/skills.dashboard.test.ts +++ b/convex/skills.dashboard.test.ts @@ -208,7 +208,24 @@ describe("skills.listDashboardPaginated", () => { it("paginates user dashboard skills through an active owner index", async () => { vi.mocked(getAuthUserId).mockResolvedValue("users:owner" as never); const { ctx, indexCalls } = makeCtx({ - by_owner_active_updated: [makeSkill("slack")], + by_owner_active_updated: [ + makeSkill("slack", { + statsDownloads: 12, + statsSkillsShInstalls: 8, + statsGithubStars: 99, + statsInstallsCurrent: 3, + statsInstallsAllTime: 7, + statsStars: 4, + stats: { + downloads: 12, + installsCurrent: 3, + installsAllTime: 7, + stars: 4, + versions: 1, + comments: 0, + }, + }), + ], }); const result = await handler( @@ -220,7 +237,20 @@ describe("skills.listDashboardPaginated", () => { ); expect(indexCalls).toContain("by_owner_active_updated"); - expect(result.page).toEqual([expect.objectContaining({ slug: "slack" })]); + expect(result.page).toEqual([ + expect.objectContaining({ + slug: "slack", + stats: expect.objectContaining({ downloads: 20 }), + metricSources: { + clawHubDownloads: 12, + skillsShInstalls: 8, + openClawInstallsCurrent: 3, + openClawInstallsAllTime: 7, + githubStars: 99, + bookmarks: 4, + }, + }), + ]); }); it("includes linked-user legacy skills when paginating a personal publisher", async () => { diff --git a/convex/skills.packageCatalog.test.ts b/convex/skills.packageCatalog.test.ts index dee1e102..2b4a8906 100644 --- a/convex/skills.packageCatalog.test.ts +++ b/convex/skills.packageCatalog.test.ts @@ -33,6 +33,7 @@ const listPackageCatalogPageHandler = ( family: "skill"; channel: "official" | "community"; isOfficial: boolean; + stats: { downloads: number }; }>; isDone: boolean; continueCursor: string; @@ -344,6 +345,34 @@ describe("skills package catalog queries", () => { ]); }); + it("presents combined downloads without changing the native download index", async () => { + const indexNames: string[] = []; + const result = await listPackageCatalogPageHandler( + makeCtx( + [ + { + page: [ + makeDigest("indexed-skill", { + statsDownloads: 12, + statsSkillsShInstalls: 8, + }), + ], + isDone: true, + continueCursor: "", + }, + ], + indexNames, + ), + { + sort: "downloads", + paginationOpts: { cursor: null, numItems: 10 }, + }, + ); + + expect(indexNames).toContain("by_active_stats_downloads"); + expect(result.page[0]?.stats.downloads).toBe(20); + }); + it("normalizes and filters skill package catalog topics", async () => { const indexNames: string[] = []; const calendarSkill = makeDigest("calendar-skill", { topics: ["calendar"] }); diff --git a/convex/skills.ts b/convex/skills.ts index 9061ec38..dbb268ce 100644 --- a/convex/skills.ts +++ b/convex/skills.ts @@ -167,7 +167,7 @@ import { upsertSkillSearchDigest, } from "./lib/skillSearchDigest"; import { assertValidSkillSlug, normalizeSkillSlug } from "./lib/skillSlugValidator"; -import { readCanonicalStat } from "./lib/skillStats"; +import { readCanonicalStat, readPublicDownloads, readSkillMetricSources } from "./lib/skillStats"; import { normalizeSkillTags } from "./lib/skillTags"; import { runStaticPublishScan } from "./lib/staticPublishScan"; import { adjustUserSkillStatsForSkillChange } from "./lib/userSkillStats"; @@ -1984,6 +1984,7 @@ type DashboardSkillListItem = { tags: Doc<"skills">["tags"]; badges: Doc<"skills">["badges"]; stats: Doc<"skills">["stats"]; + metricSources: ReturnType; moderationStatus?: Doc<"skills">["moderationStatus"]; moderationReason?: string; moderationSummary?: string; @@ -2377,7 +2378,7 @@ async function toDashboardSkillListItem( const latestVersion = skill.latestVersionId ? await ctx.db.get(skill.latestVersionId) : null; const stats = { ...skill.stats, - downloads: readCanonicalStat(skill, "downloads"), + downloads: readPublicDownloads(skill), stars: readCanonicalStat(skill, "stars"), installsCurrent: readCanonicalStat(skill, "installsCurrent"), installsAllTime: readCanonicalStat(skill, "installsAllTime"), @@ -2397,6 +2398,7 @@ async function toDashboardSkillListItem( tags: skill.tags, badges: skill.badges, stats, + metricSources: readSkillMetricSources(skill), moderationStatus: skill.moderationStatus, moderationReason: skill.moderationReason, moderationSummary: skill.moderationSummary, @@ -6432,7 +6434,9 @@ async function toPublicSkillCatalogItem( latestVersion: latestVersion?.version ?? null, verificationTier: null, stats: { - downloads: readDigestRankStat(digest, "downloads"), + // CLAW-561 changes presentation only. Download indexes and ranking remain + // native-only until a separately accepted indexed combined metric exists. + downloads: readPublicDownloads(digest), installs: readDigestRankStat(digest, "installsAllTime"), stars: readDigestRankStat(digest, "stars"), versions: digest.stats.versions, diff --git a/convex/skillsShCatalog.ts b/convex/skillsShCatalog.ts index 21056b4c..491531c1 100644 --- a/convex/skillsShCatalog.ts +++ b/convex/skillsShCatalog.ts @@ -19,6 +19,7 @@ import { isExactSkillsShCatalogAttempt, shouldPublishSkillsShCatalogEntry, } from "./lib/skillsShCatalogPublication"; +import { buildExternalSkillMetricPatch } from "./lib/skillStats"; import { validateFilePath } from "./lib/skillZip"; import { enqueueSkillsShCatalogScanRequest } from "./securityScan"; @@ -37,6 +38,10 @@ const MAX_WRITES_PER_BATCH = 100; const MAX_SCAN_ADMISSIONS_PER_BATCH = 100; const MAX_SCAN_ADMISSIONS_PER_RUN = 500; const MAX_REAL_TEST_ADMISSIONS = 10; +const CLEARED_EXTERNAL_SKILL_METRICS = { + statsSkillsShInstalls: undefined, + statsGithubStars: undefined, +}; const MAX_DETERMINISTIC_COMPLETIONS_PER_BATCH = 50; const fixtureIdValidator = v.union( @@ -69,6 +74,7 @@ const stagingLiveRowValidator = v.object({ claimPublisherHandle: v.optional(v.string()), sourceContentHash: v.string(), installs: v.number(), + githubStars: v.optional(v.number()), }); const sourceVerificationValidator = v.object({ githubOwnerId: v.number(), @@ -168,8 +174,21 @@ async function reconcileNativeSkill( const source = await ctx.db.get(skill.githubSourceId); reads += 1; if (source?.repo.trim().toLowerCase() !== `${row.owner}/${row.repo}`) continue; + const metricPatch = buildExternalSkillMetricPatch({ + skillsShInstalls: row.installs, + githubStars: row.githubStars, + }); + const metricPatchChanged = Object.entries(metricPatch).some( + ([field, value]) => skill[field as keyof typeof skill] !== value, + ); return { reads, + nativeMetricUpdate: metricPatchChanged + ? { + skillId: skill._id, + patch: metricPatch, + } + : undefined, reconciliation: { kind: "exact-native" as const, nativeSkillId: skill._id, @@ -200,6 +219,54 @@ async function reconcileNativeSkill( }; } +function nativeMetricSyncSkillIds( + existing: Doc<"skillsShCatalogEntries"> | null, + native: Awaited>, +) { + const skillIds = new Set>(); + if ("nativeMetricUpdate" in native && native.nativeMetricUpdate) { + skillIds.add(native.nativeMetricUpdate.skillId); + } + const previous = existing?.reconciliation; + if ( + previous?.kind === "exact-native" && + previous.nativeSkillId && + (native.reconciliation.kind !== "exact-native" || + native.reconciliation.nativeSkillId !== previous.nativeSkillId) + ) { + skillIds.add(previous.nativeSkillId); + } + return [...skillIds]; +} + +export const syncNativeSkillMetricsFromCatalogEntryInternal = internalMutation({ + args: { + entryId: v.id("skillsShCatalogEntries"), + skillId: v.id("skills"), + }, + handler: async (ctx, args) => { + const [entry, skill] = await Promise.all([ctx.db.get(args.entryId), ctx.db.get(args.skillId)]); + if (!entry || !skill) return { applied: false }; + const patch = + entry.reconciliation?.kind === "exact-native" && + entry.reconciliation.nativeSkillId === args.skillId + ? buildExternalSkillMetricPatch({ + skillsShInstalls: entry.installs, + githubStars: entry.githubStars, + }) + : CLEARED_EXTERNAL_SKILL_METRICS; + const changed = Object.entries(patch).some( + ([field, value]) => skill[field as keyof typeof skill] !== value, + ); + if (!changed) return { applied: false }; + + // The skills-table trigger in functions.ts refreshes the search digest from + // this patch, keeping the batch write budget independent of digest fan-out. + await ctx.db.patch(skill._id, patch); + return { applied: true }; + }, +}); + function incrementReconciliationCounts( counts: ReturnType, reconciliation: Doc<"skillsShCatalogEntries">["reconciliation"], @@ -950,6 +1017,7 @@ export const processStagingLiveBatchInternal = internalMutation({ } const native = await reconcileNativeSkill(ctx, row, now); readsUsed += native.reads; + const metricSkillIds = nativeMetricSyncSkillIds(existing, native); const observationUnchanged = existing ? sameFixtureObservation(existing, row, native.reconciliation) : false; @@ -981,6 +1049,7 @@ export const processStagingLiveBatchInternal = internalMutation({ counts.observed += 1; cursor += 1; if (shouldPlanScan) counts.scansPlanned += 1; + let entryId = existing?._id; if (existing) { if (observationUnchanged) counts.unchanged += 1; else { @@ -1001,6 +1070,7 @@ export const processStagingLiveBatchInternal = internalMutation({ githubContentHash: row.githubContentHash, sourceContentHash: row.sourceContentHash, installs: row.installs, + githubStars: row.githubStars, sourceSnapshotId: run.snapshotId, reconciliation: native.reconciliation, publicVisible: false, @@ -1017,7 +1087,7 @@ export const processStagingLiveBatchInternal = internalMutation({ } else { counts.wouldInsert += 1; counts.inserted += 1; - await ctx.db.insert("skillsShCatalogEntries", { + entryId = await ctx.db.insert("skillsShCatalogEntries", { externalId: row.externalId, sourceKind: "staging-live", githubOwnerId: row.githubOwnerId, @@ -1032,6 +1102,7 @@ export const processStagingLiveBatchInternal = internalMutation({ githubContentHash: row.githubContentHash, sourceContentHash: row.sourceContentHash, installs: row.installs, + githubStars: row.githubStars, sourceSnapshotId: run.snapshotId, reconciliation: native.reconciliation, publicVisible: false, @@ -1043,6 +1114,18 @@ export const processStagingLiveBatchInternal = internalMutation({ }); } writesUsed += 1; + if (entryId) { + for (const skillId of metricSkillIds) { + await ctx.scheduler.runAfter( + 0, + internal.skillsShCatalog.syncNativeSkillMetricsFromCatalogEntryInternal, + { + entryId, + skillId, + }, + ); + } + } } if (cursor !== args.cursor + args.rows.length) { @@ -1122,6 +1205,7 @@ export const processFixtureBatchInternal = internalMutation({ } const native = await reconcileNativeSkill(ctx, row, now); readsUsed += native.reads; + const metricSkillIds = nativeMetricSyncSkillIds(existing, native); const observationUnchanged = existing ? sameFixtureObservation(existing, row, native.reconciliation) @@ -1149,13 +1233,16 @@ export const processFixtureBatchInternal = internalMutation({ !existingAttempt && (!existing || contentChanged || existing.scanStatus !== "planned"); const entryWriteRequired = !run.dryRun; - if (entryWriteRequired && writesUsed + 2 > run.budgets.maxWritesPerBatch) break; + if (entryWriteRequired && writesUsed + 2 > run.budgets.maxWritesPerBatch) { + break; + } incrementReconciliationCounts(counts, native.reconciliation); counts.observed += 1; cursor += 1; entriesProcessed += 1; if (shouldPlanScan) counts.scansPlanned += 1; + let entryId = existing?._id; if (existing) { if (observationUnchanged) counts.unchanged += 1; else { @@ -1177,6 +1264,7 @@ export const processFixtureBatchInternal = internalMutation({ githubContentHash: row.githubContentHash, sourceContentHash: row.sourceContentHash, installs: row.installs, + githubStars: row.githubStars, sourceSnapshotId: fixture.snapshotId, reconciliation: native.reconciliation, // This gate has no publication seam; every catalog write reasserts dark visibility. @@ -1193,12 +1281,24 @@ export const processFixtureBatchInternal = internalMutation({ }); writesUsed += 1; } + if (entryId && entryWriteRequired) { + for (const skillId of metricSkillIds) { + await ctx.scheduler.runAfter( + 0, + internal.skillsShCatalog.syncNativeSkillMetricsFromCatalogEntryInternal, + { + entryId, + skillId, + }, + ); + } + } continue; } counts.wouldInsert += 1; if (!run.dryRun) { - await ctx.db.insert("skillsShCatalogEntries", { + entryId = await ctx.db.insert("skillsShCatalogEntries", { externalId: row.externalId, sourceKind: fixture.sourceKind, githubOwnerId: row.githubOwnerId, @@ -1213,6 +1313,7 @@ export const processFixtureBatchInternal = internalMutation({ githubContentHash: row.githubContentHash, sourceContentHash: row.sourceContentHash, installs: row.installs, + githubStars: row.githubStars, sourceSnapshotId: fixture.snapshotId, reconciliation: native.reconciliation, publicVisible: false, @@ -1224,6 +1325,16 @@ export const processFixtureBatchInternal = internalMutation({ }); writesUsed += 1; counts.inserted += 1; + for (const skillId of metricSkillIds) { + await ctx.scheduler.runAfter( + 0, + internal.skillsShCatalog.syncNativeSkillMetricsFromCatalogEntryInternal, + { + entryId, + skillId, + }, + ); + } } } @@ -2362,6 +2473,7 @@ export const rollbackFixtureRunInternal = internalMutation({ } const fixture = getSkillsShCatalogFixture(run.fixtureId); let deletedEntries = 0; + let nativeSkillsChanged = 0; for (let index = 0; index < fixture.length; index += 1) { const expected = normalizeIdentity(fixture.rowAt(index)); const entry = await ctx.db @@ -2384,6 +2496,21 @@ export const rollbackFixtureRunInternal = internalMutation({ `Controlled canary has retained scan history: ${expected.externalId}`, ); } + const nativeSkillId = + entry.reconciliation?.kind === "exact-native" + ? entry.reconciliation.nativeSkillId + : undefined; + if (nativeSkillId) { + const nativeSkill = await ctx.db.get(nativeSkillId); + if ( + nativeSkill && + (nativeSkill.statsSkillsShInstalls !== undefined || + nativeSkill.statsGithubStars !== undefined) + ) { + await ctx.db.patch(nativeSkillId, CLEARED_EXTERNAL_SKILL_METRICS); + nativeSkillsChanged += 1; + } + } await ctx.db.delete(entry._id); deletedEntries += 1; } @@ -2392,7 +2519,7 @@ export const rollbackFixtureRunInternal = internalMutation({ actor: args.actor.trim(), reason: args.reason.trim(), deletedEntries, - nativeSkillsChanged: 0, + nativeSkillsChanged, }; }, }); @@ -2760,6 +2887,7 @@ function sameFixtureObservation( existing.githubContentHash === row.githubContentHash && existing.sourceContentHash === row.sourceContentHash && existing.installs === row.installs && + existing.githubStars === row.githubStars && existingReconciliation?.kind === reconciliation.kind && existingReconciliation.nativeSkillId === reconciliation.nativeSkillId && existingReconciliation.nativeSlug === reconciliation.nativeSlug && diff --git a/convex/skillsShCatalogCanary.test.ts b/convex/skillsShCatalogCanary.test.ts index 01dbfe6a..8a29e8e4 100644 --- a/convex/skillsShCatalogCanary.test.ts +++ b/convex/skillsShCatalogCanary.test.ts @@ -5,6 +5,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { api, internal } from "./_generated/api"; import type { Doc, Id } from "./_generated/dataModel"; import canarySkillMarkdown from "./fixtures/patrick-html-canary-SKILL.txt?raw"; +import { extractDigestFields } from "./lib/skillSearchDigest"; import schema from "./schema"; const modules = import.meta.glob("./**/*.ts"); @@ -214,6 +215,11 @@ async function seedNativeSkill( options: { exactSource: boolean; downloads: number; + bookmarks?: number; + openClawInstalls?: number; + skillsShInstalls?: number; + githubStars?: number; + seedDigest?: boolean; }, ) { return await t.run(async (ctx) => { @@ -250,26 +256,40 @@ async function seedNativeSkill( tags: {}, moderationStatus: "active", statsDownloads: options.downloads, - statsStars: 0, - statsInstallsCurrent: 0, - statsInstallsAllTime: 0, + statsStars: options.bookmarks ?? 0, + statsInstallsCurrent: options.openClawInstalls ?? 0, + statsInstallsAllTime: options.openClawInstalls ?? 0, + statsSkillsShInstalls: options.skillsShInstalls, + statsGithubStars: options.githubStars, stats: { downloads: options.downloads, - stars: 0, - installsCurrent: 0, - installsAllTime: 0, + stars: options.bookmarks ?? 0, + installsCurrent: options.openClawInstalls ?? 0, + installsAllTime: options.openClawInstalls ?? 0, versions: 0, comments: 0, }, createdAt: 1, updatedAt: 1, }); + if (options.seedDigest) { + const skill = await ctx.db.get(skillId); + if (!skill) throw new Error("seeded native skill missing"); + await ctx.db.insert("skillSearchDigest", { + ...extractDigestFields(skill), + ownerHandle: "native-owner", + ownerKind: "user", + ownerName: "Native Owner", + ownerDisplayName: "Native Owner", + }); + } return skillId; }); } describe("skills.sh controlled hidden metadata canary", () => { afterEach(() => { + vi.useRealTimers(); vi.unstubAllEnvs(); }); @@ -324,17 +344,33 @@ describe("skills.sh controlled hidden metadata canary", () => { ).toHaveLength(0); }); - it("records an exact native match and preserves its downloads", async () => { + it("attaches upstream metrics to an exact native match without rewriting native counters", async () => { + vi.useFakeTimers(); useEnvironment(LOCAL_ENV); const t = convexTest(schema, modules); - const nativeSkillId = await seedNativeSkill(t, { exactSource: true, downloads: 143 }); + const nativeSkillId = await seedNativeSkill(t, { + exactSource: true, + downloads: 143, + bookmarks: 5, + openClawInstalls: 11, + skillsShInstalls: 2, + githubStars: 300, + seedDigest: true, + }); await configureCanary(t); const { runId, run } = await runCanary(t); + await t.finishAllScheduledFunctions(vi.runAllTimers); const readback = await t.query(internal.skillsShCatalog.getRunReconciliationInternal, { runId, }); const native = await t.run(async (ctx) => await ctx.db.get(nativeSkillId)); + const digest = await t.run(async (ctx) => + ctx.db + .query("skillSearchDigest") + .withIndex("by_skill", (q) => q.eq("skillId", nativeSkillId)) + .unique(), + ); expect(run.counts).toMatchObject({ newExternal: 0, @@ -352,10 +388,121 @@ describe("skills.sh controlled hidden metadata canary", () => { expect(native).toMatchObject({ _id: nativeSkillId, statsDownloads: 143, - stats: { downloads: 143 }, + statsStars: 5, + statsInstallsCurrent: 11, + statsInstallsAllTime: 11, + statsSkillsShInstalls: 17, + statsGithubStars: 321, + stats: { + downloads: 143, + stars: 5, + installsCurrent: 11, + installsAllTime: 11, + }, githubCurrentCommit: CANARY_COMMIT, githubCurrentContentHash: CANARY_CONTENT_HASH, }); + expect(digest).toMatchObject({ + statsDownloads: 143, + statsSkillsShInstalls: 17, + statsGithubStars: 321, + }); + }); + + it("clears exact-native upstream metrics when the controlled entry is removed", async () => { + vi.useFakeTimers(); + useEnvironment(LOCAL_ENV); + const t = convexTest(schema, modules); + const nativeSkillId = await seedNativeSkill(t, { + exactSource: true, + downloads: 143, + bookmarks: 5, + openClawInstalls: 11, + seedDigest: true, + }); + await configureCanary(t); + + const { runId } = await runCanary(t); + await t.finishAllScheduledFunctions(vi.runAllTimers); + const rollback = await t.mutation(internal.skillsShCatalog.rollbackFixtureRunInternal, { + runId, + actor: "codex-test", + reason: "remove the controlled exact-native metadata", + confirm: "rollback-skills-sh-controlled-canary", + }); + const native = await t.run(async (ctx) => await ctx.db.get(nativeSkillId)); + const digest = await t.run(async (ctx) => + ctx.db + .query("skillSearchDigest") + .withIndex("by_skill", (q) => q.eq("skillId", nativeSkillId)) + .unique(), + ); + + expect(rollback).toMatchObject({ + deletedEntries: 1, + nativeSkillsChanged: 1, + }); + expect(native?.statsSkillsShInstalls).toBeUndefined(); + expect(native?.statsGithubStars).toBeUndefined(); + expect(digest?.statsSkillsShInstalls).toBeUndefined(); + expect(digest?.statsGithubStars).toBeUndefined(); + expect(native).toMatchObject({ + statsDownloads: 143, + statsStars: 5, + statsInstallsCurrent: 11, + statsInstallsAllTime: 11, + stats: { + downloads: 143, + stars: 5, + installsCurrent: 11, + installsAllTime: 11, + }, + }); + }); + + it("clears upstream metrics when an exact native match becomes a route collision", async () => { + vi.useFakeTimers(); + useEnvironment(LOCAL_ENV); + const t = convexTest(schema, modules); + const nativeSkillId = await seedNativeSkill(t, { + exactSource: true, + downloads: 143, + skillsShInstalls: 2, + githubStars: 300, + seedDigest: true, + }); + await configureCanary(t); + + await runCanary(t); + await t.finishAllScheduledFunctions(vi.runAllTimers); + await t.run(async (ctx) => { + await ctx.db.patch(nativeSkillId, { + githubCurrentCommit: "f".repeat(40), + }); + }); + + const { run } = await runCanary(t); + await t.finishAllScheduledFunctions(vi.runAllTimers); + const native = await t.run(async (ctx) => await ctx.db.get(nativeSkillId)); + const digest = await t.run(async (ctx) => + ctx.db + .query("skillSearchDigest") + .withIndex("by_skill", (q) => q.eq("skillId", nativeSkillId)) + .unique(), + ); + + expect(run.counts).toMatchObject({ + exactNativeMatches: 0, + routeCollisions: 1, + }); + expect(native?.statsSkillsShInstalls).toBeUndefined(); + expect(native?.statsGithubStars).toBeUndefined(); + expect(digest?.statsSkillsShInstalls).toBeUndefined(); + expect(digest?.statsGithubStars).toBeUndefined(); + expect(native).toMatchObject({ + statsDownloads: 143, + stats: { downloads: 143 }, + }); }); it("records a route collision without changing or attaching the native skill", async () => { diff --git a/convex/skillsShMirror.test.ts b/convex/skillsShMirror.test.ts new file mode 100644 index 00000000..3e9932d2 --- /dev/null +++ b/convex/skillsShMirror.test.ts @@ -0,0 +1,1460 @@ +/// +/* @vitest-environment edge-runtime */ +import { convexTest } from "convex-test"; +import type { FunctionArgs } from "convex/server"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { internal } from "./_generated/api"; +import type { Doc, Id } from "./_generated/dataModel"; +import schema from "./schema"; + +const modules = import.meta.glob("./**/*.ts"); +const TEST_ENV = { + CLAWHUB_DEPLOYMENT_NAME: "academic-chihuahua-392", + CLAWHUB_DISABLE_CRONS: "1", + CLAWHUB_ENV: "test", + CLAWHUB_SKILLS_SH_ROLLOUT_MODE: "test", + CONVEX_CLOUD_URL: "https://academic-chihuahua-392.convex.cloud", +}; + +function useTestEnvironment() { + for (const [name, value] of Object.entries(TEST_ENV)) vi.stubEnv(name, value); +} + +async function sha256Hex(value: string) { + const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(value)); + return Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join(""); +} + +const githubRow = { + externalId: "vercel-labs/skills/find-skills", + sourceType: "github" as const, + upstreamSourceType: "github", + owner: "vercel-labs", + repo: "skills", + slug: "find-skills", + displayName: "Find Skills", + sourceUrl: "https://skills.sh/vercel-labs/skills/find-skills", + canonicalRepoUrl: "https://github.com/vercel-labs/skills", + upstreamInstalls: 42, + upstreamScanners: { + genAgentTrustHub: { + status: "pass", + sourceUrl: "https://skills.sh/vercel-labs/skills/find-skills/security/agent-trust-hub", + }, + socket: { + status: "pass", + sourceUrl: "https://skills.sh/vercel-labs/skills/find-skills/security/socket", + }, + snyk: { + status: "warn", + sourceUrl: "https://skills.sh/vercel-labs/skills/find-skills/security/snyk", + }, + }, + inferredCategories: ["development"], + inferredTopics: ["skill-discovery"], + inferredCategoryConfidence: "high" as const, + inferredTopicConfidence: "medium" as const, + inferredClassifierVersion: "taxonomy-prototype-v9", + inferredTopicClassifierVersion: "topic-prototype-v1", + inferredInputHash: "github-input-hash", + inferredTopicInputHash: "github-topic-input-hash", + inferredAt: 123, + sourceContentHash: "a".repeat(64), + detail: { + contentKind: "skill-md" as const, + path: "SKILL.md", + content: "# Find Skills", + contentBytes: 13, + sourceBytes: 13, + sourceFileCount: 1, + truncated: false, + }, +}; + +const wellKnownRow = { + externalId: "open.feishu.cn/lark-doc", + sourceType: "well-known" as const, + upstreamSourceType: "well-known", + sourceHost: "open.feishu.cn", + slug: "lark-doc", + displayName: "lark-doc", + sourceUrl: "https://www.skills.sh/site/open.feishu.cn/lark-doc", + upstreamInstalls: 7, + upstreamScanners: { + genAgentTrustHub: { status: "unavailable" }, + socket: { status: "unavailable" }, + snyk: { status: "unavailable" }, + }, + inferredCategories: ["productivity"], + inferredTopics: ["documents"], + inferredCategoryConfidence: "medium" as const, + inferredTopicConfidence: "medium" as const, + inferredClassifierVersion: "taxonomy-prototype-v9", + inferredTopicClassifierVersion: "topic-prototype-v1", + inferredInputHash: "well-known-input-hash", + inferredTopicInputHash: "well-known-topic-input-hash", + inferredAt: 123, + sourceContentHash: "b".repeat(64), + detail: { + contentKind: "readme" as const, + path: "README.md", + content: "# Lark Doc", + contentBytes: 10, + sourceBytes: 10, + sourceFileCount: 1, + truncated: false, + }, +}; + +async function configure(t: ReturnType) { + return await t.mutation(internal.skillsShMirror.configureInternal, { + actor: "codex-test", + reason: "CLAW-563 mirror test", + confirm: "enable-skills-sh-mirror-test", + enabled: true, + maxRowsPerRun: 10_000, + maxRowsPerBatch: 50, + maxDetailBytes: 64 * 1024, + }); +} + +async function startRun( + t: ReturnType, + snapshotId: string, + sourceTotal = 2, + sourceSnapshotHash?: string, +) { + return (await t.mutation(internal.skillsShMirror.startRunInternal, { + actor: "codex-test", + reason: "CLAW-563 mirror test", + snapshotId, + ...(sourceSnapshotHash ? { sourceSnapshotHash } : {}), + sourceTotal, + sourcePageSize: 500, + sourceMeasuredAt: "2026-07-22T20:14:10.881Z", + })) as { runId: Id<"skillsShMirrorRuns"> }; +} + +const mirrorLeaseRefs = internal.skillsShMirror as unknown as { + claimBatchLeaseInternal: Parameters["mutation"]>[0]; + releaseBatchLeaseInternal: Parameters["mutation"]>[0]; +}; + +let leaseSequence = 0; + +async function processBatch( + t: ReturnType, + args: Omit, "leaseToken">, +) { + const leaseToken = `test-lease:${(leaseSequence += 1)}`; + await t.mutation(mirrorLeaseRefs.claimBatchLeaseInternal, { + runId: args.runId, + page: args.page, + offset: args.offset, + leaseToken, + }); + return await t.mutation(internal.skillsShMirror.processBatchInternal, { + ...args, + leaseToken, + }); +} + +describe("skills.sh external mirror", () => { + afterEach(() => { + vi.useRealTimers(); + vi.unstubAllEnvs(); + }); + + it("returns the durable cursor summary when starting a run", async () => { + useTestEnvironment(); + const t = convexTest(schema, modules); + await configure(t); + + const started = await t.mutation(internal.skillsShMirror.startRunInternal, { + actor: "codex-test", + reason: "CLAW-563 mirror test", + snapshotId: "snapshot:start-summary", + sourceTotal: 9_571, + sourcePageSize: 500, + sourceMeasuredAt: "2026-07-22T20:14:10.881Z", + }); + + expect(started).toMatchObject({ + snapshotId: "snapshot:start-summary", + status: "running", + sourceTotal: 9_571, + sourcePageSize: 500, + page: 0, + offset: 0, + completedAt: null, + }); + expect(started.runId).toEqual(expect.any(String)); + }); + + it("records bounded source accounting above 100 MiB", async () => { + useTestEnvironment(); + const t = convexTest(schema, modules); + await configure(t); + const { runId } = await startRun(t, "snapshot:large-source-accounting", 1); + + await expect( + processBatch(t, { + runId, + page: 0, + offset: 0, + pageLength: 1, + hasMore: false, + sourceTotal: 1, + sourceRequests: 1, + sourceBytes: 50 * 8 * 1024 * 1024, + rows: [githubRow], + }), + ).resolves.toMatchObject({ + status: "reconciling", + operations: { sourceBytes: 50 * 8 * 1024 * 1024 }, + }); + }); + + it("stores immutable source pages and returns them with the exact leased cursor", async () => { + useTestEnvironment(); + const t = convexTest(schema, modules); + await configure(t); + const snapshotHash = "a".repeat(64); + const rows = [ + { + id: "vercel-labs/skills/find-skills", + installUrl: "https://github.com/vercel-labs/skills", + installs: 42, + name: "Find Skills", + slug: "find-skills", + source: "vercel-labs/skills", + sourceType: "github", + url: "https://skills.sh/vercel-labs/skills/find-skills", + }, + ]; + const identityHash = await sha256Hex(`${rows[0]!.id}\n`); + const contentHash = await sha256Hex(JSON.stringify(rows)); + const sourcePage = { + snapshotHash, + page: 0, + sourceTotal: 1, + pageLength: 1, + hasMore: false, + identityHash, + contentHash, + sourceBytes: 512, + serializedBytes: 768, + rows, + }; + + await expect( + t.mutation(internal.skillsShMirror.storeSourcePageInternal, sourcePage), + ).resolves.toEqual({ stored: true, page: 0, rows: 1 }); + await expect( + t.mutation(internal.skillsShMirror.storeSourcePageInternal, sourcePage), + ).resolves.toEqual({ stored: false, page: 0, rows: 1 }); + await expect( + t.mutation(internal.skillsShMirror.storeSourcePageInternal, { + ...sourcePage, + sourceBytes: sourcePage.sourceBytes + 1, + }), + ).rejects.toThrow("captured skills.sh source page is immutable"); + await expect( + t.mutation(internal.skillsShMirror.storeSourcePageInternal, { + ...sourcePage, + contentHash: "d".repeat(64), + }), + ).rejects.toThrow("captured skills.sh source page content hash mismatch"); + + const { runId } = await startRun(t, "skills-sh:proof:captured", 1, snapshotHash); + await expect( + t.mutation(mirrorLeaseRefs.claimBatchLeaseInternal, { + runId, + page: 0, + offset: 0, + leaseToken: "lease:captured", + }), + ).resolves.toMatchObject({ + sourcePage: { + snapshotHash, + page: 0, + sourceTotal: 1, + pageLength: 1, + hasMore: false, + identityHash, + contentHash, + rows, + }, + }); + await expect( + t.query(internal.skillsShMirror.getSourceCaptureSummaryInternal, { snapshotHash }), + ).resolves.toEqual({ + snapshotHash, + pageDocuments: 1, + rows: 1, + sourceBytes: 512, + serializedBytes: 768, + }); + }); + + it("counts a captured-page lookup even when the controlled page has no source document", async () => { + useTestEnvironment(); + const t = convexTest(schema, modules); + await configure(t); + const snapshotHash = "a".repeat(64); + const { runId } = await startRun(t, "skills-sh:proof:controlled", 1, snapshotHash); + await t.run(async (ctx) => { + await ctx.db.patch(runId, { page: 1 }); + }); + + await expect( + t.mutation(mirrorLeaseRefs.claimBatchLeaseInternal, { + runId, + page: 1, + offset: 0, + leaseToken: "lease:controlled", + }), + ).resolves.toMatchObject({ sourcePage: null }); + const run = await t.run(async (ctx) => await ctx.db.get(runId)); + expect(run?.operations.dbReads).toBe(5); + }); + + it("cancels a stale captured run so a fresh authenticated run can start", async () => { + useTestEnvironment(); + const t = convexTest(schema, modules); + await configure(t); + const stale = await startRun(t, "skills-sh-captured:missing-live-run"); + + await expect(startRun(t, "skills-sh:fresh-blocked")).rejects.toThrow( + "already has an active run", + ); + await expect( + t.mutation(internal.skillsShMirror.cancelRunInternal, { + runId: stale.runId, + actor: "codex-test", + reason: "discard stale captured recovery", + confirm: "cancel-skills-sh-mirror-test-run", + }), + ).resolves.toMatchObject({ + runId: stale.runId, + status: "canceled", + }); + await expect(startRun(t, "skills-sh:fresh-live")).resolves.toMatchObject({ + status: "running", + snapshotId: "skills-sh:fresh-live", + }); + }); + + it("allows only one active batch lease for an exact durable cursor", async () => { + useTestEnvironment(); + const t = convexTest(schema, modules); + await configure(t); + const { runId } = await startRun(t, "snapshot:lease", 1); + + await expect( + t.mutation(mirrorLeaseRefs.claimBatchLeaseInternal, { + runId, + page: 0, + offset: 0, + leaseToken: "lease:first", + }), + ).resolves.toMatchObject({ + runId, + page: 0, + offset: 0, + leaseExpiresAt: expect.any(Number), + }); + await expect( + t.mutation(mirrorLeaseRefs.claimBatchLeaseInternal, { + runId, + page: 0, + offset: 0, + leaseToken: "lease:second", + }), + ).rejects.toThrow("already leased"); + }); + + it("renews an active batch lease when the same worker heartbeats", async () => { + useTestEnvironment(); + const t = convexTest(schema, modules); + await configure(t); + const { runId } = await startRun(t, "snapshot:lease-renewal", 1); + const first = (await t.mutation(mirrorLeaseRefs.claimBatchLeaseInternal, { + runId, + page: 0, + offset: 0, + leaseToken: "lease:worker", + })) as { + leaseExpiresAt: number; + snapshotId: string; + sourcePageSize: number; + sourceTotal: number; + }; + expect(first).toMatchObject({ + snapshotId: "snapshot:lease-renewal", + sourcePageSize: 500, + sourceTotal: 1, + }); + await t.run(async (ctx) => { + await ctx.db.patch(runId, { batchLeaseExpiresAt: first.leaseExpiresAt - 60_000 }); + }); + + const renewed = (await t.mutation(mirrorLeaseRefs.claimBatchLeaseInternal, { + runId, + page: 0, + offset: 0, + leaseToken: "lease:worker", + })) as { leaseExpiresAt: number }; + + expect(renewed.leaseExpiresAt).toBeGreaterThan(first.leaseExpiresAt - 1_000); + }); + + it("permits stale lease takeover and rejects the superseded token", async () => { + useTestEnvironment(); + const t = convexTest(schema, modules); + await configure(t); + const { runId } = await startRun(t, "snapshot:stale-lease", 1); + await t.mutation(mirrorLeaseRefs.claimBatchLeaseInternal, { + runId, + page: 0, + offset: 0, + leaseToken: "lease:stale", + }); + await t.run(async (ctx) => { + await ctx.db.patch(runId, { batchLeaseExpiresAt: Date.now() - 1 }); + }); + + await expect( + t.mutation(mirrorLeaseRefs.claimBatchLeaseInternal, { + runId, + page: 0, + offset: 0, + leaseToken: "lease:fresh", + }), + ).resolves.toMatchObject({ leaseToken: "lease:fresh" }); + await expect( + t.mutation(internal.skillsShMirror.processBatchInternal, { + runId, + page: 0, + offset: 0, + leaseToken: "lease:stale", + pageLength: 1, + hasMore: false, + sourceTotal: 1, + sourceRequests: 3, + sourceBytes: 1_024, + rows: [githubRow], + }), + ).rejects.toThrow("lease token mismatch"); + }); + + it("requires the exact lease token to release or commit a batch", async () => { + useTestEnvironment(); + const t = convexTest(schema, modules); + await configure(t); + const { runId } = await startRun(t, "snapshot:lease-token", 1); + await t.mutation(mirrorLeaseRefs.claimBatchLeaseInternal, { + runId, + page: 0, + offset: 0, + leaseToken: "lease:owner", + }); + + await expect( + t.mutation(mirrorLeaseRefs.releaseBatchLeaseInternal, { + runId, + page: 0, + offset: 0, + leaseToken: "lease:wrong", + }), + ).rejects.toThrow("lease token mismatch"); + await expect( + t.mutation(internal.skillsShMirror.processBatchInternal, { + runId, + page: 0, + offset: 0, + leaseToken: "lease:wrong", + pageLength: 1, + hasMore: false, + sourceTotal: 1, + sourceRequests: 3, + sourceBytes: 1_024, + rows: [githubRow], + }), + ).rejects.toThrow("lease token mismatch"); + + await expect( + t.mutation(mirrorLeaseRefs.releaseBatchLeaseInternal, { + runId, + page: 0, + offset: 0, + leaseToken: "lease:owner", + }), + ).resolves.toMatchObject({ released: true }); + await expect( + t.mutation(mirrorLeaseRefs.claimBatchLeaseInternal, { + runId, + page: 0, + offset: 0, + leaseToken: "lease:replacement", + }), + ).resolves.toMatchObject({ leaseToken: "lease:replacement" }); + + const committed = await t.mutation(internal.skillsShMirror.processBatchInternal, { + runId, + page: 0, + offset: 0, + leaseToken: "lease:replacement", + pageLength: 1, + hasMore: false, + sourceTotal: 1, + sourceRequests: 3, + sourceBytes: 1_024, + rows: [githubRow], + }); + expect(committed).toMatchObject({ status: "reconciling", page: 1, offset: 0 }); + const storedRun = await t.run(async (ctx) => await ctx.db.get(runId)); + expect(storedRun).not.toHaveProperty("batchLeaseToken"); + expect(storedRun).not.toHaveProperty("batchLeaseExpiresAt"); + }); + + it("processes durable source cursors without creating scan work", async () => { + useTestEnvironment(); + const t = convexTest(schema, modules); + await configure(t); + const { runId } = await startRun(t, "skills-sh:proof:compact.evidence-hash.evidence"); + + const result = await processBatch(t, { + runId, + page: 0, + offset: 0, + pageLength: 2, + hasMore: false, + sourceTotal: 2, + sourceRequests: 3, + sourceBytes: 1_024, + rows: [githubRow, wellKnownRow], + }); + + expect(result).toMatchObject({ + status: "reconciling", + page: 1, + offset: 0, + counts: { + observed: 2, + inserted: 2, + conflicts: 0, + scansPlanned: 0, + scansAdmitted: 0, + }, + }); + expect( + await t.run(async (ctx) => await ctx.db.query("skillsShCatalogScanAttempts").collect()), + ).toEqual([]); + expect(await t.run(async (ctx) => await ctx.db.query("securityScanJobs").collect())).toEqual( + [], + ); + const storedSourceReferences = await t.run(async (ctx) => { + const digest = await ctx.db + .query("skillsShMirrorDigests") + .withIndex("by_external_id", (q) => q.eq("externalId", githubRow.externalId)) + .unique(); + const detail = await ctx.db + .query("skillsShMirrorDetails") + .withIndex("by_external_id", (q) => q.eq("externalId", githubRow.externalId)) + .unique(); + return { + digest: digest?.sourceSnapshotId, + detail: detail?.sourceSnapshotId, + }; + }); + expect(storedSourceReferences).toEqual({ + digest: "skills-sh:proof:compact.evidence-hash", + detail: "skills-sh:proof:compact.evidence-hash", + }); + expect( + await t.query(internal.skillsShMirror.getByExternalIdInternal, { + externalId: githubRow.externalId, + }), + ).toMatchObject({ + normalizedSlug: "find-skills", + normalizedSlugFirstToken: "find", + normalizedDisplayName: "find skills", + normalizedDisplayNameFirstToken: "find", + upstreamScanners: githubRow.upstreamScanners, + inferredCategories: ["development"], + inferredTopics: ["skill-discovery"], + }); + }); + + it("preserves classifier topic labels and indexes their normalized topic slugs", async () => { + useTestEnvironment(); + const t = convexTest(schema, modules); + await configure(t); + const { runId } = await startRun(t, "snapshot:topic-labels", 1); + const topicRow = { + ...githubRow, + inferredTopics: ["Code Review", "股票分析"], + }; + + await expect( + processBatch(t, { + runId, + page: 0, + offset: 0, + pageLength: 1, + hasMore: false, + sourceTotal: 1, + sourceRequests: 3, + sourceBytes: 1_024, + rows: [topicRow], + }), + ).resolves.toMatchObject({ + counts: { + inserted: 1, + rejected: 0, + conflicts: 0, + }, + }); + await expect( + t.query(internal.skillsShMirror.getByExternalIdInternal, { + externalId: topicRow.externalId, + }), + ).resolves.toMatchObject({ + inferredTopics: ["Code Review", "股票分析"], + }); + + for (const topic of ["Code Review", "股票分析"]) { + const result = await t.query(internal.skillsShMirror.listActiveByTopicInternal, { + topic, + paginationOpts: { cursor: null, numItems: 10 }, + }); + expect(result.page.map((digest) => digest.externalId)).toEqual([topicRow.externalId]); + } + await t.mutation(internal.skillsShMirror.reconcileBatchInternal, { + runId, + limit: 250, + }); + await t.run(async (ctx) => { + const digest = await ctx.db + .query("skillsShMirrorDigests") + .withIndex("by_external_id", (q) => q.eq("externalId", topicRow.externalId)) + .unique(); + if (!digest) throw new Error("topic digest missing"); + const canonical = await ctx.db + .query("skillsShMirrorFacets") + .withIndex("by_digest_id_and_kind_and_term", (q) => q.eq("digestId", digest._id)) + .filter((q) => q.eq(q.field("term"), "code-review")) + .unique(); + if (!canonical) throw new Error("canonical topic facet missing"); + await ctx.db.delete(canonical._id); + await ctx.db.insert("skillsShMirrorFacets", { + digestId: canonical.digestId, + externalId: canonical.externalId, + kind: "topic", + term: "code review", + active: true, + installs: canonical.installs, + createdAt: canonical.createdAt, + updatedAt: canonical.updatedAt, + }); + }); + const replay = await startRun(t, "snapshot:topic-label-replay", 1); + await processBatch(t, { + runId: replay.runId, + page: 0, + offset: 0, + pageLength: 1, + hasMore: false, + sourceTotal: 1, + sourceRequests: 3, + sourceBytes: 1_024, + rows: [topicRow], + }); + expect( + await t.run(async (ctx) => + (await ctx.db.query("skillsShMirrorFacets").collect()) + .filter((facet) => facet.kind === "topic" && facet.active) + .map((facet) => facet.term) + .sort(), + ), + ).toEqual(["code-review", "股票分析"]); + }); + + it("serves bounded active exact, prefix, first-token, and full-text recall", async () => { + useTestEnvironment(); + const t = convexTest(schema, modules); + await configure(t); + const { runId } = await startRun(t, "snapshot:search"); + await processBatch(t, { + runId, + page: 0, + offset: 0, + pageLength: 2, + hasMore: false, + sourceTotal: 2, + sourceRequests: 5, + sourceBytes: 1_024, + rows: [githubRow, wellKnownRow], + }); + + const externalIds = (rows: Doc<"skillsShMirrorDigests">[]) => rows.map((row) => row.externalId); + expect( + externalIds( + await t.query(internal.skillsShMirror.listActiveByNormalizedSlugInternal, { + value: "find-skills", + limit: 10, + }), + ), + ).toEqual([githubRow.externalId]); + expect( + externalIds( + await t.query(internal.skillsShMirror.listActiveByNormalizedDisplayNameInternal, { + value: "find skills", + limit: 10, + }), + ), + ).toEqual([githubRow.externalId]); + expect( + externalIds( + await t.query(internal.skillsShMirror.listActiveByNormalizedSlugPrefixInternal, { + prefix: "find", + limit: 10, + }), + ), + ).toEqual([githubRow.externalId]); + expect( + externalIds( + await t.query(internal.skillsShMirror.listActiveByNormalizedDisplayNamePrefixInternal, { + prefix: "find", + limit: 10, + }), + ), + ).toEqual([githubRow.externalId]); + expect( + externalIds( + await t.query(internal.skillsShMirror.listActiveByNormalizedSlugFirstTokenPrefixInternal, { + prefix: "fi", + limit: 10, + }), + ), + ).toEqual([githubRow.externalId]); + expect( + externalIds( + await t.query( + internal.skillsShMirror.listActiveByNormalizedDisplayNameFirstTokenPrefixInternal, + { + prefix: "fi", + limit: 10, + }, + ), + ), + ).toEqual([githubRow.externalId]); + + const fullText = (await t.query(internal.skillsShMirror.searchActiveBySearchTextInternal, { + query: "vercel find", + limit: 10, + })) as Doc<"skillsShMirrorDigests">[]; + expect(fullText.map((row) => row.externalId)).toEqual([githubRow.externalId]); + const byOwner = await t.query(internal.skillsShMirror.listActiveGithubByOwnerInternal, { + owner: " VERCEL-LABS ", + paginationOpts: { cursor: null, numItems: 10 }, + }); + expect(byOwner.page.map((row) => row.externalId)).toEqual([githubRow.externalId]); + expect(byOwner.isDone).toBe(true); + const byCategory = await t.query(internal.skillsShMirror.listActiveByCategoryInternal, { + categorySlug: " DEVELOPMENT ", + paginationOpts: { cursor: null, numItems: 10 }, + }); + expect(byCategory.page.map((row) => row.externalId)).toEqual([githubRow.externalId]); + const byTopic = await t.query(internal.skillsShMirror.listActiveByTopicInternal, { + topic: "skill-discovery", + paginationOpts: { cursor: null, numItems: 10 }, + }); + expect(byTopic.page.map((row) => row.externalId)).toEqual([githubRow.externalId]); + const byPopularity = await t.query( + internal.skillsShMirror.listActiveByUpstreamInstallsInternal, + { limit: 10 }, + ); + expect(byPopularity.map((row) => row.externalId)).toEqual([ + githubRow.externalId, + wellKnownRow.externalId, + ]); + const classificationStates = await t.query( + internal.skillsShMirror.getClassificationStatesInternal, + { externalIds: [githubRow.externalId, "missing/repo/skill"] }, + ); + expect(classificationStates).toEqual([ + expect.objectContaining({ + externalId: githubRow.externalId, + sourceContentHash: githubRow.sourceContentHash, + inferredClassifierVersion: githubRow.inferredClassifierVersion, + }), + ]); + const replayRows = await t.query(internal.skillsShMirror.getReplayRowsInternal, { + externalIds: [githubRow.externalId], + }); + expect(replayRows).toEqual([ + { + digest: expect.objectContaining({ + externalId: githubRow.externalId, + active: true, + }), + detail: expect.objectContaining({ + externalId: githubRow.externalId, + content: githubRow.detail.content, + }), + }, + ]); + await expect( + t.query(internal.skillsShMirror.listActiveByNormalizedSlugPrefixInternal, { + prefix: "", + limit: 10, + }), + ).rejects.toThrow("prefix is required"); + }); + + it("records a quarantined source row and continues the batch cursor", async () => { + useTestEnvironment(); + const t = convexTest(schema, modules); + await configure(t); + const { runId } = await startRun(t, "snapshot:quarantine", 2); + + const result = await processBatch(t, { + runId, + page: 0, + offset: 0, + pageLength: 2, + hasMore: false, + sourceTotal: 2, + sourceRequests: 4, + sourceBytes: 2_048, + rows: [ + { + quarantined: true, + externalId: "larksuite/cli/lark-doc", + upstreamSourceType: "well-known", + reason: "identity-page-fetch-failed", + }, + githubRow, + ], + }); + + expect(result).toMatchObject({ + status: "reconciling", + page: 1, + offset: 0, + counts: { + observed: 2, + inserted: 1, + rejected: 1, + quarantined: 1, + scansPlanned: 0, + scansAdmitted: 0, + }, + }); + expect( + await t.run(async (ctx) => await ctx.db.query("skillsShMirrorConflicts").collect()), + ).toEqual([ + expect.objectContaining({ + externalId: "larksuite/cli/lark-doc", + kind: "source-quarantine", + reason: "identity-page-fetch-failed", + }), + ]); + expect( + await t.query(internal.skillsShMirror.getByExternalIdInternal, { + externalId: "larksuite/cli/lark-doc", + }), + ).toBeNull(); + expect(await t.query(internal.skillsShMirror.getStatusInternal, {})).toMatchObject({ + latestRunConflicts: [ + { + externalId: "larksuite/cli/lark-doc", + kind: "source-quarantine", + reason: "identity-page-fetch-failed", + }, + ], + }); + expect( + await t.query(internal.skillsShMirror.listConflictsByRunInternal, { + runId, + limit: 50, + }), + ).toEqual([ + expect.objectContaining({ + runId, + externalId: "larksuite/cli/lark-doc", + kind: "source-quarantine", + }), + ]); + }); + + it("removes stale detail when an available observation becomes missing before replay", async () => { + useTestEnvironment(); + const t = convexTest(schema, modules); + await configure(t); + const firstRun = await startRun(t, "snapshot:detail-available", 1); + await processBatch(t, { + runId: firstRun.runId, + page: 0, + offset: 0, + pageLength: 1, + hasMore: false, + sourceTotal: 1, + sourceRequests: 3, + sourceBytes: 1_024, + rows: [githubRow], + }); + await t.mutation(internal.skillsShMirror.reconcileBatchInternal, { + runId: firstRun.runId, + limit: 10, + }); + + const secondRun = await startRun(t, "snapshot:detail-missing", 1); + await processBatch(t, { + runId: secondRun.runId, + page: 0, + offset: 0, + pageLength: 1, + hasMore: false, + sourceTotal: 1, + sourceRequests: 3, + sourceBytes: 512, + rows: [{ ...githubRow, detail: undefined }], + }); + + expect( + await t.query(internal.skillsShMirror.getByExternalIdInternal, { + externalId: githubRow.externalId, + }), + ).toMatchObject({ detailStatus: "missing", lastObservedRunId: secondRun.runId }); + expect( + await t.query(internal.skillsShMirror.getDetailByExternalIdInternal, { + externalId: githubRow.externalId, + }), + ).toBeNull(); + expect( + await t.query(internal.skillsShMirror.getReplayRowsInternal, { + externalIds: [githubRow.externalId], + }), + ).toEqual([ + { + digest: expect.objectContaining({ detailStatus: "missing" }), + detail: null, + }, + ]); + }); + + it("preserves an existing digest when identity-page transport is quarantined", async () => { + useTestEnvironment(); + const t = convexTest(schema, modules); + await configure(t); + const firstRun = await startRun(t, "snapshot:before-transient-quarantine", 1); + await processBatch(t, { + runId: firstRun.runId, + page: 0, + offset: 0, + pageLength: 1, + hasMore: false, + sourceTotal: 1, + sourceRequests: 3, + sourceBytes: 1_024, + rows: [githubRow], + }); + await t.mutation(internal.skillsShMirror.reconcileBatchInternal, { + runId: firstRun.runId, + limit: 10, + }); + await t.run(async (ctx) => { + const existing = await ctx.db + .query("skillsShMirrorDigests") + .withIndex("by_external_id", (q) => q.eq("externalId", githubRow.externalId)) + .unique(); + expect(existing).not.toBeNull(); + await ctx.db.patch(existing!._id, { upstreamSourceType: undefined }); + }); + + const secondRun = await startRun(t, "snapshot:transient-quarantine", 1); + const result = await processBatch(t, { + runId: secondRun.runId, + page: 0, + offset: 0, + pageLength: 1, + hasMore: false, + sourceTotal: 1, + sourceRequests: 2, + sourceBytes: 1_024, + rows: [ + { + quarantined: true, + externalId: githubRow.externalId, + upstreamSourceType: "well-known", + reason: "identity-page-fetch-failed", + }, + ], + }); + await t.mutation(internal.skillsShMirror.reconcileBatchInternal, { + runId: secondRun.runId, + limit: 10, + }); + + expect(result.counts).toMatchObject({ + quarantined: 1, + quarantinedPreserved: 1, + tombstoned: 0, + }); + expect( + await t.query(internal.skillsShMirror.getByExternalIdInternal, { + externalId: githubRow.externalId, + }), + ).toMatchObject({ + active: true, + lastObservedRunId: secondRun.runId, + sourceFreshnessStatus: "stale", + staleQuarantineReason: "identity-page-fetch-failed", + upstreamSourceType: "well-known", + }); + expect( + await t.query(internal.skillsShMirror.getDetailByExternalIdInternal, { + externalId: githubRow.externalId, + }), + ).toMatchObject({ + lastObservedRunId: firstRun.runId, + }); + + const disappearanceRun = await startRun(t, "snapshot:disappearance-before-quarantine", 1); + await processBatch(t, { + runId: disappearanceRun.runId, + page: 0, + offset: 0, + pageLength: 1, + hasMore: false, + sourceTotal: 1, + sourceRequests: 3, + sourceBytes: 1_024, + rows: [wellKnownRow], + }); + await t.mutation(internal.skillsShMirror.reconcileBatchInternal, { + runId: disappearanceRun.runId, + limit: 10, + }); + + const inactiveQuarantineRun = await startRun(t, "snapshot:inactive-quarantine", 1); + const inactiveResult = await processBatch(t, { + runId: inactiveQuarantineRun.runId, + page: 0, + offset: 0, + pageLength: 1, + hasMore: false, + sourceTotal: 1, + sourceRequests: 2, + sourceBytes: 1_024, + rows: [ + { + quarantined: true, + externalId: githubRow.externalId, + upstreamSourceType: "well-known", + reason: "identity-page-fetch-failed", + }, + ], + }); + expect(inactiveResult.counts.quarantinedPreserved).toBe(0); + expect( + await t.query(internal.skillsShMirror.getByExternalIdInternal, { + externalId: githubRow.externalId, + }), + ).toMatchObject({ + active: false, + lastObservedRunId: secondRun.runId, + }); + }); + + it("replays a preserved stale digest as the same quarantine observation", async () => { + useTestEnvironment(); + const t = convexTest(schema, modules); + await configure(t); + const firstRun = await startRun(t, "snapshot:before-stale-replay", 1); + await processBatch(t, { + runId: firstRun.runId, + page: 0, + offset: 0, + pageLength: 1, + hasMore: false, + sourceTotal: 1, + sourceRequests: 3, + sourceBytes: 1_024, + rows: [githubRow], + }); + await t.mutation(internal.skillsShMirror.reconcileBatchInternal, { + runId: firstRun.runId, + limit: 10, + }); + + const quarantineRun = await startRun(t, "snapshot:stale-replay-source", 1); + await processBatch(t, { + runId: quarantineRun.runId, + page: 0, + offset: 0, + pageLength: 1, + hasMore: false, + sourceTotal: 1, + sourceRequests: 2, + sourceBytes: 1_024, + rows: [ + { + quarantined: true, + externalId: githubRow.externalId, + upstreamSourceType: "well-known", + reason: "identity-page-fetch-failed", + }, + ], + }); + await t.mutation(internal.skillsShMirror.reconcileBatchInternal, { + runId: quarantineRun.runId, + limit: 10, + }); + const [replayRow] = await t.query(internal.skillsShMirror.getReplayRowsInternal, { + externalIds: [githubRow.externalId], + }); + + expect(replayRow).toEqual({ + quarantined: true, + externalId: githubRow.externalId, + upstreamSourceType: "well-known", + reason: "identity-page-fetch-failed", + }); + if (!replayRow || replayRow.quarantined !== true) { + throw new Error("stale replay row was not quarantined"); + } + + const replayRun = await startRun(t, "snapshot:stale-replay", 1); + const replayResult = await processBatch(t, { + runId: replayRun.runId, + page: 0, + offset: 0, + pageLength: 1, + hasMore: false, + sourceTotal: 1, + sourceRequests: 0, + sourceBytes: 0, + rows: [replayRow], + }); + await t.mutation(internal.skillsShMirror.reconcileBatchInternal, { + runId: replayRun.runId, + limit: 10, + }); + + expect(replayResult.counts).toMatchObject({ + rejected: 1, + quarantined: 1, + quarantinedPreserved: 1, + }); + expect( + await t.query(internal.skillsShMirror.getByExternalIdInternal, { + externalId: githubRow.externalId, + }), + ).toMatchObject({ + active: true, + lastObservedRunId: replayRun.runId, + sourceFreshnessStatus: "stale", + staleQuarantineReason: "identity-page-fetch-failed", + }); + expect( + await t.query(internal.skillsShMirror.getDetailByExternalIdInternal, { + externalId: githubRow.externalId, + }), + ).toMatchObject({ + lastObservedRunId: firstRun.runId, + content: githubRow.detail.content, + }); + }); + + it("keeps a successful same-run observation authoritative over a later quarantine", async () => { + useTestEnvironment(); + const t = convexTest(schema, modules); + await configure(t); + const { runId } = await startRun(t, "snapshot:same-run-quarantine", 2); + + const result = await processBatch(t, { + runId, + page: 0, + offset: 0, + pageLength: 2, + hasMore: false, + sourceTotal: 2, + sourceRequests: 4, + sourceBytes: 2_048, + rows: [ + githubRow, + { + quarantined: true, + externalId: githubRow.externalId, + upstreamSourceType: "well-known", + reason: "identity-page-http-404", + }, + ], + }); + + expect(result.counts).toMatchObject({ + inserted: 1, + quarantined: 1, + quarantinedPreserved: 0, + }); + expect( + await t.query(internal.skillsShMirror.getByExternalIdInternal, { + externalId: githubRow.externalId, + }), + ).toMatchObject({ + active: true, + lastObservedRunId: runId, + sourceFreshnessStatus: "observed-only", + }); + }); + + it("accepts a valid observation after preserving an earlier-run quarantined digest", async () => { + useTestEnvironment(); + const t = convexTest(schema, modules); + await configure(t); + const firstRun = await startRun(t, "snapshot:before-quarantine-first", 1); + await processBatch(t, { + runId: firstRun.runId, + page: 0, + offset: 0, + pageLength: 1, + hasMore: false, + sourceTotal: 1, + sourceRequests: 3, + sourceBytes: 1_024, + rows: [githubRow], + }); + await t.mutation(internal.skillsShMirror.reconcileBatchInternal, { + runId: firstRun.runId, + limit: 10, + }); + + const secondRun = await startRun(t, "snapshot:quarantine-first", 2); + const result = await processBatch(t, { + runId: secondRun.runId, + page: 0, + offset: 0, + pageLength: 2, + hasMore: false, + sourceTotal: 2, + sourceRequests: 4, + sourceBytes: 2_048, + rows: [ + { + quarantined: true, + externalId: githubRow.externalId, + upstreamSourceType: "well-known", + reason: "identity-page-fetch-failed", + }, + { ...githubRow, upstreamInstalls: githubRow.upstreamInstalls + 1 }, + ], + }); + + expect(result.counts).toMatchObject({ + updated: 1, + quarantined: 1, + quarantinedPreserved: 0, + }); + expect( + await t.query(internal.skillsShMirror.getByExternalIdInternal, { + externalId: githubRow.externalId, + }), + ).toMatchObject({ + active: true, + lastObservedRunId: secondRun.runId, + sourceFreshnessStatus: "observed-only", + upstreamInstalls: githubRow.upstreamInstalls + 1, + }); + }); + + it("pauses and resumes from the exact page and offset", async () => { + useTestEnvironment(); + const t = convexTest(schema, modules); + await configure(t); + const { runId } = await startRun(t, "snapshot:pause", 3); + + await processBatch(t, { + runId, + page: 0, + offset: 0, + pageLength: 3, + hasMore: false, + sourceTotal: 3, + sourceRequests: 2, + sourceBytes: 512, + rows: [githubRow], + }); + await t.mutation(internal.skillsShMirror.setPausedInternal, { + runId, + paused: true, + actor: "codex-test", + reason: "prove pause", + confirm: "set-skills-sh-mirror-pause", + }); + await expect( + processBatch(t, { + runId, + page: 0, + offset: 1, + pageLength: 3, + hasMore: false, + sourceTotal: 3, + sourceRequests: 2, + sourceBytes: 512, + rows: [wellKnownRow], + }), + ).rejects.toThrow("paused"); + await t.mutation(internal.skillsShMirror.setPausedInternal, { + runId, + paused: false, + actor: "codex-test", + reason: "resume exact cursor", + confirm: "set-skills-sh-mirror-pause", + }); + const resumed = await processBatch(t, { + runId, + page: 0, + offset: 1, + pageLength: 3, + hasMore: false, + sourceTotal: 3, + sourceRequests: 2, + sourceBytes: 512, + rows: [wellKnownRow, { ...githubRow, externalId: "vercel-labs/skills/other", slug: "other" }], + }); + expect(resumed).toMatchObject({ status: "reconciling", page: 1, offset: 0 }); + }); + + it("records conflicting same-run observations instead of overwriting them", async () => { + useTestEnvironment(); + const t = convexTest(schema, modules); + await configure(t); + const { runId } = await startRun(t, "snapshot:conflict", 2); + + await processBatch(t, { + runId, + page: 0, + offset: 0, + pageLength: 2, + hasMore: false, + sourceTotal: 2, + sourceRequests: 2, + sourceBytes: 512, + rows: [githubRow], + }); + const conflicted = await processBatch(t, { + runId, + page: 0, + offset: 1, + pageLength: 2, + hasMore: false, + sourceTotal: 2, + sourceRequests: 2, + sourceBytes: 512, + rows: [{ ...githubRow, upstreamInstalls: 99 }], + }); + + expect(conflicted).toMatchObject({ + status: "reconciling", + counts: { observed: 2, conflicts: 1, rejected: 1 }, + }); + expect( + await t.run(async (ctx) => await ctx.db.query("skillsShMirrorConflicts").collect()), + ).toHaveLength(1); + }); + + it("tombstones disappeared rows and restores them on a later run", async () => { + useTestEnvironment(); + const t = convexTest(schema, modules); + await configure(t); + const first = await startRun(t, "snapshot:all"); + await processBatch(t, { + runId: first.runId, + page: 0, + offset: 0, + pageLength: 2, + hasMore: false, + sourceTotal: 2, + sourceRequests: 3, + sourceBytes: 1_024, + rows: [githubRow, wellKnownRow], + }); + await t.mutation(internal.skillsShMirror.reconcileBatchInternal, { + runId: first.runId, + limit: 100, + }); + + const second = await startRun(t, "snapshot:missing", 1); + await processBatch(t, { + runId: second.runId, + page: 0, + offset: 0, + pageLength: 1, + hasMore: false, + sourceTotal: 1, + sourceRequests: 2, + sourceBytes: 512, + rows: [githubRow], + }); + const reconciled = await t.mutation(internal.skillsShMirror.reconcileBatchInternal, { + runId: second.runId, + limit: 100, + }); + expect(reconciled).toMatchObject({ status: "completed", counts: { tombstoned: 1 } }); + const activeFacets = await t.query(internal.skillsShMirror.listFacetsPageInternal, { + cursor: null, + limit: 500, + }); + expect(activeFacets.page.every((facet) => facet.active)).toBe(true); + expect(activeFacets.page.some((facet) => facet.externalId === wellKnownRow.externalId)).toBe( + false, + ); + + const third = await startRun(t, "snapshot:return", 2); + await processBatch(t, { + runId: third.runId, + page: 0, + offset: 0, + pageLength: 2, + hasMore: false, + sourceTotal: 2, + sourceRequests: 3, + sourceBytes: 1_024, + rows: [githubRow, wellKnownRow], + }); + await t.mutation(internal.skillsShMirror.reconcileBatchInternal, { + runId: third.runId, + limit: 100, + }); + const restored = (await t.query(internal.skillsShMirror.getByExternalIdInternal, { + externalId: wellKnownRow.externalId, + })) as Doc<"skillsShMirrorDigests"> | null; + expect(restored).toMatchObject({ active: true }); + expect(restored).not.toHaveProperty("tombstonedAt"); + expect( + await t.query(internal.skillsShMirror.getRunInternal, { runId: third.runId }), + ).toMatchObject({ counts: { reactivated: 1 } }); + }); + + it("bounds detail proof pages below the worst-case response byte limit", async () => { + useTestEnvironment(); + const t = convexTest(schema, modules); + + await expect( + t.query(internal.skillsShMirror.listDetailsPageInternal, { + cursor: null, + limit: 50, + }), + ).resolves.toMatchObject({ page: [] }); + await expect( + t.query(internal.skillsShMirror.listDetailsPageInternal, { + cursor: null, + limit: 51, + }), + ).rejects.toThrow("limit must be an integer between 1 and 50"); + }); +}); diff --git a/convex/skillsShMirror.ts b/convex/skillsShMirror.ts new file mode 100644 index 00000000..1881644c --- /dev/null +++ b/convex/skillsShMirror.ts @@ -0,0 +1,1803 @@ +import { + getCatalogTopicSlugs, + normalizeCatalogTopic, + normalizeCatalogTopics, +} from "clawhub-schema"; +import { paginationOptsValidator } from "convex/server"; +import { ConvexError, type Infer, v } from "convex/values"; +import type { Doc, Id } from "./_generated/dataModel"; +import type { MutationCtx, QueryCtx } from "./_generated/server"; +import { internalMutation, internalQuery } from "./functions"; +import { tokenize } from "./lib/searchText"; +import { assertSkillsShFixtureEnvironmentAllowed } from "./lib/skillsShCatalogEnvironment"; + +const CONTROL_KEY = "global"; +const ENABLE_CONFIRM = "enable-skills-sh-mirror-test"; +const PAUSE_CONFIRM = "set-skills-sh-mirror-pause"; +const CANCEL_CONFIRM = "cancel-skills-sh-mirror-test-run"; +const MAX_ROWS_PER_RUN = 50_000; +const MAX_ROWS_PER_BATCH = 50; +const MAX_DETAIL_BYTES = 64 * 1024; +const MAX_DETAIL_PAGE_ROWS = 50; +const MAX_RECONCILE_ROWS = 250; +const MAX_SOURCE_ATTEMPTS = 4; +// Accounting includes consumed upstream bodies, including at most one bounded +// 8 MiB GitHub tree per distinct repository. Retryable API/identity responses +// and non-OK tree responses are canceled unread, so retries do not multiply +// tree bytes. This is not the Convex mutation payload size. +const MAX_ACCOUNTED_SOURCE_BYTES_PER_BATCH = 512 * 1024 * 1024; +const MAX_SEARCH_ROWS = 50; +const MAX_SCANNER_STATUS_LENGTH = 32; +const MAX_SCANNER_URL_LENGTH = 2_048; +const MAX_UPSTREAM_SOURCE_TYPE_LENGTH = 64; +const MAX_QUARANTINE_REASON_LENGTH = 64; +const MAX_INFERRED_CATEGORIES = 3; +const MAX_INFERRED_TOPICS = 5; +const MAX_INFERENCE_METADATA_LENGTH = 128; +const BATCH_LEASE_DURATION_MS = 5 * 60 * 1_000; +const MAX_BATCH_LEASE_TOKEN_LENGTH = 128; +const SKILLS_SH_PROOF_SNAPSHOT_PREFIX = "skills-sh:proof:"; +const PRESERVE_EXISTING_QUARANTINE_REASONS = new Set(["identity-page-fetch-failed"]); + +const detailValidator = v.object({ + contentKind: v.union(v.literal("skill-md"), v.literal("readme")), + path: v.string(), + content: v.string(), + contentBytes: v.number(), + sourceBytes: v.number(), + sourceFileCount: v.number(), + truncated: v.boolean(), +}); + +const upstreamScannerValidator = v.object({ + status: v.string(), + sourceCheckedAt: v.optional(v.string()), + sourceUrl: v.optional(v.string()), +}); + +const upstreamScannersValidator = v.object({ + genAgentTrustHub: upstreamScannerValidator, + socket: upstreamScannerValidator, + snyk: upstreamScannerValidator, +}); + +const classificationConfidenceValidator = v.union( + v.literal("high"), + v.literal("medium"), + v.literal("low"), +); + +const sourceListRowValidator = v.object({ + id: v.string(), + installUrl: v.union(v.string(), v.null()), + installs: v.number(), + name: v.string(), + slug: v.string(), + source: v.string(), + sourceType: v.string(), + url: v.string(), +}); + +const rowValidator = v.object({ + externalId: v.string(), + sourceType: v.union(v.literal("github"), v.literal("well-known")), + upstreamSourceType: v.string(), + owner: v.optional(v.string()), + repo: v.optional(v.string()), + sourceHost: v.optional(v.string()), + slug: v.string(), + displayName: v.string(), + sourceUrl: v.string(), + canonicalRepoUrl: v.optional(v.string()), + githubPath: v.optional(v.string()), + githubCommit: v.optional(v.string()), + sourceContentHash: v.optional(v.string()), + upstreamInstalls: v.number(), + upstreamScanners: upstreamScannersValidator, + inferredCategories: v.array(v.string()), + inferredTopics: v.array(v.string()), + inferredCategoryConfidence: classificationConfidenceValidator, + inferredTopicConfidence: classificationConfidenceValidator, + inferredClassifierVersion: v.string(), + inferredTopicClassifierVersion: v.string(), + inferredInputHash: v.string(), + inferredTopicInputHash: v.string(), + inferredAt: v.number(), + detail: v.optional(detailValidator), +}); + +type MirrorRow = Infer; + +const quarantinedRowValidator = v.object({ + quarantined: v.literal(true), + externalId: v.string(), + upstreamSourceType: v.string(), + reason: v.string(), +}); + +type QuarantinedRow = Infer; +type BatchRow = MirrorRow | QuarantinedRow; + +function assertIntegerInRange(name: string, value: number, min: number, max: number) { + if (!Number.isInteger(value) || value < min || value > max) { + throw new ConvexError(`${name} must be an integer between ${min} and ${max}`); + } +} + +function normalizedSourceSnapshotHash(value: string) { + const normalized = value.trim().toLowerCase(); + if (!/^[a-f0-9]{64}$/.test(normalized)) { + throw new ConvexError("sourceSnapshotHash must be a SHA-256 hex digest"); + } + return normalized; +} + +async function sha256Hex(value: string) { + const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(value)); + let hex = ""; + for (const byte of new Uint8Array(digest)) { + hex += byte.toString(16).padStart(2, "0"); + } + return hex; +} + +function compactSourceSnapshotId(snapshotId: string) { + if (!snapshotId.startsWith(SKILLS_SH_PROOF_SNAPSHOT_PREFIX)) return snapshotId; + const evidenceHashSeparator = snapshotId.indexOf(".", SKILLS_SH_PROOF_SNAPSHOT_PREFIX.length); + if (evidenceHashSeparator === -1) return snapshotId; + const evidenceSeparator = snapshotId.indexOf(".", evidenceHashSeparator + 1); + // The retained hash identifies the exact measured pagination evidence without copying it to rows. + return evidenceSeparator === -1 ? snapshotId : snapshotId.slice(0, evidenceSeparator); +} + +function emptyCounts(): Doc<"skillsShMirrorRuns">["counts"] { + return { + observed: 0, + inserted: 0, + updated: 0, + unchanged: 0, + rejected: 0, + quarantined: 0, + quarantinedPreserved: 0, + conflicts: 0, + detailsInserted: 0, + detailsUpdated: 0, + detailsUnchanged: 0, + detailsMissing: 0, + detailsTruncated: 0, + tombstoned: 0, + reactivated: 0, + scansPlanned: 0, + scansAdmitted: 0, + }; +} + +function addOperations( + current: Doc<"skillsShMirrorRuns">["operations"], + delta: Partial["operations"]>, +) { + return { + functionCalls: current.functionCalls + (delta.functionCalls ?? 0), + dbReads: current.dbReads + (delta.dbReads ?? 0), + dbWrites: current.dbWrites + (delta.dbWrites ?? 0), + sourceRequests: current.sourceRequests + (delta.sourceRequests ?? 0), + sourceBytes: current.sourceBytes + (delta.sourceBytes ?? 0), + }; +} + +async function getControl(ctx: Pick) { + return await ctx.db + .query("skillsShMirrorControls") + .withIndex("by_key", (q) => q.eq("key", CONTROL_KEY)) + .unique(); +} + +function requireActiveControl(control: Doc<"skillsShMirrorControls"> | null) { + if (!control?.enabled) throw new ConvexError("skills.sh mirror is disabled"); + if (control.paused) throw new ConvexError("skills.sh mirror is paused"); + return control; +} + +function normalizedLeaseToken(value: string) { + const token = value.trim(); + if (!token || token.length > MAX_BATCH_LEASE_TOKEN_LENGTH) { + throw new ConvexError( + `leaseToken must be between 1 and ${MAX_BATCH_LEASE_TOKEN_LENGTH} characters`, + ); + } + return token; +} + +function requireExactRunCursor(run: Doc<"skillsShMirrorRuns">, page: number, offset: number) { + if (page !== run.page || offset !== run.offset) { + throw new ConvexError(`skills.sh mirror cursor mismatch: expected ${run.page}:${run.offset}`); + } +} + +function normalizeRow(row: MirrorRow): MirrorRow { + const externalId = row.externalId.trim().toLowerCase(); + const slug = row.slug.trim().toLowerCase(); + return { + ...row, + externalId, + slug, + displayName: row.displayName.trim() || slug, + sourceUrl: row.sourceUrl.trim(), + owner: row.owner?.trim().toLowerCase(), + repo: row.repo?.trim().toLowerCase(), + sourceHost: row.sourceHost?.trim().toLowerCase(), + canonicalRepoUrl: row.canonicalRepoUrl?.trim(), + githubPath: row.githubPath?.trim(), + githubCommit: row.githubCommit?.trim().toLowerCase(), + sourceContentHash: row.sourceContentHash?.trim().toLowerCase(), + upstreamSourceType: row.upstreamSourceType.trim().toLowerCase(), + upstreamScanners: { + genAgentTrustHub: normalizeScanner(row.upstreamScanners.genAgentTrustHub), + socket: normalizeScanner(row.upstreamScanners.socket), + snyk: normalizeScanner(row.upstreamScanners.snyk), + }, + inferredCategories: row.inferredCategories.map(normalizedSearchText), + inferredTopics: row.inferredTopics.map(normalizedTopicLabel), + inferredClassifierVersion: row.inferredClassifierVersion.trim(), + inferredTopicClassifierVersion: row.inferredTopicClassifierVersion.trim(), + inferredInputHash: row.inferredInputHash.trim(), + inferredTopicInputHash: row.inferredTopicInputHash.trim(), + }; +} + +function normalizeQuarantinedRow(row: QuarantinedRow) { + return { + externalId: row.externalId.trim().toLowerCase(), + upstreamSourceType: row.upstreamSourceType.trim().toLowerCase(), + reason: row.reason.trim().toLowerCase(), + }; +} + +function normalizeScanner(scanner: MirrorRow["upstreamScanners"]["socket"]) { + return { + status: scanner.status.trim().toLowerCase(), + ...(scanner.sourceCheckedAt ? { sourceCheckedAt: scanner.sourceCheckedAt.trim() } : {}), + ...(scanner.sourceUrl ? { sourceUrl: scanner.sourceUrl.trim() } : {}), + }; +} + +function validScanner(scanner: MirrorRow["upstreamScanners"]["socket"]) { + if ( + !scanner.status || + scanner.status.length > MAX_SCANNER_STATUS_LENGTH || + !/^[a-z0-9][a-z0-9-]*$/.test(scanner.status) + ) { + return false; + } + if (scanner.sourceCheckedAt !== undefined && Number.isNaN(Date.parse(scanner.sourceCheckedAt))) { + return false; + } + if (scanner.sourceUrl !== undefined) { + if (scanner.sourceUrl.length > MAX_SCANNER_URL_LENGTH) return false; + try { + const url = new URL(scanner.sourceUrl); + if (url.protocol !== "https:" || !["skills.sh", "www.skills.sh"].includes(url.hostname)) { + return false; + } + } catch { + return false; + } + } + return true; +} + +function normalizedSearchText(value: string) { + return value.trim().toLowerCase(); +} + +function normalizedTopicLabel(value: string) { + return value.normalize("NFKC").trim().replace(/\s+/g, " "); +} + +function firstSearchToken(value: string) { + return tokenize(value)[0] ?? normalizedSearchText(value); +} + +function requiredSearchValue(name: string, value: string) { + const normalized = normalizedSearchText(value); + if (!normalized) throw new ConvexError(`${name} is required`); + return normalized; +} + +function searchLimit(limit: number) { + assertIntegerInRange("limit", limit, 1, MAX_SEARCH_ROWS); + return limit; +} + +function prefixUpperBound(prefix: string) { + return `${prefix}\uffff`; +} + +function searchFields(row: MirrorRow) { + const normalizedSlug = normalizedSearchText(row.slug); + const normalizedDisplayName = normalizedSearchText(row.displayName); + return { + normalizedSlug, + normalizedSlugFirstToken: firstSearchToken(row.slug), + normalizedDisplayName, + normalizedDisplayNameFirstToken: firstSearchToken(row.displayName), + searchText: [ + row.displayName, + row.slug, + row.owner, + row.repo, + row.sourceHost, + ...row.inferredCategories, + ...row.inferredTopics, + ] + .filter((value): value is string => Boolean(value)) + .join(" "), + }; +} + +function validCategoryInferenceTerms(values: string[], min: number, max: number) { + return ( + values.length >= min && + values.length <= max && + new Set(values).size === values.length && + values.every( + (value) => value.length > 0 && value.length <= 64 && /^[a-z0-9][a-z0-9-]*$/.test(value), + ) + ); +} + +function validTopicInferenceTerms(values: string[]) { + if (values.length > MAX_INFERRED_TOPICS) return false; + try { + const normalized = normalizeCatalogTopics(values); + return ( + normalized.length === values.length && + normalized.every((value, index) => value === values[index]) + ); + } catch { + return false; + } +} + +function validInference(row: MirrorRow) { + return ( + validCategoryInferenceTerms(row.inferredCategories, 1, MAX_INFERRED_CATEGORIES) && + validTopicInferenceTerms(row.inferredTopics) && + [ + row.inferredClassifierVersion, + row.inferredTopicClassifierVersion, + row.inferredInputHash, + row.inferredTopicInputHash, + ].every((value) => value.length > 0 && value.length <= MAX_INFERENCE_METADATA_LENGTH) && + Number.isSafeInteger(row.inferredAt) && + row.inferredAt > 0 + ); +} + +function validIdentity(row: MirrorRow) { + if ( + !row.externalId || + !row.slug || + !row.sourceUrl || + !row.upstreamSourceType || + row.upstreamSourceType.length > MAX_UPSTREAM_SOURCE_TYPE_LENGTH || + !/^[a-z0-9][a-z0-9._-]*$/.test(row.upstreamSourceType) + ) { + return false; + } + if (row.sourceType === "github") { + return ( + Boolean(row.owner && row.repo && row.canonicalRepoUrl) && + !row.sourceHost && + row.externalId === `${row.owner}/${row.repo}/${row.slug}` + ); + } + return ( + Boolean(row.sourceHost) && + !row.owner && + !row.repo && + !row.canonicalRepoUrl && + row.externalId === `${row.sourceHost}/${row.slug}` + ); +} + +function observationFingerprint(row: MirrorRow) { + return JSON.stringify({ + externalId: row.externalId, + sourceType: row.sourceType, + upstreamSourceType: row.upstreamSourceType, + owner: row.owner ?? null, + repo: row.repo ?? null, + sourceHost: row.sourceHost ?? null, + slug: row.slug, + displayName: row.displayName, + sourceUrl: row.sourceUrl, + canonicalRepoUrl: row.canonicalRepoUrl ?? null, + githubPath: row.githubPath ?? null, + githubCommit: row.githubCommit ?? null, + sourceContentHash: row.sourceContentHash ?? null, + upstreamInstalls: row.upstreamInstalls, + upstreamScanners: row.upstreamScanners, + inferredCategories: row.inferredCategories, + inferredTopics: row.inferredTopics, + inferredCategoryConfidence: row.inferredCategoryConfidence, + inferredTopicConfidence: row.inferredTopicConfidence, + inferredClassifierVersion: row.inferredClassifierVersion, + inferredTopicClassifierVersion: row.inferredTopicClassifierVersion, + inferredInputHash: row.inferredInputHash, + inferredTopicInputHash: row.inferredTopicInputHash, + detail: row.detail ?? null, + }); +} + +function sameDetail(detail: Doc<"skillsShMirrorDetails">, row: MirrorRow) { + const next = row.detail; + return ( + next !== undefined && + detail.contentKind === next.contentKind && + detail.path === next.path && + detail.content === next.content && + detail.contentBytes === next.contentBytes && + detail.sourceBytes === next.sourceBytes && + detail.sourceFileCount === next.sourceFileCount && + detail.truncated === next.truncated && + detail.sourceContentHash === row.sourceContentHash + ); +} + +async function syncFacets( + ctx: MutationCtx, + digestId: Id<"skillsShMirrorDigests">, + row: MirrorRow, + now: number, +) { + const desired = new Map( + [ + ...row.inferredCategories.map((term) => ({ + key: `category:${term}`, + kind: "category" as const, + term, + })), + // Full mirror replay migrates legacy topic-label facets by replacing them with canonical slugs. + ...getCatalogTopicSlugs(row.inferredTopics).map((term) => ({ + key: `topic:${term}`, + kind: "topic" as const, + term, + })), + ].map((facet) => [facet.key, facet]), + ); + const existing = await ctx.db + .query("skillsShMirrorFacets") + .withIndex("by_digest_id_and_kind_and_term", (q) => q.eq("digestId", digestId)) + .collect(); + let writes = 0; + for (const facet of existing) { + const key = `${facet.kind}:${facet.term}`; + if (!desired.delete(key)) { + if (facet.active) { + await ctx.db.patch(facet._id, { + active: false, + updatedAt: now, + }); + writes += 1; + } + continue; + } + if (!facet.active || facet.installs !== row.upstreamInstalls) { + await ctx.db.patch(facet._id, { + active: true, + installs: row.upstreamInstalls, + updatedAt: now, + }); + writes += 1; + } + } + for (const facet of desired.values()) { + await ctx.db.insert("skillsShMirrorFacets", { + digestId, + externalId: row.externalId, + kind: facet.kind, + term: facet.term, + active: true, + installs: row.upstreamInstalls, + createdAt: now, + updatedAt: now, + }); + writes += 1; + } + return { reads: existing.length + 1, writes }; +} + +function runCounts(counts: Doc<"skillsShMirrorRuns">["counts"]) { + return { + ...counts, + quarantined: counts.quarantined ?? 0, + quarantinedPreserved: counts.quarantinedPreserved ?? 0, + }; +} + +type SummarizableMirrorRun = Pick< + Doc<"skillsShMirrorRuns">, + | "_id" + | "snapshotId" + | "sourceSnapshotHash" + | "sourceCaptureWrites" + | "status" + | "sourceTotal" + | "sourcePageSize" + | "sourceMeasuredAt" + | "page" + | "offset" + | "counts" + | "operations" + | "startedAt" + | "completedAt" + | "updatedAt" +>; + +function summarizeRun(run: SummarizableMirrorRun) { + return { + runId: run._id, + snapshotId: run.snapshotId, + sourceSnapshotHash: run.sourceSnapshotHash ?? null, + sourceCaptureWrites: run.sourceCaptureWrites ?? 0, + status: run.status, + sourceTotal: run.sourceTotal, + sourcePageSize: run.sourcePageSize, + sourceMeasuredAt: run.sourceMeasuredAt, + page: run.page, + offset: run.offset, + counts: runCounts(run.counts), + operations: run.operations, + startedAt: run.startedAt, + completedAt: run.completedAt ?? null, + updatedAt: run.updatedAt, + }; +} + +export const configureInternal = internalMutation({ + args: { + actor: v.string(), + reason: v.string(), + confirm: v.string(), + enabled: v.boolean(), + maxRowsPerRun: v.number(), + maxRowsPerBatch: v.number(), + maxDetailBytes: v.number(), + }, + handler: async (ctx, args) => { + assertSkillsShFixtureEnvironmentAllowed(); + if (args.confirm !== ENABLE_CONFIRM) { + throw new ConvexError(`Pass confirm="${ENABLE_CONFIRM}" to configure the mirror.`); + } + assertIntegerInRange("maxRowsPerRun", args.maxRowsPerRun, 1, MAX_ROWS_PER_RUN); + assertIntegerInRange("maxRowsPerBatch", args.maxRowsPerBatch, 1, MAX_ROWS_PER_BATCH); + assertIntegerInRange("maxDetailBytes", args.maxDetailBytes, 1, MAX_DETAIL_BYTES); + const now = Date.now(); + const existing = await getControl(ctx); + const next = { + enabled: args.enabled, + paused: !args.enabled, + maxRowsPerRun: args.maxRowsPerRun, + maxRowsPerBatch: args.maxRowsPerBatch, + maxDetailBytes: args.maxDetailBytes, + updatedBy: args.actor.trim(), + reason: args.reason.trim(), + updatedAt: now, + }; + if (existing) await ctx.db.patch(existing._id, next); + else await ctx.db.insert("skillsShMirrorControls", { key: CONTROL_KEY, ...next }); + return { + ...next, + environment: assertSkillsShFixtureEnvironmentAllowed().environment, + publicVisible: false as const, + installable: false as const, + scanPlanningEnabled: false as const, + scanAdmissionEnabled: false as const, + }; + }, +}); + +export const startRunInternal = internalMutation({ + args: { + actor: v.string(), + reason: v.string(), + snapshotId: v.string(), + sourceSnapshotHash: v.optional(v.string()), + sourceCaptureWrites: v.optional(v.number()), + sourceTotal: v.number(), + sourcePageSize: v.number(), + sourceMeasuredAt: v.string(), + }, + handler: async (ctx, args) => { + assertSkillsShFixtureEnvironmentAllowed(); + const control = requireActiveControl(await getControl(ctx)); + assertIntegerInRange("sourceTotal", args.sourceTotal, 1, control.maxRowsPerRun); + assertIntegerInRange("sourcePageSize", args.sourcePageSize, 1, 500); + if (Number.isNaN(Date.parse(args.sourceMeasuredAt))) { + throw new ConvexError("sourceMeasuredAt must be an ISO timestamp"); + } + assertIntegerInRange("sourceCaptureWrites", args.sourceCaptureWrites ?? 0, 0, 100); + const activeRuns = await ctx.db + .query("skillsShMirrorRuns") + .withIndex("by_started_at") + .order("desc") + .take(20); + if ( + activeRuns.some( + (run) => + run.status === "running" || run.status === "paused" || run.status === "reconciling", + ) + ) { + throw new ConvexError("skills.sh mirror already has an active run"); + } + const now = Date.now(); + const run = { + snapshotId: args.snapshotId.trim(), + ...(args.sourceSnapshotHash + ? { sourceSnapshotHash: normalizedSourceSnapshotHash(args.sourceSnapshotHash) } + : {}), + sourceCaptureWrites: args.sourceCaptureWrites ?? 0, + status: "running" as const, + sourceTotal: args.sourceTotal, + sourcePageSize: args.sourcePageSize, + sourceMeasuredAt: args.sourceMeasuredAt, + page: 0, + offset: 0, + counts: emptyCounts(), + operations: { + functionCalls: 1, + dbReads: 2, + dbWrites: 1, + sourceRequests: 0, + sourceBytes: 0, + }, + actor: args.actor.trim(), + reason: args.reason.trim(), + startedAt: now, + updatedAt: now, + }; + const runId = await ctx.db.insert("skillsShMirrorRuns", run); + return summarizeRun({ _id: runId, ...run }); + }, +}); + +export const storeSourcePageInternal = internalMutation({ + args: { + snapshotHash: v.string(), + page: v.number(), + sourceTotal: v.number(), + pageLength: v.number(), + hasMore: v.boolean(), + identityHash: v.string(), + contentHash: v.string(), + sourceBytes: v.number(), + serializedBytes: v.number(), + rows: v.array(sourceListRowValidator), + }, + handler: async (ctx, args) => { + assertSkillsShFixtureEnvironmentAllowed(); + const control = requireActiveControl(await getControl(ctx)); + const snapshotHash = normalizedSourceSnapshotHash(args.snapshotHash); + const identityHash = normalizedSourceSnapshotHash(args.identityHash); + const contentHash = normalizedSourceSnapshotHash(args.contentHash); + assertIntegerInRange("page", args.page, 0, Math.ceil(control.maxRowsPerRun / 500) - 1); + assertIntegerInRange("sourceTotal", args.sourceTotal, 1, control.maxRowsPerRun); + assertIntegerInRange("pageLength", args.pageLength, 1, 500); + assertIntegerInRange("rows.length", args.rows.length, 1, 500); + assertIntegerInRange("sourceBytes", args.sourceBytes, 1, 1024 * 1024); + assertIntegerInRange("serializedBytes", args.serializedBytes, 1, 1024 * 1024); + if (args.rows.length !== args.pageLength) { + throw new ConvexError("captured skills.sh source page length mismatch"); + } + const [computedIdentityHash, computedContentHash] = await Promise.all([ + sha256Hex(args.rows.map((row) => `${row.id.trim().toLowerCase()}\n`).join("")), + sha256Hex(JSON.stringify(args.rows)), + ]); + if (computedIdentityHash !== identityHash) { + throw new ConvexError("captured skills.sh source page identity hash mismatch"); + } + if (computedContentHash !== contentHash) { + throw new ConvexError("captured skills.sh source page content hash mismatch"); + } + const existing = await ctx.db + .query("skillsShMirrorSourcePages") + .withIndex("by_snapshot_hash_and_page", (q) => + q.eq("snapshotHash", snapshotHash).eq("page", args.page), + ) + .unique(); + const value = { + snapshotHash, + page: args.page, + sourceTotal: args.sourceTotal, + pageLength: args.pageLength, + hasMore: args.hasMore, + identityHash, + contentHash, + sourceBytes: args.sourceBytes, + serializedBytes: args.serializedBytes, + rows: args.rows, + }; + if (existing) { + const comparable = { + snapshotHash: existing.snapshotHash, + page: existing.page, + sourceTotal: existing.sourceTotal, + pageLength: existing.pageLength, + hasMore: existing.hasMore, + identityHash: existing.identityHash, + contentHash: existing.contentHash, + sourceBytes: existing.sourceBytes, + serializedBytes: existing.serializedBytes, + rows: existing.rows, + }; + if (JSON.stringify(comparable) !== JSON.stringify(value)) { + throw new ConvexError("captured skills.sh source page is immutable"); + } + return { stored: false as const, page: existing.page, rows: existing.pageLength }; + } + await ctx.db.insert("skillsShMirrorSourcePages", { + ...value, + createdAt: Date.now(), + }); + return { stored: true as const, page: args.page, rows: args.pageLength }; + }, +}); + +export const getSourceCaptureSummaryInternal = internalQuery({ + args: { snapshotHash: v.string() }, + handler: async (ctx, args) => { + assertSkillsShFixtureEnvironmentAllowed(); + const snapshotHash = normalizedSourceSnapshotHash(args.snapshotHash); + const pages = await ctx.db + .query("skillsShMirrorSourcePages") + .withIndex("by_snapshot_hash_and_page", (q) => q.eq("snapshotHash", snapshotHash)) + .take(Math.ceil(MAX_ROWS_PER_RUN / 500) + 1); + if (pages.length > Math.ceil(MAX_ROWS_PER_RUN / 500)) { + throw new ConvexError("captured skills.sh source exceeds the page limit"); + } + return { + snapshotHash, + pageDocuments: pages.length, + rows: pages.reduce((sum, page) => sum + page.pageLength, 0), + sourceBytes: pages.reduce((sum, page) => sum + page.sourceBytes, 0), + serializedBytes: pages.reduce((sum, page) => sum + page.serializedBytes, 0), + }; + }, +}); + +export const setPausedInternal = internalMutation({ + args: { + runId: v.id("skillsShMirrorRuns"), + paused: v.boolean(), + actor: v.string(), + reason: v.string(), + confirm: v.string(), + }, + handler: async (ctx, args) => { + assertSkillsShFixtureEnvironmentAllowed(); + if (args.confirm !== PAUSE_CONFIRM) { + throw new ConvexError(`Pass confirm="${PAUSE_CONFIRM}" to change mirror pause state.`); + } + const [control, run] = await Promise.all([getControl(ctx), ctx.db.get(args.runId)]); + if (!control || !run) throw new ConvexError("skills.sh mirror state not found"); + if (run.status !== "running" && run.status !== "paused") { + throw new ConvexError(`Cannot change pause state for ${run.status} run`); + } + if (!args.paused && !control.enabled) { + throw new ConvexError("skills.sh mirror is disabled"); + } + const now = Date.now(); + await ctx.db.patch(control._id, { + paused: args.paused, + updatedBy: args.actor.trim(), + reason: args.reason.trim(), + updatedAt: now, + }); + await ctx.db.patch(run._id, { + status: args.paused ? "paused" : "running", + batchLeaseToken: undefined, + batchLeaseExpiresAt: undefined, + operations: addOperations(run.operations, { + functionCalls: 1, + dbReads: 2, + dbWrites: 2, + }), + updatedAt: now, + }); + return { runId: run._id, status: args.paused ? ("paused" as const) : ("running" as const) }; + }, +}); + +export const cancelRunInternal = internalMutation({ + args: { + runId: v.id("skillsShMirrorRuns"), + actor: v.string(), + reason: v.string(), + confirm: v.string(), + }, + handler: async (ctx, args) => { + assertSkillsShFixtureEnvironmentAllowed(); + if (args.confirm !== CANCEL_CONFIRM) { + throw new ConvexError(`Pass confirm="${CANCEL_CONFIRM}" to cancel a mirror run.`); + } + const run = await ctx.db.get(args.runId); + if (!run) throw new ConvexError("skills.sh mirror run not found"); + if (!["running", "paused", "reconciling"].includes(run.status)) { + throw new ConvexError(`Cannot cancel a ${run.status} skills.sh mirror run`); + } + const now = Date.now(); + const patch = { + status: "canceled" as const, + batchLeaseToken: undefined, + batchLeaseExpiresAt: undefined, + operations: addOperations(run.operations, { + functionCalls: 1, + dbReads: 1, + dbWrites: 1, + }), + actor: args.actor.trim(), + reason: args.reason.trim(), + completedAt: now, + updatedAt: now, + }; + await ctx.db.patch(run._id, patch); + return summarizeRun({ ...run, ...patch }); + }, +}); + +export const claimBatchLeaseInternal = internalMutation({ + args: { + runId: v.id("skillsShMirrorRuns"), + page: v.number(), + offset: v.number(), + leaseToken: v.string(), + }, + handler: async (ctx, args) => { + assertSkillsShFixtureEnvironmentAllowed(); + requireActiveControl(await getControl(ctx)); + const run = await ctx.db.get(args.runId); + if (!run) throw new ConvexError("skills.sh mirror run not found"); + if (run.status === "paused") throw new ConvexError("skills.sh mirror run is paused"); + if (run.status !== "running") { + throw new ConvexError(`Cannot lease a ${run.status} skills.sh mirror run`); + } + requireExactRunCursor(run, args.page, args.offset); + const leaseToken = normalizedLeaseToken(args.leaseToken); + const now = Date.now(); + if ( + run.batchLeaseToken && + run.batchLeaseExpiresAt !== undefined && + run.batchLeaseExpiresAt > now + ) { + if (run.batchLeaseToken !== leaseToken) { + throw new ConvexError( + `skills.sh mirror cursor ${run.page}:${run.offset} is already leased`, + ); + } + } + const leaseExpiresAt = now + BATCH_LEASE_DURATION_MS; + const sourcePage = run.sourceSnapshotHash + ? await ctx.db + .query("skillsShMirrorSourcePages") + .withIndex("by_snapshot_hash_and_page", (q) => + q.eq("snapshotHash", run.sourceSnapshotHash!).eq("page", run.page), + ) + .unique() + : null; + await ctx.db.patch(run._id, { + batchLeaseToken: leaseToken, + batchLeaseExpiresAt: leaseExpiresAt, + operations: addOperations(run.operations, { + functionCalls: 1, + dbReads: run.sourceSnapshotHash ? 3 : 2, + dbWrites: 1, + }), + updatedAt: now, + }); + return { + runId: run._id, + snapshotId: run.snapshotId, + sourceTotal: run.sourceTotal, + sourcePageSize: run.sourcePageSize, + sourcePage, + page: run.page, + offset: run.offset, + leaseToken, + leaseExpiresAt, + }; + }, +}); + +export const releaseBatchLeaseInternal = internalMutation({ + args: { + runId: v.id("skillsShMirrorRuns"), + page: v.number(), + offset: v.number(), + leaseToken: v.string(), + }, + handler: async (ctx, args) => { + assertSkillsShFixtureEnvironmentAllowed(); + const run = await ctx.db.get(args.runId); + if (!run) throw new ConvexError("skills.sh mirror run not found"); + requireExactRunCursor(run, args.page, args.offset); + const leaseToken = normalizedLeaseToken(args.leaseToken); + if (!run.batchLeaseToken) return { released: false as const }; + if (run.batchLeaseToken !== leaseToken) { + throw new ConvexError("skills.sh mirror lease token mismatch"); + } + const now = Date.now(); + await ctx.db.patch(run._id, { + batchLeaseToken: undefined, + batchLeaseExpiresAt: undefined, + operations: addOperations(run.operations, { + functionCalls: 1, + dbReads: 1, + dbWrites: 1, + }), + updatedAt: now, + }); + return { released: true as const }; + }, +}); + +export const processBatchInternal = internalMutation({ + args: { + runId: v.id("skillsShMirrorRuns"), + page: v.number(), + offset: v.number(), + leaseToken: v.string(), + pageLength: v.number(), + hasMore: v.boolean(), + sourceTotal: v.number(), + sourceRequests: v.number(), + sourceBytes: v.number(), + rows: v.array(v.union(rowValidator, quarantinedRowValidator)), + }, + handler: async (ctx, args) => { + assertSkillsShFixtureEnvironmentAllowed(); + const [controlDoc, run] = await Promise.all([getControl(ctx), ctx.db.get(args.runId)]); + const control = requireActiveControl(controlDoc); + if (!run) throw new ConvexError("skills.sh mirror run not found"); + if (run.status === "paused") throw new ConvexError("skills.sh mirror run is paused"); + if (run.status !== "running") return summarizeRun(run); + requireExactRunCursor(run, args.page, args.offset); + const leaseToken = normalizedLeaseToken(args.leaseToken); + if (run.batchLeaseToken !== leaseToken) { + throw new ConvexError("skills.sh mirror lease token mismatch"); + } + if (run.batchLeaseExpiresAt === undefined || run.batchLeaseExpiresAt <= Date.now()) { + throw new ConvexError("skills.sh mirror batch lease expired"); + } + if (args.sourceTotal !== run.sourceTotal) { + throw new ConvexError("skills.sh mirror source total changed during the run"); + } + assertIntegerInRange("page", args.page, 0, 100_000); + assertIntegerInRange("offset", args.offset, 0, run.sourcePageSize); + assertIntegerInRange("pageLength", args.pageLength, 1, run.sourcePageSize); + assertIntegerInRange("rows.length", args.rows.length, 1, control.maxRowsPerBatch); + assertIntegerInRange( + "sourceRequests", + args.sourceRequests, + 0, + MAX_SOURCE_ATTEMPTS * (1 + 5 * args.rows.length), + ); + assertIntegerInRange("sourceBytes", args.sourceBytes, 0, MAX_ACCOUNTED_SOURCE_BYTES_PER_BATCH); + if (args.offset + args.rows.length > args.pageLength) { + throw new ConvexError("skills.sh mirror batch exceeds the source page"); + } + + const counts = runCounts(run.counts); + let reads = 2; + let writes = 0; + const now = Date.now(); + const sourceSnapshotId = compactSourceSnapshotId(run.snapshotId); + for (let index = 0; index < args.rows.length; index += 1) { + const batchRow: BatchRow = args.rows[index]!; + counts.observed += 1; + if ("quarantined" in batchRow) { + const row = normalizeQuarantinedRow(batchRow); + if ( + !row.externalId || + row.externalId.length > 512 || + !row.upstreamSourceType || + row.upstreamSourceType.length > MAX_UPSTREAM_SOURCE_TYPE_LENGTH || + !/^[a-z0-9][a-z0-9._-]*$/.test(row.upstreamSourceType) || + !row.reason || + row.reason.length > MAX_QUARANTINE_REASON_LENGTH || + !/^[a-z0-9][a-z0-9-]*$/.test(row.reason) + ) { + throw new ConvexError("skills.sh mirror quarantine record is invalid"); + } + await ctx.db.insert("skillsShMirrorConflicts", { + runId: run._id, + externalId: row.externalId, + kind: "source-quarantine", + reason: row.reason, + upstreamSourceType: row.upstreamSourceType, + observedFingerprint: JSON.stringify(row), + page: args.page, + offset: args.offset + index, + createdAt: now, + }); + counts.rejected += 1; + counts.quarantined += 1; + counts.conflicts += 1; + writes += 1; + if (PRESERVE_EXISTING_QUARANTINE_REASONS.has(row.reason)) { + const existing = await ctx.db + .query("skillsShMirrorDigests") + .withIndex("by_external_id", (q) => q.eq("externalId", row.externalId)) + .unique(); + reads += 1; + if (existing?.active && existing.lastObservedRunId !== run._id) { + counts.quarantinedPreserved += 1; + await ctx.db.patch(existing._id, { + upstreamSourceType: row.upstreamSourceType, + lastObservedRunId: run._id, + sourceFreshnessStatus: "stale", + staleQuarantineReason: row.reason, + updatedAt: now, + }); + writes += 1; + } + } + continue; + } + const row = normalizeRow(batchRow); + const fingerprint = observationFingerprint(row); + if ( + !validIdentity(row) || + !validInference(row) || + !Number.isSafeInteger(row.upstreamInstalls) || + row.upstreamInstalls < 0 || + (row.sourceContentHash !== undefined && !/^[a-f0-9]{64}$/.test(row.sourceContentHash)) || + !validScanner(row.upstreamScanners.genAgentTrustHub) || + !validScanner(row.upstreamScanners.socket) || + !validScanner(row.upstreamScanners.snyk) || + (row.detail !== undefined && + (row.detail.contentBytes > control.maxDetailBytes || + new TextEncoder().encode(row.detail.content).byteLength !== row.detail.contentBytes)) + ) { + await ctx.db.insert("skillsShMirrorConflicts", { + runId: run._id, + externalId: row.externalId, + kind: "identity-mismatch", + observedFingerprint: fingerprint, + page: args.page, + offset: args.offset + index, + createdAt: now, + }); + counts.rejected += 1; + counts.conflicts += 1; + writes += 1; + continue; + } + + const existing = await ctx.db + .query("skillsShMirrorDigests") + .withIndex("by_external_id", (q) => q.eq("externalId", row.externalId)) + .unique(); + reads += 1; + if (existing?.lastObservedRunId === run._id && existing.sourceFreshnessStatus === "stale") { + counts.quarantinedPreserved = Math.max(0, counts.quarantinedPreserved - 1); + } + if ( + existing?.lastObservedRunId === run._id && + existing.sourceFreshnessStatus !== "stale" && + existing.observationFingerprint !== fingerprint + ) { + await ctx.db.insert("skillsShMirrorConflicts", { + runId: run._id, + externalId: row.externalId, + kind: "same-run-drift", + previousFingerprint: existing.observationFingerprint, + observedFingerprint: fingerprint, + page: args.page, + offset: args.offset + index, + createdAt: now, + }); + counts.rejected += 1; + counts.conflicts += 1; + writes += 1; + continue; + } + + let digestId: Id<"skillsShMirrorDigests">; + const normalizedSearchFields = searchFields(row); + if (!existing) { + digestId = await ctx.db.insert("skillsShMirrorDigests", { + externalId: row.externalId, + sourceType: row.sourceType, + upstreamSourceType: row.upstreamSourceType, + ...(row.owner ? { owner: row.owner } : {}), + ...(row.repo ? { repo: row.repo } : {}), + ...(row.sourceHost ? { sourceHost: row.sourceHost } : {}), + slug: row.slug, + ...normalizedSearchFields, + displayName: row.displayName, + sourceUrl: row.sourceUrl, + ...(row.canonicalRepoUrl ? { canonicalRepoUrl: row.canonicalRepoUrl } : {}), + ...(row.githubPath ? { githubPath: row.githubPath } : {}), + ...(row.githubCommit ? { githubCommit: row.githubCommit } : {}), + ...(row.sourceContentHash ? { sourceContentHash: row.sourceContentHash } : {}), + upstreamInstalls: row.upstreamInstalls, + upstreamScanners: row.upstreamScanners, + inferredCategories: row.inferredCategories, + inferredTopics: row.inferredTopics, + inferredCategoryConfidence: row.inferredCategoryConfidence, + inferredTopicConfidence: row.inferredTopicConfidence, + inferredClassifierVersion: row.inferredClassifierVersion, + inferredTopicClassifierVersion: row.inferredTopicClassifierVersion, + inferredInputHash: row.inferredInputHash, + inferredTopicInputHash: row.inferredTopicInputHash, + inferredAt: row.inferredAt, + sourceFreshnessStatus: "observed-only", + staleQuarantineReason: undefined, + detailStatus: row.detail ? "available" : "missing", + observationFingerprint: fingerprint, + sourceSnapshotId, + lastObservedRunId: run._id, + active: true, + publicVisible: false, + installable: false, + firstObservedAt: now, + lastObservedAt: now, + createdAt: now, + updatedAt: now, + }); + counts.inserted += 1; + writes += 1; + } else if ( + existing.observationFingerprint === fingerprint && + existing.lastObservedRunId === run._id && + existing.sourceFreshnessStatus === "observed-only" + ) { + digestId = existing._id; + counts.unchanged += 1; + } else { + digestId = existing._id; + if (existing.observationFingerprint === fingerprint) counts.unchanged += 1; + else counts.updated += 1; + if (!existing.active) counts.reactivated += 1; + await ctx.db.patch(existing._id, { + sourceType: row.sourceType, + upstreamSourceType: row.upstreamSourceType, + owner: row.owner, + repo: row.repo, + sourceHost: row.sourceHost, + slug: row.slug, + ...normalizedSearchFields, + displayName: row.displayName, + sourceUrl: row.sourceUrl, + canonicalRepoUrl: row.canonicalRepoUrl, + githubPath: row.githubPath, + githubCommit: row.githubCommit, + sourceContentHash: row.sourceContentHash, + upstreamInstalls: row.upstreamInstalls, + upstreamScanners: row.upstreamScanners, + inferredCategories: row.inferredCategories, + inferredTopics: row.inferredTopics, + inferredCategoryConfidence: row.inferredCategoryConfidence, + inferredTopicConfidence: row.inferredTopicConfidence, + inferredClassifierVersion: row.inferredClassifierVersion, + inferredTopicClassifierVersion: row.inferredTopicClassifierVersion, + inferredInputHash: row.inferredInputHash, + inferredTopicInputHash: row.inferredTopicInputHash, + inferredAt: row.inferredAt, + sourceFreshnessStatus: "observed-only", + staleQuarantineReason: undefined, + detailStatus: row.detail ? "available" : "missing", + observationFingerprint: fingerprint, + sourceSnapshotId, + lastObservedRunId: run._id, + active: true, + publicVisible: false, + installable: false, + tombstonedAt: undefined, + lastObservedAt: now, + updatedAt: now, + }); + writes += 1; + } + + const facetOperations = await syncFacets(ctx, digestId, row, now); + reads += facetOperations.reads; + writes += facetOperations.writes; + + const existingDetail = await ctx.db + .query("skillsShMirrorDetails") + .withIndex("by_external_id", (q) => q.eq("externalId", row.externalId)) + .unique(); + reads += 1; + if (!row.detail) { + counts.detailsMissing += 1; + if (existingDetail) { + await ctx.db.delete(existingDetail._id); + writes += 1; + } + } else if (!existingDetail) { + await ctx.db.insert("skillsShMirrorDetails", { + externalId: row.externalId, + digestId, + ...row.detail, + ...(row.sourceContentHash ? { sourceContentHash: row.sourceContentHash } : {}), + sourceSnapshotId, + lastObservedRunId: run._id, + createdAt: now, + updatedAt: now, + }); + counts.detailsInserted += 1; + if (row.detail.truncated) counts.detailsTruncated += 1; + writes += 1; + } else if (sameDetail(existingDetail, row)) { + counts.detailsUnchanged += 1; + if (row.detail.truncated) counts.detailsTruncated += 1; + if (existingDetail.lastObservedRunId !== run._id) { + await ctx.db.patch(existingDetail._id, { + sourceSnapshotId, + lastObservedRunId: run._id, + updatedAt: now, + }); + writes += 1; + } + } else { + await ctx.db.patch(existingDetail._id, { + digestId, + ...row.detail, + sourceContentHash: row.sourceContentHash, + sourceSnapshotId, + lastObservedRunId: run._id, + updatedAt: now, + }); + counts.detailsUpdated += 1; + if (row.detail.truncated) counts.detailsTruncated += 1; + writes += 1; + } + } + + const nextOffset = args.offset + args.rows.length; + const pageComplete = nextOffset === args.pageLength; + const sourceComplete = pageComplete && !args.hasMore; + if (sourceComplete && counts.observed !== run.sourceTotal) { + throw new ConvexError( + `skills.sh mirror observed ${counts.observed} rows but source declared ${run.sourceTotal}`, + ); + } + const nextPage = pageComplete ? args.page + 1 : args.page; + const storedOffset = pageComplete ? 0 : nextOffset; + const patch = { + status: sourceComplete ? ("reconciling" as const) : ("running" as const), + page: nextPage, + offset: storedOffset, + batchLeaseToken: undefined, + batchLeaseExpiresAt: undefined, + counts, + operations: addOperations(run.operations, { + functionCalls: 1, + dbReads: reads, + dbWrites: writes + 1, + sourceRequests: args.sourceRequests, + sourceBytes: args.sourceBytes, + }), + updatedAt: now, + }; + await ctx.db.patch(run._id, patch); + return summarizeRun({ ...run, ...patch }); + }, +}); + +export const reconcileBatchInternal = internalMutation({ + args: { + runId: v.id("skillsShMirrorRuns"), + limit: v.number(), + }, + handler: async (ctx, args) => { + assertSkillsShFixtureEnvironmentAllowed(); + const run = await ctx.db.get(args.runId); + if (!run) throw new ConvexError("skills.sh mirror run not found"); + if (run.status !== "reconciling") return summarizeRun(run); + assertIntegerInRange("limit", args.limit, 1, MAX_RECONCILE_ROWS); + const page = await ctx.db + .query("skillsShMirrorDigests") + .withIndex("by_external_id") + .paginate({ cursor: run.reconcileCursor ?? null, numItems: args.limit }); + const counts = runCounts(run.counts); + const now = Date.now(); + let reads = page.page.length + 1; + let writes = 0; + for (const digest of page.page) { + if (digest.lastObservedRunId === run._id || !digest.active) continue; + await ctx.db.patch(digest._id, { + active: false, + publicVisible: false, + installable: false, + tombstonedAt: now, + updatedAt: now, + }); + counts.tombstoned += 1; + writes += 1; + const facets = await ctx.db + .query("skillsShMirrorFacets") + .withIndex("by_digest_id_and_kind_and_term", (q) => q.eq("digestId", digest._id)) + .collect(); + reads += facets.length + 1; + for (const facet of facets) { + if (!facet.active) continue; + await ctx.db.patch(facet._id, { + active: false, + updatedAt: now, + }); + writes += 1; + } + } + const completed = page.isDone; + const patch = { + status: completed ? ("completed" as const) : ("reconciling" as const), + reconcileCursor: completed ? undefined : page.continueCursor, + counts, + operations: addOperations(run.operations, { + functionCalls: 1, + dbReads: reads, + dbWrites: writes + 1, + }), + ...(completed ? { completedAt: now } : {}), + updatedAt: now, + }; + await ctx.db.patch(run._id, patch); + return summarizeRun({ ...run, ...patch }); + }, +}); + +export const getByExternalIdInternal = internalQuery({ + args: { + externalId: v.string(), + }, + handler: async (ctx, args) => { + return await ctx.db + .query("skillsShMirrorDigests") + .withIndex("by_external_id", (q) => q.eq("externalId", args.externalId.trim().toLowerCase())) + .unique(); + }, +}); + +export const getDetailByExternalIdInternal = internalQuery({ + args: { + externalId: v.string(), + }, + handler: async (ctx, args) => { + return await ctx.db + .query("skillsShMirrorDetails") + .withIndex("by_external_id", (q) => q.eq("externalId", args.externalId.trim().toLowerCase())) + .unique(); + }, +}); + +export const listActiveByNormalizedSlugInternal = internalQuery({ + args: { + value: v.string(), + limit: v.number(), + }, + handler: async (ctx, args) => { + const value = requiredSearchValue("value", args.value); + return await ctx.db + .query("skillsShMirrorDigests") + .withIndex("by_active_and_normalized_slug", (q) => + q.eq("active", true).eq("normalizedSlug", value), + ) + .take(searchLimit(args.limit)); + }, +}); + +export const listActiveByNormalizedDisplayNameInternal = internalQuery({ + args: { + value: v.string(), + limit: v.number(), + }, + handler: async (ctx, args) => { + const value = requiredSearchValue("value", args.value); + return await ctx.db + .query("skillsShMirrorDigests") + .withIndex("by_active_and_normalized_display_name", (q) => + q.eq("active", true).eq("normalizedDisplayName", value), + ) + .take(searchLimit(args.limit)); + }, +}); + +export const listActiveByNormalizedSlugPrefixInternal = internalQuery({ + args: { + prefix: v.string(), + limit: v.number(), + }, + handler: async (ctx, args) => { + const prefix = requiredSearchValue("prefix", args.prefix); + return await ctx.db + .query("skillsShMirrorDigests") + .withIndex("by_active_and_normalized_slug", (q) => + q + .eq("active", true) + .gte("normalizedSlug", prefix) + .lt("normalizedSlug", prefixUpperBound(prefix)), + ) + .take(searchLimit(args.limit)); + }, +}); + +export const listActiveByNormalizedDisplayNamePrefixInternal = internalQuery({ + args: { + prefix: v.string(), + limit: v.number(), + }, + handler: async (ctx, args) => { + const prefix = requiredSearchValue("prefix", args.prefix); + return await ctx.db + .query("skillsShMirrorDigests") + .withIndex("by_active_and_normalized_display_name", (q) => + q + .eq("active", true) + .gte("normalizedDisplayName", prefix) + .lt("normalizedDisplayName", prefixUpperBound(prefix)), + ) + .take(searchLimit(args.limit)); + }, +}); + +export const listActiveByNormalizedSlugFirstTokenPrefixInternal = internalQuery({ + args: { + prefix: v.string(), + limit: v.number(), + }, + handler: async (ctx, args) => { + const prefix = requiredSearchValue("prefix", args.prefix); + return await ctx.db + .query("skillsShMirrorDigests") + .withIndex("by_active_and_normalized_slug_first_token", (q) => + q + .eq("active", true) + .gte("normalizedSlugFirstToken", prefix) + .lt("normalizedSlugFirstToken", prefixUpperBound(prefix)), + ) + .take(searchLimit(args.limit)); + }, +}); + +export const listActiveByNormalizedDisplayNameFirstTokenPrefixInternal = internalQuery({ + args: { + prefix: v.string(), + limit: v.number(), + }, + handler: async (ctx, args) => { + const prefix = requiredSearchValue("prefix", args.prefix); + return await ctx.db + .query("skillsShMirrorDigests") + .withIndex("by_active_and_normalized_display_name_first_token", (q) => + q + .eq("active", true) + .gte("normalizedDisplayNameFirstToken", prefix) + .lt("normalizedDisplayNameFirstToken", prefixUpperBound(prefix)), + ) + .take(searchLimit(args.limit)); + }, +}); + +export const searchActiveBySearchTextInternal = internalQuery({ + args: { + query: v.string(), + limit: v.number(), + }, + handler: async (ctx, args) => { + const query = requiredSearchValue("query", args.query); + return await ctx.db + .query("skillsShMirrorDigests") + .withSearchIndex("search_by_search_text", (q) => + q.search("searchText", query).eq("active", true), + ) + .take(searchLimit(args.limit)); + }, +}); + +export const listActiveGithubByOwnerInternal = internalQuery({ + args: { + owner: v.string(), + paginationOpts: paginationOptsValidator, + }, + handler: async (ctx, args) => { + const owner = requiredSearchValue("owner", args.owner); + return await ctx.db + .query("skillsShMirrorDigests") + .withIndex("by_active_and_source_type_and_owner_and_repo_and_external_id", (q) => + q.eq("active", true).eq("sourceType", "github").eq("owner", owner), + ) + .paginate(args.paginationOpts); + }, +}); + +export const listActiveByUpstreamInstallsInternal = internalQuery({ + args: { + limit: v.number(), + }, + handler: async (ctx, args) => { + return await ctx.db + .query("skillsShMirrorDigests") + .withIndex("by_active_and_upstream_installs", (q) => q.eq("active", true)) + .order("desc") + .take(searchLimit(args.limit)); + }, +}); + +async function listActiveByFacet( + ctx: QueryCtx, + args: { + kind: "category" | "topic"; + term: string; + paginationOpts: Infer; + }, +) { + const facets = await ctx.db + .query("skillsShMirrorFacets") + .withIndex("by_active_and_kind_and_term_and_installs_and_external_id", (q) => + q.eq("active", true).eq("kind", args.kind).eq("term", args.term), + ) + .order("desc") + .paginate(args.paginationOpts); + const page = await Promise.all(facets.page.map((facet) => ctx.db.get(facet.digestId))); + if (page.some((digest) => !digest?.active)) { + throw new ConvexError("skills.sh mirror facet references an inactive or missing digest"); + } + return { + ...facets, + page: page as Doc<"skillsShMirrorDigests">[], + }; +} + +export const listActiveByCategoryInternal = internalQuery({ + args: { + categorySlug: v.string(), + paginationOpts: paginationOptsValidator, + }, + handler: async (ctx, args) => { + return await listActiveByFacet(ctx, { + kind: "category", + term: requiredSearchValue("categorySlug", args.categorySlug), + paginationOpts: args.paginationOpts, + }); + }, +}); + +export const listActiveByTopicInternal = internalQuery({ + args: { + topic: v.string(), + paginationOpts: paginationOptsValidator, + }, + handler: async (ctx, args) => { + const topic = normalizeCatalogTopic(args.topic); + if (!topic) throw new ConvexError("topic is required"); + return await listActiveByFacet(ctx, { + kind: "topic", + term: topic, + paginationOpts: args.paginationOpts, + }); + }, +}); + +export const getClassificationStatesInternal = internalQuery({ + args: { + externalIds: v.array(v.string()), + }, + handler: async (ctx, args) => { + assertIntegerInRange("externalIds.length", args.externalIds.length, 1, MAX_ROWS_PER_BATCH); + const externalIds = Array.from( + new Set(args.externalIds.map((externalId) => externalId.trim().toLowerCase())), + ); + const digests = await Promise.all( + externalIds.map((externalId) => + ctx.db + .query("skillsShMirrorDigests") + .withIndex("by_external_id", (q) => q.eq("externalId", externalId)) + .unique(), + ), + ); + return digests.flatMap((digest) => { + if ( + !digest || + !digest.inferredCategories || + !digest.inferredTopics || + !digest.inferredCategoryConfidence || + !digest.inferredTopicConfidence || + !digest.inferredClassifierVersion || + !digest.inferredTopicClassifierVersion || + !digest.inferredInputHash || + !digest.inferredTopicInputHash || + digest.inferredAt === undefined + ) { + return []; + } + return [ + { + externalId: digest.externalId, + slug: digest.slug, + displayName: digest.displayName, + ...(digest.sourceContentHash ? { sourceContentHash: digest.sourceContentHash } : {}), + inferredCategories: digest.inferredCategories, + inferredTopics: digest.inferredTopics, + inferredCategoryConfidence: digest.inferredCategoryConfidence, + inferredTopicConfidence: digest.inferredTopicConfidence, + inferredClassifierVersion: digest.inferredClassifierVersion, + inferredTopicClassifierVersion: digest.inferredTopicClassifierVersion, + inferredInputHash: digest.inferredInputHash, + inferredTopicInputHash: digest.inferredTopicInputHash, + inferredAt: digest.inferredAt, + }, + ]; + }); + }, +}); + +export const getReplayRowsInternal = internalQuery({ + args: { + externalIds: v.array(v.string()), + }, + handler: async (ctx, args) => { + assertIntegerInRange("externalIds.length", args.externalIds.length, 1, MAX_ROWS_PER_BATCH); + const externalIds = Array.from( + new Set(args.externalIds.map((externalId) => externalId.trim().toLowerCase())), + ); + return await Promise.all( + externalIds.map(async (externalId) => { + const digest = await ctx.db + .query("skillsShMirrorDigests") + .withIndex("by_external_id", (q) => q.eq("externalId", externalId)) + .unique(); + if (!digest?.active) { + throw new ConvexError( + `skills.sh mirror replay row is missing or inactive: ${externalId}`, + ); + } + if (digest.sourceFreshnessStatus === "stale") { + return { + quarantined: true as const, + externalId: digest.externalId, + upstreamSourceType: digest.upstreamSourceType ?? digest.sourceType, + reason: digest.staleQuarantineReason ?? "identity-page-fetch-failed", + }; + } + const detail = await ctx.db + .query("skillsShMirrorDetails") + .withIndex("by_external_id", (q) => q.eq("externalId", externalId)) + .unique(); + const currentDetail = + digest.detailStatus === "available" && + detail?.lastObservedRunId === digest.lastObservedRunId + ? detail + : null; + return { digest, detail: currentDetail }; + }), + ); + }, +}); + +export const getRunInternal = internalQuery({ + args: { + runId: v.id("skillsShMirrorRuns"), + }, + handler: async (ctx, args) => { + const run = await ctx.db.get(args.runId); + return run ? summarizeRun(run) : null; + }, +}); + +export const listDigestsPageInternal = internalQuery({ + args: { + cursor: v.union(v.string(), v.null()), + limit: v.number(), + }, + handler: async (ctx, args) => { + assertIntegerInRange("limit", args.limit, 1, 500); + return await ctx.db + .query("skillsShMirrorDigests") + .withIndex("by_active_and_upstream_installs", (q) => q.eq("active", true)) + .paginate({ cursor: args.cursor, numItems: args.limit }); + }, +}); + +export const listDetailsPageInternal = internalQuery({ + args: { + cursor: v.union(v.string(), v.null()), + limit: v.number(), + }, + handler: async (ctx, args) => { + assertIntegerInRange("limit", args.limit, 1, MAX_DETAIL_PAGE_ROWS); + return await ctx.db + .query("skillsShMirrorDetails") + .withIndex("by_external_id") + .paginate({ cursor: args.cursor, numItems: args.limit }); + }, +}); + +export const listFacetsPageInternal = internalQuery({ + args: { + cursor: v.union(v.string(), v.null()), + limit: v.number(), + }, + handler: async (ctx, args) => { + assertIntegerInRange("limit", args.limit, 1, 500); + return await ctx.db + .query("skillsShMirrorFacets") + .withIndex("by_active_and_kind_and_term_and_installs_and_external_id", (q) => + q.eq("active", true), + ) + .paginate({ cursor: args.cursor, numItems: args.limit }); + }, +}); + +export const listConflictsByRunInternal = internalQuery({ + args: { + runId: v.id("skillsShMirrorRuns"), + limit: v.number(), + }, + handler: async (ctx, args) => { + assertIntegerInRange("limit", args.limit, 1, 50); + return await ctx.db + .query("skillsShMirrorConflicts") + .withIndex("by_run_id", (q) => q.eq("runId", args.runId)) + .take(args.limit); + }, +}); + +export const getStatusInternal = internalQuery({ + args: {}, + handler: async (ctx) => { + const [control, runs, sampleDigests] = await Promise.all([ + getControl(ctx), + ctx.db.query("skillsShMirrorRuns").withIndex("by_started_at").order("desc").take(20), + ctx.db.query("skillsShMirrorDigests").withIndex("by_external_id").take(50), + ]); + const latestRunConflicts = runs[0] + ? await ctx.db + .query("skillsShMirrorConflicts") + .withIndex("by_run_id", (q) => q.eq("runId", runs[0]!._id)) + .take(50) + : []; + return { + environment: assertSkillsShFixtureEnvironmentAllowed(), + control, + runs: runs.map(summarizeRun), + sampleDigests, + sampleConflicts: latestRunConflicts, + latestRunConflicts, + invariants: { + publicVisible: false, + installable: false, + scanPlanningEnabled: false, + scanAdmissionEnabled: false, + publisherAttachmentEnabled: false, + }, + }; + }, +}); + +export const getIsolationInternal = internalQuery({ + args: {}, + handler: async (ctx) => { + const [catalogAttempts, nativeScanJobs] = await Promise.all([ + ctx.db.query("skillsShCatalogScanAttempts").take(1_001), + ctx.db.query("securityScanJobs").take(1_001), + ]); + return { + catalogScanAttempts: { + count: catalogAttempts.length, + isEstimate: catalogAttempts.length > 1_000, + }, + nativeScanJobs: { + count: nativeScanJobs.length, + isEstimate: nativeScanJobs.length > 1_000, + updatedAtSum: nativeScanJobs.reduce((sum, job) => sum + job.updatedAt, 0), + statuses: nativeScanJobs.reduce>((counts, job) => { + counts[job.status] = (counts[job.status] ?? 0) + 1; + return counts; + }, {}), + }, + }; + }, +}); diff --git a/docs/cli.md b/docs/cli.md index f2589bcf..de8688bd 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -96,7 +96,8 @@ Stores your API token + cached registry URL. ### `star ` / `unstar ` -- Adds/removes a skill from your highlights. +- Adds/removes a skill from your Bookmarks. Command names remain `star` and + `unstar` for compatibility. - Calls `POST /api/v1/stars/` and `DELETE /api/v1/stars/`. - `--yes` skips confirmation. diff --git a/docs/http-api.md b/docs/http-api.md index 97333a81..34b0464a 100644 --- a/docs/http-api.md +++ b/docs/http-api.md @@ -1590,7 +1590,8 @@ Response: ### `POST /api/v1/stars/{slug}` / `DELETE /api/v1/stars/{slug}` -Add/remove a star (highlights). Both endpoints are idempotent. +Add/remove a Bookmark. The legacy `stars` route and response field names remain +for compatibility. Both endpoints are idempotent. Responses: diff --git a/e2e/local-auth/skill-star-sync.pw.test.ts b/e2e/local-auth/skill-star-sync.pw.test.ts index bd5b2a00..912fb0b5 100644 --- a/e2e/local-auth/skill-star-sync.pw.test.ts +++ b/e2e/local-auth/skill-star-sync.pw.test.ts @@ -22,7 +22,7 @@ test.skip( test.setTimeout(180_000); async function gotoUntilStarButtonReady(page: Page, detailPath: string): Promise { - const starButton = page.getByRole("button", { name: "Star skill" }); + const starButton = page.getByRole("button", { name: "Bookmark skill" }); let lastError: unknown; for (let attempt = 1; attempt <= 4; attempt += 1) { @@ -96,14 +96,14 @@ test("starring a skill survives refresh with the synchronized count", async ({ await starButton.click(); - const unstarButton = page.getByRole("button", { name: "Unstar skill" }); + const unstarButton = page.getByRole("button", { name: "Remove bookmark" }); await expect(unstarButton).toBeVisible({ timeout: 30_000 }); await expect(unstarButton).toContainText("1", { timeout: 30_000 }); await page.reload({ waitUntil: "domcontentloaded" }); await waitForHydration(page); - const refreshedUnstarButton = page.getByRole("button", { name: "Unstar skill" }); + const refreshedUnstarButton = page.getByRole("button", { name: "Remove bookmark" }); await expect(refreshedUnstarButton).toBeVisible({ timeout: 30_000 }); await expect(refreshedUnstarButton).toContainText("1", { timeout: 30_000 }); diff --git a/package.json b/package.json index 59c8c709..1ef06f1a 100644 --- a/package.json +++ b/package.json @@ -73,6 +73,7 @@ "skills-sh:capture-500": "bun scripts/skills-sh-catalog/capture-frozen-snapshot.ts", "skills-sh:prove-500": "bun scripts/skills-sh-catalog/prove-500.ts", "skills-sh:prove-canary": "CLAWHUB_TEST_CATALOG_MODE=controlled-canary bun scripts/skills-sh-catalog/run-test-gate.ts", + "skills-sh:prove-mirror": "bun scripts/skills-sh-catalog/prove-mirror-test.ts", "skills:install": "npx --yes skills@1.5.16 add openclaw/design-system --skill openclaw-design openclaw-brand openclaw-design-system openclaw-marketing-pages openclaw-design-audit --agent codex --copy --yes", "test": "vitest run", "test:e2e": "vitest run -c vitest.e2e.config.ts", @@ -128,6 +129,7 @@ "mermaid": "^11.16.0", "mime": "4.1.0", "monaco-editor": "0.56.0", + "parse5": "8.0.1", "pino": "10.3.1", "react": "19.2.7", "react-dom": "19.2.7", diff --git a/scripts/skills-sh-catalog/prove-mirror-request.test.ts b/scripts/skills-sh-catalog/prove-mirror-request.test.ts new file mode 100644 index 00000000..a6d041a7 --- /dev/null +++ b/scripts/skills-sh-catalog/prove-mirror-request.test.ts @@ -0,0 +1,352 @@ +import { describe, expect, it } from "vitest"; +import { + buildMirrorStepRequest, + buildMirrorProofHeaders, + capturedMirrorSourceRunId, + findCompletedLiveMirrorRun, + findRecoverableMirrorRun, + mirrorRateLimitRetryDelayMs, + reconcileMirrorRunToCompletion, + resolveCompletedLiveMirrorRun, + mirrorRunFromPayload, + mirrorRunAccounting, +} from "./prove-mirror-request"; + +describe("skills.sh mirror proof request headers", () => { + it("carries the Test deployment protection bypass without changing operator auth", () => { + expect(buildMirrorProofHeaders("operator-token", " bypass-secret ")).toEqual({ + Authorization: "Bearer operator-token", + "Content-Type": "application/json", + "x-vercel-protection-bypass": "bypass-secret", + }); + }); + + it("omits the bypass header outside protected deployments", () => { + expect(buildMirrorProofHeaders("operator-token")).toEqual({ + Authorization: "Bearer operator-token", + "Content-Type": "application/json", + }); + }); + + it("selects the newest durable active run for interruption recovery", () => { + expect( + findRecoverableMirrorRun({ + runs: [ + { + runId: "completed-run", + status: "completed", + page: 20, + offset: 0, + sourceTotal: 9_571, + sourcePageSize: 500, + sourceMeasuredAt: "2026-07-22T20:14:00.000Z", + startedAt: 1, + }, + { + runId: "active-run", + snapshotId: "skills-sh:2026-07-22T21:18:13.365Z:9571", + status: "running", + page: 0, + offset: 50, + sourceTotal: 9_571, + sourcePageSize: 500, + sourceMeasuredAt: "2026-07-22T21:18:13.365Z", + startedAt: 2, + }, + ], + }), + ).toEqual({ + runId: "active-run", + snapshotId: "skills-sh:2026-07-22T21:18:13.365Z:9571", + status: "running", + page: 0, + offset: 50, + sourceTotal: 9_571, + sourcePageSize: 500, + sourceMeasuredAt: "2026-07-22T21:18:13.365Z", + startedAt: 2, + }); + }); + + it("normalizes direct and nested mirror run responses", () => { + expect(mirrorRunFromPayload({ runId: "run", status: "completed" }, "reconcile")).toEqual({ + runId: "run", + status: "completed", + }); + expect( + mirrorRunFromPayload( + { run: { runId: "run", status: "reconciling" }, cursor: "next" }, + "reconcile", + ), + ).toEqual({ + runId: "run", + status: "reconciling", + }); + expect(() => mirrorRunFromPayload({ runId: "run" }, "start-replay")).toThrow( + 'start-replay mirror response lacks run status: {"runId":"run"}', + ); + }); + + it("keeps a resumed captured run on the replay operation and exact cursor", () => { + expect( + buildMirrorStepRequest({ + runId: "captured-run", + page: 1, + offset: 50, + capturedSource: { + externalIds: Array.from({ length: 175 }, (_, index) => `owner/repo/skill-${index}`), + sourcePageSize: 100, + }, + }), + ).toMatchObject({ + operation: "step-replay", + runId: "captured-run", + page: 1, + offset: 50, + pageLength: 75, + hasMore: false, + sourceTotal: 175, + externalIds: Array.from({ length: 25 }, (_, index) => `owner/repo/skill-${index + 150}`), + }); + }); + + it("keeps live source steps bound to the server-side run", () => { + expect( + buildMirrorStepRequest({ + runId: "live-run", + page: 3, + offset: 50, + }), + ).toEqual({ + operation: "step", + runId: "live-run", + page: 3, + offset: 50, + }); + }); + + it("normalizes reconciliation responses through completion", async () => { + const responses = [ + { run: { runId: "run", status: "reconciling", page: 20, offset: 0 } }, + { runId: "run", status: "completed", page: 20, offset: 0 }, + ]; + const reconcile = vi.fn(async () => responses.shift()!); + + await expect( + reconcileMirrorRunToCompletion( + { runId: "run", status: "reconciling", page: 20, offset: 0 }, + reconcile, + ), + ).resolves.toEqual({ + run: { runId: "run", status: "completed", page: 20, offset: 0 }, + reconciliationBatches: 2, + }); + expect(reconcile).toHaveBeenCalledTimes(2); + }); + + it("ties a captured replay recovery to its completed authenticated source run", () => { + const liveRun = { + runId: "live-run", + snapshotId: "skills-sh:2026-07-22T21:18:13.365Z:9571", + status: "completed", + page: 20, + offset: 0, + sourceTotal: 9_571, + sourcePageSize: 500, + sourceMeasuredAt: "2026-07-22T21:18:13.365Z", + startedAt: 1, + completedAt: 2, + counts: { observed: 9_571 }, + operations: { sourceRequests: 18_360 }, + }; + const payload = { runs: [{ ...liveRun, status: "running" }, liveRun] }; + + expect(capturedMirrorSourceRunId("skills-sh-captured:live-run")).toBe("live-run"); + expect(capturedMirrorSourceRunId("skills-sh:live-run")).toBeNull(); + expect(findCompletedLiveMirrorRun(payload, "live-run")).toEqual(liveRun); + expect(findCompletedLiveMirrorRun(liveRun, "live-run")).toEqual(liveRun); + expect(findCompletedLiveMirrorRun(payload, "missing")).toBeNull(); + }); + + it("resolves a completed captured ancestor to its authenticated live source run", async () => { + const liveRun = { + runId: "live-run", + snapshotId: "skills-sh:2026-07-22T21:18:13.365Z:9571", + status: "completed" as const, + page: 20, + offset: 0, + sourceTotal: 9_571, + sourcePageSize: 500, + sourceMeasuredAt: "2026-07-22T21:18:13.365Z", + startedAt: 1, + completedAt: 2, + counts: { observed: 9_571 }, + operations: { sourceRequests: 18_360 }, + }; + const capturedRun = { + ...liveRun, + runId: "captured-run", + snapshotId: "skills-sh-captured:live-run", + startedAt: 3, + completedAt: 4, + }; + const readRun = vi.fn(async (runId: string) => { + expect(runId).toBe("live-run"); + return liveRun; + }); + + await expect( + resolveCompletedLiveMirrorRun({ + payload: { runs: [capturedRun] }, + runId: "captured-run", + readRun, + }), + ).resolves.toEqual(liveRun); + expect(readRun).toHaveBeenCalledTimes(1); + }); + + it("bounds cyclic captured-run lineage", async () => { + const completedRun = { + status: "completed" as const, + page: 20, + offset: 0, + sourceTotal: 9_571, + sourcePageSize: 500, + sourceMeasuredAt: "2026-07-22T21:18:13.365Z", + startedAt: 1, + completedAt: 2, + counts: { observed: 9_571 }, + operations: { sourceRequests: 18_360 }, + }; + const first = { + ...completedRun, + runId: "first", + snapshotId: "skills-sh-captured:second", + }; + const second = { + ...completedRun, + runId: "second", + snapshotId: "skills-sh-captured:first", + }; + const readRun = vi.fn(async (runId: string) => (runId === "first" ? first : second)); + + await expect( + resolveCompletedLiveMirrorRun({ + payload: { runs: [first] }, + runId: "first", + readRun, + }), + ).resolves.toBeNull(); + expect(readRun).toHaveBeenCalledTimes(1); + }); + + it("treats a missing captured ancestor as stale lineage", async () => { + const capturedRun = { + runId: "captured-run", + snapshotId: "skills-sh-captured:missing-live-run", + status: "completed" as const, + page: 20, + offset: 0, + sourceTotal: 9_571, + sourcePageSize: 500, + sourceMeasuredAt: "2026-07-22T21:18:13.365Z", + startedAt: 1, + completedAt: 2, + counts: { observed: 9_571 }, + operations: { sourceRequests: 18_360 }, + }; + const readRun = vi.fn(async () => null); + + await expect( + resolveCompletedLiveMirrorRun({ + payload: { runs: [capturedRun] }, + runId: "captured-run", + readRun, + }), + ).resolves.toBeNull(); + }); + + it("fails closed without canceling when captured lineage exceeds the read bound", async () => { + const completedRun = { + status: "completed" as const, + page: 20, + offset: 0, + sourceTotal: 9_571, + sourcePageSize: 500, + sourceMeasuredAt: "2026-07-22T21:18:13.365Z", + startedAt: 1, + completedAt: 2, + counts: { observed: 9_571 }, + operations: { sourceRequests: 18_360 }, + }; + const runs = new Map( + Array.from({ length: 9 }, (_, index) => [ + `captured-${index}`, + { + ...completedRun, + runId: `captured-${index}`, + snapshotId: `skills-sh-captured:captured-${index + 1}`, + }, + ]), + ); + + await expect( + resolveCompletedLiveMirrorRun({ + payload: { runs: [runs.get("captured-0")] }, + runId: "captured-0", + readRun: async (runId) => runs.get(runId) ?? null, + }), + ).rejects.toThrow("captured mirror lineage exceeded 8 runs"); + }); + + it("bounds rate-limit recovery delays while preserving Retry-After", () => { + expect(mirrorRateLimitRetryDelayMs(429, "17", 0)).toBe(17_000); + expect(mirrorRateLimitRetryDelayMs(429, "120", 0)).toBe(120_000); + expect(mirrorRateLimitRetryDelayMs(429, null, 3)).toBe(8_000); + expect(mirrorRateLimitRetryDelayMs(502, "17", 0)).toBeNull(); + }); + + it("accounts fail-closed identity conflicts separately from source quarantines", () => { + expect( + mirrorRunAccounting(9_571, { + conflicts: 205, + rejected: 205, + quarantined: 166, + }), + ).toEqual({ + accepted: 9_366, + rejected: 205, + quarantined: 166, + }); + }); + + it("rejects unrecorded failures and impossible quarantine counts", () => { + expect(() => + mirrorRunAccounting(100, { + rejected: 0, + quarantined: 0, + }), + ).toThrow("mirror conflicts must be a nonnegative integer"); + expect(() => + mirrorRunAccounting(100, { + conflicts: 0, + rejected: Number.NaN, + quarantined: 0, + }), + ).toThrow("mirror rejected must be a nonnegative integer"); + expect(() => + mirrorRunAccounting(100, { + conflicts: 4, + rejected: 5, + quarantined: 3, + }), + ).toThrow("mirror conflict accounting"); + expect(() => + mirrorRunAccounting(100, { + conflicts: 5, + rejected: 5, + quarantined: 6, + }), + ).toThrow("mirror quarantine accounting"); + }); +}); diff --git a/scripts/skills-sh-catalog/prove-mirror-request.ts b/scripts/skills-sh-catalog/prove-mirror-request.ts new file mode 100644 index 00000000..69755bae --- /dev/null +++ b/scripts/skills-sh-catalog/prove-mirror-request.ts @@ -0,0 +1,268 @@ +export function buildMirrorProofHeaders( + operatorAuthorization: string, + vercelAutomationBypassSecret?: string, +) { + return { + Authorization: `Bearer ${operatorAuthorization}`, + "Content-Type": "application/json", + ...(vercelAutomationBypassSecret?.trim() + ? { "x-vercel-protection-bypass": vercelAutomationBypassSecret.trim() } + : {}), + }; +} + +export function mirrorRateLimitRetryDelayMs( + status: number, + retryAfterHeader: string | null, + attempt: number, +) { + if (status !== 429) return null; + const retryAfterSeconds = retryAfterHeader === null ? Number.NaN : Number(retryAfterHeader); + const requestedMs = + Number.isFinite(retryAfterSeconds) && retryAfterSeconds >= 0 + ? retryAfterSeconds * 1_000 + : 1_000 * 2 ** attempt; + return Math.max(1_000, requestedMs); +} + +export function mirrorRunAccounting(total: number, counts: Record) { + if (!Number.isSafeInteger(total) || total < 0) { + throw new Error("mirror source total must be a nonnegative integer"); + } + const requiredCount = (name: "conflicts" | "rejected" | "quarantined") => { + const value = counts[name]; + if (!Number.isSafeInteger(value) || value < 0) { + throw new Error(`mirror ${name} must be a nonnegative integer`); + } + return value; + }; + const rejected = requiredCount("rejected"); + const quarantined = requiredCount("quarantined"); + const conflicts = requiredCount("conflicts"); + if (conflicts !== rejected) { + throw new Error("mirror conflict accounting does not equal rejected rows"); + } + if (quarantined > rejected) { + throw new Error("mirror quarantine accounting exceeds rejected rows"); + } + const accepted = total - rejected; + if (accepted < 0) { + throw new Error("mirror rejected rows exceed the source total"); + } + return { accepted, rejected, quarantined }; +} + +export function mirrorRunFromPayload( + payload: Record, + operation: string, +): Record { + const nested = payload.run; + const candidates = [ + payload, + nested && typeof nested === "object" && !Array.isArray(nested) + ? (nested as Record) + : null, + ]; + for (const candidate of candidates) { + if ( + candidate && + ["running", "paused", "reconciling", "completed", "failed"].includes(String(candidate.status)) + ) { + return candidate; + } + } + const diagnostic = JSON.stringify(payload).slice(0, 1_000); + throw new Error(`${operation} mirror response lacks run status: ${diagnostic}`); +} + +type CapturedMirrorStepSource = { + externalIds: string[]; + sourcePageSize: number; +}; + +export function buildMirrorStepRequest(args: { + runId: string; + page: number; + offset: number; + capturedSource?: CapturedMirrorStepSource | null; +}) { + if (!args.capturedSource) { + return { + operation: "step" as const, + runId: args.runId, + page: args.page, + offset: args.offset, + }; + } + const { externalIds, sourcePageSize } = args.capturedSource; + const pageStart = args.page * sourcePageSize; + const pageLength = Math.min(sourcePageSize, externalIds.length - pageStart); + const rowStart = pageStart + args.offset; + const rows = externalIds.slice(rowStart, rowStart + 50); + if (pageLength < 1 || rows.length < 1) { + throw new Error(`captured mirror replay has no rows for cursor ${args.page}:${args.offset}`); + } + return { + operation: "step-replay" as const, + runId: args.runId, + page: args.page, + offset: args.offset, + pageLength, + hasMore: pageStart + sourcePageSize < externalIds.length, + sourceTotal: externalIds.length, + externalIds: rows, + }; +} + +export async function reconcileMirrorRunToCompletion( + initialRun: Record, + reconcile: () => Promise>, +) { + let run = initialRun; + let reconciliationBatches = 0; + while (run.status === "reconciling") { + if (reconciliationBatches >= 1_000) { + throw new Error("mirror reconciliation exceeded 1000 bounded batches"); + } + run = mirrorRunFromPayload(await reconcile(), "reconcile"); + reconciliationBatches += 1; + } + return { run, reconciliationBatches }; +} + +export type RecoverableMirrorRun = { + runId: string; + snapshotId: string; + status: "running" | "paused" | "reconciling"; + page: number; + offset: number; + sourceTotal: number; + sourcePageSize: number; + sourceMeasuredAt: string; + startedAt: number; +}; + +export function findRecoverableMirrorRun( + payload: Record, +): RecoverableMirrorRun | null { + if (!Array.isArray(payload.runs)) return null; + for (const value of payload.runs) { + if (!value || typeof value !== "object" || Array.isArray(value)) continue; + const run = value as Record; + if ( + !["running", "paused", "reconciling"].includes(String(run.status)) || + typeof run.runId !== "string" || + typeof run.snapshotId !== "string" || + typeof run.page !== "number" || + typeof run.offset !== "number" || + typeof run.sourceTotal !== "number" || + typeof run.sourcePageSize !== "number" || + typeof run.sourceMeasuredAt !== "string" || + typeof run.startedAt !== "number" + ) { + continue; + } + return run as RecoverableMirrorRun; + } + return null; +} + +export type CompletedLiveMirrorRun = { + runId: string; + snapshotId: string; + status: "completed"; + page: number; + offset: number; + sourceTotal: number; + sourcePageSize: number; + sourceMeasuredAt: string; + startedAt: number; + completedAt: number; + counts: Record; + operations: Record; +}; + +type CompletedMirrorRun = CompletedLiveMirrorRun; + +export function capturedMirrorSourceRunId(snapshotId: string) { + const prefix = "skills-sh-captured:"; + return snapshotId.startsWith(prefix) ? snapshotId.slice(prefix.length) || null : null; +} + +function findCompletedMirrorRun( + payload: unknown, + runId?: string | null, +): CompletedMirrorRun | null { + const root = + payload && typeof payload === "object" && !Array.isArray(payload) + ? (payload as Record) + : null; + const candidates = root ? [root, ...(Array.isArray(root.runs) ? root.runs : [])] : []; + for (const value of candidates) { + if (!value || typeof value !== "object" || Array.isArray(value)) continue; + const run = value as Record; + if ( + run.status !== "completed" || + typeof run.runId !== "string" || + (runId && run.runId !== runId) || + typeof run.snapshotId !== "string" || + typeof run.page !== "number" || + typeof run.offset !== "number" || + typeof run.sourceTotal !== "number" || + typeof run.sourcePageSize !== "number" || + typeof run.sourceMeasuredAt !== "string" || + typeof run.startedAt !== "number" || + typeof run.completedAt !== "number" || + !run.counts || + typeof run.counts !== "object" || + Array.isArray(run.counts) || + !run.operations || + typeof run.operations !== "object" || + Array.isArray(run.operations) + ) { + continue; + } + return run as CompletedMirrorRun; + } + return null; +} + +export function findCompletedLiveMirrorRun( + payload: unknown, + runId?: string | null, +): CompletedLiveMirrorRun | null { + const run = findCompletedMirrorRun(payload, runId); + if ( + !run || + !run.snapshotId.startsWith("skills-sh:") || + run.snapshotId.startsWith("skills-sh-captured:") + ) { + return null; + } + return run; +} + +export async function resolveCompletedLiveMirrorRun(args: { + payload: unknown; + runId: string; + readRun: (runId: string) => Promise; +}) { + const seen = new Set(); + let runId = args.runId; + let payload = args.payload; + for (let depth = 0; depth < 8; depth += 1) { + if (seen.has(runId)) return null; + seen.add(runId); + let run = findCompletedMirrorRun(payload, runId); + if (!run) { + payload = await args.readRun(runId); + run = findCompletedMirrorRun(payload, runId); + } + if (!run) return null; + const capturedRunId = capturedMirrorSourceRunId(run.snapshotId); + if (!capturedRunId) return findCompletedLiveMirrorRun(run, runId); + runId = capturedRunId; + payload = {}; + } + throw new Error("captured mirror lineage exceeded 8 runs"); +} diff --git a/scripts/skills-sh-catalog/prove-mirror-test.ts b/scripts/skills-sh-catalog/prove-mirror-test.ts new file mode 100644 index 00000000..033a78c7 --- /dev/null +++ b/scripts/skills-sh-catalog/prove-mirror-test.ts @@ -0,0 +1,776 @@ +#!/usr/bin/env bun + +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { + SKILLS_SH_MIRROR_CONTROLLED_EXTERNAL_IDS, + parseSkillsShMirrorProofSnapshotId, +} from "../../server/skillsShCatalogSource"; +import { + buildMirrorStepRequest, + buildMirrorProofHeaders, + capturedMirrorSourceRunId, + findRecoverableMirrorRun, + mirrorRateLimitRetryDelayMs, + reconcileMirrorRunToCompletion, + resolveCompletedLiveMirrorRun, + mirrorRunFromPayload, + mirrorRunAccounting, + type CompletedLiveMirrorRun, + type RecoverableMirrorRun, +} from "./prove-mirror-request"; + +const OUTPUT_PATH = resolve("proof/claw-563/skills-sh-mirror-test-proof.json"); +const PROJECTED_SCALE = 700_000; +const MAX_RATE_LIMIT_RETRIES_PER_RUN = 30; +const MAX_RATE_LIMIT_WAIT_MS_PER_RUN = 30 * 60 * 1_000; +const MAX_DETAIL_PAGE_ROWS = 50; + +function requireEnv(name: string) { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`${name} is required`); + return value; +} + +const targetUrl = requireEnv("CLAWHUB_TEST_MIRROR_GATE_URL"); +const operatorAuthorization = requireEnv("CLAWHUB_TEST_OPERATOR_TOKEN"); +const vercelAutomationBypassSecret = process.env.VERCEL_AUTOMATION_BYPASS_SECRET; + +async function callRaw(body: Record) { + const startedAt = performance.now(); + const response = await fetch(targetUrl, { + method: "POST", + headers: buildMirrorProofHeaders(operatorAuthorization, vercelAutomationBypassSecret), + body: JSON.stringify(body), + }); + const text = await response.text(); + let payload: Record; + try { + payload = JSON.parse(text) as Record; + } catch { + payload = { text }; + } + return { + ok: response.ok, + status: response.status, + elapsedMs: performance.now() - startedAt, + payload, + retryAfter: response.headers.get("retry-after"), + }; +} + +function callFailure(body: Record, result: Awaited>) { + return new Error( + `${String(body.operation)} returned HTTP ${result.status}: ${JSON.stringify(result.payload)}`, + ); +} + +async function call(body: Record) { + const result = await callRaw(body); + if (!result.ok) throw callFailure(body, result); + return result; +} + +function requireRunId(payload: Record) { + if (typeof payload.runId !== "string") throw new Error("mirror start did not return runId"); + return payload.runId; +} + +async function runMirror( + reason: string, + provePauseResume: boolean, + recoverableRun: RecoverableMirrorRun | null = null, + capturedSource: CapturedMirrorSource | null = null, +) { + const attemptStartedAt = Date.now(); + const start = recoverableRun + ? { payload: recoverableRun } + : capturedSource + ? await call({ + operation: "start-replay", + reason, + capturedRunId: capturedSource.capturedRunId, + sourceTotal: capturedSource.externalIds.length, + sourcePageSize: capturedSource.sourcePageSize, + sourceMeasuredAt: capturedSource.sourceMeasuredAt, + }) + : await call({ operation: "start", reason }); + const startRun = mirrorRunFromPayload( + start.payload, + recoverableRun ? "recover" : capturedSource ? "start-replay" : "start", + ); + const startPayload = start.payload as Record; + const runId = requireRunId(startRun); + const proofSource = + capturedSource === null + ? parseSkillsShMirrorProofSnapshotId(String(startRun.snapshotId)) + : null; + let page = recoverableRun?.page ?? 0; + let offset = recoverableRun?.offset ?? 0; + let steps = 0; + let pauseProof: Record | null = null; + let run = startRun; + let recovery: Record | null = null; + let rateLimitRetries = 0; + let rateLimitWaitMs = 0; + if (recoverableRun) { + recovery = { + runId, + status: recoverableRun.status, + cursor: { page, offset }, + interruptedAtLeastOnce: true, + }; + if (recoverableRun.status === "paused") { + run = ( + await call({ + operation: "resume", + runId, + reason: "CLAW-563 interrupted run recovery", + }) + ).payload; + } + } + while (run.status === "running") { + const stepRequest = buildMirrorStepRequest({ + runId, + page, + offset, + capturedSource, + }); + const step = await callRaw(stepRequest); + if (!step.ok) { + const delayMs = mirrorRateLimitRetryDelayMs(step.status, step.retryAfter, rateLimitRetries); + if ( + delayMs === null || + rateLimitRetries >= MAX_RATE_LIMIT_RETRIES_PER_RUN || + rateLimitWaitMs + delayMs > MAX_RATE_LIMIT_WAIT_MS_PER_RUN + ) { + throw callFailure(stepRequest, step); + } + rateLimitRetries += 1; + rateLimitWaitMs += delayMs; + await new Promise((resolve) => setTimeout(resolve, delayMs)); + continue; + } + run = mirrorRunFromPayload(step.payload, String(stepRequest.operation)); + steps += 1; + if (typeof run.page !== "number" || typeof run.offset !== "number") { + throw new Error("mirror step did not return a durable cursor"); + } + page = run.page; + offset = run.offset; + if (provePauseResume && steps === 1) { + const pause = await call({ + operation: "pause", + runId, + reason: "CLAW-563 deliberate pause proof", + }); + const blocked = await callRaw( + buildMirrorStepRequest({ + runId, + page, + offset, + capturedSource, + }), + ); + if (blocked.ok || !JSON.stringify(blocked.payload).includes("paused")) { + throw new Error("paused mirror step did not fail closed"); + } + const resume = await call({ + operation: "resume", + runId, + reason: "CLAW-563 exact cursor resume proof", + }); + pauseProof = { + pause: pause.payload, + blockedStatus: blocked.status, + blockedPayload: blocked.payload, + resume: resume.payload, + resumeCursor: { page, offset }, + }; + } + } + const reconciliation = await reconcileMirrorRunToCompletion(run, async () => { + const result = await call({ operation: "reconcile", runId, limit: 250 }); + return result.payload; + }); + run = reconciliation.run; + const reconciliationBatches = reconciliation.reconciliationBatches; + if (run.status !== "completed") { + throw new Error(`mirror run ended in unexpected status ${String(run.status)}`); + } + return { + runId, + run, + source: { + total: startPayload.sourceTotal, + catalogTotal: proofSource?.catalogTotal ?? startPayload.sourceTotal, + controlledExternalIds: proofSource?.controlledExternalIds ?? [], + controlledOverlayExternalIds: proofSource?.controlledOverlayExternalIds ?? [], + controlledSupplementExternalIds: proofSource?.controlledSupplementExternalIds ?? [], + evidence: proofSource?.evidence ?? null, + capture: + startPayload.sourceCapture && + typeof startPayload.sourceCapture === "object" && + !Array.isArray(startPayload.sourceCapture) + ? startPayload.sourceCapture + : null, + measuredAt: startPayload.sourceMeasuredAt, + pageSize: startPayload.sourcePageSize, + }, + recovery, + steps, + reconciliationBatches, + pauseProof, + rateLimitRecovery: { + retries: rateLimitRetries, + waitedMs: rateLimitWaitMs, + retryBudget: MAX_RATE_LIMIT_RETRIES_PER_RUN, + waitBudgetMs: MAX_RATE_LIMIT_WAIT_MS_PER_RUN, + }, + elapsedMs: Date.now() - (recoverableRun?.startedAt ?? attemptStartedAt), + attemptElapsedMs: Date.now() - attemptStartedAt, + }; +} + +async function collectPages( + operation: "page" | "detail-page" | "facet-page", + validate: (document: Record) => void, +) { + const documents: Record[] = []; + let cursor: string | null = null; + let calls = 0; + let count = 0; + let serializedBytes = 0; + do { + const result = await call({ + operation, + cursor, + limit: operation === "detail-page" ? MAX_DETAIL_PAGE_ROWS : 500, + }); + const page = result.payload.page; + if (!Array.isArray(page)) throw new Error(`${operation} did not return a page`); + for (const document of page as Record[]) { + validate(document); + count += 1; + serializedBytes += Buffer.byteLength(JSON.stringify(document), "utf8"); + if (operation === "page") documents.push(document); + } + calls += 1; + cursor = + result.payload.isDone === true + ? null + : typeof result.payload.continueCursor === "string" + ? result.payload.continueCursor + : null; + if (result.payload.isDone !== true && cursor === null) { + throw new Error(`${operation} did not return a continuation cursor`); + } + } while (cursor); + return { + count, + calls, + serializedBytes, + documents, + }; +} + +type MirrorRunProof = Awaited>; +type CapturedMirrorSource = { + capturedRunId: string; + externalIds: string[]; + sourceMeasuredAt: string; + sourcePageSize: number; +}; + +function completedLiveRunProof(run: CompletedLiveMirrorRun): MirrorRunProof { + const proofSource = parseSkillsShMirrorProofSnapshotId(run.snapshotId); + return { + runId: run.runId, + run, + source: { + total: run.sourceTotal, + catalogTotal: proofSource.catalogTotal, + controlledExternalIds: proofSource.controlledExternalIds, + controlledOverlayExternalIds: proofSource.controlledOverlayExternalIds, + controlledSupplementExternalIds: proofSource.controlledSupplementExternalIds, + evidence: proofSource.evidence ?? null, + capture: null, + measuredAt: run.sourceMeasuredAt, + pageSize: run.sourcePageSize, + }, + recovery: { + runId: run.runId, + status: run.status, + cursor: { page: run.page, offset: run.offset }, + completedBeforeProofContinuation: true, + }, + steps: 0, + reconciliationBatches: 0, + pauseProof: null, + rateLimitRecovery: { + retries: 0, + waitedMs: 0, + retryBudget: MAX_RATE_LIMIT_RETRIES_PER_RUN, + waitBudgetMs: MAX_RATE_LIMIT_WAIT_MS_PER_RUN, + }, + elapsedMs: run.completedAt - run.startedAt, + attemptElapsedMs: 0, + }; +} + +function runCounts(runResult: MirrorRunProof) { + const counts = runResult.run.counts; + if (!counts || typeof counts !== "object") throw new Error("mirror run lacks counts"); + return counts as Record; +} + +function assertZeroCounts(counts: Record, names: string[]) { + for (const name of names) { + if (counts[name] !== 0) throw new Error(`expected ${name}=0, received ${String(counts[name])}`); + } +} + +function assertRunProof(runResult: MirrorRunProof, mode: "first" | "identical") { + const counts = runCounts(runResult); + const total = Number(runResult.source.total); + const { accepted } = mirrorRunAccounting(total, counts); + if (counts.observed !== total) { + throw new Error(`mirror observed ${String(counts.observed)} of ${total} source rows`); + } + assertZeroCounts(counts, ["scansPlanned", "scansAdmitted"]); + if ((counts.inserted ?? 0) + (counts.updated ?? 0) + (counts.unchanged ?? 0) !== accepted) { + throw new Error(`${mode} run digest accounting does not equal the source total`); + } + if ( + (counts.detailsInserted ?? 0) + + (counts.detailsUpdated ?? 0) + + (counts.detailsUnchanged ?? 0) + + (counts.detailsMissing ?? 0) !== + accepted + ) { + throw new Error(`${mode} run detail accounting does not equal the source total`); + } + if (mode === "identical") { + assertZeroCounts(counts, [ + "inserted", + "updated", + "detailsInserted", + "detailsUpdated", + "tombstoned", + "reactivated", + ]); + if (counts.unchanged !== accepted) throw new Error("identical rerun changed a digest"); + } +} + +function validateDigestIsolation(document: Record) { + if ( + document.active !== true || + document.publicVisible !== false || + document.installable !== false + ) { + throw new Error(`mirror digest isolation failed: ${String(document.externalId)}`); + } +} + +function validateDigest(document: Record) { + validateDigestIsolation(document); + for (const field of [ + "normalizedSlug", + "normalizedSlugFirstToken", + "normalizedDisplayName", + "normalizedDisplayNameFirstToken", + "searchText", + "upstreamSourceType", + "inferredClassifierVersion", + "inferredTopicClassifierVersion", + "inferredInputHash", + "inferredTopicInputHash", + ]) { + if (typeof document[field] !== "string" || !document[field]) { + throw new Error(`mirror digest lacks ${field}: ${String(document.externalId)}`); + } + } + if ( + !Array.isArray(document.inferredCategories) || + document.inferredCategories.length < 1 || + document.inferredCategories.length > 3 || + !Array.isArray(document.inferredTopics) || + document.inferredTopics.length > 5 || + !["high", "medium", "low"].includes(String(document.inferredCategoryConfidence)) || + !["high", "medium", "low"].includes(String(document.inferredTopicConfidence)) || + typeof document.inferredAt !== "number" + ) { + throw new Error(`mirror digest lacks bounded inference: ${String(document.externalId)}`); + } + const scanners = document.upstreamScanners; + if (!scanners || typeof scanners !== "object" || Array.isArray(scanners)) { + throw new Error(`mirror digest lacks upstream scanners: ${String(document.externalId)}`); + } + for (const provider of ["genAgentTrustHub", "socket", "snyk"]) { + const scanner = (scanners as Record)[provider]; + if ( + !scanner || + typeof scanner !== "object" || + Array.isArray(scanner) || + typeof (scanner as Record).status !== "string" + ) { + throw new Error(`mirror digest lacks ${provider} status: ${String(document.externalId)}`); + } + } +} + +function validateFacet(document: Record) { + if ( + document.active !== true || + !["category", "topic"].includes(String(document.kind)) || + typeof document.term !== "string" || + !document.term || + typeof document.externalId !== "string" || + typeof document.installs !== "number" + ) { + throw new Error(`mirror facet is invalid: ${String(document.externalId)}`); + } +} + +function validateDetail(document: Record) { + const content = document.content; + const contentBytes = document.contentBytes; + if ( + typeof content !== "string" || + typeof contentBytes !== "number" || + contentBytes > 64 * 1024 || + Buffer.byteLength(content, "utf8") !== contentBytes + ) { + throw new Error(`mirror detail exceeds its boundary: ${String(document.externalId)}`); + } +} + +function percentile(values: number[], percentileValue: number) { + const sorted = [...values].sort((left, right) => left - right); + return sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * percentileValue))] ?? 0; +} + +async function measureReads(externalIds: string[]) { + const results = []; + for (const externalId of externalIds) { + const samples = []; + for (let attempt = 0; attempt < 5; attempt += 1) { + const read = await call({ operation: "read", externalId }); + if (!read.payload.digest) throw new Error(`indexed read missed ${externalId}`); + samples.push(read.elapsedMs); + } + results.push({ + externalId, + samplesMs: samples, + medianMs: percentile(samples, 0.5), + p95Ms: percentile(samples, 0.95), + }); + } + return results; +} + +const cronSource = await readFile(resolve("convex/crons.ts"), "utf8"); +if (cronSource.includes("skillsShMirror") || cronSource.includes("skills-sh/mirror")) { + throw new Error("skills.sh mirror scheduler reference exists"); +} + +const proofStartedAt = Date.now(); +let proof: Record; +await call({ + operation: "configure", + enabled: true, + reason: "CLAW-563 authenticated leaderboard foundation proof", +}); +try { + const isolationBefore = (await call({ operation: "isolation" })).payload; + const statusBefore = (await call({ operation: "status" })).payload; + let recoverableRun = findRecoverableMirrorRun(statusBefore); + const recoveredLiveRunId = recoverableRun + ? capturedMirrorSourceRunId(recoverableRun.snapshotId) + : null; + let recoveredNormalizationRun = recoveredLiveRunId ? recoverableRun : null; + const completedLiveRun = recoveredLiveRunId + ? await resolveCompletedLiveMirrorRun({ + payload: statusBefore, + runId: recoveredLiveRunId, + readRun: async (runId) => (await call({ operation: "run", runId })).payload, + }) + : null; + let discardedStaleRecovery: Record | null = null; + if (recoveredNormalizationRun && !completedLiveRun) { + const discarded = ( + await call({ + operation: "discard", + runId: recoveredNormalizationRun.runId, + reason: `discard stale captured recovery without live source ${recoveredLiveRunId}`, + }) + ).payload; + if (discarded.status !== "canceled" || discarded.runId !== recoveredNormalizationRun.runId) { + throw new Error( + `discard stale captured recovery returned unexpected payload: ${JSON.stringify(discarded)}`, + ); + } + discardedStaleRecovery = { + runId: recoveredNormalizationRun.runId, + missingLiveRunId: recoveredLiveRunId, + status: discarded.status, + }; + recoverableRun = null; + recoveredNormalizationRun = null; + } + const firstRun = completedLiveRun + ? completedLiveRunProof(completedLiveRun) + : await runMirror( + "CLAW-563 complete authenticated leaderboard enumeration", + true, + recoveredNormalizationRun ? null : recoverableRun, + ); + const conflictsAfterFirstRun = ( + await call({ operation: "conflicts", runId: firstRun.runId, limit: 50 }) + ).payload; + const liveRunConflicts = Array.isArray(conflictsAfterFirstRun.conflicts) + ? (conflictsAfterFirstRun.conflicts as Record[]) + : []; + const capturedAfterLive = await collectPages("page", validateDigestIsolation); + const liveCapturedSource: CapturedMirrorSource = { + capturedRunId: firstRun.runId, + externalIds: capturedAfterLive.documents.map((document) => { + if (typeof document.externalId !== "string") { + throw new Error("captured mirror digest lacks externalId"); + } + return document.externalId; + }), + sourceMeasuredAt: String(firstRun.source.measuredAt), + sourcePageSize: Number(firstRun.source.pageSize), + }; + const normalizationRun = await runMirror( + "CLAW-563 authenticated captured snapshot normalization", + false, + recoveredNormalizationRun, + liveCapturedSource, + ); + const capturedAfterNormalization = await collectPages("page", validateDigest); + const normalizedCapturedSource: CapturedMirrorSource = { + ...liveCapturedSource, + capturedRunId: normalizationRun.runId, + externalIds: capturedAfterNormalization.documents.map((document) => { + if (typeof document.externalId !== "string") { + throw new Error("normalized mirror digest lacks externalId"); + } + return document.externalId; + }), + }; + const identicalRerun = await runMirror( + "CLAW-563 authenticated captured snapshot identical rerun", + false, + null, + normalizedCapturedSource, + ); + assertRunProof(firstRun, "first"); + assertRunProof(normalizationRun, "first"); + assertRunProof(identicalRerun, "identical"); + const [digests, details, facets] = await Promise.all([ + collectPages("page", validateDigest), + collectPages("detail-page", validateDetail), + collectPages("facet-page", validateFacet), + ]); + const firstCounts = runCounts(firstRun); + const sourceEvidence = firstRun.source.evidence; + if (!sourceEvidence) { + throw new Error("live mirror run lacks durable source pagination and field evidence"); + } + if (sourceEvidence.fields.normalizedUpstreamTaxonomyFields.length > 0) { + throw new Error( + "skills.sh now exposes normalized taxonomy fields; mirror ingestion must prefer them", + ); + } + const sourceSnapshotHash = parseSkillsShMirrorProofSnapshotId( + String(firstRun.run.snapshotId), + ).sourceSnapshotHash; + if (!sourceSnapshotHash) { + throw new Error("live mirror run lacks a durable source snapshot hash"); + } + const sourceCaptureSummary = (firstRun.source.capture ?? + (await call({ operation: "source-summary", snapshotHash: sourceSnapshotHash })) + .payload) as Record; + const sourceCapture = { + ...sourceCaptureSummary, + actualDbWrites: Number( + sourceCaptureSummary.requestDbWrites ?? firstRun.run.sourceCaptureWrites ?? 0, + ), + }; + const firstAccounting = mirrorRunAccounting(Number(firstRun.source.total), firstCounts); + const quarantinedCount = firstAccounting.quarantined; + const quarantinedPreservedCount = firstCounts.quarantinedPreserved ?? 0; + const acceptedCount = firstAccounting.accepted; + const expectedDigestCount = acceptedCount + quarantinedPreservedCount; + if (digests.count !== expectedDigestCount) { + throw new Error( + `mirror digest count ${digests.count} does not match accepted or preserved total ${expectedDigestCount}`, + ); + } + if (facets.count < digests.count) { + throw new Error("mirror facets do not cover every digest category fallback"); + } + const digestExternalIds = new Set(digests.documents.map((document) => document.externalId)); + for (const externalId of SKILLS_SH_MIRROR_CONTROLLED_EXTERNAL_IDS) { + if (!digestExternalIds.has(externalId)) { + throw new Error(`mirror proof lacks controlled supplement ${externalId}`); + } + } + const quarantineSamples = liveRunConflicts + .filter( + (conflict) => conflict.runId === firstRun.runId && conflict.kind === "source-quarantine", + ) + .slice(0, 50); + if (quarantineSamples.length !== Math.min(quarantinedCount, 50)) { + throw new Error("mirror status did not expose the latest run quarantine samples"); + } + const sampleIndexes = [0, Math.floor(digests.count / 2), digests.count - 1]; + const sampleExternalIds = sampleIndexes.map((index) => { + const externalId = digests.documents[index]?.externalId; + if (typeof externalId !== "string") throw new Error("mirror digest lacks externalId"); + return externalId; + }); + const indexedReads = await measureReads(sampleExternalIds); + const isolationAfter = (await call({ operation: "isolation" })).payload; + if (JSON.stringify(isolationBefore) !== JSON.stringify(isolationAfter)) { + throw new Error("mirror proof changed scan isolation state"); + } + const firstOperations = firstRun.run.operations as Record; + const serializedStorageBytes = + digests.serializedBytes + + details.serializedBytes + + facets.serializedBytes + + Number(sourceCaptureSummary.serializedBytes ?? 0); + const rows = digests.count; + const sourceRows = Number(firstRun.source.total); + const sourceMeasurementRequests = sourceEvidence.pagination.requestedPages.length + 4; + const perRow = { + serializedStorageBytes: serializedStorageBytes / rows, + dbWrites: (firstOperations.dbWrites + sourceCapture.actualDbWrites) / sourceRows, + dbReads: + (firstOperations.dbReads + Number(sourceCaptureSummary.pageDocuments ?? 0)) / sourceRows, + sourceRequests: (firstOperations.sourceRequests + sourceMeasurementRequests) / sourceRows, + sourceBytes: + (firstOperations.sourceBytes + Number(sourceCaptureSummary.sourceBytes ?? 0)) / sourceRows, + }; + proof = { + generatedAt: new Date().toISOString(), + target: { + environment: "permanent Test", + gateUrl: targetUrl, + productionWrites: 0, + schedules: 0, + publicVisibility: false, + installability: false, + publisherAttachment: false, + scanPlanning: false, + scanAdmission: false, + }, + sourceContract: { + authenticated: "Vercel OIDC", + scope: "paginated skills.sh leaderboard", + databaseEnumeration: "out of v1 scope; exhaustive feed not required", + endpointExhausted: true, + databaseCoverageProven: false, + measuredLeaderboardTotal: firstRun.source.catalogTotal, + proofSourceTotal: firstRun.source.total, + measurementRequests: sourceMeasurementRequests, + endpointExhaustion: sourceEvidence.pagination, + rawFieldSets: sourceEvidence.fields, + taxonomyEnrichment: { + upstreamNormalizedFields: sourceEvidence.fields.normalizedUpstreamTaxonomyFields, + fallbackApplied: true, + fallbackGenerator: "ClawHub local deterministic catalog classifier", + llmGenerated: false, + }, + controlledSupplement: { + authority: "immutable GitHub content", + environment: "permanent Test only", + productionFullSyncStrategy: false, + requiredCount: firstRun.source.controlledExternalIds.length, + observedRequiredExternalIds: firstRun.source.controlledExternalIds, + overlayExternalIds: firstRun.source.controlledOverlayExternalIds, + supplementExternalIds: firstRun.source.controlledSupplementExternalIds, + contractRequiredExternalIds: SKILLS_SH_MIRROR_CONTROLLED_EXTERNAL_IDS, + }, + durableSourceCapture: sourceCapture, + pageSize: firstRun.source.pageSize, + measuredAt: firstRun.source.measuredAt, + documentedRateLimitRequestsPerMinute: 600, + fetchPolicy: { + mirrorBatchRows: 50, + sourcePageRows: 500, + detailAndPageConcurrency: 8, + minimumApiRequestIntervalMs: 125, + identityPage: + "512 KiB structural HTML fallback only for exact no-install well-known owner/repo rows", + recovery: "durable cursor retry with bounded Retry-After recovery", + }, + }, + discardedStaleRecovery, + firstRun, + normalizationRun, + identicalRerun, + quarantine: { + rejected: firstCounts.rejected, + quarantined: quarantinedCount, + preserved: quarantinedPreservedCount, + samples: quarantineSamples, + }, + storage: { + digests: { + count: digests.count, + serializedBytes: digests.serializedBytes, + pageReads: digests.calls, + }, + details: { + count: details.count, + serializedBytes: details.serializedBytes, + pageReads: details.calls, + }, + facets: { + count: facets.count, + serializedBytes: facets.serializedBytes, + pageReads: facets.calls, + }, + totalSerializedBytes: serializedStorageBytes, + bytesPerSourceRow: perRow.serializedStorageBytes, + }, + isolation: { + before: isolationBefore, + after: isolationAfter, + unchanged: true, + }, + indexedReads, + projectedHypotheticalDatabaseScale: { + assumedRows: PROJECTED_SCALE, + enumerationAvailable: false, + proofStatus: "cost projection only, not a completed database sync", + serializedStorageBytes: Math.ceil(perRow.serializedStorageBytes * PROJECTED_SCALE), + dbWrites: Math.ceil(perRow.dbWrites * PROJECTED_SCALE), + dbReads: Math.ceil(perRow.dbReads * PROJECTED_SCALE), + sourceRequests: Math.ceil(perRow.sourceRequests * PROJECTED_SCALE), + sourceBytes: Math.ceil(perRow.sourceBytes * PROJECTED_SCALE), + }, + runtime: { + elapsedMs: Date.now() - proofStartedAt, + }, + }; +} finally { + const disabled = await callRaw({ + operation: "configure", + enabled: false, + reason: "CLAW-563 proof cleanup: retain mirror hidden and paused", + }); + if (!disabled.ok) { + throw new Error(`mirror cleanup failed: ${JSON.stringify(disabled.payload)}`); + } +} + +await mkdir(dirname(OUTPUT_PATH), { recursive: true }); +await writeFile(OUTPUT_PATH, `${JSON.stringify(proof, null, 2)}\n`); +console.log(JSON.stringify({ outputPath: OUTPUT_PATH, ...proof })); diff --git a/server/routes/ops/skills-sh/mirror-test.post.ts b/server/routes/ops/skills-sh/mirror-test.post.ts new file mode 100644 index 00000000..9af1877f --- /dev/null +++ b/server/routes/ops/skills-sh/mirror-test.post.ts @@ -0,0 +1,638 @@ +import { getVercelOidcToken } from "@vercel/oidc"; +import { defineEventHandler, getHeader, readBody } from "h3"; +import { + buildSkillsShMirrorProofSnapshotId, + fetchSkillsShMirrorBatch, + fetchSkillsShMirrorControlledBatch, + getSkillsShCatalogTestSourcePolicy, + measureSkillsShMirrorProofSource, + parseSkillsShMirrorProofSnapshotId, + skillsShSourceRetryAfterSeconds, +} from "../../../skillsShCatalogSource"; +import { + buildSkillsShMirrorReplayRows, + enrichSkillsShMirrorClassifications, + type SkillsShMirrorClassificationState, +} from "../../../skillsShMirrorClassification"; + +const TEST_CONVEX_SITE_URL = "https://academic-chihuahua-392.convex.site"; +const OPERATOR_PATH = "/api/v1/operator/skills-sh/catalog-test"; +const SOURCE_PAGE_SIZE = 500; +const MIRROR_BATCH_SIZE = 50; +const MAX_TEST_SOURCE_ROWS = 50_000; +const MAX_DETAIL_BYTES = 64 * 1024; +const MAX_DETAIL_PAGE_ROWS = 50; +const BATCH_LEASE_HEARTBEAT_INTERVAL_MS = 60_000; + +type MirrorRequest = { + operation?: + | "configure" + | "start" + | "start-replay" + | "run" + | "step" + | "step-replay" + | "pause" + | "resume" + | "discard" + | "reconcile" + | "conflicts" + | "status" + | "isolation" + | "read" + | "source-summary" + | "page" + | "detail-page" + | "facet-page"; + enabled?: boolean; + externalId?: string; + externalIds?: string[]; + cursor?: string | null; + capturedRunId?: string; + hasMore?: boolean; + limit?: number; + offset?: number; + page?: number; + pageLength?: number; + reason?: string; + runId?: string; + snapshotHash?: string; + sourceMeasuredAt?: string; + sourcePageSize?: number; + sourceTotal?: number; +}; + +function jsonResponse(payload: unknown, status = 200, extraHeaders: Record = {}) { + return new Response(JSON.stringify(payload), { + status, + headers: { + "Cache-Control": "no-store", + "Content-Type": "application/json; charset=utf-8", + ...extraHeaders, + }, + }); +} + +function requireString(value: string | undefined, name: string) { + const normalized = value?.trim(); + if (!normalized) throw new Error(`${name} is required`); + return normalized; +} + +function requireInteger(value: number | undefined, name: string, min: number, max: number) { + if (!Number.isInteger(value) || value === undefined || value < min || value > max) { + throw new Error(`${name} must be an integer between ${min} and ${max}`); + } + return value; +} + +function assertNoActiveMirrorRun(status: Record) { + if (!Array.isArray(status.runs)) { + throw new Error("skills.sh mirror status response lacks runs"); + } + const activeRun = status.runs.find((value) => { + if (value === null || typeof value !== "object") return false; + const runStatus = (value as Record).status; + return runStatus === "running" || runStatus === "paused" || runStatus === "reconciling"; + }); + if (activeRun === undefined) return; + const runId = + activeRun !== null && + typeof activeRun === "object" && + typeof (activeRun as Record).runId === "string" + ? `: ${(activeRun as Record).runId}` + : ""; + throw new Error(`skills.sh mirror already has an active run${runId}`); +} + +async function callConvexOperator(authorization: string, body: Record) { + const response = await fetch(`${TEST_CONVEX_SITE_URL}${OPERATOR_PATH}`, { + method: "POST", + headers: { + Accept: "application/json", + Authorization: authorization, + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + }); + const text = await response.text(); + if (!response.ok) { + throw new Error(`Convex Test mirror operator returned HTTP ${response.status}: ${text}`); + } + return JSON.parse(text) as Record; +} + +function createBatchLeaseHeartbeat(args: { + authorization: string; + runId: string; + page: number; + offset: number; + leaseToken: string; +}) { + let nextHeartbeatAt = Date.now() + BATCH_LEASE_HEARTBEAT_INTERVAL_MS; + let pending: Promise | null = null; + return async () => { + if (Date.now() < nextHeartbeatAt) return; + if (pending) return await pending; + pending = callConvexOperator(args.authorization, { + operation: "mirror-batch-claim", + runId: args.runId, + page: args.page, + offset: args.offset, + leaseToken: args.leaseToken, + }) + .then(() => undefined) + .finally(() => { + nextHeartbeatAt = Date.now() + BATCH_LEASE_HEARTBEAT_INTERVAL_MS; + pending = null; + }); + await pending; + }; +} + +export default defineEventHandler(async (event) => { + const policy = getSkillsShCatalogTestSourcePolicy(process.env); + if (!policy.allowed) return jsonResponse({ error: "not_found" }, 404); + const authorization = getHeader(event, "authorization")?.trim() ?? ""; + if (!authorization.toLowerCase().startsWith("bearer ")) { + return jsonResponse({ error: "unauthorized" }, 401); + } + + try { + const body = (await readBody(event)) as MirrorRequest; + const operation = body.operation; + if (operation === "status") { + return jsonResponse(await callConvexOperator(authorization, { operation: "mirror-status" })); + } + if (operation === "run") { + return jsonResponse( + await callConvexOperator(authorization, { + operation: "mirror-run", + runId: requireString(body.runId, "runId"), + }), + ); + } + if (operation === "isolation") { + return jsonResponse( + await callConvexOperator(authorization, { operation: "mirror-isolation" }), + ); + } + if (operation === "read") { + return jsonResponse( + await callConvexOperator(authorization, { + operation: "mirror-read", + externalId: requireString(body.externalId, "externalId"), + }), + ); + } + if (operation === "source-summary") { + return jsonResponse( + await callConvexOperator(authorization, { + operation: "mirror-source-summary", + snapshotHash: requireString(body.snapshotHash, "snapshotHash"), + }), + ); + } + if (operation === "conflicts") { + return jsonResponse( + await callConvexOperator(authorization, { + operation: "mirror-conflicts", + runId: requireString(body.runId, "runId"), + limit: requireInteger(body.limit, "limit", 1, 50), + }), + ); + } + if (operation === "page") { + return jsonResponse( + await callConvexOperator(authorization, { + operation: "mirror-page", + cursor: body.cursor ?? null, + limit: requireInteger(body.limit, "limit", 1, 500), + }), + ); + } + if (operation === "detail-page") { + return jsonResponse( + await callConvexOperator(authorization, { + operation: "mirror-detail-page", + cursor: body.cursor ?? null, + limit: requireInteger(body.limit, "limit", 1, MAX_DETAIL_PAGE_ROWS), + }), + ); + } + if (operation === "facet-page") { + return jsonResponse( + await callConvexOperator(authorization, { + operation: "mirror-facet-page", + cursor: body.cursor ?? null, + limit: requireInteger(body.limit, "limit", 1, 500), + }), + ); + } + if (operation === "configure") { + return jsonResponse( + await callConvexOperator(authorization, { + operation: "mirror-configure", + enabled: body.enabled === true, + reason: requireString(body.reason, "reason"), + confirm: "enable-skills-sh-mirror-test", + maxRowsPerRun: MAX_TEST_SOURCE_ROWS, + maxRowsPerBatch: MIRROR_BATCH_SIZE, + maxDetailBytes: MAX_DETAIL_BYTES, + }), + ); + } + if (operation === "start") { + assertNoActiveMirrorRun( + await callConvexOperator(authorization, { operation: "mirror-status" }), + ); + const oidcToken = await getVercelOidcToken(); + const sourceMeasuredAt = new Date().toISOString(); + const source = await measureSkillsShMirrorProofSource({ oidcToken }); + if (source.catalogTotal < 1 || source.catalogTotal > MAX_TEST_SOURCE_ROWS) { + throw new Error( + `skills.sh source total ${source.catalogTotal} exceeds the Test mirror capacity`, + ); + } + const sourceTotal = source.catalogTotal + source.controlledSupplementExternalIds.length; + if (sourceTotal > MAX_TEST_SOURCE_ROWS) { + throw new Error(`skills.sh proof source total ${sourceTotal} exceeds the Test capacity`); + } + const snapshotId = buildSkillsShMirrorProofSnapshotId(source); + const snapshot = parseSkillsShMirrorProofSnapshotId(snapshotId); + const sourceSnapshotHash = requireString(snapshot.sourceSnapshotHash, "sourceSnapshotHash"); + let sourceCaptureWrites = 0; + for (const page of source.sourcePages) { + const stored = await callConvexOperator(authorization, { + operation: "mirror-source-page-store", + snapshotHash: sourceSnapshotHash, + ...page, + }); + if (stored.stored === true) sourceCaptureWrites += 1; + } + const sourceCapture = await callConvexOperator(authorization, { + operation: "mirror-source-summary", + snapshotHash: sourceSnapshotHash, + }); + const result = await callConvexOperator(authorization, { + operation: "mirror-start", + reason: requireString(body.reason, "reason"), + snapshotId, + sourceSnapshotHash, + sourceCaptureWrites, + sourceTotal, + sourcePageSize: SOURCE_PAGE_SIZE, + sourceMeasuredAt, + }); + return jsonResponse({ + ...result, + sourceTotal, + sourceCatalogTotal: source.catalogTotal, + controlledOverlayTotal: source.controlledOverlayExternalIds.length, + controlledSupplementTotal: source.controlledSupplementExternalIds.length, + sourceMeasurementRequests: source.sourceRequests, + sourceCapture: { + ...sourceCapture, + requestDbWrites: sourceCaptureWrites, + }, + sourceMeasuredAt, + sourcePageSize: SOURCE_PAGE_SIZE, + }); + } + if (operation === "start-replay") { + const sourceMeasuredAt = requireString(body.sourceMeasuredAt, "sourceMeasuredAt"); + if (Number.isNaN(Date.parse(sourceMeasuredAt))) { + throw new Error("sourceMeasuredAt must be an ISO timestamp"); + } + const sourceTotal = requireInteger(body.sourceTotal, "sourceTotal", 1, MAX_TEST_SOURCE_ROWS); + const sourcePageSize = requireInteger( + body.sourcePageSize, + "sourcePageSize", + 1, + SOURCE_PAGE_SIZE, + ); + const result = await callConvexOperator(authorization, { + operation: "mirror-start", + reason: requireString(body.reason, "reason"), + snapshotId: `skills-sh-captured:${requireString(body.capturedRunId, "capturedRunId")}`, + sourceTotal, + sourcePageSize, + sourceMeasuredAt, + }); + return jsonResponse({ + ...result, + sourceTotal, + sourceMeasuredAt, + sourcePageSize, + captured: true, + }); + } + if (operation === "step") { + const runId = requireString(body.runId, "runId"); + const page = requireInteger(body.page, "page", 0, 100_000); + const offset = requireInteger(body.offset, "offset", 0, SOURCE_PAGE_SIZE - 1); + const leaseToken = crypto.randomUUID(); + const lease = await callConvexOperator(authorization, { + operation: "mirror-batch-claim", + runId, + page, + offset, + leaseToken, + }); + try { + const sourceTotal = requireInteger( + typeof lease.sourceTotal === "number" ? lease.sourceTotal : undefined, + "lease.sourceTotal", + 1, + MAX_TEST_SOURCE_ROWS, + ); + const snapshot = parseSkillsShMirrorProofSnapshotId( + requireString( + typeof lease.snapshotId === "string" ? lease.snapshotId : undefined, + "lease.snapshotId", + ), + ); + if ( + sourceTotal !== + snapshot.catalogTotal + snapshot.controlledSupplementExternalIds.length + ) { + throw new Error("skills.sh mirror run proof source metadata is inconsistent"); + } + const controlledPage = Math.ceil(snapshot.catalogTotal / SOURCE_PAGE_SIZE); + const beforeRequest = createBatchLeaseHeartbeat({ + authorization, + runId, + page, + offset, + leaseToken, + }); + const batch = + page === controlledPage && snapshot.controlledSupplementExternalIds.length > 0 + ? await fetchSkillsShMirrorControlledBatch( + { + page, + offset, + limit: MIRROR_BATCH_SIZE, + maxDetailBytes: MAX_DETAIL_BYTES, + sourceTotal, + externalIds: snapshot.controlledSupplementExternalIds, + }, + { beforeRequest }, + ) + : page < controlledPage + ? await (async () => { + const capturedPage = + lease.sourcePage && + typeof lease.sourcePage === "object" && + !Array.isArray(lease.sourcePage) + ? (lease.sourcePage as Record) + : null; + const expectedPage = snapshot.evidence?.pagination.requestedPages.find( + (entry) => entry.page === page, + ); + if ( + !capturedPage || + !expectedPage || + capturedPage.page !== page || + capturedPage.sourceTotal !== snapshot.catalogTotal || + capturedPage.pageLength !== expectedPage.count || + capturedPage.hasMore !== expectedPage.hasMore || + capturedPage.identityHash !== expectedPage.identityHash || + capturedPage.contentHash !== expectedPage.contentHash || + !Array.isArray(capturedPage.rows) + ) { + throw new Error( + `captured skills.sh leaderboard page does not match the proof: ${page}`, + ); + } + const oidcToken = await getVercelOidcToken(); + const catalogBatch = await fetchSkillsShMirrorBatch( + { page, offset, limit: MIRROR_BATCH_SIZE, maxDetailBytes: MAX_DETAIL_BYTES }, + { + oidcToken, + beforeRequest, + sourcePage: { + data: capturedPage.rows as never, + pagination: { + page, + perPage: SOURCE_PAGE_SIZE, + total: snapshot.catalogTotal, + hasMore: expectedPage.hasMore, + }, + }, + }, + ); + if (catalogBatch.sourceTotal !== snapshot.catalogTotal) { + throw new Error("skills.sh catalog source total changed during the run"); + } + if ( + expectedPage.count !== catalogBatch.pageLength || + expectedPage.hasMore !== catalogBatch.hasMore || + expectedPage.identityHash !== catalogBatch.sourcePageIdentityHash + ) { + throw new Error( + `skills.sh ordered leaderboard page changed during the run: ${page}`, + ); + } + const controlledOverlayExternalIds = new Set( + snapshot.controlledOverlayExternalIds, + ); + const overlayExternalIds = catalogBatch.rows.flatMap((row) => + controlledOverlayExternalIds.has(row.externalId) ? [row.externalId] : [], + ); + const overlay = + overlayExternalIds.length > 0 + ? await fetchSkillsShMirrorControlledBatch( + { + page, + offset: 0, + limit: overlayExternalIds.length, + maxDetailBytes: MAX_DETAIL_BYTES, + sourceTotal, + externalIds: overlayExternalIds, + }, + { beforeRequest }, + ) + : null; + const overlayByExternalId = new Map( + overlay?.rows.map((row) => [row.externalId, row]), + ); + return { + ...catalogBatch, + sourceTotal, + hasMore: + catalogBatch.hasMore || snapshot.controlledSupplementExternalIds.length > 0, + sourceRequests: catalogBatch.sourceRequests + (overlay?.sourceRequests ?? 0), + sourceBytes: catalogBatch.sourceBytes + (overlay?.sourceBytes ?? 0), + rows: catalogBatch.rows.map((row) => { + const controlled = overlayByExternalId.get(row.externalId); + if (!controlled) return row; + if ("quarantined" in row) return controlled; + return { + ...controlled, + upstreamSourceType: row.upstreamSourceType, + upstreamInstalls: row.upstreamInstalls, + upstreamScanners: row.upstreamScanners, + }; + }), + }; + })() + : (() => { + throw new Error("skills.sh mirror cursor is beyond the proof source"); + })(); + const externalIds = batch.rows.flatMap((row) => + "quarantined" in row ? [] : [row.externalId], + ); + const classificationState = + externalIds.length === 0 + ? { states: [] } + : await callConvexOperator(authorization, { + operation: "mirror-classification-states", + externalIds, + }); + if (!Array.isArray(classificationState.states)) { + throw new Error("Convex Test mirror classification state is invalid"); + } + const rows = enrichSkillsShMirrorClassifications( + batch.rows as Parameters[0], + classificationState.states as SkillsShMirrorClassificationState[], + ); + return jsonResponse( + await callConvexOperator(authorization, { + operation: "mirror-batch", + runId, + leaseToken, + ...batch, + rows, + }), + ); + } catch (error) { + try { + await callConvexOperator(authorization, { + operation: "mirror-batch-release", + runId, + page, + offset, + leaseToken, + }); + } catch { + // The five-minute durable lease remains the crash/outage recovery path. + } + throw error; + } + } + if (operation === "step-replay") { + const runId = requireString(body.runId, "runId"); + const page = requireInteger(body.page, "page", 0, 100_000); + const offset = requireInteger(body.offset, "offset", 0, SOURCE_PAGE_SIZE - 1); + const pageLength = requireInteger(body.pageLength, "pageLength", 1, SOURCE_PAGE_SIZE); + const sourceTotal = requireInteger(body.sourceTotal, "sourceTotal", 1, MAX_TEST_SOURCE_ROWS); + if (typeof body.hasMore !== "boolean") throw new Error("hasMore is required"); + if ( + !Array.isArray(body.externalIds) || + body.externalIds.length < 1 || + body.externalIds.length > MIRROR_BATCH_SIZE || + body.externalIds.some((externalId) => typeof externalId !== "string" || !externalId.trim()) + ) { + throw new Error(`externalIds must contain between 1 and ${MIRROR_BATCH_SIZE} strings`); + } + const leaseToken = crypto.randomUUID(); + await callConvexOperator(authorization, { + operation: "mirror-batch-claim", + runId, + page, + offset, + leaseToken, + }); + try { + const captured = await callConvexOperator(authorization, { + operation: "mirror-replay-rows", + externalIds: body.externalIds, + }); + if (!Array.isArray(captured.rows)) { + throw new Error("Convex Test mirror replay rows are invalid"); + } + const rows = buildSkillsShMirrorReplayRows(captured.rows as never); + return jsonResponse( + await callConvexOperator(authorization, { + operation: "mirror-batch", + runId, + page, + offset, + leaseToken, + pageLength, + hasMore: body.hasMore, + sourceTotal, + sourceRequests: 0, + sourceBytes: 0, + rows, + }), + ); + } catch (error) { + try { + await callConvexOperator(authorization, { + operation: "mirror-batch-release", + runId, + page, + offset, + leaseToken, + }); + } catch { + // The five-minute durable lease remains the crash/outage recovery path. + } + throw error; + } + } + if (operation === "pause" || operation === "resume") { + return jsonResponse( + await callConvexOperator(authorization, { + operation: "mirror-pause", + runId: requireString(body.runId, "runId"), + paused: operation === "pause", + reason: requireString(body.reason, "reason"), + confirm: "set-skills-sh-mirror-pause", + }), + ); + } + if (operation === "discard") { + return jsonResponse( + await callConvexOperator(authorization, { + operation: "mirror-cancel", + runId: requireString(body.runId, "runId"), + reason: requireString(body.reason, "reason"), + confirm: "cancel-skills-sh-mirror-test-run", + }), + ); + } + if (operation === "reconcile") { + return jsonResponse( + await callConvexOperator(authorization, { + operation: "mirror-reconcile", + runId: requireString(body.runId, "runId"), + limit: requireInteger(body.limit ?? 250, "limit", 1, 250), + }), + ); + } + return jsonResponse({ error: "unknown_operation" }, 400); + } catch (error) { + const retryAfterSeconds = skillsShSourceRetryAfterSeconds(error); + if (retryAfterSeconds !== null) { + return jsonResponse( + { + error: "skills_sh_source_rate_limited", + message: error instanceof Error ? error.message : "skills.sh source rate limited", + retryAfterSeconds, + }, + 429, + { "Retry-After": String(retryAfterSeconds) }, + ); + } + return jsonResponse( + { + error: "skills_sh_mirror_test_failed", + message: error instanceof Error ? error.message : "Unknown Test mirror failure", + }, + 502, + ); + } +}); diff --git a/server/skillsShCatalogSource.test.ts b/server/skillsShCatalogSource.test.ts index 96261eae..e84886a0 100644 --- a/server/skillsShCatalogSource.test.ts +++ b/server/skillsShCatalogSource.test.ts @@ -1,17 +1,1577 @@ /* @vitest-environment node */ -import { describe, expect, it, vi } from "vitest"; +import { createHash } from "node:crypto"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { + buildSkillsShMirrorProofSnapshotId, + buildSkillsShMirrorObservation, + buildSkillsShMirrorControlledObservation, + buildSkillsShMirrorDetail, + buildSkillsShMirrorUpstreamScanners, captureSkillsShCatalogTestSnapshot, fetchSkillsShCatalogDetail, + fetchSkillsShMirrorBatch, + fetchSkillsShMirrorControlledBatch, fetchSkillsShCatalogPage, fetchSkillsShCatalogTestPage, getSkillsShCatalogTestSourcePolicy, + measureSkillsShMirrorProofSource, + parseSkillsShMirrorProofSnapshotId, + resolveSkillsShMirrorGitHubLocators, + skillsShSourceRetryAfterSeconds, SkillsShCatalogOwnerProofRequiredError, validateSkillsShCatalogGitHubOwnerProof, } from "./skillsShCatalogSource"; describe("skills.sh Vercel source boundary", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it("retains each upstream scanner status and source link independently", () => { + expect( + buildSkillsShMirrorUpstreamScanners( + { + id: "vercel-labs/skills/find-skills", + source: "vercel-labs/skills", + slug: "find-skills", + audits: [ + { + provider: "Runlayer", + slug: "runlayer", + status: "pass", + auditedAt: "2026-07-22T20:00:00.000Z", + }, + { + provider: "Gen Agent Trust Hub", + slug: "agent-trust-hub", + status: "pass", + auditedAt: "2026-07-22T20:01:00.000Z", + }, + { + provider: "Socket", + slug: "socket", + status: "pass", + auditedAt: "2026-07-22T20:02:00.000Z", + }, + { + provider: "Snyk", + slug: "snyk", + status: "warn", + auditedAt: "2026-07-22T20:03:00.000Z", + }, + { + provider: "ZeroLeaks", + slug: "zeroleaks", + status: "fail", + auditedAt: "2026-07-22T20:04:00.000Z", + }, + ], + }, + "https://skills.sh/vercel-labs/skills/find-skills", + ), + ).toEqual({ + genAgentTrustHub: { + status: "pass", + sourceUrl: "https://skills.sh/vercel-labs/skills/find-skills/security/agent-trust-hub", + sourceCheckedAt: "2026-07-22T20:01:00.000Z", + }, + socket: { + status: "pass", + sourceUrl: "https://skills.sh/vercel-labs/skills/find-skills/security/socket", + sourceCheckedAt: "2026-07-22T20:02:00.000Z", + }, + snyk: { + status: "warn", + sourceUrl: "https://skills.sh/vercel-labs/skills/find-skills/security/snyk", + sourceCheckedAt: "2026-07-22T20:03:00.000Z", + }, + }); + + expect( + buildSkillsShMirrorUpstreamScanners(null, "https://skills.sh/open.feishu.cn/lark-doc"), + ).toEqual({ + genAgentTrustHub: { status: "unavailable" }, + socket: { status: "unavailable" }, + snyk: { status: "unavailable" }, + }); + }); + + it("builds a hidden controlled observation only from exact immutable content", () => { + const fullContent = "# Controlled\n\nThis content is longer than the retained prefix."; + const supplement = { + externalId: "owner/repo/controlled", + owner: "owner", + repo: "repo", + slug: "controlled", + displayName: "Controlled", + sourceUrl: "https://www.skills.sh/owner/repo/controlled", + githubPath: "skills/controlled", + detailPath: "skills/controlled/SKILL.md", + githubCommit: "0123456789abcdef0123456789abcdef01234567", + sourceContentHash: createHash("sha256").update(fullContent).digest("hex"), + }; + + expect( + buildSkillsShMirrorControlledObservation( + supplement, + fullContent, + Buffer.byteLength(fullContent), + 16, + ), + ).toMatchObject({ + externalId: "owner/repo/controlled", + sourceType: "github", + upstreamSourceType: "controlled-github", + canonicalRepoUrl: "https://github.com/owner/repo", + githubPath: "skills/controlled", + githubCommit: "0123456789abcdef0123456789abcdef01234567", + sourceContentHash: supplement.sourceContentHash, + upstreamInstalls: 0, + upstreamScanners: { + genAgentTrustHub: { status: "unavailable" }, + socket: { status: "unavailable" }, + snyk: { status: "unavailable" }, + }, + detail: { + path: "skills/controlled/SKILL.md", + contentBytes: 16, + sourceBytes: Buffer.byteLength(fullContent), + sourceFileCount: 1, + truncated: true, + }, + }); + expect(() => + buildSkillsShMirrorControlledObservation( + { ...supplement, sourceContentHash: "0".repeat(64) }, + fullContent, + Buffer.byteLength(fullContent), + 16, + ), + ).toThrow("controlled skills.sh mirror source hash changed"); + }); + + it("delegates long controlled-source Retry-After waits to durable recovery", async () => { + const fetchImpl = vi.fn(async () => { + return new Response("rate limited", { + status: 429, + headers: { "retry-after": "120" }, + }); + }); + + const error = await fetchSkillsShMirrorControlledBatch( + { + page: 0, + offset: 0, + limit: 1, + maxDetailBytes: 64, + sourceTotal: 1, + externalIds: ["patrick-erichsen/skills/html"], + }, + { fetchImpl: fetchImpl as typeof fetch }, + ).catch((value: unknown) => value); + + expect(skillsShSourceRetryAfterSeconds(error)).toBe(120); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + it("does not sleep after the final controlled-source rate-limit attempt", async () => { + vi.useFakeTimers(); + let canceledResponses = 0; + const fetchImpl = vi.fn(async () => { + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("rate limited")); + }, + cancel() { + canceledResponses += 1; + }, + }), + { + status: 429, + headers: { "retry-after": "1" }, + }, + ); + }); + let settled = false; + const resultPromise = fetchSkillsShMirrorControlledBatch( + { + page: 0, + offset: 0, + limit: 1, + maxDetailBytes: 64, + sourceTotal: 1, + externalIds: ["patrick-erichsen/skills/html"], + }, + { fetchImpl: fetchImpl as typeof fetch }, + ).catch((value: unknown) => value); + void resultPromise.then(() => { + settled = true; + }); + + await vi.advanceTimersByTimeAsync(0); + expect(canceledResponses).toBe(1); + await vi.advanceTimersByTimeAsync(3_000); + + expect(fetchImpl).toHaveBeenCalledTimes(4); + expect(canceledResponses).toBe(4); + expect(settled).toBe(true); + expect(skillsShSourceRetryAfterSeconds(await resultPromise)).toBe(1); + }); + + it("supplements only controlled identities absent from the authenticated catalog", async () => { + const controlledRow = { + id: "patrick-erichsen/skills/html", + installUrl: null, + installs: 1, + name: "HTML", + slug: "html", + source: "patrick-erichsen/skills", + sourceType: "github", + url: "https://www.skills.sh/patrick-erichsen/skills/html", + }; + const pages = [ + { + data: Array.from({ length: 500 }, (_, index) => + index === 0 + ? controlledRow + : { + id: `owner/repo/other-${index}`, + installUrl: null, + installs: 1, + name: `Other ${index}`, + slug: `other-${index}`, + source: "owner/repo", + sourceType: "github", + url: `https://www.skills.sh/owner/repo/other-${index}`, + }, + ), + pagination: { page: 0, perPage: 500, total: 501, hasMore: true }, + }, + { + data: [ + { + id: "owner/repo/other", + installUrl: null, + installs: 1, + name: "Other", + slug: "other", + source: "owner/repo", + sourceType: "github", + url: "https://www.skills.sh/owner/repo/other", + }, + ], + pagination: { page: 1, perPage: 500, total: 501, hasMore: false }, + }, + { + data: [], + pagination: { page: 2, perPage: 500, total: 501, hasMore: false }, + }, + ]; + const fetchImpl = vi.fn(async (input: string | URL | Request) => { + const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url; + if (url.includes("/api/v1/skills?page=")) { + return new Response(JSON.stringify(pages.shift())); + } + if (url.includes("/api/v1/skills/search?")) { + return new Response( + JSON.stringify({ + count: 1, + data: [controlledRow], + durationMs: 3, + query: "html", + searchType: "full-text", + }), + ); + } + if (url.endsWith("/api/v1/skills/patrick-erichsen/skills/html")) { + return new Response( + JSON.stringify({ + files: [{ contents: "# HTML", path: "skills/html/SKILL.md" }], + hash: "a".repeat(64), + id: controlledRow.id, + installs: 1, + slug: "html", + source: "patrick-erichsen/skills", + }), + ); + } + if (url.includes("?_rsc=")) { + return new Response( + '0:["$","div",null,{"children":"HTML","className":"skill"}]\n1:{"prompt":"use html"}\n', + { headers: { "Content-Type": "text/x-component" } }, + ); + } + if (url === controlledRow.url) { + return new Response( + '", + { headers: { "Content-Type": "text/html; charset=utf-8" } }, + ); + } + throw new Error(`unexpected URL ${url}`); + }); + + const measured = await measureSkillsShMirrorProofSource({ + oidcToken: "oidc-token", + fetchImpl: fetchImpl as typeof fetch, + minimumApiRequestIntervalMs: 0, + }); + + expect(measured).toMatchObject({ + catalogTotal: 501, + controlledExternalIds: ["patrick-erichsen/skills/html", "steipete/clawdis/discrawl"], + controlledOverlayExternalIds: ["patrick-erichsen/skills/html"], + controlledSupplementExternalIds: ["steipete/clawdis/discrawl"], + pageSize: 500, + sourceRequests: 7, + sourcePages: [ + { + page: 0, + sourceTotal: 501, + pageLength: 500, + hasMore: true, + }, + { + page: 1, + sourceTotal: 501, + pageLength: 1, + hasMore: false, + }, + ], + evidence: { + pagination: { + endpointExhausted: true, + databaseCoverage: "leaderboard-only", + page0: { page: 0, perPage: 500, total: 501, hasMore: true }, + requestedPages: [ + { page: 0, count: 500, hasMore: true }, + { page: 1, count: 1, hasMore: false }, + { page: 2, count: 0, hasMore: false }, + ], + finalNonemptyPage: { + page: 1, + count: 1, + pagination: { page: 1, perPage: 500, total: 501, hasMore: false }, + }, + firstBeyondEndPage: { + page: 2, + count: 0, + pagination: { page: 2, perPage: 500, total: 501, hasMore: false }, + }, + uniqueIds: 501, + duplicateIds: 0, + }, + fields: { + sampledExternalId: "patrick-erichsen/skills/html", + leaderboard: { + topLevelKeys: ["data", "pagination"], + paginationKeys: ["hasMore", "page", "perPage", "total"], + rowKeys: [ + "id", + "installUrl", + "installs", + "name", + "slug", + "source", + "sourceType", + "url", + ], + taxonomyFields: [], + }, + search: { + topLevelKeys: ["count", "data", "durationMs", "query", "searchType"], + rowKeys: [ + "id", + "installUrl", + "installs", + "name", + "slug", + "source", + "sourceType", + "url", + ], + taxonomyFields: [], + }, + detail: { + topLevelKeys: ["files", "hash", "id", "installs", "slug", "source"], + fileKeys: ["contents", "path"], + taxonomyFields: [], + }, + page: { + url: controlledRow.url, + jsonLdDocuments: [ + { + type: "SoftwareApplication", + keys: ["@context", "@type", "applicationCategory", "name"], + }, + ], + taxonomyFields: [], + }, + rsc: { + objectKeys: ["children", "className", "prompt"], + taxonomyFields: [], + }, + normalizedUpstreamTaxonomyFields: [], + }, + }, + }); + expect(fetchImpl).toHaveBeenCalledTimes(7); + }); + + it("delegates long proof-metadata Retry-After waits to durable recovery", async () => { + const row = { + id: "owner/repo/skill", + installUrl: "https://github.com/owner/repo", + installs: 1, + name: "Skill", + slug: "skill", + source: "owner/repo", + sourceType: "github", + url: "https://www.skills.sh/owner/repo/skill", + }; + let metadataAttempts = 0; + const fetchImpl = vi.fn(async (input: string | URL | Request) => { + const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url; + if (url.includes("/api/v1/skills?page=0")) { + return new Response( + JSON.stringify({ + data: [row], + pagination: { page: 0, perPage: 500, total: 1, hasMore: false }, + }), + ); + } + if (url.includes("/api/v1/skills?page=1")) { + return new Response( + JSON.stringify({ + data: [], + pagination: { page: 1, perPage: 500, total: 1, hasMore: false }, + }), + ); + } + if (url.includes("/api/v1/skills/search?")) { + return new Response(JSON.stringify({ data: [row] })); + } + if (url.endsWith("/api/v1/skills/owner/repo/skill")) { + return new Response( + JSON.stringify({ + files: [], + hash: "a".repeat(64), + id: row.id, + installs: row.installs, + slug: row.slug, + source: row.source, + }), + ); + } + if (url === row.url) { + metadataAttempts += 1; + return new Response("rate limited", { + status: 429, + headers: { "retry-after": "120" }, + }); + } + throw new Error(`unexpected URL ${url}`); + }); + + const error = await measureSkillsShMirrorProofSource({ + oidcToken: "oidc-token", + fetchImpl: fetchImpl as typeof fetch, + minimumApiRequestIntervalMs: 0, + }).catch((value: unknown) => value); + + expect(skillsShSourceRetryAfterSeconds(error)).toBe(120); + expect(metadataAttempts).toBe(1); + }); + + it("hashes captured leaderboard rows independently of upstream object key order", async () => { + const row = { + url: "https://www.skills.sh/owner/repo/skill", + sourceType: "github", + source: "owner/repo", + slug: "skill", + name: "Skill", + installs: 42, + installUrl: "https://github.com/owner/repo", + id: "owner/repo/skill", + }; + const pages = [ + { + data: [row], + pagination: { page: 0, perPage: 500, total: 1, hasMore: false }, + }, + { + data: [], + pagination: { page: 1, perPage: 500, total: 1, hasMore: false }, + }, + ]; + const fetchImpl = vi.fn(async (input: string | URL | Request) => { + const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url; + if (url.includes("/api/v1/skills?page=")) { + return new Response(JSON.stringify(pages.shift())); + } + if (url.includes("/api/v1/skills/search?")) { + return new Response( + JSON.stringify({ + count: 0, + data: [], + durationMs: 1, + query: "skill", + searchType: "full-text", + }), + ); + } + if (url.endsWith("/api/v1/skills/owner/repo/skill")) { + return new Response( + JSON.stringify({ + files: [], + hash: "a".repeat(64), + id: row.id, + installs: row.installs, + slug: row.slug, + source: row.source, + }), + ); + } + if (url.includes("?_rsc=")) { + return new Response("0:{}", { headers: { "Content-Type": "text/x-component" } }); + } + if (url === row.url) { + return new Response("", { + headers: { "Content-Type": "text/html; charset=utf-8" }, + }); + } + throw new Error(`unexpected URL ${url}`); + }); + + const measured = await measureSkillsShMirrorProofSource({ + oidcToken: "oidc-token", + fetchImpl: fetchImpl as typeof fetch, + minimumApiRequestIntervalMs: 0, + }); + + const capturedRows = measured.sourcePages[0]!.rows; + expect(Object.keys(capturedRows[0]!)).toEqual([ + "id", + "installUrl", + "installs", + "name", + "slug", + "source", + "sourceType", + "url", + ]); + expect(measured.sourcePages[0]!.contentHash).toBe( + createHash("sha256").update(JSON.stringify(capturedRows)).digest("hex"), + ); + expect(measured.evidence.pagination.requestedPages[0]).toMatchObject({ + sourceBytes: expect.any(Number), + serializedBytes: expect.any(Number), + }); + }); + + it("round-trips immutable proof source metadata through the run snapshot", () => { + const evidence = { + pagination: { + endpointExhausted: true as const, + databaseCoverage: "leaderboard-only" as const, + page0: { page: 0, perPage: 500, total: 9_571, hasMore: true }, + requestedPages: [ + { + page: 0, + count: 500, + hasMore: true, + identityHash: "page-0", + contentHash: "content-page-0", + sourceBytes: 1_000, + serializedBytes: 750, + }, + ], + finalNonemptyPage: { + page: 19, + count: 71, + pagination: { page: 19, perPage: 500, total: 9_571, hasMore: false }, + }, + firstBeyondEndPage: { + page: 20, + count: 0, + pagination: { page: 20, perPage: 500, total: 9_571, hasMore: false }, + }, + uniqueIds: 9_571, + duplicateIds: 0, + }, + fields: { + sampledExternalId: "vercel-labs/skills/find-skills", + leaderboard: { + topLevelKeys: ["data", "pagination"], + paginationKeys: ["hasMore", "page", "perPage", "total"], + rowKeys: ["id"], + taxonomyFields: [], + }, + search: { topLevelKeys: ["data"], rowKeys: ["id"], taxonomyFields: [] }, + detail: { topLevelKeys: ["id"], fileKeys: [], taxonomyFields: [] }, + page: { + url: "https://www.skills.sh/vercel-labs/skills/find-skills", + jsonLdDocuments: [], + taxonomyFields: [], + }, + rsc: { objectKeys: [], taxonomyFields: [] }, + normalizedUpstreamTaxonomyFields: [], + }, + }; + const snapshotId = buildSkillsShMirrorProofSnapshotId({ + catalogTotal: 9_571, + controlledExternalIds: ["patrick-erichsen/skills/html", "steipete/clawdis/discrawl"], + controlledOverlayExternalIds: ["patrick-erichsen/skills/html"], + controlledSupplementExternalIds: ["steipete/clawdis/discrawl"], + evidence, + }); + expect(parseSkillsShMirrorProofSnapshotId(snapshotId)).toEqual({ + catalogTotal: 9_571, + controlledExternalIds: ["patrick-erichsen/skills/html", "steipete/clawdis/discrawl"], + controlledOverlayExternalIds: ["patrick-erichsen/skills/html"], + controlledSupplementExternalIds: ["steipete/clawdis/discrawl"], + sourceSnapshotHash: expect.stringMatching(/^[a-f0-9]{64}$/), + evidence, + }); + const changedSnapshotId = buildSkillsShMirrorProofSnapshotId({ + catalogTotal: 9_571, + controlledExternalIds: ["patrick-erichsen/skills/html", "steipete/clawdis/discrawl"], + controlledOverlayExternalIds: ["patrick-erichsen/skills/html"], + controlledSupplementExternalIds: ["steipete/clawdis/discrawl"], + evidence: { + ...evidence, + pagination: { + ...evidence.pagination, + requestedPages: [ + { + page: 0, + count: 500, + hasMore: true, + identityHash: "different-page-0", + contentHash: "different-content-page-0", + }, + ], + }, + }, + }); + expect(snapshotId.split(".").slice(0, 2).join(".")).not.toBe( + changedSnapshotId.split(".").slice(0, 2).join("."), + ); + const changedTransportSnapshotId = buildSkillsShMirrorProofSnapshotId({ + catalogTotal: 9_571, + controlledExternalIds: ["patrick-erichsen/skills/html", "steipete/clawdis/discrawl"], + controlledOverlayExternalIds: ["patrick-erichsen/skills/html"], + controlledSupplementExternalIds: ["steipete/clawdis/discrawl"], + evidence: { + ...evidence, + pagination: { + ...evidence.pagination, + requestedPages: [ + { + ...evidence.pagination.requestedPages[0]!, + sourceBytes: 1_024, + }, + ], + }, + }, + }); + expect(snapshotId.split(".").slice(0, 2).join(".")).not.toBe( + changedTransportSnapshotId.split(".").slice(0, 2).join("."), + ); + const changedPartitionSnapshotId = buildSkillsShMirrorProofSnapshotId({ + catalogTotal: 9_571, + controlledExternalIds: ["patrick-erichsen/skills/html", "steipete/clawdis/discrawl"], + controlledOverlayExternalIds: ["steipete/clawdis/discrawl"], + controlledSupplementExternalIds: ["patrick-erichsen/skills/html"], + evidence, + }); + const [changedPayload] = changedPartitionSnapshotId.split("."); + const [, snapshotHash, encodedEvidence] = snapshotId.split("."); + expect(() => + parseSkillsShMirrorProofSnapshotId([changedPayload, snapshotHash, encodedEvidence].join(".")), + ).toThrow("skills.sh mirror proof source metadata is invalid"); + const tamperedSnapshotId = snapshotId.replace(/.$/, (value) => (value === "A" ? "B" : "A")); + expect(() => parseSkillsShMirrorProofSnapshotId(tamperedSnapshotId)).toThrow( + "skills.sh mirror proof source metadata is invalid", + ); + }); + + it("rejects proof metadata redirects outside the exact skills.sh route", async () => { + const row = { + id: "owner/repo/skill", + installUrl: "https://github.com/owner/repo", + installs: 1, + name: "Skill", + slug: "skill", + source: "owner/repo", + sourceType: "github", + url: "https://www.skills.sh/owner/repo/skill", + }; + const fetchImpl = vi.fn(async (input: string | URL | Request) => { + const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url; + if (url.includes("/api/v1/skills?page=0")) { + return new Response( + JSON.stringify({ + data: [row], + pagination: { page: 0, perPage: 500, total: 1, hasMore: false }, + }), + ); + } + if (url.includes("/api/v1/skills?page=1")) { + return new Response( + JSON.stringify({ + data: [], + pagination: { page: 1, perPage: 500, total: 1, hasMore: false }, + }), + ); + } + if (url.includes("/api/v1/skills/search?")) { + return new Response(JSON.stringify({ data: [row] })); + } + if (url.endsWith("/api/v1/skills/owner/repo/skill")) { + return new Response( + JSON.stringify({ + id: row.id, + source: row.source, + slug: row.slug, + installs: row.installs, + hash: "a".repeat(64), + files: [], + }), + ); + } + if (url === row.url) { + return new Response(null, { + status: 302, + headers: { Location: "http://127.0.0.1/internal" }, + }); + } + throw new Error(`unexpected URL ${url}`); + }); + + await expect( + measureSkillsShMirrorProofSource({ + oidcToken: "oidc-token", + fetchImpl: fetchImpl as typeof fetch, + minimumApiRequestIntervalMs: 0, + }), + ).rejects.toThrow("outside the exact skills.sh route"); + expect(fetchImpl).not.toHaveBeenCalledWith("http://127.0.0.1/internal", expect.anything()); + }); + + it("rejects duplicate identities before declaring the leaderboard exhausted", async () => { + const duplicateRow = { + id: "owner/repo/duplicate", + installUrl: "https://github.com/owner/repo", + installs: 1, + name: "Duplicate", + slug: "duplicate", + source: "owner/repo", + sourceType: "github", + url: "https://www.skills.sh/owner/repo/duplicate", + }; + const pages = [ + { + data: [duplicateRow], + pagination: { page: 0, perPage: 500, total: 2, hasMore: true }, + }, + { + data: [duplicateRow], + pagination: { page: 1, perPage: 500, total: 2, hasMore: false }, + }, + ]; + const fetchImpl = vi.fn(async () => new Response(JSON.stringify(pages.shift()))); + + await expect( + measureSkillsShMirrorProofSource({ + oidcToken: "oidc-token", + fetchImpl, + minimumApiRequestIntervalMs: 0, + }), + ).rejects.toThrow("duplicate identities"); + expect(fetchImpl).toHaveBeenCalledTimes(2); + }); + + it("rejects an over-capacity leaderboard from the first measured page", async () => { + const fetchImpl = vi.fn(async () => { + return new Response( + JSON.stringify({ + data: [], + pagination: { page: 0, perPage: 500, total: 50_001, hasMore: true }, + }), + ); + }); + + await expect( + measureSkillsShMirrorProofSource({ + oidcToken: "oidc-token", + fetchImpl: fetchImpl as typeof fetch, + minimumApiRequestIntervalMs: 0, + }), + ).rejects.toThrow("skills.sh proof source total 50001 exceeds 50000 rows"); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + it("normalizes GitHub and well-known rows without inventing repository identity", () => { + expect( + buildSkillsShMirrorObservation({ + id: "Vercel-Labs/Skills/Find-Skills", + installUrl: "https://github.com/vercel-labs/skills", + installs: 42, + name: "Find Skills", + slug: "Find-Skills", + source: "Vercel-Labs/Skills", + sourceType: "github", + url: "https://skills.sh/vercel-labs/skills/find-skills", + }), + ).toMatchObject({ + externalId: "vercel-labs/skills/find-skills", + sourceType: "github", + upstreamSourceType: "github", + owner: "vercel-labs", + repo: "skills", + slug: "find-skills", + canonicalRepoUrl: "https://github.com/vercel-labs/skills", + upstreamInstalls: 42, + }); + + expect( + buildSkillsShMirrorObservation({ + id: "open.feishu.cn/lark-doc", + installUrl: null, + installs: 7, + name: "lark-doc", + slug: "lark-doc", + source: "open.feishu.cn", + sourceType: "Well-Known", + url: "https://www.skills.sh/site/open.feishu.cn/lark-doc", + }), + ).toMatchObject({ + externalId: "open.feishu.cn/lark-doc", + sourceType: "well-known", + upstreamSourceType: "well-known", + sourceHost: "open.feishu.cn", + slug: "lark-doc", + upstreamInstalls: 7, + }); + }); + + it("uses an exact GitHub install identity when the source type marker drifts", () => { + const liveRow = { + id: "larksuite/cli/lark-doc", + installUrl: "https://github.com/larksuite/cli.git", + installs: 383_123, + name: "lark-doc", + slug: "lark-doc", + source: "larksuite/cli", + sourceType: " GitHub-Repository ", + url: "https://skills.sh/larksuite/cli/lark-doc", + }; + expect(buildSkillsShMirrorObservation(liveRow)).toMatchObject({ + externalId: "larksuite/cli/lark-doc", + sourceType: "github", + upstreamSourceType: "github-repository", + owner: "larksuite", + repo: "cli", + slug: "lark-doc", + canonicalRepoUrl: "https://github.com/larksuite/cli", + upstreamInstalls: 383_123, + }); + }); + + it("normalizes advisory source types to the persisted token grammar", () => { + const liveRow = { + id: "larksuite/cli/lark-doc", + installUrl: "https://github.com/larksuite/cli", + installs: 383_123, + name: "lark-doc", + slug: "lark-doc", + source: "larksuite/cli", + sourceType: ".GitHub_Repository.", + url: "https://skills.sh/larksuite/cli/lark-doc", + }; + expect(buildSkillsShMirrorObservation(liveRow)).toMatchObject({ + sourceType: "github", + upstreamSourceType: "github_repository", + }); + expect(buildSkillsShMirrorObservation({ ...liveRow, sourceType: "__" })).toMatchObject({ + sourceType: "github", + upstreamSourceType: "missing", + }); + }); + + it("uses only the labeled Repository section for an ambiguous GitHub-backed row", () => { + const liveRow = { + id: "larksuite/cli/lark-doc", + installUrl: null, + installs: 383_123, + name: "lark-doc", + slug: "lark-doc", + source: "larksuite/cli", + sourceType: "well-known", + url: "https://www.skills.sh/larksuite/cli/lark-doc", + }; + const html = ` +
+ +
+
Repository
+ larksuite/cli +
+
+ `; + expect(buildSkillsShMirrorObservation(liveRow, html)).toMatchObject({ + externalId: "larksuite/cli/lark-doc", + sourceType: "github", + upstreamSourceType: "well-known", + owner: "larksuite", + repo: "cli", + canonicalRepoUrl: "https://github.com/larksuite/cli", + }); + + const invalidPages = [ + `
Repository
other/cli
`, + `
Repository
larksuite/cli
`, + `
Repository
larksuite/cliother/repo
`, + ]; + for (const invalidPage of invalidPages) { + expect(() => buildSkillsShMirrorObservation(liveRow, invalidPage)).toThrow( + "Unsupported skills.sh mirror identity", + ); + } + expect(() => + buildSkillsShMirrorObservation( + { ...liveRow, url: "https://www.skills.sh/larksuite/cli/other" }, + html, + ), + ).toThrow("Unsupported skills.sh mirror identity"); + }); + + it("rejects conflicting GitHub structural identity without exposing the install URL", () => { + const liveRow = { + id: "larksuite/cli/lark-doc", + installUrl: "https://github.com/larksuite/cli", + installs: 383_123, + name: "lark-doc", + slug: "lark-doc", + source: "larksuite/cli", + sourceType: "repository", + url: "https://skills.sh/larksuite/cli/lark-doc", + }; + const invalidRows = [ + { ...liveRow, installUrl: "https://github.com/larksuite/docs" }, + { ...liveRow, installUrl: "https://github.com/larksuite/cli/tree/main" }, + { ...liveRow, id: "larksuite/docs/lark-doc" }, + { ...liveRow, source: "larksuite/docs" }, + { ...liveRow, slug: "lark-sheets" }, + { ...liveRow, installUrl: null, url: "https://skills.sh/larksuite/cli/other-skill" }, + { ...liveRow, installUrl: "https://example.com/larksuite/cli" }, + { ...liveRow, installUrl: "https://github.com:8443/larksuite/cli" }, + { ...liveRow, installUrl: null, url: "https://skills.sh:8443/larksuite/cli/lark-doc" }, + ]; + for (const row of invalidRows) { + try { + buildSkillsShMirrorObservation(row); + throw new Error("expected invalid skills.sh identity to be rejected"); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + expect(message).toContain("Unsupported skills.sh mirror identity"); + expect(message).toContain("sourceType=repository"); + expect(message).toContain(`installUrlPresent=${row.installUrl !== null}`); + if (row.installUrl) expect(message).not.toContain(row.installUrl); + } + } + }); + + it("requires the exact skills.sh site route for well-known identity", () => { + const liveRow = { + id: "open.feishu.cn/lark-doc", + installUrl: null, + installs: 7, + name: "lark-doc", + slug: "lark-doc", + source: "open.feishu.cn", + sourceType: "well-known", + url: "https://skills.sh/site/open.feishu.cn/lark-doc", + }; + expect(() => + buildSkillsShMirrorObservation({ + ...liveRow, + url: "https://skills.sh/open.feishu.cn/lark-doc", + }), + ).toThrow("Unsupported skills.sh mirror identity"); + expect(() => + buildSkillsShMirrorObservation({ + ...liveRow, + installUrl: "https://github.com/attacker/repo", + sourceType: "github", + }), + ).toThrow("Unsupported skills.sh mirror identity"); + }); + + it("stores only one bounded detail document from the upstream file tree", () => { + const detail = buildSkillsShMirrorDetail( + { + id: "vercel-labs/skills/find-skills", + source: "vercel-labs/skills", + slug: "find-skills", + installs: 42, + hash: "a".repeat(64), + files: [ + { path: "references/notes.md", contents: "do not retain" }, + { path: "README.md", contents: "readme" }, + { path: "SKILL.md", contents: "1234567890" }, + ], + }, + 8, + ); + + expect(detail).toEqual({ + sourceContentHash: "a".repeat(64), + sourceFileCount: 3, + contentKind: "skill-md", + path: "SKILL.md", + content: "12345678", + contentBytes: 8, + sourceBytes: 10, + truncated: true, + }); + }); + + it("derives a deterministic content hash from the full detail before truncation", () => { + const detail = buildSkillsShMirrorDetail( + { + id: "patrick-erichsen/skills/html", + source: "patrick-erichsen/skills", + slug: "html", + installs: 42, + hash: null, + files: [{ path: "SKILL.md", contents: "abcdef" }], + }, + 3, + ); + + expect(detail).toMatchObject({ + sourceContentHash: "bef57ec7f53a6d40beb640a780a639c83bc29ac8a9816f1fc6c5c6dcd93c4721", + content: "abc", + truncated: true, + }); + }); + + it("resolves an immutable GitHub path and commit from the exact full detail blob", async () => { + const content = "# HTML Artifact Chooser\n"; + const boundedContent = content.slice(0, 8); + const blobSha = createHash("sha1") + .update(`blob ${Buffer.byteLength(content)}\0`) + .update(content) + .digest("hex"); + const commit = "050daba89f6b6636470add5cb300aac46a412cf8"; + const fetchImpl = vi.fn(async (input: string | URL | Request) => { + const url = String(input); + if (url === "https://github.com/patrick-erichsen/skills/archive/HEAD.zip") { + return new Response(null, { + status: 302, + headers: { + location: `https://codeload.github.com/Patrick-Erichsen/skills/zip/${commit}`, + }, + }); + } + if (url.includes(`/git/trees/${commit}?recursive=1`)) { + return new Response( + JSON.stringify({ + truncated: false, + tree: [ + { path: "skills/html/SKILL.md", type: "blob", sha: blobSha }, + { path: "README.md", type: "blob", sha: "f".repeat(40) }, + ], + }), + ); + } + return new Response("not found", { status: 404 }); + }); + + await expect( + resolveSkillsShMirrorGitHubLocators( + [ + { + externalId: "patrick-erichsen/skills/html", + sourceType: "github", + owner: "patrick-erichsen", + repo: "skills", + slug: "html", + detail: { + path: "SKILL.md", + content: boundedContent, + truncated: true, + }, + }, + ], + { + fetchImpl: fetchImpl as typeof fetch, + fullDetailContentByExternalId: new Map([["patrick-erichsen/skills/html", content]]), + }, + ), + ).resolves.toEqual({ + rows: [ + expect.objectContaining({ + githubPath: "skills/html", + githubCommit: commit, + }), + ], + sourceRequests: 2, + sourceBytes: expect.any(Number), + }); + }); + + it("derives the GitHub folder from a repository-relative detail path", async () => { + const content = "# Lark Doc\n"; + const blobSha = createHash("sha1") + .update(`blob ${Buffer.byteLength(content)}\0`) + .update(content) + .digest("hex"); + const commit = "050daba89f6b6636470add5cb300aac46a412cf8"; + const fetchImpl = vi.fn(async (input: string | URL | Request) => { + const url = String(input); + if (url === "https://github.com/larksuite/cli/archive/HEAD.zip") { + return new Response(null, { + status: 302, + headers: { location: `https://codeload.github.com/larksuite/cli/zip/${commit}` }, + }); + } + if (url.includes(`/git/trees/${commit}?recursive=1`)) { + return new Response( + JSON.stringify({ + truncated: false, + tree: [{ path: "skills/lark-doc/SKILL.md", type: "blob", sha: blobSha }], + }), + ); + } + return new Response("not found", { status: 404 }); + }); + + const result = await resolveSkillsShMirrorGitHubLocators( + [ + { + externalId: "larksuite/cli/lark-doc", + sourceType: "github", + owner: "larksuite", + repo: "cli", + slug: "lark-doc", + detail: { + path: "skills/lark-doc/SKILL.md", + content, + truncated: false, + }, + }, + ], + { fetchImpl: fetchImpl as typeof fetch }, + ); + + expect(result.rows).toEqual([ + expect.objectContaining({ + githubPath: "skills/lark-doc", + githubCommit: commit, + }), + ]); + }); + + it("propagates lease heartbeat failures during GitHub locator resolution", async () => { + const beforeRequest = vi + .fn<() => Promise>() + .mockResolvedValueOnce() + .mockRejectedValueOnce(new Error("mirror batch lease expired")); + const fetchImpl = vi.fn(async (input: string | URL | Request) => { + const url = String(input); + if (url === "https://github.com/patrick-erichsen/skills/archive/HEAD.zip") { + return new Response(null, { + status: 302, + headers: { + location: + "https://codeload.github.com/patrick-erichsen/skills/zip/050daba89f6b6636470add5cb300aac46a412cf8", + }, + }); + } + throw new Error(`unexpected request: ${url}`); + }); + + await expect( + resolveSkillsShMirrorGitHubLocators( + [ + { + externalId: "patrick-erichsen/skills/html", + sourceType: "github", + owner: "patrick-erichsen", + repo: "skills", + slug: "html", + detail: { + path: "SKILL.md", + content: "# HTML Artifact Chooser\n", + truncated: false, + }, + }, + ], + { fetchImpl: fetchImpl as typeof fetch, beforeRequest }, + ), + ).rejects.toThrow("mirror batch lease expired"); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + it("refreshes a repository HEAD snapshot for each mirror batch", async () => { + const content = "# HTML Artifact Chooser\n"; + const blobSha = createHash("sha1") + .update(`blob ${Buffer.byteLength(content)}\0`) + .update(content) + .digest("hex"); + const commits = ["0".repeat(39) + "1", "0".repeat(39) + "2"]; + let archiveRequest = 0; + const fetchImpl = vi.fn(async (input: string | URL | Request) => { + const url = String(input); + if (url === "https://github.com/patrick-erichsen/skills/archive/HEAD.zip") { + const commit = commits[archiveRequest++]!; + return new Response(null, { + status: 302, + headers: { + location: `https://codeload.github.com/patrick-erichsen/skills/zip/${commit}`, + }, + }); + } + if (url.includes("/git/trees/")) { + return new Response( + JSON.stringify({ + truncated: false, + tree: [{ path: "skills/html/SKILL.md", type: "blob", sha: blobSha }], + }), + ); + } + return new Response("not found", { status: 404 }); + }); + const rows = [ + { + externalId: "patrick-erichsen/skills/html", + sourceType: "github", + owner: "patrick-erichsen", + repo: "skills", + slug: "html", + detail: { path: "SKILL.md", content, truncated: false }, + }, + ]; + + const first = await resolveSkillsShMirrorGitHubLocators(rows, { + fetchImpl: fetchImpl as typeof fetch, + }); + const second = await resolveSkillsShMirrorGitHubLocators(rows, { + fetchImpl: fetchImpl as typeof fetch, + }); + + expect(first.rows[0]).toMatchObject({ githubCommit: commits[0] }); + expect(second.rows[0]).toMatchObject({ githubCommit: commits[1] }); + expect(fetchImpl).toHaveBeenCalledTimes(4); + }); + + it("forwards authoritative GitHub locators into a live mirror batch", async () => { + const content = "# HTML Artifact Chooser\n"; + const commit = "050daba89f6b6636470add5cb300aac46a412cf8"; + const blobSha = createHash("sha1") + .update(`blob ${Buffer.byteLength(content)}\0`) + .update(content) + .digest("hex"); + const fetchImpl = vi.fn(async (input: string | URL | Request) => { + const url = String(input); + if (url.includes("?page=0&per_page=500")) { + return new Response( + JSON.stringify({ + data: [ + { + id: "patrick-erichsen/skills/html", + installUrl: "https://github.com/patrick-erichsen/skills", + installs: 42, + name: "HTML Artifact Chooser", + slug: "html", + source: "patrick-erichsen/skills", + sourceType: "github", + url: "https://skills.sh/patrick-erichsen/skills/html", + }, + ], + pagination: { page: 0, perPage: 500, total: 1, hasMore: false }, + }), + ); + } + if (url.includes("/api/v1/skills/audit/")) { + return new Response(JSON.stringify({ audits: [] })); + } + if (url.includes("/api/v1/skills/patrick-erichsen/skills/html")) { + return new Response( + JSON.stringify({ + id: "patrick-erichsen/skills/html", + source: "patrick-erichsen/skills", + slug: "html", + installs: 42, + hash: null, + files: [{ path: "SKILL.md", contents: content }], + }), + ); + } + if (url === "https://github.com/patrick-erichsen/skills/archive/HEAD.zip") { + return new Response(null, { + status: 302, + headers: { + location: `https://codeload.github.com/Patrick-Erichsen/skills/zip/${commit}`, + }, + }); + } + if (url.includes(`/git/trees/${commit}?recursive=1`)) { + return new Response( + JSON.stringify({ + truncated: false, + tree: [{ path: "skills/html/SKILL.md", type: "blob", sha: blobSha }], + }), + ); + } + return new Response("unexpected request", { status: 500 }); + }); + + const batch = await fetchSkillsShMirrorBatch( + { page: 0, offset: 0, limit: 1, maxDetailBytes: 64 * 1024 }, + { + oidcToken: "request-bound-oidc", + fetchImpl: fetchImpl as typeof fetch, + }, + ); + + expect(batch).toMatchObject({ + sourceRequests: 5, + rows: [ + { + externalId: "patrick-erichsen/skills/html", + githubPath: "skills/html", + githubCommit: commit, + }, + ], + }); + expect(fetchImpl).toHaveBeenCalledTimes(5); + }); + + it("fetches one bounded mirror batch from an exact source page and offset", async () => { + const fetchImpl = vi.fn(async (urlInput: string | URL | Request) => { + const url = String(urlInput); + if (url.includes("?page=3&per_page=500")) { + return new Response( + JSON.stringify({ + data: [ + { + id: "vercel-labs/skills/find-skills", + installUrl: "https://github.com/vercel-labs/skills", + installs: 42, + name: "Find Skills", + slug: "find-skills", + source: "vercel-labs/skills", + sourceType: "github", + url: "https://skills.sh/vercel-labs/skills/find-skills", + }, + { + id: "open.feishu.cn/lark-doc", + installUrl: null, + installs: 7, + name: "lark-doc", + slug: "lark-doc", + source: "open.feishu.cn", + sourceType: "well-known", + url: "https://www.skills.sh/site/open.feishu.cn/lark-doc", + }, + ], + pagination: { page: 3, perPage: 500, total: 1_002, hasMore: false }, + }), + ); + } + if (url.includes("/api/v1/skills/audit/")) { + return new Response( + JSON.stringify({ + id: "open.feishu.cn/lark-doc", + source: "open.feishu.cn", + slug: "lark-doc", + audits: [ + { + provider: "Socket", + slug: "socket", + status: "pass", + auditedAt: "2026-07-22T20:02:00.000Z", + }, + ], + }), + ); + } + const id = decodeURIComponent(url.split("/api/v1/skills/")[1] ?? ""); + return new Response( + JSON.stringify({ + id, + source: id.split("/").slice(0, -1).join("/"), + slug: id.split("/").at(-1), + installs: 1, + hash: "a".repeat(64), + files: [{ path: "SKILL.md", contents: `# ${id}` }], + }), + ); + }); + + const batch = await fetchSkillsShMirrorBatch( + { page: 3, offset: 1, limit: 1, maxDetailBytes: 64 }, + { + env: { VERCEL_OIDC_TOKEN: "request-token" }, + fetchImpl: fetchImpl as typeof fetch, + githubLocatorResolver: null, + }, + ); + + expect(batch).toMatchObject({ + page: 3, + offset: 1, + pageLength: 2, + sourceTotal: 1_002, + hasMore: false, + rows: [ + { + externalId: "open.feishu.cn/lark-doc", + sourceType: "well-known", + sourceHost: "open.feishu.cn", + sourceContentHash: "a".repeat(64), + upstreamScanners: { + genAgentTrustHub: { status: "unavailable" }, + socket: { + status: "pass", + sourceUrl: "https://www.skills.sh/site/open.feishu.cn/lark-doc/security/socket", + sourceCheckedAt: "2026-07-22T20:02:00.000Z", + }, + snyk: { status: "unavailable" }, + }, + detail: { contentKind: "skill-md", path: "SKILL.md" }, + }, + ], + sourceRequests: 3, + }); + }); + + it("accounts for exact downloaded list, detail, and audit JSON bytes", async () => { + const row = { + id: "owner/repo/skill", + installUrl: "https://github.com/owner/repo", + installs: 42, + name: "Skill", + slug: "skill", + source: "owner/repo", + sourceType: "github", + url: "https://www.skills.sh/owner/repo/skill", + }; + const listJson = JSON.stringify( + { + data: [row], + pagination: { page: 0, perPage: 500, total: 1, hasMore: false }, + }, + null, + 2, + ); + const detailJson = JSON.stringify( + { + id: row.id, + source: row.source, + slug: row.slug, + installs: row.installs, + hash: "a".repeat(64), + files: [{ path: "SKILL.md", contents: "# Skill" }], + }, + null, + 4, + ); + const auditJson = JSON.stringify( + { + id: row.id, + source: row.source, + slug: row.slug, + audits: [], + }, + null, + 3, + ); + const fetchImpl = vi.fn(async (input: string | URL | Request) => { + const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url; + if (url.includes("?page=0&per_page=500")) return new Response(listJson); + if (url.includes("/api/v1/skills/audit/")) return new Response(auditJson); + if (url.endsWith("/api/v1/skills/owner/repo/skill")) return new Response(detailJson); + throw new Error(`unexpected URL ${url}`); + }); + + const batch = await fetchSkillsShMirrorBatch( + { page: 0, offset: 0, limit: 1, maxDetailBytes: 64 }, + { + oidcToken: "request-bound-oidc", + fetchImpl: fetchImpl as typeof fetch, + githubLocatorResolver: null, + }, + ); + + expect(batch.sourceBytes).toBe( + Buffer.byteLength(listJson) + Buffer.byteLength(detailJson) + Buffer.byteLength(auditJson), + ); + }); + + it("uses the durable captured page without refetching the mutable leaderboard", async () => { + const row = { + id: "vercel-labs/skills/find-skills", + installUrl: "https://github.com/vercel-labs/skills", + installs: 42, + name: "Find Skills", + slug: "find-skills", + source: "vercel-labs/skills", + sourceType: "github", + url: "https://skills.sh/vercel-labs/skills/find-skills", + }; + const fetchImpl = vi.fn(async (input: string | URL | Request) => { + const url = String(input); + if (url.includes("?page=")) throw new Error("leaderboard refetch is forbidden"); + if (url.includes("/api/v1/skills/audit/")) { + return new Response(JSON.stringify({ audits: [] })); + } + return new Response( + JSON.stringify({ + id: row.id, + source: row.source, + slug: row.slug, + installs: row.installs, + hash: "a".repeat(64), + files: [{ path: "SKILL.md", contents: "# Find Skills" }], + }), + ); + }); + + const batch = await fetchSkillsShMirrorBatch( + { page: 3, offset: 0, limit: 1, maxDetailBytes: 64 * 1024 }, + { + oidcToken: "request-bound-oidc", + fetchImpl: fetchImpl as typeof fetch, + githubLocatorResolver: null, + sourcePage: { + data: [row], + pagination: { page: 3, perPage: 500, total: 1, hasMore: false }, + }, + }, + ); + + expect(batch).toMatchObject({ + page: 3, + pageLength: 1, + sourceTotal: 1, + sourceRequests: 2, + rows: [{ externalId: row.id }], + }); + expect(fetchImpl).toHaveBeenCalledTimes(2); + }); + it("uses only the injected Vercel OIDC token for source authentication", async () => { const fetchImpl = vi.fn(async () => { return new Response( @@ -64,6 +1624,987 @@ describe("skills.sh Vercel source boundary", () => { }); }); + it("keeps a mirror row when the authenticated audit endpoint has no audits", async () => { + const fetchImpl = vi.fn(async (urlInput: string | URL | Request) => { + const url = String(urlInput); + if (url.includes("?page=0&per_page=500")) { + return new Response( + JSON.stringify({ + data: [ + { + id: "vercel-labs/skills/find-skills", + installUrl: "https://github.com/vercel-labs/skills", + installs: 42, + name: "Find Skills", + slug: "find-skills", + source: "vercel-labs/skills", + sourceType: "github", + url: "https://skills.sh/vercel-labs/skills/find-skills", + }, + ], + pagination: { page: 0, perPage: 500, total: 1, hasMore: false }, + }), + ); + } + if (url.includes("/api/v1/skills/audit/")) { + return new Response(JSON.stringify({ error: "not_found" }), { status: 404 }); + } + if (url.includes("/api/v1/skills/")) { + return new Response( + JSON.stringify({ + id: "vercel-labs/skills/find-skills", + source: "vercel-labs/skills", + slug: "find-skills", + installs: 42, + hash: "a".repeat(64), + files: [{ path: "SKILL.md", contents: "# Find Skills" }], + }), + ); + } + return new Response("unexpected request", { status: 500 }); + }); + + const batch = await fetchSkillsShMirrorBatch( + { page: 0, offset: 0, limit: 1, maxDetailBytes: 64 }, + { + oidcToken: "request-bound-oidc", + fetchImpl: fetchImpl as typeof fetch, + githubLocatorResolver: null, + }, + ); + + expect(batch.rows).toEqual([ + expect.objectContaining({ + externalId: "vercel-labs/skills/find-skills", + upstreamScanners: { + genAgentTrustHub: { status: "unavailable" }, + socket: { status: "unavailable" }, + snyk: { status: "unavailable" }, + }, + detail: expect.objectContaining({ + contentKind: "skill-md", + path: "SKILL.md", + }), + }), + ]); + expect(batch.sourceRequests).toBe(3); + expect(fetchImpl).toHaveBeenCalledWith( + "https://skills.sh/api/v1/skills/audit/vercel-labs/skills/find-skills", + { + headers: { + Accept: "application/json", + Authorization: "Bearer request-bound-oidc", + }, + }, + ); + }); + + it("fetches the exact ambiguous source page only for identity resolution", async () => { + let identityPageAttempts = 0; + const fetchImpl = vi.fn(async (urlInput: string | URL | Request) => { + const url = String(urlInput); + if (url.includes("?page=0&per_page=500")) { + return new Response( + JSON.stringify({ + data: [ + { + id: "larksuite/cli/lark-doc", + installUrl: null, + installs: 383_123, + name: "lark-doc", + slug: "lark-doc", + source: "larksuite/cli", + sourceType: "well-known", + url: "https://www.skills.sh/larksuite/cli/lark-doc", + }, + ], + pagination: { page: 0, perPage: 500, total: 1, hasMore: false }, + }), + ); + } + if (url === "https://www.skills.sh/larksuite/cli/lark-doc") { + identityPageAttempts += 1; + if (identityPageAttempts === 1) { + return new Response("retry", { + status: 503, + headers: { "retry-after": "0" }, + }); + } + return new Response( + `
Repository
larksuite/cli
`, + { headers: { "content-type": "text/html; charset=utf-8" } }, + ); + } + if (url.includes("/api/v1/skills/audit/")) { + return new Response(JSON.stringify({ error: "not_found" }), { status: 404 }); + } + if (url.includes("/api/v1/skills/larksuite/cli/lark-doc")) { + return new Response( + JSON.stringify({ + id: "larksuite/cli/lark-doc", + source: "larksuite/cli", + slug: "lark-doc", + installs: 383_123, + hash: "c".repeat(64), + files: [{ path: "skills/lark-doc/SKILL.md", contents: "# Lark Doc" }], + }), + ); + } + return new Response("unexpected request", { status: 500 }); + }); + + const batch = await fetchSkillsShMirrorBatch( + { page: 0, offset: 0, limit: 1, maxDetailBytes: 64 }, + { + oidcToken: "request-bound-oidc", + fetchImpl: fetchImpl as typeof fetch, + githubLocatorResolver: null, + }, + ); + + expect(batch.rows).toEqual([ + expect.objectContaining({ + externalId: "larksuite/cli/lark-doc", + sourceType: "github", + upstreamSourceType: "well-known", + owner: "larksuite", + repo: "cli", + canonicalRepoUrl: "https://github.com/larksuite/cli", + }), + ]); + expect(batch.sourceRequests).toBe(5); + expect(fetchImpl).toHaveBeenCalledWith("https://www.skills.sh/larksuite/cli/lark-doc", { + headers: { Accept: "text/html" }, + redirect: "manual", + }); + }); + + it("delegates long identity-page Retry-After waits to durable recovery", async () => { + vi.useFakeTimers(); + let identityPageAttempts = 0; + const fetchImpl = vi.fn(async (urlInput: string | URL | Request) => { + const url = String(urlInput); + if (url.includes("?page=0&per_page=500")) { + return new Response( + JSON.stringify({ + data: [ + { + id: "larksuite/cli/lark-doc", + installUrl: null, + installs: 383_123, + name: "lark-doc", + slug: "lark-doc", + source: "larksuite/cli", + sourceType: "well-known", + url: "https://www.skills.sh/larksuite/cli/lark-doc", + }, + ], + pagination: { page: 0, perPage: 500, total: 1, hasMore: false }, + }), + ); + } + if (url === "https://www.skills.sh/larksuite/cli/lark-doc") { + identityPageAttempts += 1; + return new Response("rate limited", { + status: 429, + headers: { "retry-after": "120" }, + }); + } + return new Response("unexpected request", { status: 500 }); + }); + + let error: unknown; + void fetchSkillsShMirrorBatch( + { page: 0, offset: 0, limit: 1, maxDetailBytes: 64 }, + { + oidcToken: "request-bound-oidc", + fetchImpl: fetchImpl as typeof fetch, + githubLocatorResolver: null, + }, + ).catch((value: unknown) => { + error = value; + }); + await vi.advanceTimersByTimeAsync(0); + + expect(skillsShSourceRetryAfterSeconds(error)).toBe(120); + expect(identityPageAttempts).toBe(1); + }); + + it("quarantines an identity page redirect outside the exact skills.sh route", async () => { + let redirectBodyCanceled = false; + const fetchImpl = vi.fn(async (urlInput: string | URL | Request) => { + const url = String(urlInput); + if (url.includes("?page=0&per_page=500")) { + return new Response( + JSON.stringify({ + data: [ + { + id: "larksuite/cli/lark-doc", + installUrl: null, + installs: 383_123, + name: "lark-doc", + slug: "lark-doc", + source: "larksuite/cli", + sourceType: "well-known", + url: "https://skills.sh/larksuite/cli/lark-doc", + }, + ], + pagination: { page: 0, perPage: 500, total: 1, hasMore: false }, + }), + ); + } + if (url === "https://skills.sh/larksuite/cli/lark-doc") { + return new Response( + new ReadableStream({ + pull(controller) { + controller.enqueue(new TextEncoder().encode("redirecting")); + }, + cancel() { + redirectBodyCanceled = true; + }, + }), + { + status: 302, + headers: { location: "http://169.254.169.254/latest/meta-data/" }, + }, + ); + } + return new Response("unexpected request", { status: 500 }); + }); + + const batch = await fetchSkillsShMirrorBatch( + { page: 0, offset: 0, limit: 1, maxDetailBytes: 64 }, + { + oidcToken: "request-bound-oidc", + fetchImpl: fetchImpl as typeof fetch, + githubLocatorResolver: null, + }, + ); + + expect(batch.rows).toEqual([ + { + quarantined: true, + externalId: "larksuite/cli/lark-doc", + upstreamSourceType: "well-known", + reason: "identity-page-redirect", + }, + ]); + expect(redirectBodyCanceled).toBe(true); + expect(fetchImpl).toHaveBeenCalledTimes(2); + }); + + it("quarantines an ambiguous HTML 404 and continues the source batch", async () => { + const fetchImpl = vi.fn(async (urlInput: string | URL | Request) => { + const url = String(urlInput); + if (url.includes("?page=0&per_page=500")) { + return new Response( + JSON.stringify({ + data: [ + { + id: "larksuite/cli/lark-doc", + installUrl: null, + installs: 383_123, + name: "lark-doc", + slug: "lark-doc", + source: "larksuite/cli", + sourceType: "well-known", + url: "https://www.skills.sh/larksuite/cli/lark-doc", + }, + { + id: "vercel-labs/skills/find-skills", + installUrl: "https://github.com/vercel-labs/skills", + installs: 42, + name: "Find Skills", + slug: "find-skills", + source: "vercel-labs/skills", + sourceType: "github", + url: "https://skills.sh/vercel-labs/skills/find-skills", + }, + ], + pagination: { page: 0, perPage: 500, total: 2, hasMore: false }, + }), + ); + } + if (url === "https://www.skills.sh/larksuite/cli/lark-doc") { + return new Response("not found", { status: 404 }); + } + if (url.includes("/api/v1/skills/audit/")) { + return new Response(JSON.stringify({ error: "not_found" }), { status: 404 }); + } + if (url.includes("/api/v1/skills/vercel-labs/skills/find-skills")) { + return new Response( + JSON.stringify({ + id: "vercel-labs/skills/find-skills", + source: "vercel-labs/skills", + slug: "find-skills", + installs: 42, + hash: "a".repeat(64), + files: [{ path: "SKILL.md", contents: "# Find Skills" }], + }), + ); + } + return new Response("unexpected request", { status: 500 }); + }); + + const batch = await fetchSkillsShMirrorBatch( + { page: 0, offset: 0, limit: 2, maxDetailBytes: 64 }, + { + oidcToken: "request-bound-oidc", + fetchImpl: fetchImpl as typeof fetch, + githubLocatorResolver: null, + }, + ); + + expect(batch.rows).toEqual([ + { + quarantined: true, + externalId: "larksuite/cli/lark-doc", + upstreamSourceType: "well-known", + reason: "identity-page-http-404", + }, + expect.objectContaining({ + externalId: "vercel-labs/skills/find-skills", + sourceType: "github", + upstreamSourceType: "github", + }), + ]); + expect(batch.sourceRequests).toBe(4); + expect(fetchImpl).not.toHaveBeenCalledWith( + "https://skills.sh/api/v1/skills/larksuite/cli/lark-doc", + expect.anything(), + ); + expect(fetchImpl).not.toHaveBeenCalledWith( + "https://skills.sh/api/v1/skills/audit/larksuite/cli/lark-doc", + expect.anything(), + ); + }); + + it("cancels an oversized chunked identity page before buffering the full body", async () => { + let canceled = false; + let chunkIndex = 0; + const fetchImpl = vi.fn(async (urlInput: string | URL | Request) => { + const url = String(urlInput); + if (url.includes("?page=0&per_page=500")) { + return new Response( + JSON.stringify({ + data: [ + { + id: "larksuite/cli/lark-doc", + installUrl: null, + installs: 383_123, + name: "lark-doc", + slug: "lark-doc", + source: "larksuite/cli", + sourceType: "well-known", + url: "https://www.skills.sh/larksuite/cli/lark-doc", + }, + ], + pagination: { page: 0, perPage: 500, total: 1, hasMore: false }, + }), + ); + } + if (url === "https://www.skills.sh/larksuite/cli/lark-doc") { + return new Response( + new ReadableStream({ + pull(controller) { + chunkIndex += 1; + if (chunkIndex <= 4) { + controller.enqueue(new Uint8Array(256 * 1024)); + } else { + controller.close(); + } + }, + cancel() { + canceled = true; + }, + }), + { headers: { "content-type": "text/html" } }, + ); + } + return new Response("unexpected request", { status: 500 }); + }); + + const batch = await fetchSkillsShMirrorBatch( + { page: 0, offset: 0, limit: 1, maxDetailBytes: 64 }, + { + oidcToken: "request-bound-oidc", + fetchImpl: fetchImpl as typeof fetch, + githubLocatorResolver: null, + }, + ); + + expect(batch.rows).toEqual([ + { + quarantined: true, + externalId: "larksuite/cli/lark-doc", + upstreamSourceType: "well-known", + reason: "identity-page-too-large", + }, + ]); + expect(canceled).toBe(true); + }); + + it("retries a failed identity response stream without aborting the source batch", async () => { + let identityPageAttempts = 0; + const fetchImpl = vi.fn(async (urlInput: string | URL | Request) => { + const url = String(urlInput); + if (url.includes("?page=0&per_page=500")) { + return new Response( + JSON.stringify({ + data: [ + { + id: "larksuite/cli/lark-doc", + installUrl: null, + installs: 383_123, + name: "lark-doc", + slug: "lark-doc", + source: "larksuite/cli", + sourceType: "well-known", + url: "https://www.skills.sh/larksuite/cli/lark-doc", + }, + ], + pagination: { page: 0, perPage: 500, total: 1, hasMore: false }, + }), + ); + } + if (url === "https://www.skills.sh/larksuite/cli/lark-doc") { + identityPageAttempts += 1; + if (identityPageAttempts === 1) { + let sentChunk = false; + return new Response( + new ReadableStream({ + pull(controller) { + if (!sentChunk) { + sentChunk = true; + controller.enqueue(new Uint8Array(64 * 1024)); + } else { + controller.error(new Error("connection reset")); + } + }, + }), + { headers: { "content-type": "text/html" } }, + ); + } + return new Response( + `
Repository
larksuite/cli
`, + { headers: { "content-type": "text/html" } }, + ); + } + if (url.includes("/api/v1/skills/audit/")) { + return new Response(JSON.stringify({ error: "not_found" }), { status: 404 }); + } + if (url.includes("/api/v1/skills/larksuite/cli/lark-doc")) { + return new Response( + JSON.stringify({ + id: "larksuite/cli/lark-doc", + source: "larksuite/cli", + slug: "lark-doc", + installs: 383_123, + hash: "c".repeat(64), + files: [{ path: "skills/lark-doc/SKILL.md", contents: "# Lark Doc" }], + }), + ); + } + return new Response("unexpected request", { status: 500 }); + }); + + const batch = await fetchSkillsShMirrorBatch( + { page: 0, offset: 0, limit: 1, maxDetailBytes: 64 }, + { + oidcToken: "request-bound-oidc", + fetchImpl: fetchImpl as typeof fetch, + githubLocatorResolver: null, + }, + ); + + expect(batch.rows).toEqual([ + expect.objectContaining({ + externalId: "larksuite/cli/lark-doc", + sourceType: "github", + upstreamSourceType: "well-known", + }), + ]); + expect(identityPageAttempts).toBe(2); + expect(batch.sourceRequests).toBe(5); + expect(batch.sourceBytes).toBeGreaterThan(64 * 1024); + }); + + it("quarantines exhausted identity-page 5xx retries as a transient fetch failure", async () => { + let identityPageAttempts = 0; + const fetchImpl = vi.fn(async (urlInput: string | URL | Request) => { + const url = String(urlInput); + if (url.includes("?page=0&per_page=500")) { + return new Response( + JSON.stringify({ + data: [ + { + id: "larksuite/cli/lark-doc", + installUrl: null, + installs: 383_123, + name: "lark-doc", + slug: "lark-doc", + source: "larksuite/cli", + sourceType: "well-known", + url: "https://www.skills.sh/larksuite/cli/lark-doc", + }, + ], + pagination: { page: 0, perPage: 500, total: 1, hasMore: false }, + }), + ); + } + if (url === "https://www.skills.sh/larksuite/cli/lark-doc") { + identityPageAttempts += 1; + return new Response("unavailable", { + status: 503, + headers: { "retry-after": "0" }, + }); + } + return new Response("unexpected request", { status: 500 }); + }); + + const batch = await fetchSkillsShMirrorBatch( + { page: 0, offset: 0, limit: 1, maxDetailBytes: 64 }, + { + oidcToken: "request-bound-oidc", + fetchImpl: fetchImpl as typeof fetch, + githubLocatorResolver: null, + }, + ); + + expect(batch.rows).toEqual([ + { + quarantined: true, + externalId: "larksuite/cli/lark-doc", + upstreamSourceType: "well-known", + reason: "identity-page-fetch-failed", + }, + ]); + expect(identityPageAttempts).toBe(4); + expect(batch.sourceRequests).toBe(5); + }); + + it("quarantines an identity page without an explicit HTML content type", async () => { + const fetchImpl = vi.fn(async (urlInput: string | URL | Request) => { + const url = String(urlInput); + if (url.includes("?page=0&per_page=500")) { + return new Response( + JSON.stringify({ + data: [ + { + id: "larksuite/cli/lark-doc", + installUrl: null, + installs: 383_123, + name: "lark-doc", + slug: "lark-doc", + source: "larksuite/cli", + sourceType: "well-known", + url: "https://www.skills.sh/larksuite/cli/lark-doc", + }, + ], + pagination: { page: 0, perPage: 500, total: 1, hasMore: false }, + }), + ); + } + if (url === "https://www.skills.sh/larksuite/cli/lark-doc") { + return new Response( + new TextEncoder().encode( + `
Repository
larksuite/cli
`, + ), + ); + } + return new Response("unexpected request", { status: 500 }); + }); + + const batch = await fetchSkillsShMirrorBatch( + { page: 0, offset: 0, limit: 1, maxDetailBytes: 64 }, + { + oidcToken: "request-bound-oidc", + fetchImpl: fetchImpl as typeof fetch, + githubLocatorResolver: null, + }, + ); + + expect(batch.rows).toEqual([ + { + quarantined: true, + externalId: "larksuite/cli/lark-doc", + upstreamSourceType: "well-known", + reason: "identity-page-content-type", + }, + ]); + }); + + it("quarantines a slash-bearing detail slug and continues the source batch", async () => { + const fetchImpl = vi.fn(async (urlInput: string | URL | Request) => { + const url = String(urlInput); + if (url.includes("?page=0&per_page=500")) { + return new Response( + JSON.stringify({ + data: [ + { + id: "owner/repo/nested/skill", + installUrl: "https://github.com/owner/repo", + installs: 1, + name: "Nested Skill", + slug: "nested/skill", + source: "owner/repo", + sourceType: "github", + url: "https://skills.sh/owner/repo/nested/skill", + }, + { + id: "vercel-labs/skills/find-skills", + installUrl: "https://github.com/vercel-labs/skills", + installs: 42, + name: "Find Skills", + slug: "find-skills", + source: "vercel-labs/skills", + sourceType: "github", + url: "https://skills.sh/vercel-labs/skills/find-skills", + }, + ], + pagination: { page: 0, perPage: 500, total: 2, hasMore: false }, + }), + ); + } + if (url.includes("/api/v1/skills/audit/")) { + return new Response(JSON.stringify({ error: "not_found" }), { status: 404 }); + } + if (url.includes("/api/v1/skills/vercel-labs/skills/find-skills")) { + return new Response( + JSON.stringify({ + id: "vercel-labs/skills/find-skills", + source: "vercel-labs/skills", + slug: "find-skills", + installs: 42, + hash: "a".repeat(64), + files: [{ path: "SKILL.md", contents: "# Find Skills" }], + }), + ); + } + return new Response("unexpected request", { status: 500 }); + }); + + const batch = await fetchSkillsShMirrorBatch( + { page: 0, offset: 0, limit: 2, maxDetailBytes: 64 }, + { + oidcToken: "request-bound-oidc", + fetchImpl: fetchImpl as typeof fetch, + githubLocatorResolver: null, + }, + ); + + expect(batch.rows).toEqual([ + { + quarantined: true, + externalId: "owner/repo/nested/skill", + upstreamSourceType: "github", + reason: "unsupported-identity", + }, + expect.objectContaining({ + externalId: "vercel-labs/skills/find-skills", + sourceType: "github", + sourceContentHash: "a".repeat(64), + }), + ]); + expect(batch.sourceRequests).toBe(3); + expect(fetchImpl).not.toHaveBeenCalledWith( + "https://skills.sh/api/v1/skills/owner/repo/nested/skill", + expect.anything(), + ); + }); + + it("quarantines a malformed upstream source type instead of aborting the batch", async () => { + const fetchImpl = vi.fn(async (urlInput: string | URL | Request) => { + const url = String(urlInput); + if (url.includes("?page=0&per_page=500")) { + return new Response( + JSON.stringify({ + data: [ + { + id: "owner/repo/skill", + installUrl: null, + installs: 1, + name: "Skill", + slug: "skill", + source: "owner/repo", + sourceType: null, + url: "https://www.skills.sh/owner/repo/skill", + }, + ], + pagination: { page: 0, perPage: 500, total: 1, hasMore: false }, + }), + ); + } + return new Response("unexpected request", { status: 500 }); + }); + + const batch = await fetchSkillsShMirrorBatch( + { page: 0, offset: 0, limit: 1, maxDetailBytes: 64 }, + { + oidcToken: "request-bound-oidc", + fetchImpl: fetchImpl as typeof fetch, + githubLocatorResolver: null, + }, + ); + + expect(batch.rows).toEqual([ + { + quarantined: true, + externalId: "owner/repo/skill", + upstreamSourceType: "missing", + reason: "unsupported-identity", + }, + ]); + }); + + it("fails closed when the authenticated audit endpoint rejects authorization", async () => { + const fetchImpl = vi.fn(async (urlInput: string | URL | Request) => { + const url = String(urlInput); + if (url.includes("?page=0&per_page=500")) { + return new Response( + JSON.stringify({ + data: [ + { + id: "vercel-labs/skills/find-skills", + installUrl: "https://github.com/vercel-labs/skills", + installs: 42, + name: "Find Skills", + slug: "find-skills", + source: "vercel-labs/skills", + sourceType: "github", + url: "https://skills.sh/vercel-labs/skills/find-skills", + }, + ], + pagination: { page: 0, perPage: 500, total: 1, hasMore: false }, + }), + ); + } + if (url.includes("/api/v1/skills/audit/")) { + return new Response(JSON.stringify({ error: "unauthorized" }), { status: 401 }); + } + return new Response( + JSON.stringify({ + id: "vercel-labs/skills/find-skills", + source: "vercel-labs/skills", + slug: "find-skills", + installs: 42, + hash: "a".repeat(64), + files: [{ path: "SKILL.md", contents: "# Find Skills" }], + }), + ); + }); + + await expect( + fetchSkillsShMirrorBatch( + { page: 0, offset: 0, limit: 1, maxDetailBytes: 64 }, + { + oidcToken: "request-bound-oidc", + fetchImpl: fetchImpl as typeof fetch, + githubLocatorResolver: null, + }, + ), + ).rejects.toThrow("skills.sh catalog source returned HTTP 401"); + }); + + it("retries transient audit responses and counts every request", async () => { + let auditAttempts = 0; + const fetchImpl = vi.fn(async (urlInput: string | URL | Request) => { + const url = String(urlInput); + if (url.includes("?page=0&per_page=500")) { + return new Response( + JSON.stringify({ + data: [ + { + id: "vercel-labs/skills/find-skills", + installUrl: "https://github.com/vercel-labs/skills", + installs: 42, + name: "Find Skills", + slug: "find-skills", + source: "vercel-labs/skills", + sourceType: "github", + url: "https://skills.sh/vercel-labs/skills/find-skills", + }, + ], + pagination: { page: 0, perPage: 500, total: 1, hasMore: false }, + }), + ); + } + if (url.includes("/api/v1/skills/audit/")) { + auditAttempts += 1; + if (auditAttempts === 1) { + return new Response("busy", { + status: 503, + headers: { "retry-after": "0" }, + }); + } + return new Response( + JSON.stringify({ + id: "vercel-labs/skills/find-skills", + source: "vercel-labs/skills", + slug: "find-skills", + audits: [], + }), + ); + } + return new Response( + JSON.stringify({ + id: "vercel-labs/skills/find-skills", + source: "vercel-labs/skills", + slug: "find-skills", + installs: 42, + hash: "a".repeat(64), + files: [{ path: "SKILL.md", contents: "# Find Skills" }], + }), + ); + }); + + const batch = await fetchSkillsShMirrorBatch( + { page: 0, offset: 0, limit: 1, maxDetailBytes: 64 }, + { + oidcToken: "request-bound-oidc", + fetchImpl: fetchImpl as typeof fetch, + githubLocatorResolver: null, + }, + ); + + expect(batch.sourceRequests).toBe(4); + expect(fetchImpl).toHaveBeenCalledTimes(4); + }); + + it("paces authenticated mirror API requests below the upstream minute limit", async () => { + vi.useFakeTimers(); + const requestTimes: number[] = []; + const fetchImpl = vi.fn(async (urlInput: string | URL | Request) => { + requestTimes.push(Date.now()); + const url = String(urlInput); + if (url.includes("?page=0&per_page=500")) { + return new Response( + JSON.stringify({ + data: [ + { + id: "vercel-labs/skills/find-skills", + installUrl: "https://github.com/vercel-labs/skills", + installs: 42, + name: "Find Skills", + slug: "find-skills", + source: "vercel-labs/skills", + sourceType: "github", + url: "https://skills.sh/vercel-labs/skills/find-skills", + }, + ], + pagination: { page: 0, perPage: 500, total: 1, hasMore: false }, + }), + ); + } + if (url.includes("/api/v1/skills/audit/")) { + return new Response(JSON.stringify({ audits: [] })); + } + return new Response( + JSON.stringify({ + hash: "a".repeat(64), + files: [{ path: "SKILL.md", contents: "# Find Skills" }], + }), + ); + }); + + const batchPromise = fetchSkillsShMirrorBatch( + { page: 0, offset: 0, limit: 1, maxDetailBytes: 64 }, + { + oidcToken: "request-bound-oidc", + fetchImpl: fetchImpl as typeof fetch, + minimumApiRequestIntervalMs: 125, + githubLocatorResolver: null, + }, + ); + await vi.runAllTimersAsync(); + await batchPromise; + + expect(requestTimes).toHaveLength(3); + expect(requestTimes[1]! - requestTimes[0]!).toBeGreaterThanOrEqual(125); + expect(requestTimes[2]! - requestTimes[1]!).toBeGreaterThanOrEqual(125); + vi.useRealTimers(); + }); + + it("honors the full Retry-After delay before retrying a 429", async () => { + vi.useFakeTimers(); + let attempts = 0; + let canceledResponses = 0; + const fetchImpl = vi.fn(async () => { + attempts += 1; + if (attempts === 1) { + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("rate limited")); + }, + cancel() { + canceledResponses += 1; + }, + }), + { + status: 429, + headers: { "retry-after": "17" }, + }, + ); + } + return new Response( + JSON.stringify({ + data: [], + pagination: { page: 0, perPage: 500, total: 1, hasMore: false }, + }), + ); + }); + + const pagePromise = fetchSkillsShCatalogPage( + { page: 0, perPage: 500 }, + { oidcToken: "request-bound-oidc", fetchImpl: fetchImpl as typeof fetch }, + ); + await vi.advanceTimersByTimeAsync(16_999); + expect(attempts).toBe(1); + expect(canceledResponses).toBe(1); + await vi.advanceTimersByTimeAsync(1); + await pagePromise; + expect(attempts).toBe(2); + }); + + it("preserves Retry-After when bounded source retries are exhausted", async () => { + vi.useFakeTimers(); + const fetchImpl = vi.fn(async () => { + return new Response("rate limited", { + status: 429, + headers: { "retry-after": "17" }, + }); + }); + + const pagePromise = fetchSkillsShCatalogPage( + { page: 0, perPage: 500 }, + { oidcToken: "request-bound-oidc", fetchImpl: fetchImpl as typeof fetch }, + ).catch((error: unknown) => error); + await vi.runAllTimersAsync(); + const error = await pagePromise; + + expect(error).toBeInstanceOf(Error); + expect(skillsShSourceRetryAfterSeconds(error)).toBe(17); + expect(fetchImpl).toHaveBeenCalledTimes(4); + }); + + it("delegates long Retry-After waits without shortening the upstream cooldown", async () => { + const fetchImpl = vi.fn(async () => { + return new Response("rate limited", { + status: 429, + headers: { "retry-after": "120" }, + }); + }); + + const error = await fetchSkillsShCatalogPage( + { page: 0, perPage: 500 }, + { oidcToken: "request-bound-oidc", fetchImpl: fetchImpl as typeof fetch }, + ).catch((value: unknown) => value); + + expect(skillsShSourceRetryAfterSeconds(error)).toBe(120); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + it("fails closed without OIDC and above the 500-row boundary", async () => { await expect( fetchSkillsShCatalogPage({ page: 0, perPage: 500 }, { env: {}, fetchImpl: vi.fn() }), @@ -101,7 +2642,7 @@ describe("skills.sh Vercel source boundary", () => { ); }); - it("requires the Test build, Preview runtime, baked backend, and explicit enable", () => { + it("requires the Test build, Vercel Test runtime, baked backend, and explicit enable", () => { expect( getSkillsShCatalogTestSourcePolicy({ VERCEL_ENV: "preview", @@ -124,6 +2665,22 @@ describe("skills.sh Vercel source boundary", () => { }), ).toMatchObject({ allowed: false }); + expect( + getSkillsShCatalogTestSourcePolicy({ + CLAWHUB_SKILLS_SH_ROLLOUT_MODE: "test", + VERCEL_ENV: "test", + VERCEL_TARGET_ENV: "test", + VITE_CLAWHUB_DEPLOY_ENV: "test", + VITE_CONVEX_URL: "https://academic-chihuahua-392.convex.cloud", + CLAWHUB_SKILLS_SH_TEST_LIVE_FETCH_ENABLED: "1", + }), + ).toEqual({ + allowed: true, + environment: "test", + maxDiscoveryRows: 500, + maxRealScanAdmissions: 10, + }); + expect( getSkillsShCatalogTestSourcePolicy({ CLAWHUB_SKILLS_SH_ROLLOUT_MODE: "test", @@ -139,6 +2696,20 @@ describe("skills.sh Vercel source boundary", () => { maxDiscoveryRows: 500, maxRealScanAdmissions: 10, }); + + expect( + getSkillsShCatalogTestSourcePolicy({ + CLAWHUB_SKILLS_SH_ROLLOUT_MODE: "test", + VERCEL_ENV: "production", + VERCEL_TARGET_ENV: "test", + VITE_CLAWHUB_DEPLOY_ENV: "test", + VITE_CONVEX_URL: "https://academic-chihuahua-392.convex.cloud", + CLAWHUB_SKILLS_SH_TEST_LIVE_FETCH_ENABLED: "1", + }), + ).toMatchObject({ + allowed: false, + environment: "production", + }); }); it("requires exact authenticated immutable owner coverage for the selected live set", () => { diff --git a/server/skillsShCatalogSource.ts b/server/skillsShCatalogSource.ts index c4f59810..edf6864e 100644 --- a/server/skillsShCatalogSource.ts +++ b/server/skillsShCatalogSource.ts @@ -1,15 +1,221 @@ import { createHash } from "node:crypto"; import { getVercelOidcToken, verifyVercelOidcToken, type VercelOidcPayload } from "@vercel/oidc"; import { getClawHubRolloutCapabilities } from "clawhub-schema"; +import { parse, type DefaultTreeAdapterMap } from "parse5"; +import { buildGitHubApiHeaders } from "../convex/lib/githubAuth"; const SKILLS_SH_API_BASE = "https://skills.sh/api/v1"; const MAX_SOURCE_PAGE_SIZE = 500; +const MAX_SOURCE_ATTEMPTS = 4; const MAX_TEST_SCAN_ADMISSIONS = 10; const DETAIL_CONCURRENCY = 8; +const MAX_UPSTREAM_SCANNER_STATUS_LENGTH = 32; +const MAX_UPSTREAM_SCANNER_URL_LENGTH = 2_048; +const MAX_UPSTREAM_SOURCE_TYPE_LENGTH = 64; +const MAX_IDENTITY_PAGE_BYTES = 512 * 1024; +const MAX_IDENTITY_PAGE_REDIRECTS = 2; +const MAX_GITHUB_TREE_BYTES = 8 * 1024 * 1024; +const MAX_GITHUB_TREE_ENTRIES = 100_000; +const MAX_CONTROLLED_DETAIL_BYTES = 1024 * 1024; +const MAX_PROOF_METADATA_BYTES = 1024 * 1024; +const MAX_PROOF_SNAPSHOT_BYTES = 32 * 1024; +const MAX_PROOF_SOURCE_ROWS = 50_000; +const GITHUB_LOCATOR_CONCURRENCY = 8; +const MINIMUM_API_REQUEST_INTERVAL_MS = 125; +const MAX_INLINE_RETRY_AFTER_MS = 30_000; const CLAWHUB_VERCEL_OWNER_ID = "team_pLdjXbfy0XvPRiNmAygTjTSH"; const CLAWHUB_VERCEL_PROJECT_ID = "prj_UVAJPNPYrBwTEkPJwkpEySsge8Mc"; const CLAWHUB_TEST_CONVEX_URL = "https://academic-chihuahua-392.convex.cloud"; +const SKILLS_SH_MIRROR_CONTROLLED_SUPPLEMENTS = [ + { + externalId: "patrick-erichsen/skills/html", + owner: "patrick-erichsen", + repo: "skills", + slug: "html", + displayName: "HTML Artifact Chooser", + sourceUrl: "https://www.skills.sh/patrick-erichsen/skills/html", + githubPath: "skills/html", + detailPath: "skills/html/SKILL.md", + githubCommit: "050daba89f6b6636470add5cb300aac46a412cf8", + sourceContentHash: "42d2e89358ea927441dfede45c3b0cf89a21603bc7c32246f098d24a9cbea1ff", + }, + { + externalId: "steipete/clawdis/discrawl", + owner: "steipete", + repo: "clawdis", + slug: "discrawl", + displayName: "Discrawl", + sourceUrl: "https://www.skills.sh/steipete/clawdis/discrawl", + githubPath: ".agents/skills/discrawl", + detailPath: ".agents/skills/discrawl/SKILL.md", + githubCommit: "690ed564419291ca6e832dc69b53061300075b62", + sourceContentHash: "889dc43180b210dbca12f8291e007feb231250ecfdba90c4d3938a18125efb6d", + }, +] as const; + +type SkillsShMirrorControlledSupplement = { + externalId: string; + owner: string; + repo: string; + slug: string; + displayName: string; + sourceUrl: string; + githubPath: string; + detailPath: string; + githubCommit: string; + sourceContentHash: string; +}; + +export const SKILLS_SH_MIRROR_CONTROLLED_SUPPLEMENT_COUNT = + SKILLS_SH_MIRROR_CONTROLLED_SUPPLEMENTS.length; +export const SKILLS_SH_MIRROR_CONTROLLED_EXTERNAL_IDS = SKILLS_SH_MIRROR_CONTROLLED_SUPPLEMENTS.map( + (row) => row.externalId, +); +const SKILLS_SH_MIRROR_PROOF_SNAPSHOT_PREFIX = "skills-sh:proof:"; + +function normalizeControlledSupplementIds(values: unknown[]) { + const allowed = new Set(SKILLS_SH_MIRROR_CONTROLLED_EXTERNAL_IDS); + const normalized = values.map((value) => + typeof value === "string" ? value.trim().toLowerCase() : "", + ); + if ( + normalized.some((value) => !allowed.has(value)) || + new Set(normalized).size !== normalized.length + ) { + throw new Error("skills.sh mirror controlled identities are invalid"); + } + return SKILLS_SH_MIRROR_CONTROLLED_EXTERNAL_IDS.filter((externalId) => + normalized.includes(externalId), + ); +} + +export function buildSkillsShMirrorProofSnapshotId(args: { + catalogTotal: number; + controlledExternalIds: string[]; + controlledOverlayExternalIds?: string[]; + controlledSupplementExternalIds?: string[]; + evidence?: SkillsShMirrorProofEvidence; +}) { + assertIntegerInRange("catalogTotal", args.catalogTotal, 1, MAX_PROOF_SOURCE_ROWS); + const controlledExternalIds = normalizeControlledSupplementIds(args.controlledExternalIds); + const controlledOverlayExternalIds = normalizeControlledSupplementIds( + args.controlledOverlayExternalIds ?? [], + ); + const controlledSupplementExternalIds = normalizeControlledSupplementIds( + args.controlledSupplementExternalIds ?? controlledExternalIds, + ); + if ( + controlledOverlayExternalIds.some((externalId) => + controlledSupplementExternalIds.includes(externalId), + ) || + controlledExternalIds.some( + (externalId) => + !controlledOverlayExternalIds.includes(externalId) && + !controlledSupplementExternalIds.includes(externalId), + ) || + controlledOverlayExternalIds.some( + (externalId) => !controlledExternalIds.includes(externalId), + ) || + controlledSupplementExternalIds.some( + (externalId) => !controlledExternalIds.includes(externalId), + ) + ) { + throw new Error("skills.sh mirror controlled proof partition is invalid"); + } + const encodedPayload = Buffer.from( + JSON.stringify({ + catalogTotal: args.catalogTotal, + controlledExternalIds, + controlledOverlayExternalIds, + controlledSupplementExternalIds, + }), + ).toString("base64url"); + const compact = `${SKILLS_SH_MIRROR_PROOF_SNAPSHOT_PREFIX}${encodedPayload}`; + if (!args.evidence) return compact; + const evidence = validateSkillsShMirrorProofEvidence(args.evidence); + const serializedEvidence = JSON.stringify(evidence); + const snapshotId = + `${compact}.${sha256Hex(`${encodedPayload}.${serializedEvidence}`)}.` + + Buffer.from(serializedEvidence).toString("base64url"); + if (Buffer.byteLength(snapshotId, "utf8") > MAX_PROOF_SNAPSHOT_BYTES) { + throw new Error("skills.sh mirror proof source metadata is too large"); + } + return snapshotId; +} + +export function parseSkillsShMirrorProofSnapshotId(snapshotId: string) { + if (!snapshotId.startsWith(SKILLS_SH_MIRROR_PROOF_SNAPSHOT_PREFIX)) { + throw new Error("skills.sh mirror run lacks proof source metadata"); + } + let payload: unknown; + let evidence: SkillsShMirrorProofEvidence | undefined; + let sourceSnapshotHash: string | undefined; + try { + const [encodedPayload, snapshotHash, encodedEvidence, ...unexpected] = snapshotId + .slice(SKILLS_SH_MIRROR_PROOF_SNAPSHOT_PREFIX.length) + .split("."); + if ( + unexpected.length > 0 || + (snapshotHash !== undefined && encodedEvidence === undefined) || + (snapshotHash === undefined && encodedEvidence !== undefined) + ) { + throw new Error("invalid proof snapshot segments"); + } + payload = JSON.parse(Buffer.from(encodedPayload ?? "", "base64url").toString("utf8")); + if (snapshotHash && encodedEvidence) { + const serializedEvidence = Buffer.from(encodedEvidence, "base64url").toString("utf8"); + if ( + !/^[a-f0-9]{64}$/.test(snapshotHash) || + sha256Hex(`${encodedPayload}.${serializedEvidence}`) !== snapshotHash + ) { + throw new Error("invalid proof snapshot hash"); + } + evidence = validateSkillsShMirrorProofEvidence(JSON.parse(serializedEvidence)); + sourceSnapshotHash = snapshotHash; + } + } catch { + throw new Error("skills.sh mirror proof source metadata is invalid"); + } + if (!payload || typeof payload !== "object" || Array.isArray(payload)) { + throw new Error("skills.sh mirror proof source metadata is invalid"); + } + const record = payload as Record; + if ( + !Number.isInteger(record.catalogTotal) || + Number(record.catalogTotal) < 1 || + Number(record.catalogTotal) > MAX_PROOF_SOURCE_ROWS + ) { + throw new Error("skills.sh mirror proof catalog total is invalid"); + } + if (!Array.isArray(record.controlledExternalIds)) { + throw new Error("skills.sh mirror proof controlled identities are invalid"); + } + const controlledExternalIds = normalizeControlledSupplementIds(record.controlledExternalIds); + const controlledOverlayExternalIds = normalizeControlledSupplementIds( + Array.isArray(record.controlledOverlayExternalIds) ? record.controlledOverlayExternalIds : [], + ); + const controlledSupplementExternalIds = normalizeControlledSupplementIds( + Array.isArray(record.controlledSupplementExternalIds) + ? record.controlledSupplementExternalIds + : controlledExternalIds, + ); + buildSkillsShMirrorProofSnapshotId({ + catalogTotal: Number(record.catalogTotal), + controlledExternalIds, + controlledOverlayExternalIds, + controlledSupplementExternalIds, + }); + return { + catalogTotal: Number(record.catalogTotal), + controlledExternalIds, + controlledOverlayExternalIds, + controlledSupplementExternalIds, + ...(sourceSnapshotHash ? { sourceSnapshotHash } : {}), + ...(evidence ? { evidence } : {}), + }; +} + export type SkillsShCatalogSourceEnv = { CLAWHUB_SKILLS_SH_ROLLOUT_MODE?: string; CLAWHUB_SKILLS_SH_TEST_LIVE_FETCH_ENABLED?: string; @@ -40,9 +246,30 @@ export type SkillsShCatalogDetail = { files: Array<{ name?: unknown; content?: unknown; + path?: unknown; + contents?: unknown; }> | null; }; +export type SkillsShCatalogAudit = { + id?: unknown; + source?: unknown; + slug?: unknown; + audits?: unknown; +}; + +export type SkillsShMirrorUpstreamScanner = { + status: string; + sourceCheckedAt?: string; + sourceUrl?: string; +}; + +export type SkillsShMirrorUpstreamScanners = { + genAgentTrustHub: SkillsShMirrorUpstreamScanner; + socket: SkillsShMirrorUpstreamScanner; + snyk: SkillsShMirrorUpstreamScanner; +}; + type SkillsShCatalogPage = { data: SkillsShCatalogListRow[]; pagination: { @@ -57,10 +284,1023 @@ type SkillsShCatalogSearch = { data: SkillsShCatalogListRow[]; }; +type SkillsShCatalogPagination = SkillsShCatalogPage["pagination"]; + +export type SkillsShMirrorProofEvidence = { + pagination: { + endpointExhausted: true; + databaseCoverage: "leaderboard-only"; + page0: SkillsShCatalogPagination; + requestedPages: Array<{ + page: number; + count: number; + hasMore: boolean; + identityHash: string; + contentHash: string; + sourceBytes?: number; + serializedBytes?: number; + }>; + finalNonemptyPage: { + page: number; + count: number; + pagination: SkillsShCatalogPagination; + }; + firstBeyondEndPage: { + page: number; + count: number; + pagination: SkillsShCatalogPagination; + }; + uniqueIds: number; + duplicateIds: number; + }; + fields: { + sampledExternalId: string; + leaderboard: { + topLevelKeys: string[]; + paginationKeys: string[]; + rowKeys: string[]; + taxonomyFields: string[]; + }; + search: { + topLevelKeys: string[]; + rowKeys: string[]; + taxonomyFields: string[]; + }; + detail: { + topLevelKeys: string[]; + fileKeys: string[]; + taxonomyFields: string[]; + }; + page: { + url: string; + jsonLdDocuments: Array<{ type: string | null; keys: string[] }>; + taxonomyFields: string[]; + }; + rsc: { + objectKeys: string[]; + taxonomyFields: string[]; + }; + normalizedUpstreamTaxonomyFields: string[]; + }; +}; + +export type SkillsShMirrorCapturedSourcePage = { + page: number; + sourceTotal: number; + pageLength: number; + hasMore: boolean; + identityHash: string; + contentHash: string; + sourceBytes: number; + serializedBytes: number; + rows: SkillsShCatalogListRow[]; +}; + type HashQualifiedSkillsShCatalogDetail = SkillsShCatalogDetail & { hash: string; }; +type SkillsShMirrorQuarantineReason = + | "identity-page-content-type" + | "identity-page-fetch-failed" + | "identity-page-http-404" + | "identity-page-http-error" + | "identity-page-repository-conflict" + | "identity-page-repository-mismatch" + | "identity-page-repository-missing" + | "identity-page-redirect" + | "identity-page-required" + | "identity-page-too-large" + | "unsupported-identity"; + +type HtmlNode = DefaultTreeAdapterMap["node"]; +type HtmlElement = DefaultTreeAdapterMap["element"]; + +class SkillsShMirrorIdentityError extends Error { + constructor( + readonly reason: SkillsShMirrorQuarantineReason, + message: string = reason, + ) { + super(message); + this.name = "SkillsShMirrorIdentityError"; + } +} + +const UPSTREAM_SCANNER_PROVIDERS = { + "agent-trust-hub": "genAgentTrustHub", + socket: "socket", + snyk: "snyk", +} as const; + +function normalizeUpstreamScannerStatus(value: string) { + const status = value.trim().toLowerCase().replace(/\s+/g, "-"); + return status && + status.length <= MAX_UPSTREAM_SCANNER_STATUS_LENGTH && + /^[a-z0-9][a-z0-9-]*$/.test(status) + ? status + : "unavailable"; +} + +export function buildSkillsShMirrorUpstreamScanners( + auditPayload: SkillsShCatalogAudit | null, + sourceUrl: string, +): SkillsShMirrorUpstreamScanners { + const unavailable = (): SkillsShMirrorUpstreamScanner => ({ status: "unavailable" }); + const scanners: SkillsShMirrorUpstreamScanners = { + genAgentTrustHub: unavailable(), + socket: unavailable(), + snyk: unavailable(), + }; + if (!auditPayload || !Array.isArray(auditPayload.audits)) return scanners; + let pageUrl: URL; + try { + pageUrl = new URL(sourceUrl); + } catch { + return scanners; + } + if (pageUrl.protocol !== "https:" || !["skills.sh", "www.skills.sh"].includes(pageUrl.hostname)) { + return scanners; + } + for (const value of auditPayload.audits) { + if (!value || typeof value !== "object" || Array.isArray(value)) continue; + const audit = value as Record; + if (typeof audit.slug !== "string" || typeof audit.status !== "string") continue; + const slug = audit.slug.trim().toLowerCase(); + const provider = UPSTREAM_SCANNER_PROVIDERS[slug as keyof typeof UPSTREAM_SCANNER_PROVIDERS]; + if (!provider) continue; + const status = normalizeUpstreamScannerStatus(audit.status); + if (status === "unavailable") continue; + const detailUrl = new URL(pageUrl); + detailUrl.pathname = `${detailUrl.pathname.replace(/\/+$/, "")}/security/${slug}`; + detailUrl.search = ""; + detailUrl.hash = ""; + if (detailUrl.href.length > MAX_UPSTREAM_SCANNER_URL_LENGTH) continue; + let sourceCheckedAt: string | undefined; + if (typeof audit.auditedAt === "string" && !Number.isNaN(Date.parse(audit.auditedAt))) { + sourceCheckedAt = audit.auditedAt; + } + scanners[provider] = { + status, + sourceUrl: detailUrl.href, + ...(sourceCheckedAt ? { sourceCheckedAt } : {}), + }; + } + return scanners; +} + +type SkillsShFetchOptions = { + env?: SkillsShCatalogSourceEnv; + fetchImpl?: typeof fetch; + oidcToken?: string; + minimumApiRequestIntervalMs?: number; +}; + +type SkillsShMirrorGitHubLocatorRow = { + externalId: string; + sourceType?: string; + owner?: string; + repo?: string; + slug?: string; + githubPath?: string; + githubCommit?: string; + detail?: { + path: string; + content: string; + truncated: boolean; + }; +}; + +type GitHubRepoTreeSnapshot = { + commit: string; + blobs: Array<{ path: string; sha: string }>; +}; + +class SkillsShSourceHttpError extends Error { + constructor( + readonly status: number, + readonly retryAfterSeconds: number | null, + ) { + super(`skills.sh catalog source returned HTTP ${status}`); + this.name = "SkillsShSourceHttpError"; + } +} + +export function skillsShSourceRetryAfterSeconds(error: unknown) { + return error instanceof SkillsShSourceHttpError && error.status === 429 + ? error.retryAfterSeconds + : null; +} + +async function fetchSkillsShApiResponse( + path: string, + options: SkillsShFetchOptions, + allowNotFound = false, +) { + const env = options.env ?? process.env; + const fetchImpl = options.fetchImpl ?? fetch; + for (let attempt = 0; attempt < MAX_SOURCE_ATTEMPTS; attempt += 1) { + const response = await fetchImpl(`${SKILLS_SH_API_BASE}${path}`, { + headers: { + Accept: "application/json", + Authorization: `Bearer ${requireOidcToken(env, options.oidcToken)}`, + }, + }); + if (response.ok) return response; + if (allowNotFound && response.status === 404) { + await cancelResponseBody(response); + return null; + } + const retryAfterMs = response.status === 429 ? skillsShRetryAfterMs(response, attempt) : null; + if ( + (response.status !== 429 && response.status < 500) || + attempt === MAX_SOURCE_ATTEMPTS - 1 || + (retryAfterMs !== null && retryAfterMs > MAX_INLINE_RETRY_AFTER_MS) + ) { + await cancelResponseBody(response); + throw new SkillsShSourceHttpError( + response.status, + retryAfterMs === null ? null : Math.max(1, Math.ceil(retryAfterMs / 1_000)), + ); + } + await cancelResponseBody(response); + await waitForSkillsShRetry(response, attempt); + } + throw new Error("skills.sh catalog source exhausted retries"); +} + +function normalizeSkillsShId(id: string) { + const parts = id.split("/"); + if (parts.length < 2 || parts.length > 3 || parts.some((part) => !part.trim())) { + throw new Error("skills.sh mirror detail id must be source/skill or owner/repo/skill"); + } + return parts.map((part) => encodeURIComponent(part)).join("/"); +} + +function isSkillsShIdentitySegment(value: string) { + return Boolean(value) && !value.includes("/"); +} + +function normalizeUpstreamSourceType(value: unknown) { + const normalized = (typeof value === "string" ? value : "") + .trim() + .toLowerCase() + .replace(/[^a-z0-9._-]+/g, "-") + .replace(/^[^a-z0-9]+|[^a-z0-9]+$/g, ""); + return (normalized || "missing").slice(0, MAX_UPSTREAM_SOURCE_TYPE_LENGTH); +} + +async function fetchSkillsShMirrorAudit(id: string, options: SkillsShFetchOptions = {}) { + const response = await fetchSkillsShApiResponse( + `/skills/audit/${normalizeSkillsShId(id)}`, + options, + true, + ); + return response === null ? null : await parseSkillsShJsonResponse(response); +} + +function isHtmlElement(node: HtmlNode): node is HtmlElement { + return "tagName" in node; +} + +function htmlText(node: HtmlNode): string { + if ("value" in node) return node.value; + return "childNodes" in node ? node.childNodes.map(htmlText).join("") : ""; +} + +function htmlElements(node: HtmlNode, predicate: (element: HtmlElement) => boolean): HtmlElement[] { + const matches: HtmlElement[] = []; + if (isHtmlElement(node) && predicate(node)) matches.push(node); + if ("childNodes" in node) { + for (const child of node.childNodes) matches.push(...htmlElements(child, predicate)); + } + return matches; +} + +function htmlAttribute(element: HtmlElement, name: string) { + return element.attrs.find((attribute) => attribute.name === name)?.value; +} + +function parseExactGitHubRepositoryUrl(value: string) { + let url: URL; + try { + url = new URL(value); + } catch { + return null; + } + const pathParts = url.pathname.split("/").filter(Boolean); + if ( + url.protocol !== "https:" || + url.hostname.toLowerCase() !== "github.com" || + url.port || + url.username || + url.password || + url.search || + url.hash || + pathParts.length !== 2 + ) { + return null; + } + const owner = pathParts[0]?.toLowerCase(); + const repo = pathParts[1]?.replace(/\.git$/i, "").toLowerCase(); + if (!owner || !repo) return null; + return { + owner, + repo, + canonicalRepoUrl: `https://github.com/${owner}/${repo}`, + }; +} + +/* + * Repository identity is read only from the page's metadata row. Rendered + * README content can contain arbitrary labels and links and is not authoritative. + */ +function repositoryIdentityFromHtml(html: string, expectedOwner: string, expectedRepo: string) { + const document = parse(html); + const metadataContainers = htmlElements(document, (element) => { + const classes = new Set((htmlAttribute(element, "class") ?? "").split(/\s+/)); + return ( + classes.has("bg-background") && + classes.has("py-8") && + element.childNodes.some( + (child) => isHtmlElement(child) && htmlText(child).trim().toLowerCase() === "repository", + ) + ); + }); + const repositoryLinks = new Set(); + for (const container of metadataContainers) { + const links = container.childNodes.filter( + (node): node is HtmlElement => + isHtmlElement(node) && node.tagName === "a" && htmlAttribute(node, "href") !== undefined, + ); + for (const link of links) { + const href = htmlAttribute(link, "href"); + const repository = href ? parseExactGitHubRepositoryUrl(href) : null; + if (repository) { + repositoryLinks.add( + `${repository.owner}/${repository.repo}|${repository.canonicalRepoUrl}`, + ); + } + } + } + const candidates = Array.from(repositoryLinks, (value) => { + const [identity, canonicalRepoUrl] = value.split("|"); + const [owner, repo] = identity!.split("/"); + return { owner: owner!, repo: repo!, canonicalRepoUrl: canonicalRepoUrl! }; + }); + if (candidates.length === 0) { + throw new SkillsShMirrorIdentityError("identity-page-repository-missing"); + } + if (candidates.length > 1) { + throw new SkillsShMirrorIdentityError("identity-page-repository-conflict"); + } + const [candidate] = candidates; + if (candidate!.owner !== expectedOwner || candidate!.repo !== expectedRepo) { + throw new SkillsShMirrorIdentityError("identity-page-repository-mismatch"); + } + return candidate!; +} + +function exactSkillsShIdentityPageUrl(value: string, expectedPath: string) { + let url: URL; + try { + url = new URL(value); + } catch { + return null; + } + return url.protocol === "https:" && + ["skills.sh", "www.skills.sh"].includes(url.hostname) && + !url.port && + !url.username && + !url.password && + !url.search && + !url.hash && + url.pathname.toLowerCase() === expectedPath + ? url + : null; +} + +async function cancelResponseBody(response: Response) { + try { + await response.body?.cancel(); + } catch { + // The response may already be closed by the runtime. + } +} + +function isRedirectStatus(status: number) { + return status === 301 || status === 302 || status === 303 || status === 307 || status === 308; +} + +function hasExactSkillsShPagePath(sourceUrl: string, pathParts: string[]) { + let url: URL; + try { + url = new URL(sourceUrl); + } catch { + return false; + } + return ( + url.protocol === "https:" && + ["skills.sh", "www.skills.sh"].includes(url.hostname) && + !url.port && + !url.username && + !url.password && + !url.search && + !url.hash && + url.pathname.toLowerCase() === `/${pathParts.join("/")}` + ); +} + +function parseExactGitHubInstallIdentity(source: string, installUrl: string | null) { + const [owner, repo, ...rest] = source.split("/"); + if (!owner || !repo || rest.length > 0) return null; + if (!installUrl) return null; + const repository = parseExactGitHubRepositoryUrl(installUrl); + return repository?.owner === owner && repository.repo === repo ? repository : null; +} + +export function buildSkillsShMirrorObservation( + row: SkillsShCatalogListRow, + sourcePageHtml?: string, +) { + const externalId = row.id.trim().toLowerCase(); + const slug = row.slug.trim().toLowerCase(); + const source = row.source.trim().toLowerCase(); + const upstreamSourceType = normalizeUpstreamSourceType(row.sourceType); + const installUrl = row.installUrl?.trim() || null; + const identityError = (reason: SkillsShMirrorQuarantineReason) => + new SkillsShMirrorIdentityError( + reason, + `Unsupported skills.sh mirror identity: ${externalId} ` + + `(sourceType=${upstreamSourceType}, source=${source || "missing"}, ` + + `installUrlPresent=${installUrl !== null})`, + ); + const base = { + externalId, + slug, + displayName: row.name.trim() || slug, + sourceUrl: row.url.trim(), + upstreamInstalls: row.installs, + upstreamSourceType, + }; + const githubIdentity = parseExactGitHubInstallIdentity(source, installUrl); + if ( + githubIdentity && + isSkillsShIdentitySegment(slug) && + externalId === `${githubIdentity.owner}/${githubIdentity.repo}/${slug}` + ) { + return { + ...base, + sourceType: "github" as const, + ...githubIdentity, + }; + } + const [owner, repo, ...sourceRest] = source.split("/"); + const structurallyAmbiguousGithub = + !installUrl && + upstreamSourceType === "well-known" && + Boolean(owner && repo) && + sourceRest.length === 0 && + isSkillsShIdentitySegment(slug) && + externalId === `${owner}/${repo}/${slug}` && + hasExactSkillsShPagePath(base.sourceUrl, [owner!, repo!, slug]); + if (structurallyAmbiguousGithub) { + if (sourcePageHtml === undefined) { + throw identityError("identity-page-required"); + } + let repositoryIdentity: ReturnType; + try { + repositoryIdentity = repositoryIdentityFromHtml(sourcePageHtml, owner!, repo!); + } catch (error) { + if (error instanceof SkillsShMirrorIdentityError) throw identityError(error.reason); + throw error; + } + return { + ...base, + sourceType: "github" as const, + ...repositoryIdentity, + }; + } + if ( + !installUrl && + source && + !source.includes("/") && + isSkillsShIdentitySegment(slug) && + externalId === `${source}/${slug}` && + hasExactSkillsShPagePath(base.sourceUrl, ["site", source, slug]) + ) { + return { + ...base, + sourceType: "well-known" as const, + sourceHost: source, + }; + } + throw identityError("unsupported-identity"); +} + +function safeMirrorIdentityError(row: SkillsShCatalogListRow, error: unknown) { + const externalId = row.id.trim().toLowerCase().slice(0, 512) || "missing"; + const upstreamSourceType = normalizeUpstreamSourceType(row.sourceType); + const reason = + error instanceof SkillsShMirrorIdentityError ? error.reason : "unsupported-identity"; + const source = row.source.trim().toLowerCase().slice(0, 256) || "missing"; + const installUrl = row.installUrl?.trim() || null; + console.warn( + `Unsupported skills.sh mirror identity: ${externalId} ` + + `(sourceType=${upstreamSourceType}, source=${source}, ` + + `installUrlPresent=${installUrl !== null})`, + ); + return { + quarantined: true as const, + externalId, + upstreamSourceType, + reason, + }; +} + +async function fetchSkillsShIdentityPage( + sourceUrl: string, + options: SkillsShFetchOptions, +): Promise< + | { ok: true; html: string; sourceBytes: number } + | { ok: false; reason: SkillsShMirrorQuarantineReason; sourceBytes: number } +> { + const fetchImpl = options.fetchImpl ?? fetch; + let sourceBytes = 0; + const failure = (reason: SkillsShMirrorQuarantineReason) => ({ + ok: false as const, + reason, + sourceBytes, + }); + let expectedPath: string; + try { + expectedPath = new URL(sourceUrl).pathname.toLowerCase(); + } catch { + return failure("identity-page-redirect"); + } + const initialUrl = exactSkillsShIdentityPageUrl(sourceUrl, expectedPath); + if (!initialUrl) return failure("identity-page-redirect"); + attemptLoop: for (let attempt = 0; attempt < MAX_SOURCE_ATTEMPTS; attempt += 1) { + let requestUrl = initialUrl.href; + for (let redirects = 0; redirects <= MAX_IDENTITY_PAGE_REDIRECTS; redirects += 1) { + let response: Response; + try { + response = await fetchImpl(requestUrl, { + headers: { Accept: "text/html" }, + redirect: "manual", + }); + } catch { + if (attempt === MAX_SOURCE_ATTEMPTS - 1) { + return failure("identity-page-fetch-failed"); + } + await new Promise((resolve) => setTimeout(resolve, 250 * 2 ** attempt)); + continue attemptLoop; + } + if (isRedirectStatus(response.status)) { + const location = response.headers.get("location"); + await cancelResponseBody(response); + if (!location || redirects === MAX_IDENTITY_PAGE_REDIRECTS) { + return failure("identity-page-redirect"); + } + let target: URL; + try { + target = new URL(location, requestUrl); + } catch { + return failure("identity-page-redirect"); + } + const validatedTarget = exactSkillsShIdentityPageUrl( + target.href, + initialUrl.pathname.toLowerCase(), + ); + if (!validatedTarget) return failure("identity-page-redirect"); + requestUrl = validatedTarget.href; + continue; + } + if (response.status === 404) { + await cancelResponseBody(response); + return failure("identity-page-http-404"); + } + if (!response.ok) { + if (response.status === 429 || response.status >= 500) { + const retryAfterMs = + response.status === 429 ? skillsShRetryAfterMs(response, attempt) : null; + if ( + response.status === 429 && + (attempt === MAX_SOURCE_ATTEMPTS - 1 || + (retryAfterMs !== null && retryAfterMs > MAX_INLINE_RETRY_AFTER_MS)) + ) { + await cancelResponseBody(response); + throw new SkillsShSourceHttpError( + response.status, + retryAfterMs === null ? null : Math.max(1, Math.ceil(retryAfterMs / 1_000)), + ); + } + if (attempt < MAX_SOURCE_ATTEMPTS - 1) { + await cancelResponseBody(response); + await waitForSkillsShRetry(response, attempt); + continue attemptLoop; + } + await cancelResponseBody(response); + return failure("identity-page-fetch-failed"); + } + await cancelResponseBody(response); + return failure("identity-page-http-error"); + } + const contentType = response.headers.get("content-type")?.toLowerCase() ?? ""; + if (!contentType.includes("text/html")) { + await cancelResponseBody(response); + return failure("identity-page-content-type"); + } + const contentLength = Number(response.headers.get("content-length")); + if (Number.isFinite(contentLength) && contentLength > MAX_IDENTITY_PAGE_BYTES) { + await cancelResponseBody(response); + return failure("identity-page-too-large"); + } + const body = await readBoundedResponseBytes(response, MAX_IDENTITY_PAGE_BYTES); + sourceBytes += body.sourceBytes; + if (!body.ok) { + await cancelResponseBody(response); + if (attempt === MAX_SOURCE_ATTEMPTS - 1) { + return failure("identity-page-fetch-failed"); + } + await new Promise((resolve) => setTimeout(resolve, 250 * 2 ** attempt)); + continue attemptLoop; + } + if (body.bytes === null) { + return failure("identity-page-too-large"); + } + return { + ok: true, + html: new TextDecoder().decode(body.bytes), + sourceBytes, + }; + } + } + return failure("identity-page-fetch-failed"); +} + +async function readBoundedResponseBytes(response: Response, maximumBytes: number) { + if (!response.body) { + return { ok: true as const, bytes: new Uint8Array(), sourceBytes: 0 }; + } + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let byteLength = 0; + try { + while (true) { + const chunk = await reader.read(); + if (chunk.done) break; + byteLength += chunk.value.byteLength; + if (byteLength > maximumBytes) { + await reader.cancel(); + return { ok: true as const, bytes: null, sourceBytes: byteLength }; + } + chunks.push(chunk.value); + } + } catch { + return { ok: false as const, sourceBytes: byteLength }; + } + const bytes = new Uint8Array(byteLength); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + return { ok: true as const, bytes, sourceBytes: byteLength }; +} + +async function resolveSkillsShMirrorObservation( + row: SkillsShCatalogListRow, + options: SkillsShFetchOptions, +) { + try { + return { + row: buildSkillsShMirrorObservation(row), + identitySourceBytes: 0, + }; + } catch (error) { + if ( + !(error instanceof SkillsShMirrorIdentityError) || + error.reason !== "identity-page-required" + ) { + return { + row: safeMirrorIdentityError(row, error), + identitySourceBytes: 0, + }; + } + } + const sourcePage = await fetchSkillsShIdentityPage(row.url.trim(), options); + if (!sourcePage.ok) { + return { + row: safeMirrorIdentityError(row, new SkillsShMirrorIdentityError(sourcePage.reason)), + identitySourceBytes: sourcePage.sourceBytes, + }; + } + try { + return { + row: buildSkillsShMirrorObservation(row, sourcePage.html), + identitySourceBytes: sourcePage.sourceBytes, + }; + } catch (error) { + return { + row: safeMirrorIdentityError(row, error), + identitySourceBytes: sourcePage.sourceBytes, + }; + } +} + +function mirrorDetailFile(file: NonNullable[number]) { + const path = + typeof file.path === "string" + ? file.path.trim() + : typeof file.name === "string" + ? file.name.trim() + : ""; + const content = + typeof file.contents === "string" + ? file.contents + : typeof file.content === "string" + ? file.content + : null; + return path && content !== null ? { path, content } : null; +} + +function truncateUtf8(value: string, maxBytes: number) { + const bytes = Buffer.from(value, "utf8"); + if (bytes.byteLength <= maxBytes) return value; + let truncated = bytes.subarray(0, maxBytes).toString("utf8"); + while (Buffer.byteLength(truncated, "utf8") > maxBytes) { + truncated = truncated.slice(0, -1); + } + return truncated.replace(/\uFFFD$/u, ""); +} + +function gitBlobSha(content: string) { + const bytes = Buffer.from(content, "utf8"); + return createHash("sha1").update(`blob ${bytes.byteLength}\0`).update(bytes).digest("hex"); +} + +function normalizedRepoRelativePath(value: string) { + const normalized = value.trim().replaceAll("\\", "/").replace(/^\/+/, ""); + if ( + !normalized || + normalized.includes("\0") || + normalized.split("/").some((segment) => !segment || segment === "." || segment === "..") + ) { + return null; + } + return normalized; +} + +function githubCommitFromArchiveRedirect(value: string, owner: string, repo: string) { + let url: URL; + try { + url = new URL(value); + } catch { + return null; + } + const parts = url.pathname.split("/").filter(Boolean); + const commit = parts[3]?.toLowerCase(); + return url.protocol === "https:" && + url.hostname === "codeload.github.com" && + parts.length === 4 && + parts[0]?.toLowerCase() === owner && + parts[1]?.toLowerCase() === repo && + parts[2] === "zip" && + commit && + /^[a-f0-9]{40}$/.test(commit) + ? commit + : null; +} + +async function fetchGitHubRepoTreeSnapshot( + owner: string, + repo: string, + options: { + fetchImpl: typeof fetch; + beforeRequest?: () => Promise | void; + accountRequest: () => void; + accountBytes: (sourceBytes: number) => void; + }, +): Promise { + // GitHub resolves HEAD.zip to an immutable codeload commit URL. Fail closed + // if that redirect ever stops carrying the exact repository commit SHA. + const archiveUrl = `https://github.com/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/archive/HEAD.zip`; + await options.beforeRequest?.(); + options.accountRequest(); + let archive: Response; + try { + archive = await options.fetchImpl(archiveUrl, { + headers: { Accept: "application/zip", "User-Agent": "clawhub/skills-sh-mirror" }, + redirect: "manual", + }); + } catch { + return null; + } + const location = archive.headers.get("location"); + // Source byte accounting includes only response bodies consumed by this + // process; this redirect body is canceled unread and contributes zero. + await cancelResponseBody(archive); + const commit = location ? githubCommitFromArchiveRedirect(location, owner, repo) : null; + if (!commit) return null; + + const headers = await buildGitHubApiHeaders({ + userAgent: "clawhub/skills-sh-mirror", + fetchImpl: options.fetchImpl, + }); + await options.beforeRequest?.(); + options.accountRequest(); + let treeResponse: Response; + try { + treeResponse = await options.fetchImpl( + `https://api.github.com/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/git/trees/${commit}?recursive=1`, + { headers }, + ); + } catch { + return null; + } + if (!treeResponse.ok) { + await cancelResponseBody(treeResponse); + return null; + } + let body: Awaited>; + try { + body = await readBoundedResponseBytes(treeResponse, MAX_GITHUB_TREE_BYTES); + } catch { + return null; + } + options.accountBytes(body.sourceBytes); + if (!body.ok || body.bytes === null) return null; + let payload: unknown; + try { + payload = JSON.parse(new TextDecoder().decode(body.bytes)); + } catch { + return null; + } + if (!payload || typeof payload !== "object" || Array.isArray(payload)) return null; + const record = payload as Record; + if (record.truncated === true || !Array.isArray(record.tree)) return null; + if (record.tree.length > MAX_GITHUB_TREE_ENTRIES) return null; + const blobs = record.tree.flatMap((value) => { + if (!value || typeof value !== "object" || Array.isArray(value)) return []; + const entry = value as Record; + const path = typeof entry.path === "string" ? normalizedRepoRelativePath(entry.path) : null; + const sha = typeof entry.sha === "string" ? entry.sha.trim().toLowerCase() : ""; + return entry.type === "blob" && path && /^[a-f0-9]{40}$/.test(sha) ? [{ path, sha }] : []; + }); + return { commit, blobs }; +} + +export async function resolveSkillsShMirrorGitHubLocators( + rows: T[], + options: { + fetchImpl?: typeof fetch; + beforeRequest?: () => Promise | void; + fullDetailContentByExternalId?: ReadonlyMap; + } = {}, +) { + const fetchImpl = options.fetchImpl ?? fetch; + let sourceRequests = 0; + let sourceBytes = 0; + const accountRequest = () => { + sourceRequests += 1; + }; + const accountBytes = (bytes: number) => { + sourceBytes += bytes; + }; + const cache = new Map>(); + const nextRows = [...rows]; + const groups = new Map>(); + rows.forEach((row, index) => { + if ( + row.sourceType !== "github" || + !row.owner || + !row.repo || + !row.slug || + !row.detail || + (row.detail.truncated && !options.fullDetailContentByExternalId?.has(row.externalId)) || + (row.githubPath && row.githubCommit) + ) { + return; + } + const key = `${row.owner.toLowerCase()}/${row.repo.toLowerCase()}`; + const group = groups.get(key) ?? []; + group.push({ index, row }); + groups.set(key, group); + }); + + const groupEntries = Array.from(groups); + let nextGroup = 0; + await Promise.all( + Array.from({ length: Math.min(GITHUB_LOCATOR_CONCURRENCY, groupEntries.length) }, async () => { + while (nextGroup < groupEntries.length) { + const [key, group] = groupEntries[nextGroup++]!; + let snapshotPromise = cache.get(key); + if (!snapshotPromise) { + const [owner, repo] = key.split("/") as [string, string]; + snapshotPromise = fetchGitHubRepoTreeSnapshot(owner, repo, { + fetchImpl, + beforeRequest: options.beforeRequest, + accountRequest, + accountBytes, + }).then((snapshot) => { + if (!snapshot) cache.delete(key); + return snapshot; + }); + cache.set(key, snapshotPromise); + } + const snapshot = await snapshotPromise; + if (!snapshot) continue; + for (const { index, row } of group) { + const relativePath = normalizedRepoRelativePath(row.detail!.path); + if (!relativePath) continue; + const fullContent = + options.fullDetailContentByExternalId?.get(row.externalId) ?? row.detail!.content; + const expectedBlobSha = gitBlobSha(fullContent); + const suffix = `/${relativePath.toLowerCase()}`; + const matches = snapshot.blobs.filter((blob) => { + const path = blob.path.toLowerCase(); + if ( + blob.sha !== expectedBlobSha || + (!path.endsWith(suffix) && path !== relativePath.toLowerCase()) + ) { + return false; + } + const folder = blob.path.split("/").slice(0, -1).join("/"); + return folder.split("/").at(-1)?.toLowerCase() === row.slug!.toLowerCase(); + }); + if (matches.length !== 1) continue; + const githubPath = matches[0]!.path.split("/").slice(0, -1).join("/"); + if (!githubPath) continue; + nextRows[index] = { + ...row, + githubPath, + githubCommit: snapshot.commit, + }; + } + } + }), + ); + return { rows: nextRows, sourceRequests, sourceBytes }; +} + +export function buildSkillsShMirrorDetail(detail: SkillsShCatalogDetail, maxBytes: number) { + assertIntegerInRange("maxBytes", maxBytes, 1, 64 * 1024); + const files = (detail.files ?? []) + .map(mirrorDetailFile) + .filter((file): file is { path: string; content: string } => file !== null); + const candidates = files + .map((file) => { + const basename = file.path.split("/").at(-1)?.toLowerCase(); + const contentKind = + basename === "skill.md" + ? ("skill-md" as const) + : basename === "readme.md" + ? ("readme" as const) + : null; + return contentKind ? { ...file, contentKind } : null; + }) + .filter( + ( + file, + ): file is { + path: string; + content: string; + contentKind: "skill-md" | "readme"; + } => file !== null, + ) + .sort((left, right) => { + if (left.contentKind !== right.contentKind) { + return left.contentKind === "skill-md" ? -1 : 1; + } + return left.path.length - right.path.length || left.path.localeCompare(right.path); + }); + const upstreamSourceContentHash = + typeof detail.hash === "string" && /^[a-f0-9]{64}$/i.test(detail.hash) + ? detail.hash.toLowerCase() + : undefined; + const selected = candidates[0]; + if (!selected) { + return { + ...(upstreamSourceContentHash ? { sourceContentHash: upstreamSourceContentHash } : {}), + sourceFileCount: files.length, + contentKind: "none" as const, + }; + } + const sourceBytes = Buffer.byteLength(selected.content, "utf8"); + const content = truncateUtf8(selected.content, maxBytes); + const sourceContentHash = upstreamSourceContentHash ?? sha256Hex(selected.content); + return { + sourceContentHash, + sourceFileCount: files.length, + contentKind: selected.contentKind, + path: selected.path, + content, + contentBytes: Buffer.byteLength(content, "utf8"), + sourceBytes, + truncated: sourceBytes > maxBytes, + }; +} + function assertIntegerInRange(name: string, value: number, min: number, max: number) { if (!Number.isInteger(value) || value < min || value > max) { throw new Error(`${name} must be an integer between ${min} and ${max}`); @@ -75,26 +1315,63 @@ function requireOidcToken(env: SkillsShCatalogSourceEnv, requestOidcToken?: stri return token; } -async function fetchSkillsShJson( - path: string, - options: { - env?: SkillsShCatalogSourceEnv; - fetchImpl?: typeof fetch; - oidcToken?: string; - } = {}, -): Promise { - const env = options.env ?? process.env; - const fetchImpl = options.fetchImpl ?? fetch; - const response = await fetchImpl(`${SKILLS_SH_API_BASE}${path}`, { - headers: { - Accept: "application/json", - Authorization: `Bearer ${requireOidcToken(env, options.oidcToken)}`, - }, - }); - if (!response.ok) { - throw new Error(`skills.sh catalog source returned HTTP ${response.status}`); +async function fetchSkillsShJson(path: string, options: SkillsShFetchOptions = {}): Promise { + return (await fetchSkillsShJsonWithBytes(path, options)).value; +} + +async function fetchSkillsShJsonWithBytes(path: string, options: SkillsShFetchOptions = {}) { + const response = await fetchSkillsShApiResponse(path, options); + if (response === null) throw new Error("skills.sh catalog source returned unexpected not found"); + return await parseSkillsShJsonResponse(response); +} + +async function parseSkillsShJsonResponse(response: Response) { + const bytes = new Uint8Array(await response.arrayBuffer()); + return { + value: JSON.parse(new TextDecoder().decode(bytes)) as T, + sourceBytes: bytes.byteLength, + }; +} + +async function waitForSkillsShRetry(response: Response, attempt: number) { + const delayMs = skillsShRetryAfterMs(response, attempt); + await new Promise((resolve) => setTimeout(resolve, delayMs)); +} + +function skillsShRetryAfterMs(response: Response, attempt: number) { + const header = response.headers.get("retry-after")?.trim(); + if (header) { + const seconds = Number(header); + if (Number.isFinite(seconds) && seconds >= 0) { + return seconds * 1_000; + } + const dateMs = Date.parse(header); + if (Number.isFinite(dateMs)) { + return Math.max(0, dateMs - Date.now()); + } } - return (await response.json()) as T; + return Math.min(5_000, 250 * 2 ** attempt); +} + +function requestUrl(input: string | URL | Request) { + if (typeof input === "string") return input; + return input instanceof URL ? input.href : input.url; +} + +function createRequestPacer(minimumIntervalMs: number) { + let nextRequestAt = 0; + let queue = Promise.resolve(); + return async () => { + const turn = queue.then(async () => { + const delayMs = Math.max(0, nextRequestAt - Date.now()); + if (delayMs > 0) { + await new Promise((resolve) => setTimeout(resolve, delayMs)); + } + nextRequestAt = Date.now() + minimumIntervalMs; + }); + queue = turn.catch(() => undefined); + await turn; + }; } export async function fetchSkillsShCatalogPage( @@ -107,15 +1384,494 @@ export async function fetchSkillsShCatalogPage( fetchImpl?: typeof fetch; oidcToken?: string; } = {}, +) { + return (await fetchSkillsShCatalogPageWithBytes(args, options)).value; +} + +async function fetchSkillsShCatalogPageWithBytes( + args: { + page: number; + perPage: number; + }, + options: { + env?: SkillsShCatalogSourceEnv; + fetchImpl?: typeof fetch; + oidcToken?: string; + } = {}, ) { assertIntegerInRange("page", args.page, 0, 100_000); assertIntegerInRange("perPage", args.perPage, 1, MAX_SOURCE_PAGE_SIZE); - return await fetchSkillsShJson( + return await fetchSkillsShJsonWithBytes( `/skills?page=${args.page}&per_page=${args.perPage}`, options, ); } +function validateSkillsShMirrorProofEvidence(value: unknown): SkillsShMirrorProofEvidence { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("skills.sh mirror proof evidence is invalid"); + } + const serialized = JSON.stringify(value); + if (Buffer.byteLength(serialized, "utf8") > MAX_PROOF_SNAPSHOT_BYTES) { + throw new Error("skills.sh mirror proof evidence is too large"); + } + const record = value as Record; + if ( + !record.pagination || + typeof record.pagination !== "object" || + Array.isArray(record.pagination) || + !record.fields || + typeof record.fields !== "object" || + Array.isArray(record.fields) + ) { + throw new Error("skills.sh mirror proof evidence is invalid"); + } + return value as SkillsShMirrorProofEvidence; +} + +function sortedObjectKeys(value: unknown) { + return value && typeof value === "object" && !Array.isArray(value) + ? Object.keys(value).sort() + : []; +} + +function sortedArrayObjectKeys(value: unknown) { + if (!Array.isArray(value)) return []; + return Array.from(new Set(value.flatMap((entry) => sortedObjectKeys(entry)))).sort(); +} + +function collectObjectKeys(value: unknown, keys = new Set()) { + if (Array.isArray(value)) { + for (const entry of value) collectObjectKeys(entry, keys); + return keys; + } + if (!value || typeof value !== "object") return keys; + for (const [key, entry] of Object.entries(value)) { + keys.add(key); + collectObjectKeys(entry, keys); + } + return keys; +} + +function normalizedTaxonomyFields(value: unknown) { + return Array.from(collectObjectKeys(value)) + .filter((key) => /^(?:category|categories|topic|topics|tags)$/i.test(key)) + .sort(); +} + +function skillsShPageIdentityHash(rows: SkillsShCatalogListRow[]) { + return sha256Hex(rows.map((row) => `${row.id.trim().toLowerCase()}\n`).join("")); +} + +function capturedSkillsShCatalogRows(rows: SkillsShCatalogListRow[]) { + return rows.map((row) => ({ + id: row.id, + installUrl: row.installUrl, + installs: row.installs, + name: row.name, + slug: row.slug, + source: row.source, + sourceType: row.sourceType, + url: row.url, + })); +} + +function skillsShPageContentHash(rows: SkillsShCatalogListRow[]) { + return sha256Hex(JSON.stringify(capturedSkillsShCatalogRows(rows))); +} + +function parseSkillsShPageFieldEvidence(html: string) { + const document = parse(html); + const jsonLdValues = htmlElements( + document, + (element) => + element.tagName === "script" && + htmlAttribute(element, "type")?.toLowerCase() === "application/ld+json", + ).flatMap((element): unknown[] => { + try { + const value = JSON.parse(htmlText(element)) as unknown; + return value && typeof value === "object" && !Array.isArray(value) ? [value] : []; + } catch { + return []; + } + }); + const jsonLdDocuments = jsonLdValues.map((value) => { + const record = value as Record; + return { + type: typeof record["@type"] === "string" ? record["@type"] : null, + keys: sortedObjectKeys(value), + }; + }); + return { + jsonLdDocuments, + taxonomyFields: normalizedTaxonomyFields(jsonLdValues), + }; +} + +function parseSkillsShRscFieldEvidence(rsc: string) { + const values: unknown[] = []; + for (const line of rsc.split("\n")) { + const separator = line.indexOf(":"); + if (separator < 0) continue; + let encoded = line.slice(separator + 1); + if (encoded.startsWith("I[")) encoded = encoded.slice(1); + if (!["[", "{", '"'].includes(encoded[0] ?? "") && !/^(?:null|true|false|-?\d)/.test(encoded)) { + continue; + } + try { + values.push(JSON.parse(encoded)); + } catch { + // Flight control rows are not all standalone JSON values. + } + } + return { + objectKeys: Array.from(collectObjectKeys(values)).sort(), + taxonomyFields: normalizedTaxonomyFields(values), + }; +} + +function exactSkillsShProofMetadataUrl(value: string, expectedPath: string, rsc: boolean) { + let url: URL; + try { + url = new URL(value); + } catch { + return null; + } + const searchKeys = Array.from(url.searchParams.keys()); + const validSearch = rsc + ? searchKeys.length === 1 && + searchKeys[0] === "_rsc" && + url.searchParams.get("_rsc") === "clawhub-proof" + : searchKeys.length === 0; + return url.protocol === "https:" && + ["skills.sh", "www.skills.sh"].includes(url.hostname.toLowerCase()) && + !url.port && + !url.username && + !url.password && + !url.hash && + url.pathname.toLowerCase() === expectedPath && + validSearch + ? url + : null; +} + +async function fetchSkillsShProofText( + url: string, + options: { + fetchImpl: typeof fetch; + headers?: Record; + expectedPath: string; + rsc: boolean; + }, +) { + const initialUrl = exactSkillsShProofMetadataUrl(url, options.expectedPath, options.rsc); + if (!initialUrl) { + throw new Error("skills.sh proof metadata URL is outside the exact skills.sh route"); + } + for (let attempt = 0; attempt < MAX_SOURCE_ATTEMPTS; attempt += 1) { + let requestUrl = initialUrl; + let response: Response | null = null; + for (let redirect = 0; redirect <= MAX_IDENTITY_PAGE_REDIRECTS; redirect += 1) { + response = await options.fetchImpl(requestUrl, { + redirect: "manual", + headers: { + "User-Agent": "clawhub/skills-sh-mirror-proof", + ...options.headers, + }, + }); + if (!isRedirectStatus(response.status)) break; + const location = response.headers.get("location"); + const redirected = location + ? exactSkillsShProofMetadataUrl( + new URL(location, requestUrl).href, + options.expectedPath, + options.rsc, + ) + : null; + await cancelResponseBody(response); + if (!redirected) { + throw new Error("skills.sh proof metadata redirect is outside the exact skills.sh route"); + } + if (redirect === MAX_IDENTITY_PAGE_REDIRECTS) { + throw new Error("skills.sh proof metadata exceeded the redirect limit"); + } + requestUrl = redirected; + } + if (!response) throw new Error("skills.sh proof metadata response is missing"); + if (response.ok) { + const body = await readBoundedResponseBytes(response, MAX_PROOF_METADATA_BYTES); + if (!body.ok || body.bytes === null) { + throw new Error("skills.sh proof metadata response is too large"); + } + return new TextDecoder().decode(body.bytes); + } + const retryAfterMs = response.status === 429 ? skillsShRetryAfterMs(response, attempt) : null; + if ( + (response.status !== 429 && response.status < 500) || + attempt === MAX_SOURCE_ATTEMPTS - 1 || + (retryAfterMs !== null && retryAfterMs > MAX_INLINE_RETRY_AFTER_MS) + ) { + await cancelResponseBody(response); + if (response.status === 429) { + throw new SkillsShSourceHttpError( + response.status, + retryAfterMs === null ? null : Math.max(1, Math.ceil(retryAfterMs / 1_000)), + ); + } + throw new Error(`skills.sh proof metadata returned HTTP ${response.status}`); + } + await cancelResponseBody(response); + await waitForSkillsShRetry(response, attempt); + } + throw new Error("skills.sh proof metadata exhausted retries"); +} + +export async function measureSkillsShMirrorProofSource( + options: { + env?: SkillsShCatalogSourceEnv; + fetchImpl?: typeof fetch; + oidcToken?: string; + minimumApiRequestIntervalMs?: number; + } = {}, +) { + const minimumApiRequestIntervalMs = + options.minimumApiRequestIntervalMs ?? + (options.fetchImpl ? 0 : MINIMUM_API_REQUEST_INTERVAL_MS); + assertIntegerInRange("minimumApiRequestIntervalMs", minimumApiRequestIntervalMs, 0, 1_000); + const pace = createRequestPacer(minimumApiRequestIntervalMs); + const present = new Set(); + const rowKeys = new Set(); + const taxonomyFields = new Set(); + const requestedPages: SkillsShMirrorProofEvidence["pagination"]["requestedPages"] = []; + const sourcePages: SkillsShMirrorCapturedSourcePage[] = []; + let page = 0; + let sourceRequests = 0; + let observedRows = 0; + let catalogTotal: number | null = null; + let firstPage: SkillsShCatalogPage | null = null; + let finalNonemptyPage: SkillsShMirrorProofEvidence["pagination"]["finalNonemptyPage"] | null = + null; + let sampleRow: SkillsShCatalogListRow | null = null; + while (true) { + await pace(); + const pageResponse = await fetchSkillsShCatalogPageWithBytes( + { page, perPage: MAX_SOURCE_PAGE_SIZE }, + options, + ); + const response = pageResponse.value; + sourceRequests += 1; + if ( + response.pagination.page !== page || + response.pagination.perPage !== MAX_SOURCE_PAGE_SIZE || + response.data.length > MAX_SOURCE_PAGE_SIZE + ) { + throw new Error("skills.sh proof source returned an invalid page contract"); + } + if (catalogTotal === null) catalogTotal = response.pagination.total; + if (catalogTotal > MAX_PROOF_SOURCE_ROWS) { + throw new Error( + `skills.sh proof source total ${catalogTotal} exceeds ${MAX_PROOF_SOURCE_ROWS} rows`, + ); + } + if (response.pagination.total !== catalogTotal) { + throw new Error("skills.sh catalog source total changed during proof measurement"); + } + firstPage ??= response; + const identityHash = skillsShPageIdentityHash(response.data); + const contentHash = skillsShPageContentHash(response.data); + const capturedRows = capturedSkillsShCatalogRows(response.data); + const captured = + response.data.length > 0 + ? { + page, + sourceTotal: catalogTotal, + pageLength: response.data.length, + hasMore: response.pagination.hasMore, + identityHash, + contentHash, + sourceBytes: pageResponse.sourceBytes, + rows: capturedRows, + } + : null; + const serializedBytes = captured + ? Buffer.byteLength(JSON.stringify(captured), "utf8") + : undefined; + requestedPages.push({ + page, + count: response.data.length, + hasMore: response.pagination.hasMore, + identityHash, + contentHash, + ...(captured ? { sourceBytes: pageResponse.sourceBytes, serializedBytes } : {}), + }); + observedRows += response.data.length; + if (observedRows > catalogTotal || observedRows > MAX_PROOF_SOURCE_ROWS) { + throw new Error("skills.sh proof source exceeded its reported total"); + } + if (captured && serializedBytes !== undefined) { + finalNonemptyPage = { + page, + count: response.data.length, + pagination: response.pagination, + }; + sourcePages.push({ + ...captured, + serializedBytes, + }); + } + for (const row of response.data) { + present.add(row.id.trim().toLowerCase()); + for (const key of sortedObjectKeys(row)) rowKeys.add(key); + for (const key of normalizedTaxonomyFields(row)) taxonomyFields.add(key); + if (!sampleRow && row.id.split("/").length === 3) sampleRow = row; + } + if (!response.pagination.hasMore) break; + if ( + observedRows >= catalogTotal || + page + 1 >= Math.ceil(MAX_PROOF_SOURCE_ROWS / MAX_SOURCE_PAGE_SIZE) + ) { + throw new Error("skills.sh proof source pagination exceeds its bounded total"); + } + page += 1; + } + if (catalogTotal === null || catalogTotal < 1) { + throw new Error("skills.sh proof source is empty"); + } + if (observedRows !== catalogTotal) { + throw new Error(`skills.sh proof source observed ${observedRows} of ${catalogTotal} rows`); + } + if (present.size !== observedRows) { + throw new Error( + `skills.sh proof source contains duplicate identities: ${observedRows - present.size}`, + ); + } + if (!firstPage || !finalNonemptyPage || !sampleRow) { + throw new Error("skills.sh proof source lacks a metadata sample"); + } + const beyondEndPageNumber = page + 1; + await pace(); + const beyondEnd = await fetchSkillsShCatalogPage( + { page: beyondEndPageNumber, perPage: MAX_SOURCE_PAGE_SIZE }, + options, + ); + sourceRequests += 1; + requestedPages.push({ + page: beyondEndPageNumber, + count: beyondEnd.data.length, + hasMore: beyondEnd.pagination.hasMore, + identityHash: skillsShPageIdentityHash(beyondEnd.data), + contentHash: skillsShPageContentHash(beyondEnd.data), + }); + if ( + beyondEnd.pagination.page !== beyondEndPageNumber || + beyondEnd.pagination.perPage !== MAX_SOURCE_PAGE_SIZE || + beyondEnd.pagination.total !== catalogTotal || + beyondEnd.pagination.hasMore || + beyondEnd.data.length !== 0 + ) { + throw new Error("skills.sh proof source did not return an empty beyond-end page"); + } + await pace(); + const search = await searchSkillsShCatalog( + { + query: sampleRow.slug, + owner: sampleRow.source.split("/")[0], + limit: 10, + }, + options, + ); + sourceRequests += 1; + await pace(); + const detail = await fetchSkillsShCatalogDetail(sampleRow.id, options); + sourceRequests += 1; + const expectedPagePath = `/${sampleRow.id.trim().toLowerCase()}`; + const pageHtml = await fetchSkillsShProofText(sampleRow.url, { + fetchImpl: options.fetchImpl ?? fetch, + expectedPath: expectedPagePath, + rsc: false, + }); + sourceRequests += 1; + const rscUrl = new URL(sampleRow.url); + rscUrl.searchParams.set("_rsc", "clawhub-proof"); + const rsc = await fetchSkillsShProofText(rscUrl.href, { + fetchImpl: options.fetchImpl ?? fetch, + headers: { + Accept: "text/x-component", + RSC: "1", + }, + expectedPath: expectedPagePath, + rsc: true, + }); + sourceRequests += 1; + const pageFields = parseSkillsShPageFieldEvidence(pageHtml); + const rscFields = parseSkillsShRscFieldEvidence(rsc); + const searchTaxonomyFields = normalizedTaxonomyFields(search); + const detailTaxonomyFields = normalizedTaxonomyFields(detail); + const normalizedUpstreamTaxonomyFields = Array.from( + new Set([ + ...taxonomyFields, + ...searchTaxonomyFields, + ...detailTaxonomyFields, + ...pageFields.taxonomyFields, + ...rscFields.taxonomyFields, + ]), + ).sort(); + return { + catalogTotal, + controlledExternalIds: [...SKILLS_SH_MIRROR_CONTROLLED_EXTERNAL_IDS], + controlledOverlayExternalIds: SKILLS_SH_MIRROR_CONTROLLED_EXTERNAL_IDS.filter((externalId) => + present.has(externalId), + ), + controlledSupplementExternalIds: SKILLS_SH_MIRROR_CONTROLLED_EXTERNAL_IDS.filter( + (externalId) => !present.has(externalId), + ), + pageSize: MAX_SOURCE_PAGE_SIZE, + sourceRequests, + sourcePages, + evidence: { + pagination: { + endpointExhausted: true, + databaseCoverage: "leaderboard-only", + page0: firstPage.pagination, + requestedPages, + finalNonemptyPage, + firstBeyondEndPage: { + page: beyondEndPageNumber, + count: 0, + pagination: beyondEnd.pagination, + }, + uniqueIds: present.size, + duplicateIds: observedRows - present.size, + }, + fields: { + sampledExternalId: sampleRow.id, + leaderboard: { + topLevelKeys: sortedObjectKeys(firstPage), + paginationKeys: sortedObjectKeys(firstPage.pagination), + rowKeys: Array.from(rowKeys).sort(), + taxonomyFields: Array.from(taxonomyFields).sort(), + }, + search: { + topLevelKeys: sortedObjectKeys(search), + rowKeys: sortedArrayObjectKeys(search.data), + taxonomyFields: searchTaxonomyFields, + }, + detail: { + topLevelKeys: sortedObjectKeys(detail), + fileKeys: sortedArrayObjectKeys(detail.files), + taxonomyFields: detailTaxonomyFields, + }, + page: { + url: sampleRow.url, + ...pageFields, + }, + rsc: rscFields, + normalizedUpstreamTaxonomyFields, + }, + } satisfies SkillsShMirrorProofEvidence, + }; +} + export async function searchSkillsShCatalog( args: { query: string; @@ -158,6 +1914,325 @@ export async function fetchSkillsShCatalogDetail( return await fetchSkillsShJson(`/skills/${normalizedId}`, options); } +async function fetchSkillsShMirrorDetail(id: string, options: SkillsShFetchOptions = {}) { + return await fetchSkillsShJsonWithBytes( + `/skills/${normalizeSkillsShId(id)}`, + options, + ); +} + +export async function fetchSkillsShMirrorBatch( + args: { + page: number; + offset: number; + limit: number; + maxDetailBytes: number; + }, + options: { + env?: SkillsShCatalogSourceEnv; + fetchImpl?: typeof fetch; + oidcToken?: string; + minimumApiRequestIntervalMs?: number; + beforeRequest?: () => Promise | void; + githubLocatorResolver?: typeof resolveSkillsShMirrorGitHubLocators | null; + sourcePage?: SkillsShCatalogPage; + } = {}, +) { + assertIntegerInRange("page", args.page, 0, 100_000); + assertIntegerInRange("offset", args.offset, 0, MAX_SOURCE_PAGE_SIZE); + assertIntegerInRange("limit", args.limit, 1, 50); + assertIntegerInRange("maxDetailBytes", args.maxDetailBytes, 1, 64 * 1024); + const baseFetch = options.fetchImpl ?? fetch; + const minimumApiRequestIntervalMs = + options.minimumApiRequestIntervalMs ?? + (options.fetchImpl ? 0 : MINIMUM_API_REQUEST_INTERVAL_MS); + assertIntegerInRange("minimumApiRequestIntervalMs", minimumApiRequestIntervalMs, 0, 1_000); + const paceApiRequest = createRequestPacer(minimumApiRequestIntervalMs); + let sourceRequests = 0; + // Count every list, detail, audit, and identity-page attempt through one wrapper. + const monitoredFetch: typeof fetch = async (input, init) => { + await options.beforeRequest?.(); + sourceRequests += 1; + if (minimumApiRequestIntervalMs > 0 && requestUrl(input).startsWith(`${SKILLS_SH_API_BASE}/`)) { + await paceApiRequest(); + } + return await baseFetch(input, init); + }; + const fetchOptions = { ...options, fetchImpl: monitoredFetch }; + const sourcePageResponse = options.sourcePage + ? { value: options.sourcePage, sourceBytes: 0 } + : await fetchSkillsShCatalogPageWithBytes( + { page: args.page, perPage: MAX_SOURCE_PAGE_SIZE }, + fetchOptions, + ); + const sourcePage = sourcePageResponse.value; + if ( + sourcePage.pagination.page !== args.page || + sourcePage.pagination.perPage !== MAX_SOURCE_PAGE_SIZE || + sourcePage.data.length > MAX_SOURCE_PAGE_SIZE + ) { + throw new Error("skills.sh mirror source returned an invalid page contract"); + } + if (args.offset >= sourcePage.data.length) { + throw new Error("skills.sh mirror offset is outside the source page"); + } + const listRows = sourcePage.data.slice(args.offset, args.offset + args.limit); + const rows = Array.from< + | (ReturnType & { + upstreamScanners: SkillsShMirrorUpstreamScanners; + sourceContentHash?: string; + detail?: { + contentKind: "skill-md" | "readme"; + path: string; + content: string; + contentBytes: number; + sourceBytes: number; + sourceFileCount: number; + truncated: boolean; + }; + }) + | ReturnType + >({ length: listRows.length }); + const fullDetailContentByExternalId = new Map(); + // Count only response-body bytes the mirror consumed; canceled unread bodies contribute zero. + let rowSourceBytes = 0; + let nextIndex = 0; + await Promise.all( + Array.from({ length: Math.min(DETAIL_CONCURRENCY, listRows.length) }, async () => { + while (nextIndex < listRows.length) { + const index = nextIndex; + nextIndex += 1; + const listRow = listRows[index]!; + const identity = await resolveSkillsShMirrorObservation(listRow, fetchOptions); + rowSourceBytes += identity.identitySourceBytes; + if ("quarantined" in identity.row) { + rows[index] = identity.row; + continue; + } + try { + normalizeSkillsShId(identity.row.externalId); + } catch (error) { + rows[index] = safeMirrorIdentityError(listRow, error); + continue; + } + const [detailResponse, auditResponse] = await Promise.all([ + fetchSkillsShMirrorDetail(identity.row.externalId, fetchOptions), + fetchSkillsShMirrorAudit(identity.row.externalId, fetchOptions), + ]); + rowSourceBytes += detailResponse.sourceBytes + (auditResponse?.sourceBytes ?? 0); + const detailPayload = detailResponse.value; + const auditPayload = auditResponse?.value ?? null; + const detail = buildSkillsShMirrorDetail(detailPayload, args.maxDetailBytes); + if (detail.contentKind !== "none") { + const fullDetail = (detailPayload.files ?? []) + .map(mirrorDetailFile) + .find((file) => file?.path === detail.path); + if (fullDetail) { + fullDetailContentByExternalId.set(identity.row.externalId, fullDetail.content); + } + } + rows[index] = { + ...identity.row, + ...(detail.sourceContentHash ? { sourceContentHash: detail.sourceContentHash } : {}), + upstreamScanners: buildSkillsShMirrorUpstreamScanners(auditPayload, listRow.url), + ...(detail.contentKind === "none" + ? {} + : { + detail: { + contentKind: detail.contentKind, + path: detail.path, + content: detail.content, + contentBytes: detail.contentBytes, + sourceBytes: detail.sourceBytes, + sourceFileCount: detail.sourceFileCount, + truncated: detail.truncated, + }, + }), + }; + } + }), + ); + const githubLocatorResolver = + options.githubLocatorResolver === undefined + ? resolveSkillsShMirrorGitHubLocators + : options.githubLocatorResolver; + const located = githubLocatorResolver + ? await githubLocatorResolver(rows, { + fetchImpl: baseFetch, + beforeRequest: options.beforeRequest, + fullDetailContentByExternalId, + }) + : { rows, sourceRequests: 0, sourceBytes: 0 }; + sourceRequests += located.sourceRequests; + rowSourceBytes += located.sourceBytes; + return { + page: args.page, + offset: args.offset, + pageLength: sourcePage.data.length, + sourceTotal: sourcePage.pagination.total, + sourcePageIdentityHash: skillsShPageIdentityHash(sourcePage.data), + hasMore: sourcePage.pagination.hasMore, + sourceRequests, + sourceBytes: sourcePageResponse.sourceBytes + rowSourceBytes, + rows: located.rows, + }; +} + +export async function fetchSkillsShMirrorControlledBatch( + args: { + page: number; + offset: number; + limit: number; + maxDetailBytes: number; + sourceTotal: number; + externalIds: string[]; + }, + options: { + fetchImpl?: typeof fetch; + beforeRequest?: () => Promise | void; + } = {}, +) { + const controlledExternalIds = normalizeControlledSupplementIds(args.externalIds); + if (controlledExternalIds.length < 1) throw new Error("controlled mirror supplement is empty"); + assertIntegerInRange("offset", args.offset, 0, controlledExternalIds.length - 1); + assertIntegerInRange("limit", args.limit, 1, 50); + assertIntegerInRange("maxDetailBytes", args.maxDetailBytes, 1, 64 * 1024); + const supplementsByExternalId = new Map( + SKILLS_SH_MIRROR_CONTROLLED_SUPPLEMENTS.map((row) => [row.externalId, row]), + ); + const selected = controlledExternalIds + .slice(args.offset, args.offset + args.limit) + .map((externalId) => supplementsByExternalId.get(externalId)!); + if (selected.length < 1) throw new Error("controlled mirror supplement is exhausted"); + const fetchImpl = options.fetchImpl ?? fetch; + let sourceBytes = 0; + let sourceRequests = 0; + const rows = await Promise.all( + selected.map(async (supplement) => { + const rawUrl = + `https://raw.githubusercontent.com/${supplement.owner}/${supplement.repo}/` + + `${supplement.githubCommit}/${supplement.detailPath}`; + let response: Response | null = null; + for (let attempt = 0; attempt < MAX_SOURCE_ATTEMPTS; attempt += 1) { + await options.beforeRequest?.(); + sourceRequests += 1; + try { + response = await fetchImpl(rawUrl, { + headers: { Accept: "text/plain", "User-Agent": "clawhub/skills-sh-mirror" }, + }); + } catch { + response = null; + } + if (response?.ok) break; + if (response && response.status !== 429 && response.status < 500) { + await cancelResponseBody(response); + throw new Error( + `controlled skills.sh mirror source returned HTTP ${response.status}: ${supplement.externalId}`, + ); + } + if (response) { + const retryAfterMs = + response.status === 429 ? skillsShRetryAfterMs(response, attempt) : null; + if ( + response.status === 429 && + (attempt === MAX_SOURCE_ATTEMPTS - 1 || + (retryAfterMs !== null && retryAfterMs > MAX_INLINE_RETRY_AFTER_MS)) + ) { + await cancelResponseBody(response); + throw new SkillsShSourceHttpError( + response.status, + retryAfterMs === null ? null : Math.max(1, Math.ceil(retryAfterMs / 1_000)), + ); + } + if (attempt === MAX_SOURCE_ATTEMPTS - 1) { + await cancelResponseBody(response); + break; + } + await cancelResponseBody(response); + await waitForSkillsShRetry(response, attempt); + } else { + if (attempt === MAX_SOURCE_ATTEMPTS - 1) break; + await new Promise((resolve) => setTimeout(resolve, 250 * 2 ** attempt)); + } + } + if (!response?.ok) { + throw new Error( + `controlled skills.sh mirror source fetch failed: ${supplement.externalId}`, + ); + } + const body = await readBoundedResponseBytes(response, MAX_CONTROLLED_DETAIL_BYTES); + sourceBytes += body.sourceBytes; + if (!body.ok || body.bytes === null) { + throw new Error( + `controlled skills.sh mirror source is too large: ${supplement.externalId}`, + ); + } + const fullContent = new TextDecoder().decode(body.bytes); + return buildSkillsShMirrorControlledObservation( + supplement, + fullContent, + body.sourceBytes, + args.maxDetailBytes, + ); + }), + ); + return { + page: args.page, + offset: args.offset, + pageLength: controlledExternalIds.length, + sourceTotal: args.sourceTotal, + hasMore: false, + sourceRequests, + sourceBytes, + rows, + }; +} + +export function buildSkillsShMirrorControlledObservation( + supplement: SkillsShMirrorControlledSupplement, + fullContent: string, + sourceBytes: number, + maxDetailBytes: number, +) { + assertIntegerInRange("sourceBytes", sourceBytes, 0, MAX_CONTROLLED_DETAIL_BYTES); + assertIntegerInRange("maxDetailBytes", maxDetailBytes, 1, 64 * 1024); + const sourceContentHash = sha256Hex(fullContent); + if (sourceContentHash !== supplement.sourceContentHash) { + throw new Error(`controlled skills.sh mirror source hash changed: ${supplement.externalId}`); + } + const content = truncateUtf8(fullContent, maxDetailBytes); + const unavailable = { status: "unavailable" as const }; + return { + externalId: supplement.externalId, + sourceType: "github" as const, + upstreamSourceType: "controlled-github", + owner: supplement.owner, + repo: supplement.repo, + slug: supplement.slug, + displayName: supplement.displayName, + sourceUrl: supplement.sourceUrl, + canonicalRepoUrl: `https://github.com/${supplement.owner}/${supplement.repo}`, + githubPath: supplement.githubPath, + githubCommit: supplement.githubCommit, + sourceContentHash, + upstreamInstalls: 0, + upstreamScanners: { + genAgentTrustHub: unavailable, + socket: unavailable, + snyk: unavailable, + }, + detail: { + contentKind: "skill-md" as const, + path: supplement.detailPath, + content, + contentBytes: Buffer.byteLength(content, "utf8"), + sourceBytes, + sourceFileCount: 1, + truncated: sourceBytes > maxDetailBytes, + }, + }; +} + export function getSkillsShCatalogTestSourcePolicy(env: SkillsShCatalogSourceEnv = process.env) { const rollout = getClawHubRolloutCapabilities(env); if (!rollout.skillsSh.runtimeEnabled) { @@ -174,11 +2249,17 @@ export function getSkillsShCatalogTestSourcePolicy(env: SkillsShCatalogSourceEnv reason: "skills.sh live Test discovery requires the Test build marker", }; } - if (env.VERCEL_ENV !== "preview") { + const vercelEnvironment = env.VERCEL_ENV?.trim().toLowerCase(); + const vercelTargetEnvironment = env.VERCEL_TARGET_ENV?.trim().toLowerCase(); + if ( + rollout.environment !== "test" || + vercelTargetEnvironment !== "test" || + (vercelEnvironment !== "preview" && vercelEnvironment !== "test") + ) { return { allowed: false as const, - environment: env.VERCEL_ENV?.trim() || "unknown", - reason: "skills.sh live Test discovery requires the Vercel Preview runtime", + environment: rollout.environment, + reason: "skills.sh live Test discovery requires the Vercel Test runtime", }; } if (env.VITE_CONVEX_URL !== CLAWHUB_TEST_CONVEX_URL) { diff --git a/server/skillsShMirrorClassification.test.ts b/server/skillsShMirrorClassification.test.ts new file mode 100644 index 00000000..171ee113 --- /dev/null +++ b/server/skillsShMirrorClassification.test.ts @@ -0,0 +1,353 @@ +/* @vitest-environment node */ + +import { describe, expect, it, vi } from "vitest"; +import { CLASSIFIER_VERSION, TOPIC_CLASSIFIER_VERSION } from "../convex/lib/catalogClassifier.mjs"; +import { + buildSkillsShMirrorReplayRows, + enrichSkillsShMirrorClassifications, +} from "./skillsShMirrorClassification"; + +const row = { + externalId: "patrick-erichsen/skills/html", + slug: "html", + displayName: "HTML", + sourceContentHash: "a".repeat(64), + detail: { + content: "# HTML\n\nBuild interactive HTML artifacts and frontend prototypes.", + }, +}; + +describe("skills.sh mirror classification enrichment", () => { + it("classifies bounded mirror detail content with the native inference contract", () => { + const [classified] = enrichSkillsShMirrorClassifications([row], [], 123); + + expect(classified).toMatchObject({ + externalId: row.externalId, + inferredCategories: expect.any(Array), + inferredTopics: expect.any(Array), + inferredCategoryConfidence: expect.stringMatching(/^(high|medium|low)$/), + inferredTopicConfidence: expect.stringMatching(/^(high|medium|low)$/), + inferredClassifierVersion: CLASSIFIER_VERSION, + inferredTopicClassifierVersion: TOPIC_CLASSIFIER_VERSION, + inferredInputHash: expect.any(String), + inferredTopicInputHash: expect.any(String), + inferredAt: 123, + }); + expect(classified.inferredCategories.length).toBeGreaterThan(0); + }); + + it("reuses inference when source identity, content hash, and classifier versions match", () => { + const classify = vi.fn(); + const inference = { + inferredCategories: ["development"], + inferredTopics: ["html"], + inferredCategoryConfidence: "high" as const, + inferredTopicConfidence: "medium" as const, + inferredClassifierVersion: CLASSIFIER_VERSION, + inferredTopicClassifierVersion: TOPIC_CLASSIFIER_VERSION, + inferredInputHash: "input-hash", + inferredTopicInputHash: "topic-input-hash", + inferredAt: 100, + }; + + const [classified] = enrichSkillsShMirrorClassifications( + [row], + [ + { + externalId: row.externalId, + slug: row.slug, + displayName: row.displayName, + sourceContentHash: row.sourceContentHash, + ...inference, + }, + ], + 200, + classify, + ); + + expect(classified).toMatchObject(inference); + expect(classify).not.toHaveBeenCalled(); + }); + + it("reclassifies when the source content hash or classifier version changes", () => { + const classify = vi.fn(() => ({ + categories: [], + topics: [], + confidence: "low" as const, + topicConfidence: "low" as const, + classifierVersion: CLASSIFIER_VERSION, + topicClassifierVersion: TOPIC_CLASSIFIER_VERSION, + inputHash: "new-input", + topicInputHash: "new-topic-input", + })); + const staleState = { + externalId: row.externalId, + slug: row.slug, + displayName: row.displayName, + sourceContentHash: "b".repeat(64), + inferredCategories: ["development"], + inferredTopics: ["html"], + inferredCategoryConfidence: "high" as const, + inferredTopicConfidence: "high" as const, + inferredClassifierVersion: "taxonomy-old", + inferredTopicClassifierVersion: TOPIC_CLASSIFIER_VERSION, + inferredInputHash: "old-input", + inferredTopicInputHash: "old-topic-input", + inferredAt: 100, + }; + + const [classified] = enrichSkillsShMirrorClassifications([row], [staleState], 200, classify); + + expect(classify).toHaveBeenCalledOnce(); + expect(classified).toMatchObject({ + inferredCategories: ["other"], + inferredTopics: [], + inferredClassifierVersion: CLASSIFIER_VERSION, + inferredTopicClassifierVersion: TOPIC_CLASSIFIER_VERSION, + inferredAt: 200, + }); + }); + + it("reclassifies changed detail when a legacy state has no content hash", () => { + const classify = vi.fn(() => ({ + categories: ["development"], + topics: ["html"], + confidence: "high" as const, + topicConfidence: "high" as const, + classifierVersion: CLASSIFIER_VERSION, + topicClassifierVersion: TOPIC_CLASSIFIER_VERSION, + inputHash: "new-input", + topicInputHash: "new-topic-input", + })); + const unhashedRow = { + ...row, + sourceContentHash: undefined, + detail: { content: "# HTML\n\nChanged content." }, + }; + const state = { + externalId: row.externalId, + slug: row.slug, + displayName: row.displayName, + inferredCategories: ["other"], + inferredTopics: [], + inferredCategoryConfidence: "low" as const, + inferredTopicConfidence: "low" as const, + inferredClassifierVersion: CLASSIFIER_VERSION, + inferredTopicClassifierVersion: TOPIC_CLASSIFIER_VERSION, + inferredInputHash: "old-input", + inferredTopicInputHash: "old-topic-input", + inferredAt: 100, + }; + + const [classified] = enrichSkillsShMirrorClassifications([unhashedRow], [state], 200, classify); + + expect(classify).toHaveBeenCalledOnce(); + expect(classified).toMatchObject({ + inferredCategories: ["development"], + inferredAt: 200, + }); + }); + + it("reclassifies from the display-name stub when retained detail disappears", () => { + const [previous] = enrichSkillsShMirrorClassifications([row], [], 100); + const classify = vi.fn(() => ({ + categories: [], + topics: [], + confidence: "low" as const, + topicConfidence: "low" as const, + classifierVersion: CLASSIFIER_VERSION, + topicClassifierVersion: TOPIC_CLASSIFIER_VERSION, + inputHash: "stub-input", + topicInputHash: "stub-topic-input", + })); + const withoutDetail = { + externalId: row.externalId, + slug: row.slug, + displayName: row.displayName, + }; + + const [classified] = enrichSkillsShMirrorClassifications( + [withoutDetail], + [previous], + 200, + classify, + ); + + expect(classify).toHaveBeenCalledWith({ + slug: "html", + text: "---\nname: HTML\n---\n# HTML", + }); + expect(classified).toMatchObject({ + inferredCategories: ["other"], + inferredInputHash: "stub-input", + inferredAt: 200, + }); + }); + + it("reuses classification when the same no-detail stub is observed again", () => { + const withoutDetail = { + externalId: row.externalId, + slug: row.slug, + displayName: row.displayName, + }; + const [previous] = enrichSkillsShMirrorClassifications([withoutDetail], [], 100); + const classify = vi.fn(); + + const [classified] = enrichSkillsShMirrorClassifications( + [withoutDetail], + [previous], + 200, + classify, + ); + + expect(classify).not.toHaveBeenCalled(); + expect(classified.inferredAt).toBe(100); + }); + + it("rebuilds bounded rows from the captured digest and detail snapshot", () => { + const [replayed] = buildSkillsShMirrorReplayRows( + [ + { + digest: { + ...row, + sourceType: "github", + upstreamSourceType: "github", + owner: "patrick-erichsen", + repo: "skills", + sourceUrl: "https://skills.sh/patrick-erichsen/skills/html", + canonicalRepoUrl: "https://github.com/patrick-erichsen/skills", + upstreamInstalls: 42, + upstreamScanners: { + genAgentTrustHub: { status: "unavailable" }, + socket: { status: "unavailable" }, + snyk: { status: "unavailable" }, + }, + }, + detail: { + contentKind: "skill-md", + path: "skills/html/SKILL.md", + content: row.detail.content, + contentBytes: Buffer.byteLength(row.detail.content), + sourceBytes: Buffer.byteLength(row.detail.content), + sourceFileCount: 1, + truncated: false, + }, + }, + ], + 456, + ); + + expect(replayed).toMatchObject({ + externalId: row.externalId, + owner: "patrick-erichsen", + repo: "skills", + detail: { + path: "skills/html/SKILL.md", + content: row.detail.content, + }, + inferredCategories: expect.any(Array), + inferredAt: 456, + }); + }); + + it("preserves stale replay rows as quarantine observations", () => { + expect( + buildSkillsShMirrorReplayRows([ + { + quarantined: true, + externalId: "larksuite/cli/lark-doc", + upstreamSourceType: "well-known", + reason: "identity-page-fetch-failed", + }, + ] as never), + ).toEqual([ + { + quarantined: true, + externalId: "larksuite/cli/lark-doc", + upstreamSourceType: "well-known", + reason: "identity-page-fetch-failed", + }, + ]); + }); + + it("synthesizes the same bounded content hash while replaying legacy detail", () => { + const [replayed] = buildSkillsShMirrorReplayRows( + [ + { + digest: { + externalId: row.externalId, + sourceType: "github", + upstreamSourceType: "github", + owner: "patrick-erichsen", + repo: "skills", + slug: row.slug, + displayName: row.displayName, + sourceUrl: "https://skills.sh/patrick-erichsen/skills/html", + canonicalRepoUrl: "https://github.com/patrick-erichsen/skills", + upstreamInstalls: 42, + upstreamScanners: { + genAgentTrustHub: { status: "unavailable" }, + socket: { status: "unavailable" }, + snyk: { status: "unavailable" }, + }, + }, + detail: { + contentKind: "skill-md", + path: "SKILL.md", + content: "abc", + contentBytes: 3, + sourceBytes: 3, + sourceFileCount: 1, + truncated: false, + }, + }, + ], + 456, + ); + + if ("quarantined" in replayed) { + throw new Error("legacy detail replay was quarantined"); + } + expect(replayed.sourceContentHash).toBe( + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + ); + }); + + it("does not synthesize a full-content hash from truncated legacy detail", () => { + const [replayed] = buildSkillsShMirrorReplayRows([ + { + digest: { + externalId: row.externalId, + sourceType: "github", + upstreamSourceType: "github", + owner: "patrick-erichsen", + repo: "skills", + slug: row.slug, + displayName: row.displayName, + sourceUrl: "https://skills.sh/patrick-erichsen/skills/html", + canonicalRepoUrl: "https://github.com/patrick-erichsen/skills", + upstreamInstalls: 42, + upstreamScanners: { + genAgentTrustHub: { status: "unavailable" }, + socket: { status: "unavailable" }, + snyk: { status: "unavailable" }, + }, + }, + detail: { + contentKind: "skill-md", + path: "SKILL.md", + content: "bounded prefix", + contentBytes: 14, + sourceBytes: 128_000, + sourceFileCount: 1, + truncated: true, + }, + }, + ]); + + if ("quarantined" in replayed) { + throw new Error("legacy detail replay was quarantined"); + } + expect(replayed.sourceContentHash).toBeUndefined(); + }); +}); diff --git a/server/skillsShMirrorClassification.ts b/server/skillsShMirrorClassification.ts new file mode 100644 index 00000000..88226fa1 --- /dev/null +++ b/server/skillsShMirrorClassification.ts @@ -0,0 +1,264 @@ +import { createHash } from "node:crypto"; +import { + CLASSIFIER_VERSION, + TOPIC_CLASSIFIER_VERSION, + classifySkill, +} from "../convex/lib/catalogClassifier.mjs"; + +const MAX_CLASSIFICATION_TEXT_LENGTH = 40_000; + +type ClassificationConfidence = "high" | "medium" | "low"; + +type MirrorClassification = { + inferredCategories: string[]; + inferredTopics: string[]; + inferredCategoryConfidence: ClassificationConfidence; + inferredTopicConfidence: ClassificationConfidence; + inferredClassifierVersion: string; + inferredTopicClassifierVersion: string; + inferredInputHash: string; + inferredTopicInputHash: string; + inferredAt: number; +}; + +export type SkillsShMirrorClassificationState = MirrorClassification & { + externalId: string; + slug: string; + displayName: string; + sourceContentHash?: string; +}; + +type ClassifiableMirrorRow = { + quarantined?: never; + externalId: string; + slug: string; + displayName: string; + sourceContentHash?: string; + detail?: { content: string }; +}; + +type QuarantinedMirrorRow = { + quarantined: true; + externalId: string; +}; + +type MirrorReplayPair = { + digest: { + externalId: string; + sourceType: "github" | "well-known"; + upstreamSourceType?: string; + owner?: string; + repo?: string; + sourceHost?: string; + slug: string; + displayName: string; + sourceUrl: string; + canonicalRepoUrl?: string; + githubPath?: string; + githubCommit?: string; + sourceContentHash?: string; + upstreamInstalls: number; + upstreamScanners: { + genAgentTrustHub: { status: string; sourceCheckedAt?: string; sourceUrl?: string }; + socket: { status: string; sourceCheckedAt?: string; sourceUrl?: string }; + snyk: { status: string; sourceCheckedAt?: string; sourceUrl?: string }; + }; + inferredCategories?: string[]; + inferredTopics?: string[]; + inferredCategoryConfidence?: ClassificationConfidence; + inferredTopicConfidence?: ClassificationConfidence; + inferredClassifierVersion?: string; + inferredTopicClassifierVersion?: string; + inferredInputHash?: string; + inferredTopicInputHash?: string; + inferredAt?: number; + }; + detail: { + contentKind: "skill-md" | "readme"; + path: string; + content: string; + contentBytes: number; + sourceBytes: number; + sourceFileCount: number; + truncated: boolean; + } | null; +}; + +type MirrorReplayQuarantine = { + quarantined: true; + externalId: string; + upstreamSourceType: string; + reason: string; +}; + +type ClassifierOutput = { + categories: string[]; + topics: string[]; + confidence: ClassificationConfidence; + topicConfidence: ClassificationConfidence; + classifierVersion: string; + topicClassifierVersion: string; + inputHash: string; + topicInputHash: string; +}; + +type EnrichedMirrorRow = T extends QuarantinedMirrorRow ? T : T & MirrorClassification; + +function hasReusableClassification( + row: ClassifiableMirrorRow, + state: SkillsShMirrorClassificationState | undefined, +): state is SkillsShMirrorClassificationState { + return ( + state !== undefined && + state.slug === row.slug && + state.displayName === row.displayName && + (row.detail === undefined + ? state.inferredInputHash === classificationInputHash(row) + : row.sourceContentHash !== undefined && state.sourceContentHash === row.sourceContentHash) && + state.inferredClassifierVersion === CLASSIFIER_VERSION && + state.inferredTopicClassifierVersion === TOPIC_CLASSIFIER_VERSION + ); +} + +function boundedContentHash(content: string) { + return createHash("sha256").update(content).digest("hex"); +} + +function classificationText(row: ClassifiableMirrorRow) { + const displayName = row.displayName.replace(/[\r\n]+/g, " ").trim() || row.slug; + const content = row.detail?.content ?? `# ${displayName}`; + return `---\nname: ${displayName}\n---\n${content}`.slice(0, MAX_CLASSIFICATION_TEXT_LENGTH); +} + +function classificationInputHash(row: ClassifiableMirrorRow) { + return createHash("sha256") + .update(`${row.slug}\0${classificationText(row)}\0${JSON.stringify([])}`) + .digest("hex"); +} + +export function enrichSkillsShMirrorClassifications< + T extends ClassifiableMirrorRow | QuarantinedMirrorRow, +>( + rows: T[], + states: SkillsShMirrorClassificationState[], + inferredAt = Date.now(), + classify: (input: { slug?: string; text?: string }) => ClassifierOutput = classifySkill, +): Array> { + const statesByExternalId = new Map(states.map((state) => [state.externalId, state])); + return rows.map((row) => { + if ("quarantined" in row) return row; + const state = statesByExternalId.get(row.externalId); + if (hasReusableClassification(row, state)) { + return { + ...row, + inferredCategories: state.inferredCategories, + inferredTopics: state.inferredTopics, + inferredCategoryConfidence: state.inferredCategoryConfidence, + inferredTopicConfidence: state.inferredTopicConfidence, + inferredClassifierVersion: state.inferredClassifierVersion, + inferredTopicClassifierVersion: state.inferredTopicClassifierVersion, + inferredInputHash: state.inferredInputHash, + inferredTopicInputHash: state.inferredTopicInputHash, + inferredAt: state.inferredAt, + }; + } + const result = classify({ + slug: row.slug, + text: classificationText(row), + }); + return { + ...row, + inferredCategories: result.categories.length > 0 ? result.categories : ["other"], + inferredTopics: result.topics, + inferredCategoryConfidence: result.confidence, + inferredTopicConfidence: result.topicConfidence, + inferredClassifierVersion: result.classifierVersion, + inferredTopicClassifierVersion: result.topicClassifierVersion, + inferredInputHash: result.inputHash, + inferredTopicInputHash: result.topicInputHash, + inferredAt, + }; + }) as Array>; +} + +function replayClassificationState( + digest: MirrorReplayPair["digest"], +): SkillsShMirrorClassificationState | null { + if ( + !digest.inferredCategories || + !digest.inferredTopics || + !digest.inferredCategoryConfidence || + !digest.inferredTopicConfidence || + !digest.inferredClassifierVersion || + !digest.inferredTopicClassifierVersion || + !digest.inferredInputHash || + !digest.inferredTopicInputHash || + digest.inferredAt === undefined + ) { + return null; + } + return { + externalId: digest.externalId, + slug: digest.slug, + displayName: digest.displayName, + ...(digest.sourceContentHash ? { sourceContentHash: digest.sourceContentHash } : {}), + inferredCategories: digest.inferredCategories, + inferredTopics: digest.inferredTopics, + inferredCategoryConfidence: digest.inferredCategoryConfidence, + inferredTopicConfidence: digest.inferredTopicConfidence, + inferredClassifierVersion: digest.inferredClassifierVersion, + inferredTopicClassifierVersion: digest.inferredTopicClassifierVersion, + inferredInputHash: digest.inferredInputHash, + inferredTopicInputHash: digest.inferredTopicInputHash, + inferredAt: digest.inferredAt, + }; +} + +export function buildSkillsShMirrorReplayRows( + inputs: Array, + inferredAt = Date.now(), +) { + const pairs = inputs.filter((input): input is MirrorReplayPair => !("quarantined" in input)); + const states = pairs.flatMap((pair) => { + const state = replayClassificationState(pair.digest); + return state ? [state] : []; + }); + const rows = inputs.map((input) => { + if ("quarantined" in input) return input; + const { digest, detail } = input; + const sourceContentHash = + digest.sourceContentHash ?? + (detail && !detail.truncated ? boundedContentHash(detail.content) : undefined); + return { + externalId: digest.externalId, + sourceType: digest.sourceType, + upstreamSourceType: digest.upstreamSourceType ?? digest.sourceType, + ...(digest.owner ? { owner: digest.owner } : {}), + ...(digest.repo ? { repo: digest.repo } : {}), + ...(digest.sourceHost ? { sourceHost: digest.sourceHost } : {}), + slug: digest.slug, + displayName: digest.displayName, + sourceUrl: digest.sourceUrl, + ...(digest.canonicalRepoUrl ? { canonicalRepoUrl: digest.canonicalRepoUrl } : {}), + ...(digest.githubPath ? { githubPath: digest.githubPath } : {}), + ...(digest.githubCommit ? { githubCommit: digest.githubCommit } : {}), + ...(sourceContentHash ? { sourceContentHash } : {}), + upstreamInstalls: digest.upstreamInstalls, + upstreamScanners: digest.upstreamScanners, + ...(detail + ? { + detail: { + contentKind: detail.contentKind, + path: detail.path, + content: detail.content, + contentBytes: detail.contentBytes, + sourceBytes: detail.sourceBytes, + sourceFileCount: detail.sourceFileCount, + truncated: detail.truncated, + }, + } + : {}), + }; + }); + return enrichSkillsShMirrorClassifications(rows, states, inferredAt); +} diff --git a/server/skillsShMirrorTestRoute.test.ts b/server/skillsShMirrorTestRoute.test.ts new file mode 100644 index 00000000..6577c53e --- /dev/null +++ b/server/skillsShMirrorTestRoute.test.ts @@ -0,0 +1,1258 @@ +/* @vitest-environment node */ + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const getHeaderMock = vi.fn(); +const getVercelOidcTokenMock = vi.fn(); +const readBodyMock = vi.fn(); +const fetchPageMock = vi.fn(); +const fetchBatchMock = vi.fn(); +const fetchControlledBatchMock = vi.fn(); +const measureProofSourceMock = vi.fn(); +const buildProofSnapshotIdMock = vi.fn(); +const parseProofSnapshotIdMock = vi.fn(); +const sourcePolicyMock = vi.fn(); +const sourceRetryAfterMock = vi.fn(); +const enrichClassificationsMock = vi.fn(); +const buildReplayRowsMock = vi.fn(); + +function capturedSourcePage( + page = 3, + pageLength = 500, + hasMore = true, + identityHash = `page-${page}`, +) { + return { + page, + sourceTotal: 9_571, + pageLength, + hasMore, + identityHash, + contentHash: `content-${page}`, + rows: [ + { + id: "vercel-labs/skills/find-skills", + installUrl: "https://github.com/vercel-labs/skills", + installs: 42, + name: "Find Skills", + slug: "find-skills", + source: "vercel-labs/skills", + sourceType: "github", + url: "https://skills.sh/vercel-labs/skills/find-skills", + }, + ], + }; +} + +vi.mock("h3", () => ({ + defineEventHandler: (handler: unknown) => handler, + getHeader: (...args: unknown[]) => getHeaderMock(...args), + readBody: (...args: unknown[]) => readBodyMock(...args), +})); + +vi.mock("@vercel/oidc", () => ({ + getVercelOidcToken: (...args: unknown[]) => getVercelOidcTokenMock(...args), +})); + +vi.mock("./skillsShCatalogSource", () => ({ + buildSkillsShMirrorProofSnapshotId: (...args: unknown[]) => buildProofSnapshotIdMock(...args), + fetchSkillsShCatalogPage: (...args: unknown[]) => fetchPageMock(...args), + fetchSkillsShMirrorBatch: (...args: unknown[]) => fetchBatchMock(...args), + fetchSkillsShMirrorControlledBatch: (...args: unknown[]) => fetchControlledBatchMock(...args), + getSkillsShCatalogTestSourcePolicy: (...args: unknown[]) => sourcePolicyMock(...args), + measureSkillsShMirrorProofSource: (...args: unknown[]) => measureProofSourceMock(...args), + parseSkillsShMirrorProofSnapshotId: (...args: unknown[]) => parseProofSnapshotIdMock(...args), + skillsShSourceRetryAfterSeconds: (...args: unknown[]) => sourceRetryAfterMock(...args), +})); + +vi.mock("./skillsShMirrorClassification", () => ({ + buildSkillsShMirrorReplayRows: (...args: unknown[]) => buildReplayRowsMock(...args), + enrichSkillsShMirrorClassifications: (...args: unknown[]) => enrichClassificationsMock(...args), +})); + +describe("skills.sh permanent Test mirror route", () => { + beforeEach(() => { + getHeaderMock.mockReset(); + getVercelOidcTokenMock.mockReset(); + readBodyMock.mockReset(); + fetchPageMock.mockReset(); + fetchBatchMock.mockReset(); + fetchControlledBatchMock.mockReset(); + measureProofSourceMock.mockReset(); + buildProofSnapshotIdMock.mockReset(); + parseProofSnapshotIdMock.mockReset(); + sourcePolicyMock.mockReset(); + sourceRetryAfterMock.mockReset(); + enrichClassificationsMock.mockReset(); + enrichClassificationsMock.mockImplementation((rows) => rows); + buildReplayRowsMock.mockReset(); + sourceRetryAfterMock.mockReturnValue(null); + sourcePolicyMock.mockReturnValue({ allowed: true, environment: "test" }); + getHeaderMock.mockReturnValue("Bearer operator-token"); + getVercelOidcTokenMock.mockResolvedValue("request-oidc-token"); + buildProofSnapshotIdMock.mockReturnValue("skills-sh:proof:snapshot"); + parseProofSnapshotIdMock.mockReturnValue({ + catalogTotal: 9_571, + controlledExternalIds: ["patrick-erichsen/skills/html", "steipete/clawdis/discrawl"], + controlledOverlayExternalIds: [], + controlledSupplementExternalIds: [ + "patrick-erichsen/skills/html", + "steipete/clawdis/discrawl", + ], + sourceSnapshotHash: "a".repeat(64), + evidence: { + pagination: { + requestedPages: [ + { + page: 3, + count: 500, + hasMore: true, + identityHash: "page-3", + contentHash: "content-3", + }, + { + page: 19, + count: 71, + hasMore: false, + identityHash: "page-19", + contentHash: "content-19", + }, + ], + }, + }, + }); + }); + + afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); + }); + + it("authenticates the operator before measuring the live source", async () => { + readBodyMock.mockResolvedValue({ operation: "start", reason: "CLAW-563 proof" }); + const convexFetch = vi.fn(async (_url: string, init: RequestInit) => { + const body = JSON.parse(String(init.body)); + expect(body).toEqual({ operation: "mirror-status" }); + return new Response(JSON.stringify({ error: "unauthorized" }), { status: 401 }); + }); + vi.stubGlobal("fetch", convexFetch); + + const handler = (await import("./routes/ops/skills-sh/mirror-test.post")).default; + const response = (await handler({} as never)) as Response; + + expect(response.status).toBe(502); + expect(await response.json()).toMatchObject({ + message: expect.stringContaining("HTTP 401"), + }); + expect(convexFetch).toHaveBeenCalledTimes(1); + expect(getVercelOidcTokenMock).not.toHaveBeenCalled(); + expect(measureProofSourceMock).not.toHaveBeenCalled(); + }); + + it.each(["running", "paused", "reconciling"])( + "rejects a %s run before measuring the live source", + async (status) => { + readBodyMock.mockResolvedValue({ operation: "start", reason: "CLAW-563 proof" }); + const convexFetch = vi.fn(async (_url: string, init: RequestInit) => { + const body = JSON.parse(String(init.body)); + expect(body).toEqual({ operation: "mirror-status" }); + return new Response( + JSON.stringify({ + control: { enabled: true }, + runs: [{ runId: "skillsShMirrorRuns:active", status }], + }), + ); + }); + vi.stubGlobal("fetch", convexFetch); + + const handler = (await import("./routes/ops/skills-sh/mirror-test.post")).default; + const response = (await handler({} as never)) as Response; + + expect(response.status).toBe(502); + expect(await response.json()).toMatchObject({ + message: expect.stringContaining("already has an active run"), + }); + expect(convexFetch).toHaveBeenCalledTimes(1); + expect(getVercelOidcTokenMock).not.toHaveBeenCalled(); + expect(measureProofSourceMock).not.toHaveBeenCalled(); + }, + ); + + it("starts from a freshly measured authenticated source total", async () => { + readBodyMock.mockResolvedValue({ operation: "start", reason: "CLAW-563 proof" }); + measureProofSourceMock.mockResolvedValue({ + catalogTotal: 9_571, + controlledExternalIds: ["patrick-erichsen/skills/html", "steipete/clawdis/discrawl"], + controlledOverlayExternalIds: ["patrick-erichsen/skills/html"], + controlledSupplementExternalIds: ["steipete/clawdis/discrawl"], + pageSize: 500, + sourceRequests: 20, + sourcePages: [ + { + ...capturedSourcePage(0, 500, true, "page-0"), + sourceBytes: 10_000, + serializedBytes: 10_200, + }, + ], + }); + const convexFetch = vi.fn(async (_url: string, init: RequestInit) => { + const body = JSON.parse(String(init.body)); + if (body.operation === "mirror-status") { + return new Response(JSON.stringify({ control: { enabled: true }, runs: [] })); + } + if (body.operation === "mirror-source-page-store") { + return new Response(JSON.stringify({ stored: true, page: body.page, rows: 500 })); + } + if (body.operation === "mirror-source-summary") { + return new Response( + JSON.stringify({ + snapshotHash: "a".repeat(64), + pageDocuments: 1, + rows: 500, + sourceBytes: 10_000, + serializedBytes: 10_200, + }), + ); + } + if (body.operation === "mirror-run") { + return new Response(JSON.stringify({ status: "running", page: 3, offset: 50 })); + } + expect(body).toMatchObject({ + operation: "mirror-start", + snapshotId: "skills-sh:proof:snapshot", + sourceSnapshotHash: "a".repeat(64), + sourceCaptureWrites: 1, + sourceTotal: 9_572, + sourcePageSize: 500, + reason: "CLAW-563 proof", + }); + return new Response( + JSON.stringify({ + runId: "skillsShMirrorRuns:test", + status: "running", + page: 0, + offset: 0, + }), + ); + }); + vi.stubGlobal("fetch", convexFetch); + + const handler = (await import("./routes/ops/skills-sh/mirror-test.post")).default; + const response = (await handler({} as never)) as Response; + + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ + runId: "skillsShMirrorRuns:test", + status: "running", + page: 0, + offset: 0, + sourceTotal: 9_572, + sourceCatalogTotal: 9_571, + controlledOverlayTotal: 1, + controlledSupplementTotal: 1, + sourceMeasurementRequests: 20, + sourceCapture: { + snapshotHash: "a".repeat(64), + pageDocuments: 1, + rows: 500, + sourceBytes: 10_000, + serializedBytes: 10_200, + requestDbWrites: 1, + }, + }); + expect(measureProofSourceMock).toHaveBeenCalledWith( + expect.objectContaining({ oidcToken: "request-oidc-token" }), + ); + }); + + it("records zero capture writes when immutable source pages are reused", async () => { + readBodyMock.mockResolvedValue({ operation: "start", reason: "CLAW-563 recovered proof" }); + measureProofSourceMock.mockResolvedValue({ + catalogTotal: 9_571, + controlledExternalIds: ["patrick-erichsen/skills/html", "steipete/clawdis/discrawl"], + controlledOverlayExternalIds: [], + controlledSupplementExternalIds: [ + "patrick-erichsen/skills/html", + "steipete/clawdis/discrawl", + ], + pageSize: 500, + sourceRequests: 20, + sourcePages: [ + { + ...capturedSourcePage(0, 500, true, "page-0"), + sourceBytes: 10_000, + serializedBytes: 10_200, + }, + ], + }); + const convexFetch = vi.fn(async (_url: string, init: RequestInit) => { + const body = JSON.parse(String(init.body)); + if (body.operation === "mirror-status") { + return new Response(JSON.stringify({ control: { enabled: true }, runs: [] })); + } + if (body.operation === "mirror-source-page-store") { + return new Response(JSON.stringify({ stored: false, page: body.page, rows: 500 })); + } + if (body.operation === "mirror-source-summary") { + return new Response( + JSON.stringify({ + snapshotHash: "a".repeat(64), + pageDocuments: 1, + rows: 500, + sourceBytes: 10_000, + serializedBytes: 10_200, + }), + ); + } + expect(body).toMatchObject({ + operation: "mirror-start", + sourceCaptureWrites: 0, + }); + return new Response( + JSON.stringify({ + runId: "skillsShMirrorRuns:recovered", + status: "running", + page: 0, + offset: 0, + }), + ); + }); + vi.stubGlobal("fetch", convexFetch); + + const handler = (await import("./routes/ops/skills-sh/mirror-test.post")).default; + const response = (await handler({} as never)) as Response; + + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ + runId: "skillsShMirrorRuns:recovered", + sourceCapture: { + pageDocuments: 1, + requestDbWrites: 0, + }, + }); + }); + + it("passes through a completed reconciliation run summary", async () => { + readBodyMock.mockResolvedValue({ + operation: "reconcile", + runId: "skillsShMirrorRuns:test", + limit: 250, + }); + const convexFetch = vi.fn(async (_url: string, init: RequestInit) => { + expect(JSON.parse(String(init.body))).toEqual({ + operation: "mirror-reconcile", + runId: "skillsShMirrorRuns:test", + limit: 250, + }); + return new Response( + JSON.stringify({ + runId: "skillsShMirrorRuns:test", + status: "completed", + page: 20, + offset: 0, + }), + ); + }); + vi.stubGlobal("fetch", convexFetch); + + const handler = (await import("./routes/ops/skills-sh/mirror-test.post")).default; + const response = (await handler({} as never)) as Response; + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ + runId: "skillsShMirrorRuns:test", + status: "completed", + page: 20, + offset: 0, + }); + }); + + it("reads bounded conflicts for the requested live run", async () => { + readBodyMock.mockResolvedValue({ + operation: "conflicts", + runId: "skillsShMirrorRuns:live", + limit: 50, + }); + const convexFetch = vi.fn(async (_url: string, init: RequestInit) => { + expect(JSON.parse(String(init.body))).toEqual({ + operation: "mirror-conflicts", + runId: "skillsShMirrorRuns:live", + limit: 50, + }); + return new Response( + JSON.stringify({ + conflicts: [ + { + runId: "skillsShMirrorRuns:live", + externalId: "larksuite/cli/lark-doc", + kind: "source-quarantine", + }, + ], + }), + ); + }); + vi.stubGlobal("fetch", convexFetch); + + const handler = (await import("./routes/ops/skills-sh/mirror-test.post")).default; + const response = (await handler({} as never)) as Response; + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ + conflicts: [ + { + runId: "skillsShMirrorRuns:live", + externalId: "larksuite/cli/lark-doc", + kind: "source-quarantine", + }, + ], + }); + }); + + it("reads one exact mirror run for durable recovery", async () => { + readBodyMock.mockResolvedValue({ + operation: "run", + runId: "skillsShMirrorRuns:live", + }); + const convexFetch = vi.fn(async (_url: string, init: RequestInit) => { + expect(JSON.parse(String(init.body))).toEqual({ + operation: "mirror-run", + runId: "skillsShMirrorRuns:live", + }); + return new Response( + JSON.stringify({ + runId: "skillsShMirrorRuns:live", + snapshotId: "skills-sh:2026-07-22T21:18:13.365Z:9571", + status: "completed", + }), + ); + }); + vi.stubGlobal("fetch", convexFetch); + + const handler = (await import("./routes/ops/skills-sh/mirror-test.post")).default; + const response = (await handler({} as never)) as Response; + + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ + runId: "skillsShMirrorRuns:live", + status: "completed", + }); + }); + + it("discards one stale captured recovery before a fresh authenticated run", async () => { + readBodyMock.mockResolvedValue({ + operation: "discard", + runId: "skillsShMirrorRuns:stale", + reason: "discard stale captured recovery", + }); + const convexFetch = vi.fn(async (_url: string, init: RequestInit) => { + expect(JSON.parse(String(init.body))).toEqual({ + operation: "mirror-cancel", + runId: "skillsShMirrorRuns:stale", + reason: "discard stale captured recovery", + confirm: "cancel-skills-sh-mirror-test-run", + }); + return new Response( + JSON.stringify({ + runId: "skillsShMirrorRuns:stale", + status: "canceled", + }), + ); + }); + vi.stubGlobal("fetch", convexFetch); + + const handler = (await import("./routes/ops/skills-sh/mirror-test.post")).default; + const response = (await handler({} as never)) as Response; + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ + runId: "skillsShMirrorRuns:stale", + status: "canceled", + }); + }); + + it("fetches and commits one exact page-offset batch", async () => { + readBodyMock.mockResolvedValue({ + operation: "step", + runId: "skillsShMirrorRuns:test", + page: 3, + offset: 50, + }); + fetchBatchMock.mockResolvedValue({ + page: 3, + offset: 50, + pageLength: 500, + sourceTotal: 9_571, + hasMore: true, + sourceRequests: 101, + sourceBytes: 123_456, + sourcePageIdentityHash: "page-3", + rows: [{ externalId: "vercel-labs/skills/find-skills" }], + }); + const convexFetch = vi.fn(async (_url: string, init: RequestInit) => { + const body = JSON.parse(String(init.body)); + if (body.operation === "mirror-batch-claim") { + expect(body).toMatchObject({ + operation: "mirror-batch-claim", + runId: "skillsShMirrorRuns:test", + page: 3, + offset: 50, + leaseToken: expect.any(String), + }); + return new Response( + JSON.stringify({ + ...body, + snapshotId: "skills-sh:proof:snapshot", + sourcePageSize: 500, + sourceTotal: 9_573, + sourcePage: capturedSourcePage(), + }), + ); + } + if (body.operation === "mirror-classification-states") { + expect(body).toEqual({ + operation: "mirror-classification-states", + externalIds: ["vercel-labs/skills/find-skills"], + }); + return new Response(JSON.stringify({ states: [] })); + } + expect(body).toMatchObject({ + operation: "mirror-batch", + runId: "skillsShMirrorRuns:test", + page: 3, + offset: 50, + leaseToken: expect.any(String), + sourceRequests: 101, + }); + return new Response(JSON.stringify({ status: "running", page: 3, offset: 100 })); + }); + vi.stubGlobal("fetch", convexFetch); + + const handler = (await import("./routes/ops/skills-sh/mirror-test.post")).default; + const response = (await handler({} as never)) as Response; + + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ status: "running", page: 3, offset: 100 }); + expect(fetchBatchMock).toHaveBeenCalledWith( + { page: 3, offset: 50, limit: 50, maxDetailBytes: 65_536 }, + expect.objectContaining({ oidcToken: "request-oidc-token" }), + ); + expect(convexFetch).toHaveBeenCalledTimes(3); + const operatorBodies = convexFetch.mock.calls.map(([, init]) => JSON.parse(String(init?.body))); + expect(operatorBodies[0].leaseToken).toBe(operatorBodies[2].leaseToken); + expect(enrichClassificationsMock).toHaveBeenCalledWith( + [{ externalId: "vercel-labs/skills/find-skills" }], + [], + ); + }); + + it("replaces a live controlled row with its pinned Test observation", async () => { + parseProofSnapshotIdMock.mockReturnValue({ + catalogTotal: 9_571, + controlledExternalIds: ["patrick-erichsen/skills/html", "steipete/clawdis/discrawl"], + controlledOverlayExternalIds: ["patrick-erichsen/skills/html"], + controlledSupplementExternalIds: ["steipete/clawdis/discrawl"], + evidence: { + pagination: { + requestedPages: [ + { + page: 3, + count: 500, + hasMore: true, + identityHash: "page-3", + contentHash: "content-3", + }, + ], + }, + }, + }); + readBodyMock.mockResolvedValue({ + operation: "step", + runId: "skillsShMirrorRuns:test", + page: 3, + offset: 50, + }); + fetchBatchMock.mockResolvedValue({ + page: 3, + offset: 50, + pageLength: 500, + sourceTotal: 9_571, + hasMore: true, + sourceRequests: 5, + sourceBytes: 5_000, + sourcePageIdentityHash: "page-3", + rows: [ + { + externalId: "patrick-erichsen/skills/html", + sourceContentHash: "mutable", + upstreamSourceType: "github", + upstreamInstalls: 123, + upstreamScanners: { + genAgentTrustHub: { status: "pass" }, + socket: { status: "warn" }, + snyk: { status: "unavailable" }, + }, + }, + { externalId: "vercel-labs/skills/find-skills" }, + ], + }); + fetchControlledBatchMock.mockResolvedValue({ + page: 3, + offset: 50, + pageLength: 1, + sourceTotal: 9_572, + hasMore: false, + sourceRequests: 1, + sourceBytes: 1_000, + rows: [ + { + externalId: "patrick-erichsen/skills/html", + sourceContentHash: "pinned", + upstreamSourceType: "controlled-github", + upstreamInstalls: 0, + upstreamScanners: { + genAgentTrustHub: { status: "unavailable" }, + socket: { status: "unavailable" }, + snyk: { status: "unavailable" }, + }, + }, + ], + }); + const convexFetch = vi.fn(async (_url: string, init: RequestInit) => { + const body = JSON.parse(String(init.body)); + if (body.operation === "mirror-batch-claim") { + return new Response( + JSON.stringify({ + ...body, + snapshotId: "skills-sh:proof:snapshot", + sourcePageSize: 500, + sourceTotal: 9_572, + sourcePage: capturedSourcePage(), + }), + ); + } + if (body.operation === "mirror-classification-states") { + expect(body.externalIds).toEqual([ + "patrick-erichsen/skills/html", + "vercel-labs/skills/find-skills", + ]); + return new Response(JSON.stringify({ states: [] })); + } + expect(body).toMatchObject({ + operation: "mirror-batch", + sourceTotal: 9_572, + sourceRequests: 6, + sourceBytes: 6_000, + rows: [ + { + externalId: "patrick-erichsen/skills/html", + sourceContentHash: "pinned", + upstreamSourceType: "github", + upstreamInstalls: 123, + upstreamScanners: { + genAgentTrustHub: { status: "pass" }, + socket: { status: "warn" }, + snyk: { status: "unavailable" }, + }, + }, + { externalId: "vercel-labs/skills/find-skills" }, + ], + }); + return new Response(JSON.stringify({ status: "running", page: 3, offset: 100 })); + }); + vi.stubGlobal("fetch", convexFetch); + + const handler = (await import("./routes/ops/skills-sh/mirror-test.post")).default; + const response = (await handler({} as never)) as Response; + + expect(response.status).toBe(200); + expect(fetchControlledBatchMock).toHaveBeenCalledWith( + { + page: 3, + offset: 0, + limit: 1, + maxDetailBytes: 65_536, + sourceTotal: 9_572, + externalIds: ["patrick-erichsen/skills/html"], + }, + expect.objectContaining({ beforeRequest: expect.any(Function) }), + ); + }); + + it("releases the claimed batch lease when snapshot validation fails", async () => { + parseProofSnapshotIdMock.mockImplementation(() => { + throw new Error("invalid proof snapshot"); + }); + readBodyMock.mockResolvedValue({ + operation: "step", + runId: "skillsShMirrorRuns:test", + page: 3, + offset: 50, + }); + const convexFetch = vi.fn(async (_url: string, init: RequestInit) => { + const body = JSON.parse(String(init.body)); + if (body.operation === "mirror-batch-claim") { + return new Response( + JSON.stringify({ + ...body, + snapshotId: "invalid", + sourcePageSize: 500, + sourceTotal: 9_572, + }), + ); + } + expect(body).toMatchObject({ + operation: "mirror-batch-release", + runId: "skillsShMirrorRuns:test", + page: 3, + offset: 50, + leaseToken: expect.any(String), + }); + return new Response(JSON.stringify({ released: true })); + }); + vi.stubGlobal("fetch", convexFetch); + + const handler = (await import("./routes/ops/skills-sh/mirror-test.post")).default; + const response = (await handler({} as never)) as Response; + + expect(response.status).toBe(502); + expect(await response.json()).toMatchObject({ + message: expect.stringContaining("invalid proof snapshot"), + }); + expect(convexFetch).toHaveBeenCalledTimes(2); + expect(fetchBatchMock).not.toHaveBeenCalled(); + }); + + it("renews the durable lease during a long source batch", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-07-22T20:00:00.000Z")); + readBodyMock.mockResolvedValue({ + operation: "step", + runId: "skillsShMirrorRuns:test", + page: 3, + offset: 50, + sourceTotal: 9_573, + }); + fetchBatchMock.mockImplementation(async (_args, options) => { + await options.beforeRequest(); + vi.setSystemTime(new Date("2026-07-22T20:01:01.000Z")); + await options.beforeRequest(); + return { + page: 3, + offset: 50, + pageLength: 500, + sourceTotal: 9_571, + hasMore: true, + sourceRequests: 101, + sourceBytes: 123_456, + sourcePageIdentityHash: "page-3", + rows: [{ externalId: "vercel-labs/skills/find-skills" }], + }; + }); + const convexFetch = vi.fn(async (_url: string, init: RequestInit) => { + const body = JSON.parse(String(init.body)); + if (body.operation === "mirror-batch-claim") { + return new Response( + JSON.stringify({ + ...body, + snapshotId: "skills-sh:proof:snapshot", + sourcePageSize: 500, + sourceTotal: 9_573, + sourcePage: capturedSourcePage(), + }), + ); + } + if (body.operation === "mirror-batch-release") { + return new Response(JSON.stringify(body)); + } + if (body.operation === "mirror-classification-states") { + return new Response(JSON.stringify({ states: [] })); + } + return new Response(JSON.stringify({ status: "running", page: 3, offset: 100 })); + }); + vi.stubGlobal("fetch", convexFetch); + + const handler = (await import("./routes/ops/skills-sh/mirror-test.post")).default; + const response = (await handler({} as never)) as Response; + + expect(response.status).toBe(200); + const claims = convexFetch.mock.calls + .map(([, init]) => JSON.parse(String(init?.body))) + .filter((body) => body.operation === "mirror-batch-claim"); + expect(claims).toHaveLength(2); + expect(claims[1]).toMatchObject({ + runId: claims[0].runId, + page: claims[0].page, + offset: claims[0].offset, + leaseToken: claims[0].leaseToken, + }); + }); + + it("stops a paused run before fetching another source batch", async () => { + readBodyMock.mockResolvedValue({ + operation: "step", + runId: "skillsShMirrorRuns:test", + page: 3, + offset: 50, + }); + vi.stubGlobal( + "fetch", + vi.fn(async (_url: string, init: RequestInit) => { + const body = JSON.parse(String(init.body)); + expect(body).toMatchObject({ + operation: "mirror-batch-claim", + runId: "skillsShMirrorRuns:test", + page: 3, + offset: 50, + leaseToken: expect.any(String), + }); + return new Response(JSON.stringify({ error: "skills.sh mirror run is paused" }), { + status: 400, + }); + }), + ); + + const handler = (await import("./routes/ops/skills-sh/mirror-test.post")).default; + const response = (await handler({} as never)) as Response; + + expect(response.status).toBe(502); + expect(await response.json()).toMatchObject({ message: expect.stringContaining("paused") }); + expect(getVercelOidcTokenMock).not.toHaveBeenCalled(); + expect(fetchBatchMock).not.toHaveBeenCalled(); + }); + + it("releases the lease when the ordered leaderboard page changed", async () => { + readBodyMock.mockResolvedValue({ + operation: "step", + runId: "skillsShMirrorRuns:test", + page: 3, + offset: 50, + }); + fetchBatchMock.mockResolvedValue({ + page: 3, + offset: 50, + pageLength: 500, + sourceTotal: 9_571, + hasMore: true, + sourceRequests: 5, + sourceBytes: 5_000, + sourcePageIdentityHash: "changed-page-3", + rows: [{ externalId: "vercel-labs/skills/find-skills" }], + }); + const convexFetch = vi.fn(async (_url: string, init: RequestInit) => { + const body = JSON.parse(String(init.body)); + if (body.operation === "mirror-batch-claim") { + return new Response( + JSON.stringify({ + ...body, + snapshotId: "skills-sh:proof:snapshot", + sourcePageSize: 500, + sourceTotal: 9_573, + sourcePage: capturedSourcePage(), + }), + ); + } + expect(body.operation).toBe("mirror-batch-release"); + return new Response(JSON.stringify({ released: true })); + }); + vi.stubGlobal("fetch", convexFetch); + + const handler = (await import("./routes/ops/skills-sh/mirror-test.post")).default; + const response = (await handler({} as never)) as Response; + + expect(response.status).toBe(502); + expect(await response.json()).toMatchObject({ + message: expect.stringContaining("ordered leaderboard page changed"), + }); + expect(convexFetch).toHaveBeenCalledTimes(2); + }); + + it("releases the lease when the measured leaderboard pagination state changed", async () => { + readBodyMock.mockResolvedValue({ + operation: "step", + runId: "skillsShMirrorRuns:test", + page: 3, + offset: 50, + }); + fetchBatchMock.mockResolvedValue({ + page: 3, + offset: 50, + pageLength: 500, + sourceTotal: 9_571, + hasMore: false, + sourceRequests: 5, + sourceBytes: 5_000, + sourcePageIdentityHash: "page-3", + rows: [{ externalId: "vercel-labs/skills/find-skills" }], + }); + const convexFetch = vi.fn(async (_url: string, init: RequestInit) => { + const body = JSON.parse(String(init.body)); + if (body.operation === "mirror-batch-claim") { + return new Response( + JSON.stringify({ + ...body, + snapshotId: "skills-sh:proof:snapshot", + sourcePageSize: 500, + sourceTotal: 9_573, + sourcePage: capturedSourcePage(), + }), + ); + } + expect(body.operation).toBe("mirror-batch-release"); + return new Response(JSON.stringify({ released: true })); + }); + vi.stubGlobal("fetch", convexFetch); + + const handler = (await import("./routes/ops/skills-sh/mirror-test.post")).default; + const response = (await handler({} as never)) as Response; + + expect(response.status).toBe(502); + expect(await response.json()).toMatchObject({ + message: expect.stringContaining("ordered leaderboard page changed"), + }); + expect(convexFetch).toHaveBeenCalledTimes(2); + }); + + it("returns a retryable response without advancing the durable cursor on source rate limits", async () => { + readBodyMock.mockResolvedValue({ + operation: "step", + runId: "skillsShMirrorRuns:test", + page: 3, + offset: 50, + sourceTotal: 9_573, + }); + fetchBatchMock.mockRejectedValue(new Error("source rate limited")); + sourceRetryAfterMock.mockReturnValue(17); + const convexFetch = vi.fn(async (_url: string, init: RequestInit) => { + const body = JSON.parse(String(init.body)); + if (body.operation === "mirror-batch-claim") { + return new Response( + JSON.stringify({ + ...body, + snapshotId: "skills-sh:proof:snapshot", + sourcePageSize: 500, + sourceTotal: 9_573, + sourcePage: capturedSourcePage(), + }), + ); + } + expect(body).toMatchObject({ + operation: "mirror-batch-release", + runId: "skillsShMirrorRuns:test", + page: 3, + offset: 50, + leaseToken: expect.any(String), + }); + return new Response(JSON.stringify({ released: true })); + }); + vi.stubGlobal("fetch", convexFetch); + + const handler = (await import("./routes/ops/skills-sh/mirror-test.post")).default; + const response = (await handler({} as never)) as Response; + + expect(response.status).toBe(429); + expect(response.headers.get("Retry-After")).toBe("17"); + expect(await response.json()).toMatchObject({ + error: "skills_sh_source_rate_limited", + retryAfterSeconds: 17, + }); + expect(convexFetch).toHaveBeenCalledTimes(2); + const operatorBodies = convexFetch.mock.calls.map(([, init]) => JSON.parse(String(init?.body))); + expect(operatorBodies[0].leaseToken).toBe(operatorBodies[1].leaseToken); + }); + + it("passes through bounded facet proof pages", async () => { + readBodyMock.mockResolvedValue({ operation: "facet-page", cursor: null, limit: 500 }); + const convexFetch = vi.fn(async (_url: string, init: RequestInit) => { + expect(JSON.parse(String(init.body))).toEqual({ + operation: "mirror-facet-page", + cursor: null, + limit: 500, + }); + return new Response( + JSON.stringify({ + page: [{ kind: "category", term: "development" }], + isDone: true, + continueCursor: "", + }), + ); + }); + vi.stubGlobal("fetch", convexFetch); + + const handler = (await import("./routes/ops/skills-sh/mirror-test.post")).default; + const response = (await handler({} as never)) as Response; + + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ + page: [{ kind: "category", term: "development" }], + isDone: true, + }); + }); + + it("passes through only byte-safe detail proof pages", async () => { + readBodyMock.mockResolvedValue({ operation: "detail-page", cursor: null, limit: 50 }); + const convexFetch = vi.fn(async (_url: string, init: RequestInit) => { + expect(JSON.parse(String(init.body))).toEqual({ + operation: "mirror-detail-page", + cursor: null, + limit: 50, + }); + return new Response( + JSON.stringify({ + page: [{ externalId: "patrick-erichsen/skills/html" }], + isDone: true, + continueCursor: "", + }), + ); + }); + vi.stubGlobal("fetch", convexFetch); + + const handler = (await import("./routes/ops/skills-sh/mirror-test.post")).default; + const response = (await handler({} as never)) as Response; + + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ + page: [{ externalId: "patrick-erichsen/skills/html" }], + isDone: true, + }); + }); + + it("rejects detail proof pages that can exceed the response byte limit", async () => { + readBodyMock.mockResolvedValue({ operation: "detail-page", cursor: null, limit: 51 }); + const convexFetch = vi.fn(); + vi.stubGlobal("fetch", convexFetch); + + const handler = (await import("./routes/ops/skills-sh/mirror-test.post")).default; + const response = (await handler({} as never)) as Response; + + expect(response.status).toBe(502); + expect(await response.json()).toMatchObject({ + message: "limit must be an integer between 1 and 50", + }); + expect(convexFetch).not.toHaveBeenCalled(); + }); + + it("appends the bounded controlled GitHub proof page after the authenticated catalog", async () => { + readBodyMock.mockResolvedValue({ + operation: "step", + runId: "skillsShMirrorRuns:test", + page: 20, + offset: 0, + }); + fetchControlledBatchMock.mockResolvedValue({ + page: 20, + offset: 0, + pageLength: 2, + sourceTotal: 9_573, + hasMore: false, + sourceRequests: 2, + sourceBytes: 11_085, + rows: [ + { externalId: "patrick-erichsen/skills/html" }, + { externalId: "steipete/clawdis/discrawl" }, + ], + }); + const convexFetch = vi.fn(async (_url: string, init: RequestInit) => { + const body = JSON.parse(String(init.body)); + if (body.operation === "mirror-batch-claim") { + return new Response( + JSON.stringify({ + ...body, + snapshotId: "skills-sh:proof:snapshot", + sourcePageSize: 500, + sourceTotal: 9_573, + }), + ); + } + if (body.operation === "mirror-classification-states") { + expect(body.externalIds).toEqual([ + "patrick-erichsen/skills/html", + "steipete/clawdis/discrawl", + ]); + return new Response(JSON.stringify({ states: [] })); + } + expect(body).toMatchObject({ + operation: "mirror-batch", + page: 20, + offset: 0, + pageLength: 2, + sourceTotal: 9_573, + hasMore: false, + }); + return new Response(JSON.stringify({ status: "reconciling", page: 21, offset: 0 })); + }); + vi.stubGlobal("fetch", convexFetch); + + const handler = (await import("./routes/ops/skills-sh/mirror-test.post")).default; + const response = (await handler({} as never)) as Response; + + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ status: "reconciling" }); + expect(fetchControlledBatchMock).toHaveBeenCalledWith( + { + page: 20, + offset: 0, + limit: 50, + maxDetailBytes: 65_536, + sourceTotal: 9_573, + externalIds: ["patrick-erichsen/skills/html", "steipete/clawdis/discrawl"], + }, + expect.objectContaining({ beforeRequest: expect.any(Function) }), + ); + expect(fetchBatchMock).not.toHaveBeenCalled(); + expect(getVercelOidcTokenMock).not.toHaveBeenCalled(); + }); + + it("completes on the leaderboard page when every controlled identity is already present", async () => { + parseProofSnapshotIdMock.mockReturnValue({ + catalogTotal: 9_571, + controlledExternalIds: ["patrick-erichsen/skills/html", "steipete/clawdis/discrawl"], + controlledOverlayExternalIds: ["patrick-erichsen/skills/html", "steipete/clawdis/discrawl"], + controlledSupplementExternalIds: [], + evidence: { + pagination: { + requestedPages: [ + { + page: 19, + count: 71, + hasMore: false, + identityHash: "page-19", + contentHash: "content-19", + }, + ], + }, + }, + }); + readBodyMock.mockResolvedValue({ + operation: "step", + runId: "skillsShMirrorRuns:test", + page: 19, + offset: 70, + }); + fetchBatchMock.mockResolvedValue({ + page: 19, + offset: 70, + pageLength: 71, + sourceTotal: 9_571, + hasMore: false, + sourceRequests: 3, + sourceBytes: 1_024, + sourcePageIdentityHash: "page-19", + rows: [{ externalId: "owner/repo/final" }], + }); + const convexFetch = vi.fn(async (_url: string, init: RequestInit) => { + const body = JSON.parse(String(init.body)); + if (body.operation === "mirror-batch-claim") { + return new Response( + JSON.stringify({ + ...body, + snapshotId: "skills-sh:proof:snapshot", + sourcePageSize: 500, + sourceTotal: 9_571, + sourcePage: capturedSourcePage(19, 71, false, "page-19"), + }), + ); + } + if (body.operation === "mirror-classification-states") { + return new Response(JSON.stringify({ states: [] })); + } + expect(body).toMatchObject({ + operation: "mirror-batch", + page: 19, + offset: 70, + sourceTotal: 9_571, + hasMore: false, + }); + return new Response(JSON.stringify({ status: "reconciling", page: 20, offset: 0 })); + }); + vi.stubGlobal("fetch", convexFetch); + + const handler = (await import("./routes/ops/skills-sh/mirror-test.post")).default; + const response = (await handler({} as never)) as Response; + + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ status: "reconciling" }); + expect(fetchControlledBatchMock).not.toHaveBeenCalled(); + }); + + it("starts and steps a lease-guarded captured replay without source auth", async () => { + readBodyMock + .mockResolvedValueOnce({ + operation: "start-replay", + reason: "captured normalization", + capturedRunId: "skillsShMirrorRuns:live", + sourceTotal: 1, + sourcePageSize: 500, + sourceMeasuredAt: "2026-07-22T20:14:10.881Z", + }) + .mockResolvedValueOnce({ + operation: "step-replay", + runId: "skillsShMirrorRuns:replay", + page: 0, + offset: 0, + pageLength: 1, + hasMore: false, + sourceTotal: 1, + externalIds: ["patrick-erichsen/skills/html"], + }); + buildReplayRowsMock.mockReturnValue([ + { + externalId: "patrick-erichsen/skills/html", + inferredCategories: ["other"], + }, + ]); + const convexFetch = vi.fn(async (_url: string, init: RequestInit) => { + const body = JSON.parse(String(init.body)); + if (body.operation === "mirror-start") { + expect(body).toMatchObject({ + snapshotId: "skills-sh-captured:skillsShMirrorRuns:live", + sourceTotal: 1, + sourcePageSize: 500, + }); + return new Response( + JSON.stringify({ + runId: "skillsShMirrorRuns:replay", + status: "running", + page: 0, + offset: 0, + }), + ); + } + if (body.operation === "mirror-batch-claim") return new Response(JSON.stringify(body)); + if (body.operation === "mirror-replay-rows") { + expect(body.externalIds).toEqual(["patrick-erichsen/skills/html"]); + return new Response(JSON.stringify({ rows: [{ digest: {}, detail: null }] })); + } + expect(body).toMatchObject({ + operation: "mirror-batch", + runId: "skillsShMirrorRuns:replay", + sourceRequests: 0, + sourceBytes: 0, + rows: [ + { + externalId: "patrick-erichsen/skills/html", + inferredCategories: ["other"], + }, + ], + }); + return new Response(JSON.stringify({ status: "reconciling", page: 1, offset: 0 })); + }); + vi.stubGlobal("fetch", convexFetch); + const handler = (await import("./routes/ops/skills-sh/mirror-test.post")).default; + + const startResponse = (await handler({} as never)) as Response; + const stepResponse = (await handler({} as never)) as Response; + + expect(startResponse.status).toBe(200); + expect(await startResponse.json()).toMatchObject({ + runId: "skillsShMirrorRuns:replay", + status: "running", + page: 0, + offset: 0, + sourceTotal: 1, + }); + expect(stepResponse.status).toBe(200); + expect(await stepResponse.json()).toMatchObject({ status: "reconciling" }); + expect(getVercelOidcTokenMock).not.toHaveBeenCalled(); + expect(buildReplayRowsMock).toHaveBeenCalledWith([{ digest: {}, detail: null }]); + }); +}); diff --git a/specs/download-metering.md b/specs/download-metering.md index 66ea7f73..cab80546 100644 --- a/specs/download-metering.md +++ b/specs/download-metering.md @@ -20,17 +20,37 @@ ip: This keeps a user id and IP with the same visible string in separate hash domains for dedupe and local diagnostics. -## Counters +## Source-Attributed Counters The dedupe table does not store user-vs-IP counters. It only gates whether a download should emit the existing skill or package stat event. Public counters -still store one total: +for native ClawHub downloads still use the canonical skill download field: ```text -downloads +statsDownloads ``` -Existing historical counts are not estimated or rewritten in this phase. +Skills mirrored from skills.sh store the upstream install count separately in +`statsSkillsShInstalls`. Public skill Downloads are computed at serialization: + +```text +native skill: statsDownloads +skills.sh indexed: statsDownloads + statsSkillsShInstalls +``` + +The combined value is never written back into `statsDownloads`. Existing +historical ClawHub downloads remain intact, and search ranking continues to use +the native field. + +OpenClaw install telemetry remains in `statsInstallsCurrent` and +`statsInstallsAllTime`; it is not added to public Downloads. GitHub popularity +is stored in `statsGithubStars`. Existing `stars` rows and `statsStars` count +ClawHub Bookmarks and retain those storage/API names for compatibility. + +Source refresh, adoption, content replacement, and GitHub synchronization may +update source metadata or upstream counters, but must not reset or rewrite any +other metric source. Publisher dashboards receive the source breakdown while +ordinary public skill shapes expose only the combined Downloads value. ## Daily Package Graphs diff --git a/specs/github-backed-skills.md b/specs/github-backed-skills.md index d65b8f27..54e22ac2 100644 --- a/specs/github-backed-skills.md +++ b/specs/github-backed-skills.md @@ -98,6 +98,35 @@ not create or mutate `skills` rows during its planning gates. - `githubScanStatus`: `pending`, `clean`, `suspicious`, `malicious`, or `failed` - `githubRemovedAt` +## Permanent external skills.sh mirror + +The full authenticated skills.sh mirror is separate from both native `skills` +and the controlled scan-admission catalog. It stores source observations in +`skillsShMirrorDigests` and bounded detail content in `skillsShMirrorDetails`; +controls, durable run cursors, and conflicts remain in their own mirror tables. + +- GitHub identities are exact `owner/repo/slug` values. Well-known identities + are exact `sourceHost/slug` values and must not invent a repository owner. +- Every digest is permanently `publicVisible: false` and `installable: false`. + Mirror ingestion never creates native skills, publisher attachment, claims, + scan plans, or scan jobs. +- The digest stores normalized slug/display-name fields and a lean + `searchText`. Exact, prefix, first-token, popularity, freshness, and + full-text indexes are staged before activation on the permanent Test corpus. +- Gen Agent Trust Hub, Socket, and Snyk observations are stored independently + with a bounded status plus optional source timestamp and source link. These + are upstream claims only and must never be serialized as a ClawHub verdict. +- Detail storage retains at most one preferred `SKILL.md` or `README.md`, + capped at 64 KiB. The complete upstream file tree is never persisted for an + unclaimed mirror row. +- Snapshot ingestion uses bounded page/offset cursors. Pause is checked before + another source batch is fetched; resume continues at the exact stored cursor. + Reconciliation tombstones disappeared rows and reactivates later + observations without deleting or changing native skills. +- The mirror has no scheduler in this stage. Production activation, public + search/detail/install behavior, claims, and publisher attachment require + separately accepted work. + ## Publisher Gate The legacy `NVIDIA/skills` source keeps its existing production behavior. diff --git a/specs/spec.md b/specs/spec.md index 91143170..829f65a5 100644 --- a/specs/spec.md +++ b/specs/spec.md @@ -52,7 +52,18 @@ read_when: - `moderationFlags`: `string[]` (automatic detection) - `moderationNotes`, `moderationReason` - `hiddenAt`, `hiddenBy`, `lastReviewedAt`, `reportCount` -- `stats`: `{ downloads, stars, versions, comments }` (`comments` is retained as a historical stat field; skill comments are retired) +- `stats`: legacy nested compatibility counters; canonical migrated counters live + in top-level fields. +- `statsDownloads`: native ClawHub downloads. +- `statsSkillsShInstalls`: upstream skills.sh installs, stored separately. +- `statsInstallsCurrent`, `statsInstallsAllTime`: OpenClaw install telemetry. +- `statsGithubStars`: upstream GitHub popularity. +- `statsStars`: ClawHub Bookmarks; existing `stars` rows and API names remain for + compatibility. +- Public Downloads are `statsDownloads` for native-only skills and + `statsDownloads + statsSkillsShInstalls` for skills.sh-indexed skills. +- Search ranking continues to use native counters rather than the combined + presentation value. - `createdAt`, `updatedAt` ### SkillVersion @@ -95,9 +106,11 @@ From SKILL.md frontmatter + AgentSkills + Clawdis extensions: detail page and as hover text; 70 is a presentation rule, not a storage, publish, API, or sync constraint. -### Star +### Bookmark - `skillId`, `userId`, `createdAt` +- Stored in the legacy `stars` table and exposed through compatibility API + routes named `stars`; user-facing product language is Bookmark. ### AuditLog diff --git a/src/__tests__/deploy-test-workflow.test.ts b/src/__tests__/deploy-test-workflow.test.ts index f82cea8e..ace97d5a 100644 --- a/src/__tests__/deploy-test-workflow.test.ts +++ b/src/__tests__/deploy-test-workflow.test.ts @@ -24,6 +24,9 @@ async function readWorkflow() { }; jobs?: Record; on?: { + pull_request?: { + types?: string[]; + }; workflow_dispatch?: unknown; workflow_run?: { branches?: string[]; @@ -36,27 +39,49 @@ async function readWorkflow() { } describe("Test deploy workflow", () => { - it("runs only after successful main CI or a manual dispatch", async () => { + it("admits main CI and exact guarded CLAW-563 branch deploys", async () => { const workflow = await readWorkflow(); const job = workflow.jobs?.["deploy-test"]; const steps = job?.steps ?? []; + const revision = steps.find((step) => step.name === "Resolve deployment revision")?.run ?? ""; expect(workflow.on?.workflow_run).toEqual({ workflows: ["CI"], types: ["completed"], branches: ["main"], }); + expect(workflow.on?.pull_request).toEqual({ + types: ["synchronize", "labeled"], + }); expect(workflow.on?.workflow_dispatch).toBeDefined(); expect(workflow.concurrency).toEqual({ group: "deploy-test", "cancel-in-progress": false, }); + expect(job?.if).toContain("github.event_name == 'workflow_dispatch'"); + expect(job?.if).toContain("github.ref == 'refs/heads/main'"); + expect(job?.if).toContain("github.ref == 'refs/heads/pe/claw-563-skills-sh-mirror-10k'"); + expect(job?.if).toContain("inputs.branch_test_confirm == 'deploy-claw-563-to-permanent-test'"); + expect(job?.if).toContain("inputs.expected_sha != ''"); + expect(job?.if).toContain("github.event_name == 'pull_request'"); + expect(job?.if).toContain( + "github.event.pull_request.head.ref == 'pe/claw-563-skills-sh-mirror-10k'", + ); + expect(job?.if).toContain("github.event.pull_request.head.repo.full_name == github.repository"); + expect(job?.if).toContain("github.actor == 'Patrick-Erichsen'"); + expect(job?.if).toContain( + "contains(github.event.pull_request.labels.*.name, 'test-mirror-load')", + ); expect(job?.if).toContain("github.event.workflow_run.conclusion == 'success'"); expect(job?.if).toContain("github.event.workflow_run.event == 'push'"); - expect(job?.if).toContain("github.ref == 'refs/heads/main'"); - expect(steps.find((step) => step.name === "Resolve deployment revision")?.run).toContain( - 'deploy_sha" != "$main_sha', - ); + expect(revision).toContain('deploy_sha" != "$main_sha'); + expect(revision).toContain("refs/heads/pe/claw-563-skills-sh-mirror-10k"); + expect(revision).toContain("Patrick-Erichsen"); + expect(revision).toContain("deploy-claw-563-to-permanent-test"); + expect(revision).toContain("${{ inputs.expected_sha }}"); + expect(revision).toContain("${{ github.event.pull_request.head.sha }}"); + expect(revision).toContain("${{ github.event.pull_request.head.repo.full_name }}"); + expect(revision).toContain("$GITHUB_REPOSITORY"); }); it("uses only the Test environment and narrowly scoped secrets", async () => { @@ -114,6 +139,9 @@ describe("Test deploy workflow", () => { expect(deployStep?.run).toContain("--target=preview"); expect(deployStep?.run).toContain('--scope "$VERCEL_SCOPE"'); expect(deployStep?.run).toContain("--build-env CONVEX_DEPLOY_KEY="); + expect(deployStep?.run).not.toContain("--build-env VERCEL_ENV="); + expect(deployStep?.run).toContain("--build-env VERCEL_TARGET_ENV=test"); + expect(deployStep?.run).toContain("--env VERCEL_TARGET_ENV=test"); expect(deployStep?.run).toContain("--build-env CLAWHUB_SKILLS_SH_ROLLOUT_MODE=test"); expect(deployStep?.run).toContain("--build-env CLAWHUB_GITHUB_SKILL_SYNC_ROLLOUT_MODE=test"); expect(deployStep?.run).toContain("--env CLAWHUB_SKILLS_SH_ROLLOUT_MODE=test"); @@ -137,4 +165,19 @@ describe("Test deploy workflow", () => { expect(verify?.run).toContain(".skillsSh.runtimeEnabled == true"); expect(verify?.run).toContain(".githubSkillSync.selfServiceEnabled == true"); }); + + it("proves both immutable controlled mirror entries before cleanup", async () => { + const workflow = await readWorkflow(); + const step = workflow.jobs?.["claw563-mirror-load"]?.steps?.find( + (candidate) => + candidate.name === "Load and prove the authenticated leaderboard mirror foundation", + ); + const run = step?.run ?? ""; + + expect(run).toContain('"externalId":"patrick-erichsen/skills/html"'); + expect(run).toContain('"externalId":"steipete/clawdis/discrawl"'); + expect(run).toContain("050daba89f6b6636470add5cb300aac46a412cf8"); + expect(run).toContain("690ed564419291ca6e832dc69b53061300075b62"); + expect(run).toContain("claw563-discrawl-entry.json"); + }); }); diff --git a/src/__tests__/header.test.tsx b/src/__tests__/header.test.tsx index 65e8c908..5a32250d 100644 --- a/src/__tests__/header.test.tsx +++ b/src/__tests__/header.test.tsx @@ -687,7 +687,7 @@ describe("Header", () => { expect(labels).toEqual(["Home", "Skills", "Plugins", "Official", "Docs"]); }); - it("links profile and starred skills from the signed-in avatar menu", () => { + it("links profile and bookmarks from the signed-in avatar menu", () => { profileHandleMock.mockReturnValue("patrick-profile"); authStatusMock.mockReturnValue({ isAuthenticated: true, @@ -710,7 +710,7 @@ describe("Header", () => { expect(profile.compareDocumentPosition(dashboard) & Node.DOCUMENT_POSITION_FOLLOWING).toBe( Node.DOCUMENT_POSITION_FOLLOWING, ); - expect(screen.getByText("Stars").closest("a")?.getAttribute("href")).toBe("/stars"); + expect(screen.getByText("Bookmarks").closest("a")?.getAttribute("href")).toBe("/stars"); expect(screen.getAllByText("Dashboard").length).toBeGreaterThan(0); expect(screen.getByText("Settings")).toBeTruthy(); }); diff --git a/src/__tests__/skill-detail-page.test.tsx b/src/__tests__/skill-detail-page.test.tsx index e4c6f8ca..a7970a7b 100644 --- a/src/__tests__/skill-detail-page.test.tsx +++ b/src/__tests__/skill-detail-page.test.tsx @@ -1789,20 +1789,20 @@ describe("SkillDetailPage", () => { render(); - const starButton = await screen.findByRole("button", { name: "Star skill" }); + const starButton = await screen.findByRole("button", { name: "Bookmark skill" }); expect(starButton.textContent).toContain("8"); fireEvent.click(starButton); await waitFor(() => { - expect(screen.getByRole("button", { name: "Unstar skill" }).textContent).toContain("9"); + expect(screen.getByRole("button", { name: "Remove bookmark" }).textContent).toContain("9"); }); expect(toggleStarMock).toHaveBeenCalledWith({ skillId }); - fireEvent.click(screen.getByRole("button", { name: "Unstar skill" })); + fireEvent.click(screen.getByRole("button", { name: "Remove bookmark" })); await waitFor(() => { - expect(screen.getByRole("button", { name: "Star skill" }).textContent).toContain("8"); + expect(screen.getByRole("button", { name: "Bookmark skill" }).textContent).toContain("8"); }); expect(toggleStarMock).toHaveBeenCalledTimes(2); expect(routerInvalidateMock).toHaveBeenCalledTimes(2); @@ -1869,7 +1869,7 @@ describe("SkillDetailPage", () => { render(); - expect((await screen.findByRole("button", { name: "Unstar skill" })).textContent).toContain( + expect((await screen.findByRole("button", { name: "Remove bookmark" })).textContent).toContain( "1", ); }); diff --git a/src/__tests__/skills-index.test.tsx b/src/__tests__/skills-index.test.tsx index 064b3268..a368957f 100644 --- a/src/__tests__/skills-index.test.tsx +++ b/src/__tests__/skills-index.test.tsx @@ -107,7 +107,7 @@ describe("SkillsIndex", () => { const sortOptions = Array.from( screen.getByRole("radiogroup", { name: "Skill view" }).querySelectorAll('[role="radio"]'), ).map((option) => option.textContent); - expect(sortOptions).toEqual(["All", "Trending", "Top", "Most starred", "Featured"]); + expect(sortOptions).toEqual(["All", "Trending", "Top", "Most bookmarked", "Featured"]); }); it("offers Top without exposing downloads as a browse view", async () => { @@ -144,7 +144,7 @@ describe("SkillsIndex", () => { fireEvent.click(screen.getByRole("combobox", { name: "Sort" })); const sortOptions = screen.getAllByRole("option").map((option) => option.textContent); - expect(views).toEqual(["All", "Trending", "Top", "Most starred", "Featured"]); + expect(views).toEqual(["All", "Trending", "Top", "Most bookmarked", "Featured"]); expect(sortOptions).toEqual(["Recently updated", "Newest", "Name"]); }); diff --git a/src/components/Header.tsx b/src/components/Header.tsx index 6a9eace4..2d394d34 100644 --- a/src/components/Header.tsx +++ b/src/components/Header.tsx @@ -649,7 +649,7 @@ export default function Header() { diff --git a/src/components/SkillDetailPage.tsx b/src/components/SkillDetailPage.tsx index 9f8a6c12..1b9912bf 100644 --- a/src/components/SkillDetailPage.tsx +++ b/src/components/SkillDetailPage.tsx @@ -763,8 +763,8 @@ export function SkillDetailPage({ }); void router.invalidate(); } catch (error) { - console.error("Failed to toggle star", error); - toast.error(getUserFacingConvexError(error, "Unable to update star. Please try again.")); + console.error("Failed to toggle bookmark", error); + toast.error(getUserFacingConvexError(error, "Unable to update bookmark. Please try again.")); } }; diff --git a/src/components/SkillHeader.test.tsx b/src/components/SkillHeader.test.tsx index c3b2c17e..1f79e848 100644 --- a/src/components/SkillHeader.test.tsx +++ b/src/components/SkillHeader.test.tsx @@ -162,14 +162,14 @@ describe("SkillHeader", () => { ); } - it("keeps signed-out star and report actions visible and routes clicks to sign-in", () => { + it("keeps signed-out bookmark and report actions visible and routes clicks to sign-in", () => { const onToggleStar = vi.fn(); const onOpenReport = vi.fn(); const onRequireSignIn = vi.fn(); const { container } = renderHeader({ onToggleStar, onOpenReport, onRequireSignIn }); - fireEvent.click(screen.getByRole("button", { name: "Star skill" })); + fireEvent.click(screen.getByRole("button", { name: "Bookmark skill" })); fireEvent.click(screen.getByRole("button", { name: "Report" })); expect(onRequireSignIn).toHaveBeenCalledTimes(2); @@ -488,19 +488,19 @@ describe("SkillHeader", () => { ).toBeTruthy(); }); - it("places Star in the sidebar without an outline button", () => { + it("places Bookmark in the sidebar without an outline button", () => { const { container } = renderHeader(); const starBand = container.querySelector(".skill-sidebar-star-band"); expect(starBand).toBeTruthy(); const starButton = within(starBand as HTMLElement).getByRole("button", { - name: "Star skill", + name: "Bookmark skill", }); expect(starButton.className).toContain("skill-sidebar-star-action"); expect(container.querySelector(".skill-hero-title-row .skill-title-actions")).toBeNull(); }); - it("places Star on the creator row on mobile detail layout", () => { + it("places Bookmark on the creator row on mobile detail layout", () => { setViewportWidth(390); const { container } = renderHeader(); @@ -508,7 +508,7 @@ describe("SkillHeader", () => { const creator = container.querySelector(".skill-hero-creator"); expect(creator).toBeTruthy(); const starButton = within(creator as HTMLElement).getByRole("button", { - name: "Star skill", + name: "Bookmark skill", }); expect(starButton.className).toContain("skill-sidebar-star-action"); expect(starButton.closest(".skill-hero-creator-star")).toBeTruthy(); diff --git a/src/components/SkillHeader.tsx b/src/components/SkillHeader.tsx index 72d3457d..40dff200 100644 --- a/src/components/SkillHeader.tsx +++ b/src/components/SkillHeader.tsx @@ -201,21 +201,21 @@ export function SkillHeader({ const renderStarAction = () => ( diff --git a/src/components/UserBadge.tsx b/src/components/UserBadge.tsx index 4107f8d2..24082329 100644 --- a/src/components/UserBadge.tsx +++ b/src/components/UserBadge.tsx @@ -212,7 +212,7 @@ function UserStatsTooltipContent({ {formatCompactStat(stats.totalStars)} diff --git a/src/components/dashboard/DashboardCatalogView.test.tsx b/src/components/dashboard/DashboardCatalogView.test.tsx new file mode 100644 index 00000000..23244a68 --- /dev/null +++ b/src/components/dashboard/DashboardCatalogView.test.tsx @@ -0,0 +1,60 @@ +/* @vitest-environment jsdom */ + +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; +import { DashboardCatalogView } from "./DashboardCatalogView"; +import type { DashboardCatalogItem, DashboardSkill } from "./types"; + +const skill = { + _id: "skills:demo", + _creationTime: 1, + slug: "demo", + displayName: "Demo", + ownerUserId: "users:owner", + ownerPath: "owner", + tags: {}, + badges: {}, + stats: { + downloads: 20, + installsCurrent: 3, + installsAllTime: 7, + stars: 4, + versions: 1, + }, + metricSources: { + clawHubDownloads: 12, + skillsShInstalls: 8, + openClawInstallsCurrent: 3, + openClawInstallsAllTime: 7, + githubStars: 99, + bookmarks: 4, + }, + latestVersion: null, + createdAt: 1, + updatedAt: 1, +} as DashboardSkill; + +const item: DashboardCatalogItem = { + kind: "skill", + id: skill._id, + name: skill.displayName, + searchText: "demo", + data: skill, + updatedAt: skill.updatedAt, + installs: 7, + downloads: 20, +}; + +describe("DashboardCatalogView", () => { + it("lets publishers inspect the source breakdown behind combined downloads", () => { + render( + , + ); + + expect( + screen.getByTitle( + "20 downloads: 12 ClawHub downloads + 8 skills.sh installs. 7 OpenClaw installs; 99 GitHub stars; 4 bookmarks.", + ), + ).toBeTruthy(); + }); +}); diff --git a/src/components/dashboard/DashboardCatalogView.tsx b/src/components/dashboard/DashboardCatalogView.tsx index 985eb717..6cf8c1e0 100644 --- a/src/components/dashboard/DashboardCatalogView.tsx +++ b/src/components/dashboard/DashboardCatalogView.tsx @@ -110,6 +110,7 @@ function SkillListRow({ secondary={packageRowSecondary(skill.updatedAt)} status={skillArtifactStatus(skill)} downloads={skill.stats?.downloads ?? 0} + downloadTitle={skillMetricSourceLabel(skill)} menu={} /> ); @@ -149,6 +150,7 @@ function CatalogRow({ secondary, status, downloads, + downloadTitle, menu, }: { href: string; @@ -159,6 +161,7 @@ function CatalogRow({ secondary: string; status: ArtifactDisplayStatus; downloads: number; + downloadTitle?: string; menu: ReactNode; }) { return ( @@ -180,7 +183,10 @@ function CatalogRow({
- +