mirror of
https://github.com/openclaw/clawhub.git
synced 2026-08-14 00:47:57 +00:00
feat: add canonical Trending snapshot API (#3265)
* feat: materialize canonical trending snapshots * feat: expose canonical trending api * test: prove canonical trending in permanent Test * test: seed canonical trending Test corpus * test: retain trending sources on cleanup failure
This commit is contained in:
Patrick Erichsen
@@ -39,7 +39,9 @@ jobs:
|
||||
(github.ref == 'refs/heads/pe/claw-583-mirrored-search-journey' &&
|
||||
inputs.branch_test_confirm == 'deploy-claw-583-to-permanent-test') ||
|
||||
(github.ref == 'refs/heads/pe/claw-560-verified-adoption' &&
|
||||
inputs.branch_test_confirm == 'deploy-claw-560-to-permanent-test')) &&
|
||||
inputs.branch_test_confirm == 'deploy-claw-560-to-permanent-test') ||
|
||||
(github.ref == 'refs/heads/pe/claw-590-trending-snapshot' &&
|
||||
inputs.branch_test_confirm == 'deploy-claw-590-to-permanent-test')) &&
|
||||
github.actor == 'Patrick-Erichsen' &&
|
||||
inputs.expected_sha != '')) ||
|
||||
(github.event_name == 'pull_request' &&
|
||||
@@ -126,6 +128,14 @@ jobs:
|
||||
then
|
||||
branch_test_allowed=true
|
||||
fi
|
||||
if [[ "$GITHUB_EVENT_NAME" == workflow_dispatch ]] &&
|
||||
[[ "$GITHUB_REF" == refs/heads/pe/claw-590-trending-snapshot ]] &&
|
||||
[[ "$GITHUB_ACTOR" == Patrick-Erichsen ]] &&
|
||||
[[ "${{ inputs.branch_test_confirm }}" == deploy-claw-590-to-permanent-test ]] &&
|
||||
[[ "${{ inputs.expected_sha }}" == "$deploy_sha" ]]
|
||||
then
|
||||
branch_test_allowed=true
|
||||
fi
|
||||
if [[ "$GITHUB_EVENT_NAME" == pull_request ]] &&
|
||||
[[ "$GITHUB_HEAD_REF" == pe/claw-563-skills-sh-mirror-10k ]] &&
|
||||
[[ "$GITHUB_ACTOR" == Patrick-Erichsen ]] &&
|
||||
@@ -1120,3 +1130,217 @@ jobs:
|
||||
claw583-cleanup-status.json
|
||||
claw583-deployment-readback.json
|
||||
claw583-deployment.json
|
||||
|
||||
claw590-canonical-trending-proof:
|
||||
needs: deploy-test
|
||||
if: >-
|
||||
github.event_name == 'workflow_dispatch' &&
|
||||
github.ref == 'refs/heads/pe/claw-590-trending-snapshot' &&
|
||||
inputs.branch_test_confirm == 'deploy-claw-590-to-permanent-test' &&
|
||||
inputs.expected_sha == needs.deploy-test.outputs.deploy_sha
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
environment:
|
||||
name: Test
|
||||
steps:
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
fetch-depth: 0
|
||||
ref: ${{ inputs.expected_sha }}
|
||||
|
||||
- uses: ./.github/actions/setup-bun
|
||||
|
||||
- name: Prove canonical Trending materialization and API in permanent Test
|
||||
env:
|
||||
CONVEX_DEPLOY_KEY: ${{ secrets.CONVEX_DEPLOY_KEY }}
|
||||
DEPLOY_SHA: ${{ needs.deploy-test.outputs.deploy_sha }}
|
||||
TEST_SITE_URL: ${{ vars.SITE_URL }}
|
||||
VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
confirm=manage-claw-590-canonical-trending-test-proof
|
||||
snapshot_id="claw-590-proof-$DEPLOY_SHA"
|
||||
mkdir -p proof/claw-590
|
||||
printf '{"armed":false}\n' > proof/claw-590/active-snapshot.json
|
||||
cleanup() {
|
||||
set +e
|
||||
armed="$(jq -r '.armed // false' proof/claw-590/active-snapshot.json 2>/dev/null)"
|
||||
active_snapshot_id="$(jq -r '.snapshotId // empty' proof/claw-590/active-snapshot.json 2>/dev/null)"
|
||||
cleanup_exit=0
|
||||
if [[ "$armed" == true && -n "$active_snapshot_id" ]]; then
|
||||
cleanup_args="$(
|
||||
jq -cn \
|
||||
--arg confirm "$confirm" \
|
||||
--arg snapshotId "$active_snapshot_id" \
|
||||
'{confirm:$confirm,snapshotId:$snapshotId}'
|
||||
)"
|
||||
bunx convex run --no-push \
|
||||
canonicalTrendingTestFixtures:cleanupCanonicalTrendingProof \
|
||||
"$cleanup_args" > claw590-cleanup.json 2>&1
|
||||
cleanup_exit=$?
|
||||
bunx convex run --no-push \
|
||||
canonicalTrendingTestFixtures:readCanonicalTrendingProof \
|
||||
"$cleanup_args" > claw590-cleanup-readback.json 2>&1
|
||||
readback_exit=$?
|
||||
else
|
||||
jq -n '{ok:true,skipped:true}' > claw590-cleanup.json
|
||||
jq -n '{present:false,skipped:true}' > claw590-cleanup-readback.json
|
||||
readback_exit=0
|
||||
fi
|
||||
snapshot_removed=false
|
||||
if [[ "$cleanup_exit" -eq 0 && "$readback_exit" -eq 0 ]] && \
|
||||
jq -e '.present == false' claw590-cleanup-readback.json >/dev/null; then
|
||||
snapshot_removed=true
|
||||
fi
|
||||
source_args="$(jq -cn --arg confirm "$confirm" '{confirm:$confirm}')"
|
||||
if [[ "$snapshot_removed" == true ]]; then
|
||||
bunx convex run --no-push \
|
||||
canonicalTrendingTestFixtures:cleanupCanonicalTrendingSourceFixture \
|
||||
"$source_args" > claw590-source-cleanup.json 2>&1
|
||||
source_cleanup_exit=$?
|
||||
else
|
||||
jq -n \
|
||||
'{ok:false,skipped:true,reason:"snapshot cleanup incomplete; source retained"}' \
|
||||
> claw590-source-cleanup.json
|
||||
source_cleanup_exit=0
|
||||
fi
|
||||
bunx convex run --no-push \
|
||||
canonicalTrendingTestFixtures:readCanonicalTrendingSourceFixture \
|
||||
"$source_args" > claw590-source-cleanup-readback.json 2>&1
|
||||
source_readback_exit=$?
|
||||
jq -n \
|
||||
--argjson cleanupExit "$cleanup_exit" \
|
||||
--argjson readbackExit "$readback_exit" \
|
||||
--argjson sourceCleanupExit "$source_cleanup_exit" \
|
||||
--argjson sourceReadbackExit "$source_readback_exit" \
|
||||
'{
|
||||
cleanupExit:$cleanupExit,
|
||||
readbackExit:$readbackExit,
|
||||
sourceCleanupExit:$sourceCleanupExit,
|
||||
sourceReadbackExit:$sourceReadbackExit
|
||||
}' \
|
||||
> claw590-cleanup-status.json
|
||||
set -e
|
||||
[[ \
|
||||
"$cleanup_exit" -eq 0 && \
|
||||
"$readback_exit" -eq 0 && \
|
||||
"$source_cleanup_exit" -eq 0 && \
|
||||
"$source_readback_exit" -eq 0 \
|
||||
]]
|
||||
jq -e '.present == false' claw590-cleanup-readback.json >/dev/null
|
||||
jq -e '.present == false' claw590-source-cleanup-readback.json >/dev/null
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
[[ "$(git rev-parse HEAD)" == "$DEPLOY_SHA" ]]
|
||||
bunx convex run --no-push appMeta:getDeploymentInfo '{}' > claw590-deployment-readback.json
|
||||
jq -e --arg sha "$DEPLOY_SHA" '.appBuildSha == $sha' claw590-deployment-readback.json
|
||||
proof_args="$(
|
||||
jq -cn \
|
||||
--arg confirm "$confirm" \
|
||||
--arg snapshotId "$snapshot_id" \
|
||||
'{confirm:$confirm,snapshotId:$snapshotId}'
|
||||
)"
|
||||
bunx convex run --no-push \
|
||||
canonicalTrendingTestFixtures:readCanonicalTrendingProof \
|
||||
"$proof_args" > claw590-before.json
|
||||
if jq -e '.present == true' claw590-before.json >/dev/null; then
|
||||
bunx convex run --no-push \
|
||||
canonicalTrendingTestFixtures:cleanupCanonicalTrendingProof \
|
||||
"$proof_args" > claw590-recovery.json
|
||||
bunx convex run --no-push \
|
||||
canonicalTrendingTestFixtures:readCanonicalTrendingProof \
|
||||
"$proof_args" > claw590-recovery-readback.json
|
||||
jq -e '.present == false' claw590-recovery-readback.json >/dev/null
|
||||
else
|
||||
jq -n '{ok:true,skipped:true}' > claw590-recovery.json
|
||||
cp claw590-before.json claw590-recovery-readback.json
|
||||
fi
|
||||
jq -e '.present == false' claw590-recovery-readback.json >/dev/null
|
||||
|
||||
source_args="$(jq -cn --arg confirm "$confirm" '{confirm:$confirm}')"
|
||||
bunx convex run --no-push \
|
||||
canonicalTrendingTestFixtures:readCanonicalTrendingSourceFixture \
|
||||
"$source_args" > claw590-source-before.json
|
||||
if jq -e '.present == true' claw590-source-before.json >/dev/null; then
|
||||
bunx convex run --no-push \
|
||||
canonicalTrendingTestFixtures:cleanupCanonicalTrendingSourceFixture \
|
||||
"$source_args" > claw590-source-recovery.json
|
||||
bunx convex run --no-push \
|
||||
canonicalTrendingTestFixtures:readCanonicalTrendingSourceFixture \
|
||||
"$source_args" > claw590-source-recovery-readback.json
|
||||
jq -e '.present == false' claw590-source-recovery-readback.json >/dev/null
|
||||
else
|
||||
jq -n '{ok:true,skipped:true}' > claw590-source-recovery.json
|
||||
cp claw590-source-before.json claw590-source-recovery-readback.json
|
||||
fi
|
||||
bunx convex run --no-push \
|
||||
canonicalTrendingTestFixtures:seedCanonicalTrendingSourceFixture \
|
||||
"$source_args" > claw590-source-seed.json
|
||||
bunx convex run --no-push \
|
||||
canonicalTrendingTestFixtures:readCanonicalTrendingSourceFixture \
|
||||
"$source_args" > claw590-source-active.json
|
||||
jq -e '
|
||||
.present == true and
|
||||
.nativeCount == 12 and
|
||||
.nativePublisherCount == 6 and
|
||||
.externalCount == 8 and
|
||||
.scansPlanned == 0 and
|
||||
.scansAdmitted == 0
|
||||
' claw590-source-active.json >/dev/null
|
||||
|
||||
bun run trending:prove-test > claw590-proof-output.json
|
||||
jq -e '
|
||||
.target.environment == "permanent Test" and
|
||||
.target.deploySha == env.DEPLOY_SHA and
|
||||
.materialization.snapshotId == ("claw-590-proof-" + env.DEPLOY_SHA) and
|
||||
.materialization.totalItems >= 20 and
|
||||
.materialization.operations.documentsRead > 0 and
|
||||
.materialization.operations.documentsWritten > 0 and
|
||||
.materialization.operations.functionCalls > 0 and
|
||||
.assertions.laneCounts["clawhub-trending"] == 8 and
|
||||
.assertions.laneCounts["clawhub-rising"] == 4 and
|
||||
.assertions.laneCounts["skills-sh-trending"] == 8 and
|
||||
.assertions.maximumPublisherCount <= 2 and
|
||||
.assertions.stableSnapshot == true and
|
||||
.assertions.stableCursor == true and
|
||||
.assertions.stableOrder == true and
|
||||
.assertions.completePagination == true and
|
||||
.assertions.skillsShTrending24hInstalls == null
|
||||
' proof/claw-590/canonical-trending-test-proof.json >/dev/null
|
||||
|
||||
cleanup
|
||||
trap - EXIT
|
||||
jq -e '
|
||||
.cleanupExit == 0 and
|
||||
.readbackExit == 0 and
|
||||
.sourceCleanupExit == 0 and
|
||||
.sourceReadbackExit == 0
|
||||
' claw590-cleanup-status.json >/dev/null
|
||||
jq -e '.present == false' claw590-cleanup-readback.json >/dev/null
|
||||
jq -e '.present == false' claw590-source-cleanup-readback.json >/dev/null
|
||||
|
||||
- name: Upload permanent Test canonical Trending proof
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: claw590-canonical-trending-proof
|
||||
if-no-files-found: error
|
||||
path: |
|
||||
proof/claw-590/canonical-trending-test-proof.json
|
||||
proof/claw-590/active-snapshot.json
|
||||
claw590-proof-output.json
|
||||
claw590-before.json
|
||||
claw590-recovery.json
|
||||
claw590-recovery-readback.json
|
||||
claw590-cleanup.json
|
||||
claw590-cleanup-readback.json
|
||||
claw590-cleanup-status.json
|
||||
claw590-deployment-readback.json
|
||||
claw590-source-before.json
|
||||
claw590-source-recovery.json
|
||||
claw590-source-recovery-readback.json
|
||||
claw590-source-seed.json
|
||||
claw590-source-active.json
|
||||
claw590-source-cleanup.json
|
||||
claw590-source-cleanup-readback.json
|
||||
|
||||
Vendored
+8
@@ -11,6 +11,8 @@
|
||||
import type * as agentSkillsHttp from "../agentSkillsHttp.js";
|
||||
import type * as appMeta from "../appMeta.js";
|
||||
import type * as auth from "../auth.js";
|
||||
import type * as canonicalTrending from "../canonicalTrending.js";
|
||||
import type * as canonicalTrendingTestFixtures from "../canonicalTrendingTestFixtures.js";
|
||||
import type * as catalogClassification from "../catalogClassification.js";
|
||||
import type * as catalogClassificationNode from "../catalogClassificationNode.js";
|
||||
import type * as catalogFeed from "../catalogFeed.js";
|
||||
@@ -46,6 +48,7 @@ import type * as httpApiV1_skillsShCatalogV1 from "../httpApiV1/skillsShCatalogV
|
||||
import type * as httpApiV1_skillsV1 from "../httpApiV1/skillsV1.js";
|
||||
import type * as httpApiV1_starsV1 from "../httpApiV1/starsV1.js";
|
||||
import type * as httpApiV1_transfersV1 from "../httpApiV1/transfersV1.js";
|
||||
import type * as httpApiV1_trendingV1 from "../httpApiV1/trendingV1.js";
|
||||
import type * as httpApiV1_usersV1 from "../httpApiV1/usersV1.js";
|
||||
import type * as httpApiV1_whoamiV1 from "../httpApiV1/whoamiV1.js";
|
||||
import type * as httpPreflight from "../httpPreflight.js";
|
||||
@@ -60,6 +63,7 @@ import type * as lib_batching from "../lib/batching.js";
|
||||
import type * as lib_canonicalSkillSearch from "../lib/canonicalSkillSearch.js";
|
||||
import type * as lib_canonicalSkillSearchBounds from "../lib/canonicalSkillSearchBounds.js";
|
||||
import type * as lib_canonicalSkillSearchResponse from "../lib/canonicalSkillSearchResponse.js";
|
||||
import type * as lib_canonicalTrending from "../lib/canonicalTrending.js";
|
||||
import type * as lib_catalogClassification from "../lib/catalogClassification.js";
|
||||
import type * as lib_catalogClassifier from "../lib/catalogClassifier.js";
|
||||
import type * as lib_changelog from "../lib/changelog.js";
|
||||
@@ -208,6 +212,8 @@ declare const fullApi: ApiFromModules<{
|
||||
agentSkillsHttp: typeof agentSkillsHttp;
|
||||
appMeta: typeof appMeta;
|
||||
auth: typeof auth;
|
||||
canonicalTrending: typeof canonicalTrending;
|
||||
canonicalTrendingTestFixtures: typeof canonicalTrendingTestFixtures;
|
||||
catalogClassification: typeof catalogClassification;
|
||||
catalogClassificationNode: typeof catalogClassificationNode;
|
||||
catalogFeed: typeof catalogFeed;
|
||||
@@ -243,6 +249,7 @@ declare const fullApi: ApiFromModules<{
|
||||
"httpApiV1/skillsV1": typeof httpApiV1_skillsV1;
|
||||
"httpApiV1/starsV1": typeof httpApiV1_starsV1;
|
||||
"httpApiV1/transfersV1": typeof httpApiV1_transfersV1;
|
||||
"httpApiV1/trendingV1": typeof httpApiV1_trendingV1;
|
||||
"httpApiV1/usersV1": typeof httpApiV1_usersV1;
|
||||
"httpApiV1/whoamiV1": typeof httpApiV1_whoamiV1;
|
||||
httpPreflight: typeof httpPreflight;
|
||||
@@ -257,6 +264,7 @@ declare const fullApi: ApiFromModules<{
|
||||
"lib/canonicalSkillSearch": typeof lib_canonicalSkillSearch;
|
||||
"lib/canonicalSkillSearchBounds": typeof lib_canonicalSkillSearchBounds;
|
||||
"lib/canonicalSkillSearchResponse": typeof lib_canonicalSkillSearchResponse;
|
||||
"lib/canonicalTrending": typeof lib_canonicalTrending;
|
||||
"lib/catalogClassification": typeof lib_catalogClassification;
|
||||
"lib/catalogClassifier": typeof lib_catalogClassifier;
|
||||
"lib/changelog": typeof lib_changelog;
|
||||
|
||||
@@ -0,0 +1,592 @@
|
||||
/// <reference types="vite/client" />
|
||||
/* @vitest-environment edge-runtime */
|
||||
import { convexTest } from "convex-test";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { internal } from "./_generated/api";
|
||||
import schema from "./schema";
|
||||
|
||||
const modules = import.meta.glob("./**/*.ts");
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
function nativeCard(id: string, installs24h: number) {
|
||||
const slug = id.replace("clawhub:", "");
|
||||
return {
|
||||
id,
|
||||
source: "clawhub" as const,
|
||||
slug,
|
||||
displayName: slug,
|
||||
summary: null,
|
||||
canonicalUrl: `/patrick/skills/${slug}`,
|
||||
links: { canonical: `/patrick/skills/${slug}`, source: null },
|
||||
publisher: {
|
||||
kind: "user" as const,
|
||||
handle: "patrick",
|
||||
displayName: "Patrick",
|
||||
image: null,
|
||||
official: false,
|
||||
},
|
||||
official: false,
|
||||
featured: false,
|
||||
install: { kind: "clawhub" as const, reference: `patrick/${slug}`, sourceUrl: null },
|
||||
sourceIdentity: {
|
||||
id,
|
||||
owner: "patrick",
|
||||
repo: null,
|
||||
host: null,
|
||||
lifetimeInstalls: null,
|
||||
},
|
||||
trust: {
|
||||
visibility: "public" as const,
|
||||
installability: "installable" as const,
|
||||
clawHubVerdict: null,
|
||||
upstreamScanners: null,
|
||||
sourceFreshness: "native" as const,
|
||||
},
|
||||
metrics: {
|
||||
trending24hInstalls: installs24h,
|
||||
trending24hBookmarks: 0,
|
||||
lifetimeInstalls: null,
|
||||
lifetimeInstallsPeriod: "lifetime" as const,
|
||||
updatedAt: 1_000,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function insertEligibleNativeSource(t: ReturnType<typeof convexTest>, slug: string) {
|
||||
return await t.run(async (ctx) => {
|
||||
const now = Date.now();
|
||||
const userId = await ctx.db.insert("users", {
|
||||
handle: "patrick",
|
||||
displayName: "Patrick",
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
});
|
||||
const skillId = await ctx.db.insert("skills", {
|
||||
slug,
|
||||
displayName: slug,
|
||||
ownerUserId: userId,
|
||||
tags: {},
|
||||
stats: { downloads: 0, stars: 0, versions: 1, comments: 0 },
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
});
|
||||
const versionId = await ctx.db.insert("skillVersions", {
|
||||
skillId,
|
||||
version: "1.0.0",
|
||||
changelog: "Initial",
|
||||
files: [],
|
||||
parsed: { frontmatter: {} },
|
||||
createdBy: userId,
|
||||
createdAt: now,
|
||||
});
|
||||
const digestId = await ctx.db.insert("skillSearchDigest", {
|
||||
skillId,
|
||||
slug,
|
||||
displayName: slug,
|
||||
ownerUserId: userId,
|
||||
ownerHandle: "patrick",
|
||||
ownerKind: "user",
|
||||
ownerName: "patrick",
|
||||
ownerDisplayName: "Patrick",
|
||||
latestVersionId: versionId,
|
||||
latestVersionSkillId: skillId,
|
||||
publicVersion: { status: "available", versionId },
|
||||
tags: {},
|
||||
stats: { downloads: 0, stars: 0, versions: 1, comments: 0 },
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
});
|
||||
return { digestId, skillId };
|
||||
});
|
||||
}
|
||||
|
||||
describe("canonical Trending snapshot storage", () => {
|
||||
it("selects the newest completed Trending run even when no digest references it", async () => {
|
||||
const t = convexTest(schema, modules);
|
||||
const makeRun = (snapshotId: string, startedAt: number) => ({
|
||||
snapshotId,
|
||||
sourceView: "trending" as const,
|
||||
sourceSnapshotHash: snapshotId.padEnd(64, "a").slice(0, 64),
|
||||
status: "completed" as const,
|
||||
sourceTotal: 0,
|
||||
sourcePageSize: 100,
|
||||
sourceMeasuredAt: new Date(startedAt).toISOString(),
|
||||
page: 1,
|
||||
offset: 0,
|
||||
counts: {
|
||||
observed: 0,
|
||||
inserted: 0,
|
||||
updated: 0,
|
||||
unchanged: 0,
|
||||
rejected: 0,
|
||||
conflicts: 0,
|
||||
detailsInserted: 0,
|
||||
detailsUpdated: 0,
|
||||
detailsUnchanged: 0,
|
||||
detailsMissing: 0,
|
||||
detailsTruncated: 0,
|
||||
tombstoned: 0,
|
||||
reactivated: 0,
|
||||
scansPlanned: 0 as const,
|
||||
scansAdmitted: 0 as const,
|
||||
},
|
||||
operations: { functionCalls: 1, dbReads: 1, dbWrites: 1, sourceRequests: 1, sourceBytes: 0 },
|
||||
actor: "runtime-test",
|
||||
reason: "canonical Trending run selection test",
|
||||
startedAt,
|
||||
completedAt: startedAt + 1,
|
||||
updatedAt: startedAt + 1,
|
||||
});
|
||||
const { latestId } = await t.run(async (ctx) => {
|
||||
await ctx.db.insert("skillsShMirrorRuns", makeRun("older-run", 100));
|
||||
const insertedLatestId = await ctx.db.insert(
|
||||
"skillsShMirrorRuns",
|
||||
makeRun("empty-latest-run", 200),
|
||||
);
|
||||
return { latestId: insertedLatestId };
|
||||
});
|
||||
|
||||
const result = await t.query(
|
||||
internal.canonicalTrending.getLatestCompletedTrendingRunInternal,
|
||||
{},
|
||||
);
|
||||
|
||||
expect(result.runId).toBe(latestId);
|
||||
});
|
||||
|
||||
it("keeps pagination pinned to the snapshot encoded by the cursor", async () => {
|
||||
const t = convexTest(schema, modules);
|
||||
const source = await insertEligibleNativeSource(t, "pagination-source");
|
||||
|
||||
await t.mutation(internal.canonicalTrending.startSnapshotInternal, {
|
||||
snapshotId: "skills-1000",
|
||||
generatedAt: 1_000,
|
||||
expiresAt: Date.now() + 100_000,
|
||||
windowStartDay: 40,
|
||||
windowEndDay: 40,
|
||||
});
|
||||
await t.mutation(internal.canonicalTrending.writeItemsInternal, {
|
||||
snapshotId: "skills-1000",
|
||||
items: [
|
||||
{
|
||||
position: 0,
|
||||
lane: "clawhub-trending",
|
||||
sourceRef: { kind: "clawhub", skillId: source.skillId },
|
||||
card: nativeCard("clawhub:one", 3),
|
||||
},
|
||||
{
|
||||
position: 1,
|
||||
lane: "skills-sh-trending",
|
||||
sourceRef: { kind: "clawhub", skillId: source.skillId },
|
||||
card: nativeCard("clawhub:two", 2),
|
||||
},
|
||||
{
|
||||
position: 2,
|
||||
lane: "clawhub-rising",
|
||||
sourceRef: { kind: "clawhub", skillId: source.skillId },
|
||||
card: nativeCard("clawhub:three", 1),
|
||||
},
|
||||
],
|
||||
});
|
||||
await t.mutation(internal.canonicalTrending.finalizeSnapshotInternal, {
|
||||
snapshotId: "skills-1000",
|
||||
completedAt: 1_050,
|
||||
totalItems: 3,
|
||||
sourceCounts: { clawhubTrending: 1, clawhubRising: 1, skillsShTrending: 1 },
|
||||
operations: { documentsRead: 12, documentsWritten: 4, functionCalls: 3 },
|
||||
});
|
||||
|
||||
const firstResult = await t.query(internal.canonicalTrending.getPageInternal, {
|
||||
cursor: null,
|
||||
limit: 2,
|
||||
});
|
||||
expect(firstResult.status).toBe("ok");
|
||||
if (firstResult.status !== "ok") throw new Error("Expected a ready Trending page");
|
||||
const firstPage = firstResult.page;
|
||||
expect(firstPage).toMatchObject({
|
||||
kind: "skills",
|
||||
snapshotId: "skills-1000",
|
||||
generatedAt: "1970-01-01T00:00:01.000Z",
|
||||
windowHours: 24,
|
||||
rankingVersion: "skills-trending-v1",
|
||||
items: [
|
||||
{ id: "clawhub:one", rank: 1, lane: "clawhub-trending" },
|
||||
{ id: "clawhub:two", rank: 2, lane: "skills-sh-trending" },
|
||||
],
|
||||
});
|
||||
expect(firstPage?.nextCursor).toEqual(expect.any(String));
|
||||
|
||||
await t.mutation(internal.canonicalTrending.startSnapshotInternal, {
|
||||
snapshotId: "skills-2000",
|
||||
generatedAt: 2_000,
|
||||
expiresAt: Date.now() + 100_000,
|
||||
windowStartDay: 41,
|
||||
windowEndDay: 41,
|
||||
});
|
||||
await t.mutation(internal.canonicalTrending.writeItemsInternal, {
|
||||
snapshotId: "skills-2000",
|
||||
items: [
|
||||
{
|
||||
position: 0,
|
||||
lane: "clawhub-trending",
|
||||
sourceRef: { kind: "clawhub", skillId: source.skillId },
|
||||
card: nativeCard("clawhub:new", 10),
|
||||
},
|
||||
],
|
||||
});
|
||||
await t.mutation(internal.canonicalTrending.finalizeSnapshotInternal, {
|
||||
snapshotId: "skills-2000",
|
||||
completedAt: 2_050,
|
||||
totalItems: 1,
|
||||
sourceCounts: { clawhubTrending: 1, clawhubRising: 0, skillsShTrending: 0 },
|
||||
operations: { documentsRead: 5, documentsWritten: 2, functionCalls: 3 },
|
||||
});
|
||||
|
||||
const secondResult = await t.query(internal.canonicalTrending.getPageInternal, {
|
||||
cursor: firstPage?.nextCursor ?? null,
|
||||
limit: 2,
|
||||
});
|
||||
expect(secondResult.status).toBe("ok");
|
||||
if (secondResult.status !== "ok") throw new Error("Expected the pinned Trending page");
|
||||
const secondPage = secondResult.page;
|
||||
expect(secondPage?.snapshotId).toBe("skills-1000");
|
||||
expect(secondPage?.items).toEqual([
|
||||
expect.objectContaining({ id: "clawhub:three", rank: 3, lane: "clawhub-rising" }),
|
||||
]);
|
||||
expect(secondPage?.nextCursor).toBeNull();
|
||||
|
||||
await t.run(async (ctx) => {
|
||||
await ctx.db.patch(source.digestId, { softDeletedAt: Date.now() });
|
||||
});
|
||||
const revokedResult = await t.query(internal.canonicalTrending.getPageInternal, {
|
||||
cursor: firstPage.snapshotCursor,
|
||||
limit: 2,
|
||||
});
|
||||
expect(revokedResult.status).toBe("ok");
|
||||
if (revokedResult.status !== "ok") throw new Error("Expected the pinned Trending page");
|
||||
expect(revokedResult.page.items).toEqual([]);
|
||||
});
|
||||
|
||||
it("rejects an expired stable cursor before reading pruned item rows", async () => {
|
||||
const t = convexTest(schema, modules);
|
||||
await t.mutation(internal.canonicalTrending.startSnapshotInternal, {
|
||||
snapshotId: "skills-expired",
|
||||
generatedAt: 1_000,
|
||||
expiresAt: Date.now() - 1,
|
||||
windowStartDay: 40,
|
||||
windowEndDay: 40,
|
||||
});
|
||||
await t.mutation(internal.canonicalTrending.finalizeSnapshotInternal, {
|
||||
snapshotId: "skills-expired",
|
||||
completedAt: 1_050,
|
||||
totalItems: 0,
|
||||
sourceCounts: { clawhubTrending: 0, clawhubRising: 0, skillsShTrending: 0 },
|
||||
operations: { documentsRead: 1, documentsWritten: 2, functionCalls: 2 },
|
||||
});
|
||||
|
||||
expect(
|
||||
await t.query(internal.canonicalTrending.getPageInternal, {
|
||||
cursor: "eyJ2IjoxLCJzIjoic2tpbGxzLWV4cGlyZWQiLCJvIjowfQ",
|
||||
limit: 20,
|
||||
}),
|
||||
).toEqual({ status: "expired" });
|
||||
expect(
|
||||
await t.query(internal.canonicalTrending.getPageInternal, { cursor: null, limit: 20 }),
|
||||
).toEqual({ status: "unavailable" });
|
||||
});
|
||||
|
||||
it("returns a typed error for malformed cursors", async () => {
|
||||
const t = convexTest(schema, modules);
|
||||
|
||||
expect(
|
||||
await t.query(internal.canonicalTrending.getPageInternal, {
|
||||
cursor: "not-a-cursor",
|
||||
limit: 20,
|
||||
}),
|
||||
).toEqual({ status: "invalid-cursor" });
|
||||
});
|
||||
|
||||
it("does not read or write source state while the rollout is dark", async () => {
|
||||
vi.stubEnv("CLAWHUB_ENV", "production");
|
||||
vi.stubEnv("CLAWHUB_SKILLS_SH_ROLLOUT_MODE", "off");
|
||||
const t = convexTest(schema, modules);
|
||||
|
||||
const result = await t.action(internal.canonicalTrending.materializeInternal, {});
|
||||
const snapshots = await t.run(async (ctx) =>
|
||||
ctx.db.query("canonicalTrendingSnapshots").collect(),
|
||||
);
|
||||
|
||||
expect(result).toEqual({ status: "disabled" });
|
||||
expect(snapshots).toEqual([]);
|
||||
});
|
||||
|
||||
it("prunes expired snapshots independently while materialization is dark", async () => {
|
||||
vi.stubEnv("CLAWHUB_ENV", "production");
|
||||
vi.stubEnv("CLAWHUB_SKILLS_SH_ROLLOUT_MODE", "off");
|
||||
const t = convexTest(schema, modules);
|
||||
const source = await insertEligibleNativeSource(t, "cleanup-source");
|
||||
await t.mutation(internal.canonicalTrending.startSnapshotInternal, {
|
||||
snapshotId: "skills-expired-cleanup",
|
||||
generatedAt: 1_000,
|
||||
expiresAt: Date.now() - 1,
|
||||
windowStartDay: 40,
|
||||
windowEndDay: 40,
|
||||
});
|
||||
await t.mutation(internal.canonicalTrending.writeItemsInternal, {
|
||||
snapshotId: "skills-expired-cleanup",
|
||||
items: [
|
||||
{
|
||||
position: 0,
|
||||
lane: "clawhub-trending",
|
||||
sourceRef: { kind: "clawhub", skillId: source.skillId },
|
||||
card: nativeCard("clawhub:old", 1),
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const result = await t.action(internal.canonicalTrending.pruneExpiredActionInternal, {});
|
||||
const rows = await t.run(async (ctx) => ({
|
||||
snapshots: await ctx.db.query("canonicalTrendingSnapshots").collect(),
|
||||
items: await ctx.db.query("canonicalTrendingItems").collect(),
|
||||
}));
|
||||
|
||||
expect(result).toEqual({
|
||||
itemsDeleted: 1,
|
||||
snapshotsDeleted: 1,
|
||||
batches: 1,
|
||||
continuationScheduled: false,
|
||||
});
|
||||
expect(rows).toEqual({ snapshots: [], items: [] });
|
||||
});
|
||||
|
||||
it("materializes imported 24-hour metrics into a ready snapshot", async () => {
|
||||
vi.stubEnv("CLAWHUB_ENV", "test");
|
||||
vi.stubEnv("CLAWHUB_SKILLS_SH_ROLLOUT_MODE", "test");
|
||||
const t = convexTest(schema, modules);
|
||||
const now = Date.now();
|
||||
|
||||
await t.run(async (ctx) => {
|
||||
const userId = await ctx.db.insert("users", {
|
||||
handle: "patrick",
|
||||
displayName: "Patrick",
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
});
|
||||
const skillId = await ctx.db.insert("skills", {
|
||||
slug: "native",
|
||||
displayName: "Native",
|
||||
summary: "Native summary",
|
||||
ownerUserId: userId,
|
||||
tags: {},
|
||||
statsInstallsAllTime: 900,
|
||||
stats: { downloads: 1_000, installsAllTime: 900, stars: 20, versions: 1, comments: 0 },
|
||||
createdAt: now - 1_000,
|
||||
updatedAt: now,
|
||||
});
|
||||
const versionId = await ctx.db.insert("skillVersions", {
|
||||
skillId,
|
||||
version: "1.0.0",
|
||||
changelog: "Initial",
|
||||
files: [],
|
||||
parsed: { frontmatter: {} },
|
||||
createdBy: userId,
|
||||
createdAt: now,
|
||||
});
|
||||
await ctx.db.insert("skillSearchDigest", {
|
||||
skillId,
|
||||
slug: "native",
|
||||
displayName: "Native",
|
||||
summary: "Native summary",
|
||||
ownerUserId: userId,
|
||||
ownerHandle: "patrick",
|
||||
ownerKind: "user",
|
||||
ownerName: "patrick",
|
||||
ownerDisplayName: "Patrick",
|
||||
latestVersionId: versionId,
|
||||
latestVersionSkillId: skillId,
|
||||
publicVersion: { status: "available", versionId },
|
||||
tags: {},
|
||||
statsInstallsAllTime: 900,
|
||||
stats: { downloads: 1_000, installsAllTime: 900, stars: 20, versions: 1, comments: 0 },
|
||||
createdAt: now - 1_000,
|
||||
updatedAt: now,
|
||||
});
|
||||
await ctx.db.insert("rankingMetricImports", {
|
||||
datasetVersion: "ranking-test-v1",
|
||||
checksum: "a".repeat(64),
|
||||
generatedAt: new Date(now).toISOString(),
|
||||
importedAt: now,
|
||||
startDay: 1,
|
||||
endDay: 60,
|
||||
targetCount: 1,
|
||||
skillTargetCount: 1,
|
||||
packageTargetCount: 0,
|
||||
dailyRowCount: 1,
|
||||
importedSkillRows: 1,
|
||||
importedPackageRows: 0,
|
||||
unresolvedTargets: 0,
|
||||
skippedOverlayRows: 0,
|
||||
});
|
||||
await ctx.db.insert("skillDailyStats", {
|
||||
skillId,
|
||||
day: 60,
|
||||
downloads: 18,
|
||||
installs: 12,
|
||||
bookmarks: 4,
|
||||
rankingDatasetVersion: "ranking-test-v1",
|
||||
rankingImportedAt: now,
|
||||
updatedAt: now,
|
||||
});
|
||||
await ctx.db.insert("skillDailyStats", {
|
||||
skillId,
|
||||
day: 61,
|
||||
downloads: 999,
|
||||
installs: 999,
|
||||
bookmarks: 999,
|
||||
updatedAt: now + 1,
|
||||
});
|
||||
const trendingRunId = await ctx.db.insert("skillsShMirrorRuns", {
|
||||
snapshotId: "skills-sh-trending-runtime",
|
||||
sourceView: "trending",
|
||||
sourceSnapshotHash: "b".repeat(64),
|
||||
status: "completed",
|
||||
sourceTotal: 1,
|
||||
sourcePageSize: 100,
|
||||
sourceMeasuredAt: new Date(now).toISOString(),
|
||||
page: 1,
|
||||
offset: 0,
|
||||
counts: {
|
||||
observed: 1,
|
||||
inserted: 0,
|
||||
updated: 1,
|
||||
unchanged: 0,
|
||||
rejected: 0,
|
||||
conflicts: 0,
|
||||
detailsInserted: 0,
|
||||
detailsUpdated: 0,
|
||||
detailsUnchanged: 0,
|
||||
detailsMissing: 0,
|
||||
detailsTruncated: 0,
|
||||
tombstoned: 0,
|
||||
reactivated: 0,
|
||||
scansPlanned: 0,
|
||||
scansAdmitted: 0,
|
||||
},
|
||||
operations: {
|
||||
functionCalls: 1,
|
||||
dbReads: 1,
|
||||
dbWrites: 1,
|
||||
sourceRequests: 1,
|
||||
sourceBytes: 100,
|
||||
},
|
||||
actor: "runtime-test",
|
||||
reason: "canonical Trending runtime test",
|
||||
startedAt: now - 100,
|
||||
completedAt: now,
|
||||
updatedAt: now,
|
||||
});
|
||||
await ctx.db.insert("skillsShMirrorDigests", {
|
||||
externalId: "patrick/repo/external",
|
||||
sourceType: "github",
|
||||
owner: "patrick",
|
||||
repo: "repo",
|
||||
slug: "external",
|
||||
normalizedSlug: "external",
|
||||
normalizedSlugFirstToken: "external",
|
||||
displayName: "External",
|
||||
normalizedDisplayName: "external",
|
||||
normalizedDisplayNameFirstToken: "external",
|
||||
searchSummary: "External summary",
|
||||
searchText: "external external summary",
|
||||
sourceUrl: "https://skills.sh/patrick/repo/external",
|
||||
canonicalRepoUrl: "https://github.com/patrick/repo",
|
||||
upstreamInstalls: 4_000,
|
||||
trendingRank: 1,
|
||||
trendingLifetimeInstalls: 4_200,
|
||||
trendingObservedAt: now,
|
||||
trendingSnapshotId: "skills-sh-trending-runtime",
|
||||
trendingObservedRunId: trendingRunId,
|
||||
upstreamScanners: {
|
||||
genAgentTrustHub: { status: "pass" },
|
||||
socket: { status: "pass" },
|
||||
snyk: { status: "pass" },
|
||||
},
|
||||
sourceFreshnessStatus: "observed-only",
|
||||
detailStatus: "available",
|
||||
observationFingerprint: "c".repeat(64),
|
||||
sourceSnapshotId: "skills-sh-trending-runtime",
|
||||
lastObservedRunId: trendingRunId,
|
||||
active: true,
|
||||
publicVisible: true,
|
||||
installable: true,
|
||||
firstObservedAt: now - 500,
|
||||
lastObservedAt: now,
|
||||
createdAt: now - 500,
|
||||
updatedAt: now,
|
||||
});
|
||||
});
|
||||
|
||||
const result = await t.action(internal.canonicalTrending.materializeInternal, {});
|
||||
expect(result).toMatchObject({
|
||||
status: "ready",
|
||||
totalItems: 2,
|
||||
sourceCounts: { clawhubTrending: 1, clawhubRising: 1, skillsShTrending: 1 },
|
||||
sample: [
|
||||
{
|
||||
rank: 1,
|
||||
lane: "clawhub-trending",
|
||||
id: expect.stringMatching(/^clawhub:/),
|
||||
trending24hInstalls: 12,
|
||||
lifetimeInstalls: 900,
|
||||
},
|
||||
{
|
||||
rank: 2,
|
||||
lane: "skills-sh-trending",
|
||||
id: "skills-sh:patrick/repo/external",
|
||||
trending24hInstalls: null,
|
||||
lifetimeInstalls: 4_200,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const pageResult = await t.query(internal.canonicalTrending.getPageInternal, {
|
||||
cursor: null,
|
||||
limit: 20,
|
||||
});
|
||||
expect(pageResult.status).toBe("ok");
|
||||
if (pageResult.status !== "ok") throw new Error("Expected a materialized Trending page");
|
||||
const page = pageResult.page;
|
||||
expect(page?.items).toEqual([
|
||||
expect.objectContaining({
|
||||
source: "clawhub",
|
||||
rank: 1,
|
||||
metrics: {
|
||||
trending24hInstalls: 12,
|
||||
trending24hBookmarks: 4,
|
||||
lifetimeInstalls: 900,
|
||||
lifetimeInstallsPeriod: "lifetime",
|
||||
updatedAt: now,
|
||||
},
|
||||
}),
|
||||
expect.objectContaining({
|
||||
source: "skills-sh",
|
||||
rank: 2,
|
||||
install: {
|
||||
kind: "skills-sh",
|
||||
reference: "skills-sh:patrick/repo/external",
|
||||
sourceUrl: "https://skills.sh/patrick/repo/external",
|
||||
},
|
||||
metrics: {
|
||||
trending24hInstalls: null,
|
||||
trending24hBookmarks: null,
|
||||
lifetimeInstalls: 4_200,
|
||||
lifetimeInstallsPeriod: "lifetime",
|
||||
updatedAt: now,
|
||||
},
|
||||
}),
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,700 @@
|
||||
import { paginationOptsValidator } from "convex/server";
|
||||
import { v } from "convex/values";
|
||||
import { internal } from "./_generated/api";
|
||||
import type { Doc } from "./_generated/dataModel";
|
||||
import { internalAction, internalMutation, internalQuery } from "./_generated/server";
|
||||
import {
|
||||
CANONICAL_TRENDING_RANKING_VERSION,
|
||||
CANONICAL_TRENDING_WINDOW_HOURS,
|
||||
blendCanonicalTrendingPools,
|
||||
buildExternalCanonicalTrendingCandidate,
|
||||
buildNativeCanonicalTrendingCandidate,
|
||||
canonicalTrendingCardValidator,
|
||||
canonicalTrendingSourceRefValidator,
|
||||
decodeCanonicalTrendingCursor,
|
||||
encodeCanonicalTrendingCursor,
|
||||
type CanonicalTrendingMaterializationCandidate,
|
||||
} from "./lib/canonicalTrending";
|
||||
import { shouldExcludeSkillFromPublicBrowse } from "./lib/publicBrowse";
|
||||
import { getRuntimeRolloutCapabilities } from "./lib/rolloutCapabilities";
|
||||
import { isPublicSkillsShMirrorDigest } from "./lib/skillsShMirrorPublic";
|
||||
import { assertTestSeedAllowed } from "./lib/testSeed";
|
||||
|
||||
const SOURCE_PAGE_SIZE = 250;
|
||||
const WRITE_BATCH_SIZE = 100;
|
||||
const SNAPSHOT_RETENTION_MS = 48 * 60 * 60 * 1_000;
|
||||
const RISING_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1_000;
|
||||
const PRUNE_BATCH_SIZE = 500;
|
||||
const PRUNE_MAX_BATCHES = 20;
|
||||
|
||||
const internalRefs = internal as unknown as {
|
||||
canonicalTrending: {
|
||||
failSnapshotInternal: unknown;
|
||||
finalizeSnapshotInternal: unknown;
|
||||
getExternalSourcePageInternal: unknown;
|
||||
getMetricSourcePageInternal: unknown;
|
||||
getMetricWindowInternal: unknown;
|
||||
getNativeSourcePageInternal: unknown;
|
||||
getLatestCompletedTrendingRunInternal: unknown;
|
||||
pruneExpiredInternal: unknown;
|
||||
pruneExpiredActionInternal: unknown;
|
||||
startSnapshotInternal: unknown;
|
||||
writeItemsInternal: unknown;
|
||||
};
|
||||
};
|
||||
|
||||
async function pruneExpiredRows(
|
||||
ctx: { runMutation: (ref: never, args: never) => Promise<unknown> },
|
||||
now: number,
|
||||
) {
|
||||
let itemsDeleted = 0;
|
||||
let snapshotsDeleted = 0;
|
||||
let batches = 0;
|
||||
for (; batches < PRUNE_MAX_BATCHES; batches += 1) {
|
||||
const pruned = (await ctx.runMutation(
|
||||
internalRefs.canonicalTrending.pruneExpiredInternal as never,
|
||||
{ now, batchSize: PRUNE_BATCH_SIZE } as never,
|
||||
)) as { itemsDeleted: number; snapshotsDeleted: number; fullBatch: boolean };
|
||||
itemsDeleted += pruned.itemsDeleted;
|
||||
snapshotsDeleted += pruned.snapshotsDeleted;
|
||||
if (!pruned.fullBatch) return { itemsDeleted, snapshotsDeleted, batches: batches + 1 };
|
||||
}
|
||||
return { itemsDeleted, snapshotsDeleted, batches };
|
||||
}
|
||||
|
||||
const laneValidator = v.union(
|
||||
v.literal("clawhub-trending"),
|
||||
v.literal("clawhub-rising"),
|
||||
v.literal("skills-sh-trending"),
|
||||
);
|
||||
|
||||
const sourceCountsValidator = v.object({
|
||||
clawhubTrending: v.number(),
|
||||
clawhubRising: v.number(),
|
||||
skillsShTrending: v.number(),
|
||||
});
|
||||
|
||||
const operationsValidator = v.object({
|
||||
documentsRead: v.number(),
|
||||
documentsWritten: v.number(),
|
||||
functionCalls: v.number(),
|
||||
});
|
||||
|
||||
type SourcePage<T> = {
|
||||
page: T[];
|
||||
isDone: boolean;
|
||||
continueCursor: string;
|
||||
documentsRead: number;
|
||||
};
|
||||
|
||||
type CollectedSource<T> = {
|
||||
rows: T[];
|
||||
documentsRead: number;
|
||||
functionCalls: number;
|
||||
};
|
||||
|
||||
async function collectSourcePages(
|
||||
ctx: { runQuery: (ref: never, args: never) => Promise<unknown> },
|
||||
ref: unknown,
|
||||
args: Record<string, unknown> = {},
|
||||
) {
|
||||
const rows: unknown[] = [];
|
||||
let cursor: string | null = null;
|
||||
let documentsRead = 0;
|
||||
let functionCalls = 0;
|
||||
do {
|
||||
const result = (await ctx.runQuery(
|
||||
ref as never,
|
||||
{
|
||||
...args,
|
||||
paginationOpts: { cursor, numItems: SOURCE_PAGE_SIZE },
|
||||
} as never,
|
||||
)) as SourcePage<unknown>;
|
||||
rows.push(...result.page);
|
||||
documentsRead += result.documentsRead;
|
||||
functionCalls += 1;
|
||||
cursor = result.isDone ? null : result.continueCursor;
|
||||
} while (cursor);
|
||||
return { rows, documentsRead, functionCalls };
|
||||
}
|
||||
|
||||
export const getMetricWindowInternal = internalQuery({
|
||||
args: {},
|
||||
handler: async (ctx) => {
|
||||
const latestImport = await ctx.db
|
||||
.query("rankingMetricImports")
|
||||
.withIndex("by_imported_at")
|
||||
.order("desc")
|
||||
.first();
|
||||
return latestImport
|
||||
? {
|
||||
datasetVersion: latestImport.datasetVersion,
|
||||
importedAt: latestImport.importedAt,
|
||||
startDay: latestImport.endDay,
|
||||
endDay: latestImport.endDay,
|
||||
documentsRead: 1,
|
||||
}
|
||||
: null;
|
||||
},
|
||||
});
|
||||
|
||||
export const getNativeSourcePageInternal = internalQuery({
|
||||
args: { paginationOpts: paginationOptsValidator },
|
||||
handler: async (ctx, args) => {
|
||||
const result = await ctx.db
|
||||
.query("skillSearchDigest")
|
||||
.withIndex("by_active_updated", (q) => q.eq("softDeletedAt", undefined))
|
||||
.paginate(args.paginationOpts);
|
||||
return {
|
||||
...result,
|
||||
page: result.page.filter(
|
||||
(digest) =>
|
||||
!shouldExcludeSkillFromPublicBrowse(digest) &&
|
||||
digest.publicVersion?.status === "available",
|
||||
),
|
||||
documentsRead: result.page.length,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
export const getMetricSourcePageInternal = internalQuery({
|
||||
args: {
|
||||
day: v.number(),
|
||||
datasetVersion: v.string(),
|
||||
importedAt: v.number(),
|
||||
paginationOpts: paginationOptsValidator,
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const result = await ctx.db
|
||||
.query("skillDailyStats")
|
||||
.withIndex("by_day", (q) => q.eq("day", args.day))
|
||||
.paginate(args.paginationOpts);
|
||||
return {
|
||||
...result,
|
||||
page: result.page.filter(
|
||||
(row) =>
|
||||
row.rankingDatasetVersion === args.datasetVersion &&
|
||||
row.rankingImportedAt === args.importedAt,
|
||||
),
|
||||
documentsRead: result.page.length,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
export const getExternalSourcePageInternal = internalQuery({
|
||||
args: { paginationOpts: paginationOptsValidator },
|
||||
handler: async (ctx, args) => {
|
||||
const result = await ctx.db
|
||||
.query("skillsShMirrorDigests")
|
||||
.withIndex("by_active_visible_installable_fresh_slug", (q) =>
|
||||
q
|
||||
.eq("active", true)
|
||||
.eq("publicVisible", true)
|
||||
.eq("installable", true)
|
||||
.eq("sourceFreshnessStatus", "observed-only"),
|
||||
)
|
||||
.paginate(args.paginationOpts);
|
||||
return {
|
||||
...result,
|
||||
page: result.page.filter(
|
||||
(digest) => isPublicSkillsShMirrorDigest(digest) && digest.trendingRank !== undefined,
|
||||
),
|
||||
documentsRead: result.page.length,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
export const getLatestCompletedTrendingRunInternal = internalQuery({
|
||||
args: {},
|
||||
handler: async (ctx) => {
|
||||
const run = await ctx.db
|
||||
.query("skillsShMirrorRuns")
|
||||
.withIndex("by_started_at")
|
||||
.order("desc")
|
||||
.filter((q) =>
|
||||
q.and(q.eq(q.field("sourceView"), "trending"), q.eq(q.field("status"), "completed")),
|
||||
)
|
||||
.first();
|
||||
return { runId: run?._id ?? null, documentsRead: Number(Boolean(run)) };
|
||||
},
|
||||
});
|
||||
|
||||
export const startSnapshotInternal = internalMutation({
|
||||
args: {
|
||||
snapshotId: v.string(),
|
||||
generatedAt: v.number(),
|
||||
expiresAt: v.number(),
|
||||
windowStartDay: v.number(),
|
||||
windowEndDay: v.number(),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const existing = await ctx.db
|
||||
.query("canonicalTrendingSnapshots")
|
||||
.withIndex("by_snapshot_id", (q) => q.eq("snapshotId", args.snapshotId))
|
||||
.unique();
|
||||
if (existing) throw new Error("Trending snapshot already exists");
|
||||
return await ctx.db.insert("canonicalTrendingSnapshots", {
|
||||
snapshotId: args.snapshotId,
|
||||
kind: "skills",
|
||||
status: "building",
|
||||
rankingVersion: CANONICAL_TRENDING_RANKING_VERSION,
|
||||
generatedAt: args.generatedAt,
|
||||
expiresAt: args.expiresAt,
|
||||
windowHours: CANONICAL_TRENDING_WINDOW_HOURS,
|
||||
windowStartDay: args.windowStartDay,
|
||||
windowEndDay: args.windowEndDay,
|
||||
writtenItems: 0,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
export const writeItemsInternal = internalMutation({
|
||||
args: {
|
||||
snapshotId: v.string(),
|
||||
items: v.array(
|
||||
v.object({
|
||||
position: v.number(),
|
||||
lane: laneValidator,
|
||||
sourceRef: canonicalTrendingSourceRefValidator,
|
||||
card: canonicalTrendingCardValidator,
|
||||
}),
|
||||
),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const snapshot = await ctx.db
|
||||
.query("canonicalTrendingSnapshots")
|
||||
.withIndex("by_snapshot_id", (q) => q.eq("snapshotId", args.snapshotId))
|
||||
.unique();
|
||||
if (!snapshot || snapshot.status !== "building") {
|
||||
throw new Error("Trending snapshot is not writable");
|
||||
}
|
||||
for (const item of args.items) {
|
||||
if (!Number.isSafeInteger(item.position) || item.position < 0) {
|
||||
throw new Error("Invalid Trending position");
|
||||
}
|
||||
await ctx.db.insert("canonicalTrendingItems", {
|
||||
snapshotId: args.snapshotId,
|
||||
position: item.position,
|
||||
lane: item.lane,
|
||||
sourceRef: item.sourceRef,
|
||||
card: item.card,
|
||||
expiresAt: snapshot.expiresAt,
|
||||
});
|
||||
}
|
||||
await ctx.db.patch(snapshot._id, { writtenItems: snapshot.writtenItems + args.items.length });
|
||||
return { writtenItems: snapshot.writtenItems + args.items.length };
|
||||
},
|
||||
});
|
||||
|
||||
export const finalizeSnapshotInternal = internalMutation({
|
||||
args: {
|
||||
snapshotId: v.string(),
|
||||
completedAt: v.number(),
|
||||
totalItems: v.number(),
|
||||
sourceCounts: sourceCountsValidator,
|
||||
operations: operationsValidator,
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const snapshot = await ctx.db
|
||||
.query("canonicalTrendingSnapshots")
|
||||
.withIndex("by_snapshot_id", (q) => q.eq("snapshotId", args.snapshotId))
|
||||
.unique();
|
||||
if (!snapshot || snapshot.status !== "building") {
|
||||
throw new Error("Trending snapshot cannot be finalized");
|
||||
}
|
||||
if (snapshot.writtenItems !== args.totalItems) {
|
||||
throw new Error("Trending snapshot item count mismatch");
|
||||
}
|
||||
await ctx.db.patch(snapshot._id, {
|
||||
status: "ready",
|
||||
completedAt: args.completedAt,
|
||||
totalItems: args.totalItems,
|
||||
sourceCounts: args.sourceCounts,
|
||||
operations: args.operations,
|
||||
});
|
||||
return { snapshotId: args.snapshotId, status: "ready" as const };
|
||||
},
|
||||
});
|
||||
|
||||
export const failSnapshotInternal = internalMutation({
|
||||
args: {
|
||||
snapshotId: v.string(),
|
||||
error: v.string(),
|
||||
completedAt: v.number(),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const snapshot = await ctx.db
|
||||
.query("canonicalTrendingSnapshots")
|
||||
.withIndex("by_snapshot_id", (q) => q.eq("snapshotId", args.snapshotId))
|
||||
.unique();
|
||||
if (!snapshot || snapshot.status !== "building") return { changed: false };
|
||||
await ctx.db.patch(snapshot._id, {
|
||||
status: "failed",
|
||||
completedAt: args.completedAt,
|
||||
error: args.error.slice(0, 500),
|
||||
});
|
||||
return { changed: true };
|
||||
},
|
||||
});
|
||||
|
||||
export const pruneExpiredInternal = internalMutation({
|
||||
args: { now: v.number(), batchSize: v.number() },
|
||||
handler: async (ctx, args) => {
|
||||
const batchSize = Math.min(Math.max(Math.trunc(args.batchSize), 1), PRUNE_BATCH_SIZE);
|
||||
const items = await ctx.db
|
||||
.query("canonicalTrendingItems")
|
||||
.withIndex("by_expires_at", (q) => q.lte("expiresAt", args.now))
|
||||
.take(batchSize);
|
||||
for (const item of items) await ctx.db.delete(item._id);
|
||||
const remaining = batchSize - items.length;
|
||||
const snapshots =
|
||||
remaining > 0
|
||||
? await ctx.db
|
||||
.query("canonicalTrendingSnapshots")
|
||||
.withIndex("by_expires_at", (q) => q.lte("expiresAt", args.now))
|
||||
.take(remaining)
|
||||
: [];
|
||||
for (const snapshot of snapshots) await ctx.db.delete(snapshot._id);
|
||||
return {
|
||||
itemsDeleted: items.length,
|
||||
snapshotsDeleted: snapshots.length,
|
||||
fullBatch: items.length + snapshots.length === batchSize,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
export const pruneExpiredActionInternal = internalAction({
|
||||
args: {},
|
||||
handler: async (ctx) => {
|
||||
const result = await pruneExpiredRows(ctx, Date.now());
|
||||
const continuationScheduled = result.batches === PRUNE_MAX_BATCHES;
|
||||
if (continuationScheduled) {
|
||||
await ctx.scheduler.runAfter(
|
||||
0,
|
||||
internalRefs.canonicalTrending.pruneExpiredActionInternal as never,
|
||||
{},
|
||||
);
|
||||
}
|
||||
return { ...result, continuationScheduled };
|
||||
},
|
||||
});
|
||||
|
||||
export const materializeInternal = internalAction({
|
||||
args: { proofSnapshotId: v.optional(v.string()) },
|
||||
handler: async (ctx, args) => {
|
||||
if (args.proofSnapshotId !== undefined) {
|
||||
assertTestSeedAllowed();
|
||||
if (!/^claw-590-proof-[0-9a-f]{40}$/.test(args.proofSnapshotId)) {
|
||||
throw new Error("Invalid CLAW-590 proof snapshot ID");
|
||||
}
|
||||
}
|
||||
if (!getRuntimeRolloutCapabilities().skillsSh.runtimeEnabled) {
|
||||
return { status: "disabled" as const };
|
||||
}
|
||||
|
||||
const startedAt = Date.now();
|
||||
const snapshotId = args.proofSnapshotId ?? `skills-${startedAt}`;
|
||||
let snapshotStarted = false;
|
||||
let functionCalls = 0;
|
||||
let documentsRead = 0;
|
||||
let documentsWritten = 0;
|
||||
|
||||
try {
|
||||
const metricWindow = (await ctx.runQuery(
|
||||
internalRefs.canonicalTrending.getMetricWindowInternal as never,
|
||||
{},
|
||||
)) as {
|
||||
datasetVersion: string;
|
||||
importedAt: number;
|
||||
startDay: number;
|
||||
endDay: number;
|
||||
documentsRead: number;
|
||||
} | null;
|
||||
functionCalls += 1;
|
||||
documentsRead += metricWindow?.documentsRead ?? 0;
|
||||
if (!metricWindow) throw new Error("No 24-hour ranking metric window is available");
|
||||
|
||||
const nativeSource = (await collectSourcePages(
|
||||
ctx,
|
||||
internalRefs.canonicalTrending.getNativeSourcePageInternal,
|
||||
)) as CollectedSource<Doc<"skillSearchDigest">>;
|
||||
const metricSource = (await collectSourcePages(
|
||||
ctx,
|
||||
internalRefs.canonicalTrending.getMetricSourcePageInternal,
|
||||
{
|
||||
day: metricWindow.endDay,
|
||||
datasetVersion: metricWindow.datasetVersion,
|
||||
importedAt: metricWindow.importedAt,
|
||||
},
|
||||
)) as CollectedSource<Doc<"skillDailyStats">>;
|
||||
const latestTrendingRun = (await ctx.runQuery(
|
||||
internalRefs.canonicalTrending.getLatestCompletedTrendingRunInternal as never,
|
||||
{},
|
||||
)) as { runId: Doc<"skillsShMirrorRuns">["_id"] | null; documentsRead: number };
|
||||
documentsRead += latestTrendingRun.documentsRead;
|
||||
functionCalls += 1;
|
||||
const externalSource = (await collectSourcePages(
|
||||
ctx,
|
||||
internalRefs.canonicalTrending.getExternalSourcePageInternal,
|
||||
)) as CollectedSource<Doc<"skillsShMirrorDigests">>;
|
||||
documentsRead +=
|
||||
nativeSource.documentsRead + metricSource.documentsRead + externalSource.documentsRead;
|
||||
functionCalls +=
|
||||
nativeSource.functionCalls + metricSource.functionCalls + externalSource.functionCalls;
|
||||
|
||||
const confirmedMetricWindow = (await ctx.runQuery(
|
||||
internalRefs.canonicalTrending.getMetricWindowInternal as never,
|
||||
{},
|
||||
)) as typeof metricWindow;
|
||||
functionCalls += 1;
|
||||
documentsRead += confirmedMetricWindow?.documentsRead ?? 0;
|
||||
if (
|
||||
!confirmedMetricWindow ||
|
||||
confirmedMetricWindow.datasetVersion !== metricWindow.datasetVersion ||
|
||||
confirmedMetricWindow.importedAt !== metricWindow.importedAt
|
||||
) {
|
||||
throw new Error("24-hour ranking metric import changed during materialization");
|
||||
}
|
||||
|
||||
const confirmedTrendingRun = (await ctx.runQuery(
|
||||
internalRefs.canonicalTrending.getLatestCompletedTrendingRunInternal as never,
|
||||
{},
|
||||
)) as { runId: Doc<"skillsShMirrorRuns">["_id"] | null; documentsRead: number };
|
||||
documentsRead += confirmedTrendingRun.documentsRead;
|
||||
functionCalls += 1;
|
||||
if (confirmedTrendingRun.runId !== latestTrendingRun.runId) {
|
||||
throw new Error("skills.sh Trending run changed during materialization");
|
||||
}
|
||||
const latestTrendingRunId = latestTrendingRun.runId;
|
||||
|
||||
const usageBySkill = new Map(
|
||||
metricSource.rows.map((row) => [
|
||||
String(row.skillId),
|
||||
{
|
||||
installs: row.installs,
|
||||
bookmarks: row.bookmarks ?? 0,
|
||||
updatedAt: row.rankingImportedAt ?? row.updatedAt,
|
||||
},
|
||||
]),
|
||||
);
|
||||
const nativeCandidates = nativeSource.rows
|
||||
.map((digest) =>
|
||||
buildNativeCanonicalTrendingCandidate(
|
||||
digest,
|
||||
usageBySkill.get(String(digest.skillId)) ?? {
|
||||
installs: 0,
|
||||
bookmarks: 0,
|
||||
updatedAt: metricWindow.importedAt,
|
||||
},
|
||||
),
|
||||
)
|
||||
.filter(
|
||||
(candidate): candidate is CanonicalTrendingMaterializationCandidate => candidate !== null,
|
||||
);
|
||||
const risingCutoff = startedAt - RISING_MAX_AGE_MS;
|
||||
const risingCandidates = nativeCandidates
|
||||
.filter((candidate) => candidate.createdAt >= risingCutoff)
|
||||
.map((candidate) => ({ ...candidate, lane: "clawhub-rising" as const }));
|
||||
const externalCandidates = externalSource.rows
|
||||
.filter((digest) => digest.trendingObservedRunId === latestTrendingRunId)
|
||||
.map(buildExternalCanonicalTrendingCandidate)
|
||||
.filter(
|
||||
(candidate): candidate is CanonicalTrendingMaterializationCandidate => candidate !== null,
|
||||
);
|
||||
const blended = blendCanonicalTrendingPools({
|
||||
clawhubTrending: nativeCandidates,
|
||||
clawhubRising: risingCandidates,
|
||||
skillsShTrending: externalCandidates,
|
||||
});
|
||||
|
||||
const expiresAt = startedAt + SNAPSHOT_RETENTION_MS;
|
||||
await ctx.runMutation(
|
||||
internalRefs.canonicalTrending.startSnapshotInternal as never,
|
||||
{
|
||||
snapshotId,
|
||||
generatedAt: startedAt,
|
||||
expiresAt,
|
||||
windowStartDay: metricWindow.startDay,
|
||||
windowEndDay: metricWindow.endDay,
|
||||
} as never,
|
||||
);
|
||||
snapshotStarted = true;
|
||||
functionCalls += 1;
|
||||
documentsWritten += 1;
|
||||
|
||||
for (let index = 0; index < blended.length; index += WRITE_BATCH_SIZE) {
|
||||
const batch = blended.slice(index, index + WRITE_BATCH_SIZE);
|
||||
await ctx.runMutation(
|
||||
internalRefs.canonicalTrending.writeItemsInternal as never,
|
||||
{
|
||||
snapshotId,
|
||||
items: batch.map((candidate, batchIndex) => ({
|
||||
position: index + batchIndex,
|
||||
lane: candidate.lane,
|
||||
sourceRef: candidate.sourceRef,
|
||||
card: candidate.card,
|
||||
})),
|
||||
} as never,
|
||||
);
|
||||
functionCalls += 1;
|
||||
documentsWritten += batch.length + 1;
|
||||
}
|
||||
|
||||
const sourceCounts = {
|
||||
clawhubTrending: nativeCandidates.length,
|
||||
clawhubRising: risingCandidates.length,
|
||||
skillsShTrending: externalCandidates.length,
|
||||
};
|
||||
const operations = {
|
||||
documentsRead,
|
||||
documentsWritten: documentsWritten + 1,
|
||||
functionCalls: functionCalls + 1,
|
||||
};
|
||||
await ctx.runMutation(
|
||||
internalRefs.canonicalTrending.finalizeSnapshotInternal as never,
|
||||
{
|
||||
snapshotId,
|
||||
completedAt: Date.now(),
|
||||
totalItems: blended.length,
|
||||
sourceCounts,
|
||||
operations,
|
||||
} as never,
|
||||
);
|
||||
functionCalls += 1;
|
||||
documentsWritten += 1;
|
||||
|
||||
return {
|
||||
status: "ready" as const,
|
||||
snapshotId,
|
||||
generatedAt: new Date(startedAt).toISOString(),
|
||||
windowHours: CANONICAL_TRENDING_WINDOW_HOURS,
|
||||
rankingVersion: CANONICAL_TRENDING_RANKING_VERSION,
|
||||
totalItems: blended.length,
|
||||
sourceCounts,
|
||||
operations: {
|
||||
documentsRead,
|
||||
documentsWritten,
|
||||
functionCalls,
|
||||
},
|
||||
durationMs: Date.now() - startedAt,
|
||||
sample: blended.slice(0, 20).map((candidate, index) => ({
|
||||
rank: index + 1,
|
||||
lane: candidate.lane,
|
||||
id: candidate.card.id,
|
||||
displayName: candidate.card.displayName,
|
||||
trending24hInstalls: candidate.card.metrics.trending24hInstalls,
|
||||
lifetimeInstalls: candidate.card.metrics.lifetimeInstalls,
|
||||
})),
|
||||
};
|
||||
} catch (error) {
|
||||
if (snapshotStarted) {
|
||||
await ctx.runMutation(
|
||||
internalRefs.canonicalTrending.failSnapshotInternal as never,
|
||||
{
|
||||
snapshotId,
|
||||
completedAt: Date.now(),
|
||||
error:
|
||||
error instanceof Error ? error.message : "Unknown Trending materialization failure",
|
||||
} as never,
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
export const getPageInternal = internalQuery({
|
||||
args: {
|
||||
cursor: v.union(v.string(), v.null()),
|
||||
limit: v.number(),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
if (!Number.isSafeInteger(args.limit) || args.limit < 1 || args.limit > 100) {
|
||||
throw new Error("Invalid Trending page limit");
|
||||
}
|
||||
let decoded = null;
|
||||
if (args.cursor) {
|
||||
try {
|
||||
decoded = decodeCanonicalTrendingCursor(args.cursor);
|
||||
} catch {
|
||||
return { status: "invalid-cursor" as const };
|
||||
}
|
||||
}
|
||||
const now = Date.now();
|
||||
const snapshot = decoded
|
||||
? await ctx.db
|
||||
.query("canonicalTrendingSnapshots")
|
||||
.withIndex("by_snapshot_id", (q) => q.eq("snapshotId", decoded.snapshotId))
|
||||
.unique()
|
||||
: await ctx.db
|
||||
.query("canonicalTrendingSnapshots")
|
||||
.withIndex("by_kind_and_status_and_expires_at", (q) =>
|
||||
q.eq("kind", "skills").eq("status", "ready").gt("expiresAt", now),
|
||||
)
|
||||
.order("desc")
|
||||
.first();
|
||||
if (!snapshot && !decoded) return { status: "unavailable" as const };
|
||||
if (
|
||||
!snapshot ||
|
||||
snapshot.status !== "ready" ||
|
||||
snapshot.totalItems === undefined ||
|
||||
snapshot.expiresAt <= now
|
||||
) {
|
||||
return { status: "expired" as const };
|
||||
}
|
||||
const offset = decoded?.offset ?? 0;
|
||||
const rows = await ctx.db
|
||||
.query("canonicalTrendingItems")
|
||||
.withIndex("by_snapshot_id_and_position", (q) =>
|
||||
q
|
||||
.eq("snapshotId", snapshot.snapshotId)
|
||||
.gte("position", offset)
|
||||
.lt("position", offset + args.limit),
|
||||
)
|
||||
.take(args.limit);
|
||||
const eligibility = await Promise.all(
|
||||
rows.map(async (row) => {
|
||||
const sourceRef = row.sourceRef;
|
||||
if (sourceRef.kind === "clawhub") {
|
||||
const digest = await ctx.db
|
||||
.query("skillSearchDigest")
|
||||
.withIndex("by_skill", (q) => q.eq("skillId", sourceRef.skillId))
|
||||
.unique();
|
||||
return Boolean(
|
||||
digest &&
|
||||
!shouldExcludeSkillFromPublicBrowse(digest) &&
|
||||
digest.publicVersion?.status === "available",
|
||||
);
|
||||
}
|
||||
const digest = await ctx.db
|
||||
.query("skillsShMirrorDigests")
|
||||
.withIndex("by_external_id", (q) => q.eq("externalId", sourceRef.externalId))
|
||||
.unique();
|
||||
return Boolean(digest && isPublicSkillsShMirrorDigest(digest));
|
||||
}),
|
||||
);
|
||||
const nextOffset = offset + rows.length;
|
||||
return {
|
||||
status: "ok" as const,
|
||||
page: {
|
||||
kind: "skills" as const,
|
||||
snapshotId: snapshot.snapshotId,
|
||||
snapshotCursor: encodeCanonicalTrendingCursor({
|
||||
snapshotId: snapshot.snapshotId,
|
||||
offset: 0,
|
||||
}),
|
||||
generatedAt: new Date(snapshot.generatedAt).toISOString(),
|
||||
windowHours: snapshot.windowHours,
|
||||
rankingVersion: snapshot.rankingVersion,
|
||||
totalItems: snapshot.totalItems,
|
||||
items: rows.flatMap((row, index) =>
|
||||
eligibility[index] ? [{ ...row.card, rank: row.position + 1, lane: row.lane }] : [],
|
||||
),
|
||||
nextCursor:
|
||||
nextOffset < snapshot.totalItems
|
||||
? encodeCanonicalTrendingCursor({ snapshotId: snapshot.snapshotId, offset: nextOffset })
|
||||
: null,
|
||||
},
|
||||
};
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,179 @@
|
||||
/// <reference types="vite/client" />
|
||||
/* @vitest-environment edge-runtime */
|
||||
import { convexTest } from "convex-test";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { internal } from "./_generated/api";
|
||||
import schema from "./schema";
|
||||
|
||||
const modules = import.meta.glob("./**/*.ts");
|
||||
const CONFIRM = "manage-claw-590-canonical-trending-test-proof";
|
||||
const SNAPSHOT_ID = `claw-590-proof-${"a".repeat(40)}`;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.stubEnv("CLAWHUB_ENV", "test");
|
||||
vi.stubEnv("CLAWHUB_DISABLE_CRONS", "1");
|
||||
vi.stubEnv("CLAWHUB_DEPLOYMENT_NAME", "academic-chihuahua-392");
|
||||
vi.stubEnv("CLAWHUB_SKILLS_SH_ROLLOUT_MODE", "test");
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
describe("CLAW-590 permanent Test snapshot ownership", () => {
|
||||
it("seeds and removes an exact owned 20-row source corpus", async () => {
|
||||
const t = convexTest(schema, modules);
|
||||
await t.run(async (ctx) => {
|
||||
await ctx.db.insert("rankingMetricImports", {
|
||||
datasetVersion: "claw-590-test-dataset",
|
||||
checksum: "claw-590-test-checksum",
|
||||
generatedAt: new Date(1_000).toISOString(),
|
||||
importedAt: 2_000,
|
||||
startDay: 1,
|
||||
endDay: 1,
|
||||
targetCount: 0,
|
||||
skillTargetCount: 0,
|
||||
packageTargetCount: 0,
|
||||
dailyRowCount: 0,
|
||||
importedSkillRows: 0,
|
||||
importedPackageRows: 0,
|
||||
unresolvedTargets: 0,
|
||||
skippedOverlayRows: 0,
|
||||
});
|
||||
});
|
||||
|
||||
await expect(
|
||||
t.mutation(internal.canonicalTrendingTestFixtures.seedCanonicalTrendingSourceFixture, {
|
||||
confirm: CONFIRM,
|
||||
}),
|
||||
).resolves.toEqual({ ok: true, created: true, nativeCount: 12, externalCount: 8 });
|
||||
await expect(
|
||||
t.query(internal.canonicalTrendingTestFixtures.readCanonicalTrendingSourceFixture, {
|
||||
confirm: CONFIRM,
|
||||
}),
|
||||
).resolves.toMatchObject({
|
||||
present: true,
|
||||
nativeCount: 12,
|
||||
nativePublisherCount: 6,
|
||||
externalCount: 8,
|
||||
scansPlanned: 0,
|
||||
scansAdmitted: 0,
|
||||
});
|
||||
await expect(
|
||||
t.mutation(internal.canonicalTrendingTestFixtures.seedCanonicalTrendingSourceFixture, {
|
||||
confirm: CONFIRM,
|
||||
}),
|
||||
).resolves.toEqual({ ok: true, created: false, nativeCount: 12, externalCount: 8 });
|
||||
|
||||
const materialized = await t.action(internal.canonicalTrending.materializeInternal, {
|
||||
proofSnapshotId: SNAPSHOT_ID,
|
||||
});
|
||||
if (materialized.status !== "ready") throw new Error("Expected ready fixture materialization");
|
||||
expect(materialized).toMatchObject({
|
||||
status: "ready",
|
||||
snapshotId: SNAPSHOT_ID,
|
||||
totalItems: 20,
|
||||
sourceCounts: { clawhubTrending: 12, clawhubRising: 12, skillsShTrending: 8 },
|
||||
});
|
||||
expect(materialized.sample.map((row) => row.lane)).toEqual([
|
||||
"clawhub-trending",
|
||||
"skills-sh-trending",
|
||||
"clawhub-rising",
|
||||
"clawhub-trending",
|
||||
"skills-sh-trending",
|
||||
"clawhub-trending",
|
||||
"skills-sh-trending",
|
||||
"clawhub-rising",
|
||||
"clawhub-trending",
|
||||
"skills-sh-trending",
|
||||
"clawhub-trending",
|
||||
"skills-sh-trending",
|
||||
"clawhub-rising",
|
||||
"clawhub-trending",
|
||||
"skills-sh-trending",
|
||||
"clawhub-trending",
|
||||
"skills-sh-trending",
|
||||
"clawhub-rising",
|
||||
"clawhub-trending",
|
||||
"skills-sh-trending",
|
||||
]);
|
||||
await expect(
|
||||
t.action(internal.canonicalTrendingTestFixtures.cleanupCanonicalTrendingProof, {
|
||||
confirm: CONFIRM,
|
||||
snapshotId: SNAPSHOT_ID,
|
||||
}),
|
||||
).resolves.toMatchObject({ ok: true, itemsDeleted: 20, snapshotDeleted: true });
|
||||
|
||||
await expect(
|
||||
t.mutation(internal.canonicalTrendingTestFixtures.cleanupCanonicalTrendingSourceFixture, {
|
||||
confirm: CONFIRM,
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
ok: true,
|
||||
removed: true,
|
||||
nativeDeleted: 12,
|
||||
externalDeleted: 8,
|
||||
usersDeleted: 6,
|
||||
});
|
||||
await expect(
|
||||
t.query(internal.canonicalTrendingTestFixtures.readCanonicalTrendingSourceFixture, {
|
||||
confirm: CONFIRM,
|
||||
}),
|
||||
).resolves.toEqual({ present: false });
|
||||
});
|
||||
|
||||
it("reports an absent owned snapshot without broad reads", async () => {
|
||||
const t = convexTest(schema, modules);
|
||||
|
||||
await expect(
|
||||
t.query(internal.canonicalTrendingTestFixtures.readCanonicalTrendingProof, {
|
||||
confirm: CONFIRM,
|
||||
snapshotId: SNAPSHOT_ID,
|
||||
}),
|
||||
).resolves.toEqual({ present: false });
|
||||
});
|
||||
|
||||
it("removes an exact owned snapshot and all item batches", async () => {
|
||||
const t = convexTest(schema, modules);
|
||||
await t.run(async (ctx) => {
|
||||
await ctx.db.insert("canonicalTrendingSnapshots", {
|
||||
snapshotId: SNAPSHOT_ID,
|
||||
kind: "skills",
|
||||
status: "failed",
|
||||
rankingVersion: "skills-trending-v1",
|
||||
generatedAt: 1_000,
|
||||
completedAt: 2_000,
|
||||
expiresAt: Date.now() + 100_000,
|
||||
windowHours: 24,
|
||||
windowStartDay: 1,
|
||||
windowEndDay: 1,
|
||||
writtenItems: 0,
|
||||
error: "proof failure",
|
||||
});
|
||||
});
|
||||
|
||||
await expect(
|
||||
t.action(internal.canonicalTrendingTestFixtures.cleanupCanonicalTrendingProof, {
|
||||
confirm: CONFIRM,
|
||||
snapshotId: SNAPSHOT_ID,
|
||||
}),
|
||||
).resolves.toMatchObject({ ok: true, itemsDeleted: 0, snapshotDeleted: true, batches: 1 });
|
||||
await expect(
|
||||
t.query(internal.canonicalTrendingTestFixtures.readCanonicalTrendingProof, {
|
||||
confirm: CONFIRM,
|
||||
snapshotId: SNAPSHOT_ID,
|
||||
}),
|
||||
).resolves.toEqual({ present: false });
|
||||
});
|
||||
|
||||
it("rejects IDs outside the exact CLAW-590 proof namespace", async () => {
|
||||
const t = convexTest(schema, modules);
|
||||
|
||||
await expect(
|
||||
t.query(internal.canonicalTrendingTestFixtures.readCanonicalTrendingProof, {
|
||||
confirm: CONFIRM,
|
||||
snapshotId: "skills-123",
|
||||
}),
|
||||
).rejects.toThrow("Invalid CLAW-590 proof snapshot ID");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,719 @@
|
||||
import { v } from "convex/values";
|
||||
import { internal } from "./_generated/api";
|
||||
import type { Doc, Id } from "./_generated/dataModel";
|
||||
import {
|
||||
internalAction,
|
||||
internalMutation,
|
||||
internalQuery,
|
||||
type QueryCtx,
|
||||
} from "./_generated/server";
|
||||
import { assertTestSeedAllowed } from "./lib/testSeed";
|
||||
|
||||
const CONFIRM = "manage-claw-590-canonical-trending-test-proof";
|
||||
const PROOF_SNAPSHOT_PATTERN = /^claw-590-proof-[0-9a-f]{40}$/;
|
||||
const CLEANUP_BATCH_SIZE = 200;
|
||||
const CLEANUP_MAX_BATCHES = 100;
|
||||
const SAMPLE_SIZE = 20;
|
||||
const SOURCE_FIXTURE_ID = "claw-590-canonical-trending-source-v1";
|
||||
const SOURCE_FIXTURE_ACTOR = "CLAW-590 Test workflow";
|
||||
const NATIVE_COUNT = 12;
|
||||
const NATIVE_PUBLISHER_COUNT = 6;
|
||||
const EXTERNAL_COUNT = 8;
|
||||
|
||||
const internalRefs = internal as unknown as {
|
||||
canonicalTrendingTestFixtures: {
|
||||
cleanupCanonicalTrendingProofBatch: unknown;
|
||||
};
|
||||
};
|
||||
|
||||
const proofArgs = {
|
||||
confirm: v.literal(CONFIRM),
|
||||
snapshotId: v.string(),
|
||||
};
|
||||
|
||||
const confirmArgs = { confirm: v.literal(CONFIRM) };
|
||||
|
||||
function fixtureOrdinal(index: number) {
|
||||
return String(index + 1).padStart(2, "0");
|
||||
}
|
||||
|
||||
function nativeOwnerHandle(index: number) {
|
||||
return `claw-590-proof-owner-${fixtureOrdinal(index % NATIVE_PUBLISHER_COUNT)}`;
|
||||
}
|
||||
|
||||
function nativeSlug(index: number) {
|
||||
return `claw-590-proof-native-${fixtureOrdinal(index)}`;
|
||||
}
|
||||
|
||||
function externalOwner(index: number) {
|
||||
return `clawhub-test-${fixtureOrdinal(index)}`;
|
||||
}
|
||||
|
||||
function externalFixtureId(index: number) {
|
||||
return `${externalOwner(index)}/claw-590/trending-${fixtureOrdinal(index)}`;
|
||||
}
|
||||
|
||||
function emptySkillStats() {
|
||||
return {
|
||||
downloads: 0,
|
||||
installsCurrent: 0,
|
||||
installsAllTime: 0,
|
||||
stars: 0,
|
||||
versions: 1,
|
||||
comments: 0,
|
||||
};
|
||||
}
|
||||
|
||||
function cleanVersionScans(now: number) {
|
||||
return {
|
||||
vtAnalysis: {
|
||||
status: "clean",
|
||||
verdict: "clean",
|
||||
analysis: "Owned CLAW-590 permanent-Test fixture.",
|
||||
source: SOURCE_FIXTURE_ID,
|
||||
checkedAt: now,
|
||||
},
|
||||
llmAnalysis: {
|
||||
status: "clean",
|
||||
verdict: "clean",
|
||||
confidence: "high",
|
||||
summary: "Owned CLAW-590 permanent-Test fixture.",
|
||||
model: SOURCE_FIXTURE_ID,
|
||||
checkedAt: now,
|
||||
},
|
||||
staticScan: {
|
||||
status: "clean" as const,
|
||||
reasonCodes: [],
|
||||
findings: [],
|
||||
summary: "Owned CLAW-590 permanent-Test fixture.",
|
||||
engineVersion: SOURCE_FIXTURE_ID,
|
||||
checkedAt: now,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function assertOwnedUser(user: Doc<"users">, index: number) {
|
||||
if (
|
||||
user.handle !== nativeOwnerHandle(index) ||
|
||||
user.name !== `CLAW-590 Proof Owner ${fixtureOrdinal(index)}`
|
||||
) {
|
||||
throw new Error("CLAW-590 source fixture user ownership mismatch");
|
||||
}
|
||||
}
|
||||
|
||||
function assertOwnedSkill(skill: Doc<"skills">, index: number, ownerId: Id<"users">) {
|
||||
if (
|
||||
skill.slug !== nativeSlug(index) ||
|
||||
skill.ownerUserId !== ownerId ||
|
||||
skill.batch !== SOURCE_FIXTURE_ID ||
|
||||
skill.displayName !== `CLAW-590 Native ${fixtureOrdinal(index)}` ||
|
||||
skill.stats.versions !== 1 ||
|
||||
!skill.latestVersionId
|
||||
) {
|
||||
throw new Error("CLAW-590 source fixture skill ownership mismatch");
|
||||
}
|
||||
}
|
||||
|
||||
function assertOwnedVersion(
|
||||
version: Doc<"skillVersions">,
|
||||
skillId: Id<"skills">,
|
||||
ownerId: Id<"users">,
|
||||
) {
|
||||
if (
|
||||
version.skillId !== skillId ||
|
||||
version.createdBy !== ownerId ||
|
||||
version.version !== "1.0.0" ||
|
||||
version.changelog !== SOURCE_FIXTURE_ID ||
|
||||
version.files.length !== 0 ||
|
||||
version.vtAnalysis?.source !== SOURCE_FIXTURE_ID ||
|
||||
version.llmAnalysis?.model !== SOURCE_FIXTURE_ID ||
|
||||
version.staticScan?.engineVersion !== SOURCE_FIXTURE_ID
|
||||
) {
|
||||
throw new Error("CLAW-590 source fixture version ownership mismatch");
|
||||
}
|
||||
}
|
||||
|
||||
function assertOwnedNativeDigest(
|
||||
digest: Doc<"skillSearchDigest">,
|
||||
index: number,
|
||||
skillId: Id<"skills">,
|
||||
ownerId: Id<"users">,
|
||||
versionId: Id<"skillVersions">,
|
||||
) {
|
||||
if (
|
||||
digest.skillId !== skillId ||
|
||||
digest.ownerUserId !== ownerId ||
|
||||
digest.slug !== nativeSlug(index) ||
|
||||
digest.ownerHandle !== nativeOwnerHandle(index) ||
|
||||
digest.publicVersion?.status !== "available" ||
|
||||
digest.publicVersion.versionId !== versionId ||
|
||||
digest.latestVersionId !== versionId ||
|
||||
digest.latestVersionSkillId !== skillId ||
|
||||
digest.softDeletedAt !== undefined ||
|
||||
digest.isSuspicious !== false
|
||||
) {
|
||||
throw new Error("CLAW-590 source fixture native digest ownership mismatch");
|
||||
}
|
||||
}
|
||||
|
||||
function assertOwnedDailyStat(stat: Doc<"skillDailyStats">, index: number, skillId: Id<"skills">) {
|
||||
if (
|
||||
stat.skillId !== skillId ||
|
||||
stat.installs !== 100_000 - index ||
|
||||
stat.bookmarks !== 10_000 - index ||
|
||||
!stat.rankingDatasetVersion ||
|
||||
stat.rankingImportedAt === undefined
|
||||
) {
|
||||
throw new Error("CLAW-590 source fixture metric ownership mismatch");
|
||||
}
|
||||
}
|
||||
|
||||
function assertOwnedRun(run: Doc<"skillsShMirrorRuns">) {
|
||||
if (
|
||||
run.snapshotId !== SOURCE_FIXTURE_ID ||
|
||||
run.sourceView !== "trending" ||
|
||||
run.sourceSnapshotHash !== SOURCE_FIXTURE_ID ||
|
||||
run.status !== "completed" ||
|
||||
run.actor !== SOURCE_FIXTURE_ACTOR ||
|
||||
run.counts.scansPlanned !== 0 ||
|
||||
run.counts.scansAdmitted !== 0
|
||||
) {
|
||||
throw new Error("CLAW-590 source fixture run ownership mismatch");
|
||||
}
|
||||
}
|
||||
|
||||
function assertOwnedExternalDigest(
|
||||
digest: Doc<"skillsShMirrorDigests">,
|
||||
index: number,
|
||||
runId: Id<"skillsShMirrorRuns">,
|
||||
) {
|
||||
const id = externalFixtureId(index);
|
||||
if (
|
||||
digest.externalId !== id ||
|
||||
digest.owner !== externalOwner(index) ||
|
||||
digest.repo !== "claw-590" ||
|
||||
digest.slug !== `trending-${fixtureOrdinal(index)}` ||
|
||||
digest.trendingRank !== index + 1 ||
|
||||
digest.trendingObservedRunId !== runId ||
|
||||
digest.sourceSnapshotId !== SOURCE_FIXTURE_ID ||
|
||||
digest.observationFingerprint !== `${SOURCE_FIXTURE_ID}-${fixtureOrdinal(index)}` ||
|
||||
!digest.active ||
|
||||
!digest.publicVisible ||
|
||||
!digest.installable ||
|
||||
digest.sourceFreshnessStatus !== "observed-only" ||
|
||||
digest.tombstonedAt !== undefined
|
||||
) {
|
||||
throw new Error("CLAW-590 source fixture external digest ownership mismatch");
|
||||
}
|
||||
}
|
||||
|
||||
async function findOwnedRun(ctx: Pick<QueryCtx, "db">) {
|
||||
return await ctx.db
|
||||
.query("skillsShMirrorRuns")
|
||||
.withIndex("by_source_view_and_status_and_source_snapshot_hash", (q) =>
|
||||
q
|
||||
.eq("sourceView", "trending")
|
||||
.eq("status", "completed")
|
||||
.eq("sourceSnapshotHash", SOURCE_FIXTURE_ID),
|
||||
)
|
||||
.unique();
|
||||
}
|
||||
|
||||
async function readOwnedSourceFixture(ctx: Pick<QueryCtx, "db">) {
|
||||
const users = [];
|
||||
for (let index = 0; index < NATIVE_PUBLISHER_COUNT; index += 1) {
|
||||
users.push(
|
||||
await ctx.db
|
||||
.query("users")
|
||||
.withIndex("handle", (q) => q.eq("handle", nativeOwnerHandle(index)))
|
||||
.unique(),
|
||||
);
|
||||
}
|
||||
const skills = [];
|
||||
for (let index = 0; index < NATIVE_COUNT; index += 1) {
|
||||
skills.push(
|
||||
await ctx.db
|
||||
.query("skills")
|
||||
.withIndex("by_slug", (q) => q.eq("slug", nativeSlug(index)))
|
||||
.unique(),
|
||||
);
|
||||
}
|
||||
const externalDigests = [];
|
||||
for (let index = 0; index < EXTERNAL_COUNT; index += 1) {
|
||||
externalDigests.push(
|
||||
await ctx.db
|
||||
.query("skillsShMirrorDigests")
|
||||
.withIndex("by_external_id", (q) => q.eq("externalId", externalFixtureId(index)))
|
||||
.unique(),
|
||||
);
|
||||
}
|
||||
const run = await findOwnedRun(ctx);
|
||||
const roots = [...users, ...skills, ...externalDigests, run];
|
||||
if (roots.every((row) => row === null)) return null;
|
||||
if (roots.some((row) => row === null)) {
|
||||
throw new Error("CLAW-590 source fixture has partial root state");
|
||||
}
|
||||
|
||||
const checkedUsers = users as Doc<"users">[];
|
||||
const checkedSkills = skills as Doc<"skills">[];
|
||||
const checkedExternalDigests = externalDigests as Doc<"skillsShMirrorDigests">[];
|
||||
const checkedRun = run as Doc<"skillsShMirrorRuns">;
|
||||
checkedUsers.forEach(assertOwnedUser);
|
||||
assertOwnedRun(checkedRun);
|
||||
checkedExternalDigests.forEach((digest, index) =>
|
||||
assertOwnedExternalDigest(digest, index, checkedRun._id),
|
||||
);
|
||||
|
||||
const native = [];
|
||||
for (const [index, skill] of checkedSkills.entries()) {
|
||||
const owner = checkedUsers[index % NATIVE_PUBLISHER_COUNT]!;
|
||||
assertOwnedSkill(skill, index, owner._id);
|
||||
const [version, digest, stats] = await Promise.all([
|
||||
ctx.db.get(skill.latestVersionId!),
|
||||
ctx.db
|
||||
.query("skillSearchDigest")
|
||||
.withIndex("by_skill", (q) => q.eq("skillId", skill._id))
|
||||
.unique(),
|
||||
ctx.db
|
||||
.query("skillDailyStats")
|
||||
.withIndex("by_skill_day", (q) => q.eq("skillId", skill._id))
|
||||
.collect(),
|
||||
]);
|
||||
if (!version || !digest || stats.length !== 1) {
|
||||
throw new Error("CLAW-590 source fixture has partial native state");
|
||||
}
|
||||
assertOwnedVersion(version, skill._id, owner._id);
|
||||
assertOwnedNativeDigest(digest, index, skill._id, owner._id, version._id);
|
||||
assertOwnedDailyStat(stats[0]!, index, skill._id);
|
||||
native.push({ owner, skill, version, digest, stat: stats[0]! });
|
||||
}
|
||||
|
||||
return {
|
||||
users: checkedUsers,
|
||||
native,
|
||||
run: checkedRun,
|
||||
externalDigests: checkedExternalDigests,
|
||||
};
|
||||
}
|
||||
|
||||
function assertProofSnapshotId(snapshotId: string) {
|
||||
if (!PROOF_SNAPSHOT_PATTERN.test(snapshotId)) {
|
||||
throw new Error("Invalid CLAW-590 proof snapshot ID");
|
||||
}
|
||||
}
|
||||
|
||||
function assertOwnedSnapshot(snapshot: Doc<"canonicalTrendingSnapshots">, snapshotId: string) {
|
||||
if (
|
||||
snapshot.snapshotId !== snapshotId ||
|
||||
snapshot.kind !== "skills" ||
|
||||
snapshot.rankingVersion !== "skills-trending-v1" ||
|
||||
snapshot.windowHours !== 24
|
||||
) {
|
||||
throw new Error("CLAW-590 proof snapshot ownership mismatch");
|
||||
}
|
||||
}
|
||||
|
||||
export const seedCanonicalTrendingSourceFixture = internalMutation({
|
||||
args: confirmArgs,
|
||||
handler: async (ctx) => {
|
||||
assertTestSeedAllowed();
|
||||
const existing = await readOwnedSourceFixture(ctx);
|
||||
if (existing) {
|
||||
return {
|
||||
ok: true as const,
|
||||
created: false as const,
|
||||
nativeCount: existing.native.length,
|
||||
externalCount: existing.externalDigests.length,
|
||||
};
|
||||
}
|
||||
|
||||
const metricWindow = await ctx.db
|
||||
.query("rankingMetricImports")
|
||||
.withIndex("by_imported_at")
|
||||
.order("desc")
|
||||
.first();
|
||||
if (!metricWindow) throw new Error("CLAW-590 source fixture requires a ranking metric import");
|
||||
|
||||
const now = Date.now();
|
||||
const users: Id<"users">[] = [];
|
||||
for (let index = 0; index < NATIVE_PUBLISHER_COUNT; index += 1) {
|
||||
users.push(
|
||||
await ctx.db.insert("users", {
|
||||
handle: nativeOwnerHandle(index),
|
||||
name: `CLAW-590 Proof Owner ${fixtureOrdinal(index)}`,
|
||||
displayName: `CLAW-590 Proof Owner ${fixtureOrdinal(index)}`,
|
||||
role: "user",
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
for (let index = 0; index < NATIVE_COUNT; index += 1) {
|
||||
const ownerUserId = users[index % NATIVE_PUBLISHER_COUNT]!;
|
||||
const slug = nativeSlug(index);
|
||||
const displayName = `CLAW-590 Native ${fixtureOrdinal(index)}`;
|
||||
const skillId = await ctx.db.insert("skills", {
|
||||
slug,
|
||||
displayName,
|
||||
summary: "Owned synthetic native candidate for permanent-Test Trending proof.",
|
||||
ownerUserId,
|
||||
tags: {},
|
||||
batch: SOURCE_FIXTURE_ID,
|
||||
statsDownloads: 0,
|
||||
statsStars: 0,
|
||||
statsInstallsCurrent: 0,
|
||||
statsInstallsAllTime: 1_000 + index,
|
||||
stats: emptySkillStats(),
|
||||
createdAt: now - index,
|
||||
updatedAt: now - index,
|
||||
});
|
||||
const versionId = await ctx.db.insert("skillVersions", {
|
||||
skillId,
|
||||
version: "1.0.0",
|
||||
publicationStatus: "published",
|
||||
changelog: SOURCE_FIXTURE_ID,
|
||||
files: [],
|
||||
parsed: { frontmatter: {} },
|
||||
createdBy: ownerUserId,
|
||||
createdAt: now - index,
|
||||
...cleanVersionScans(now),
|
||||
});
|
||||
await ctx.db.patch(skillId, {
|
||||
latestVersionId: versionId,
|
||||
latestVersionSummary: {
|
||||
version: "1.0.0",
|
||||
createdAt: now - index,
|
||||
changelog: SOURCE_FIXTURE_ID,
|
||||
},
|
||||
tags: { latest: versionId },
|
||||
});
|
||||
await ctx.db.insert("skillSearchDigest", {
|
||||
skillId,
|
||||
slug,
|
||||
normalizedSlug: slug,
|
||||
normalizedSlugFirstToken: "claw",
|
||||
displayName,
|
||||
normalizedDisplayName: displayName.toLowerCase(),
|
||||
normalizedDisplayNameFirstToken: "claw",
|
||||
summary: "Owned synthetic native candidate for permanent-Test Trending proof.",
|
||||
ownerUserId,
|
||||
ownerHandle: nativeOwnerHandle(index),
|
||||
ownerKind: "user",
|
||||
ownerName: `CLAW-590 Proof Owner ${fixtureOrdinal(index % NATIVE_PUBLISHER_COUNT)}`,
|
||||
ownerDisplayName: `CLAW-590 Proof Owner ${fixtureOrdinal(index % NATIVE_PUBLISHER_COUNT)}`,
|
||||
latestVersionId: versionId,
|
||||
latestVersionSkillId: skillId,
|
||||
publicVersion: { status: "available", versionId },
|
||||
latestVersionSummary: {
|
||||
version: "1.0.0",
|
||||
createdAt: now - index,
|
||||
changelog: SOURCE_FIXTURE_ID,
|
||||
},
|
||||
tags: { latest: versionId },
|
||||
statsDownloads: 0,
|
||||
statsStars: 0,
|
||||
statsInstallsCurrent: 0,
|
||||
statsInstallsAllTime: 1_000 + index,
|
||||
stats: emptySkillStats(),
|
||||
isSuspicious: false,
|
||||
createdAt: now - index,
|
||||
updatedAt: now - index,
|
||||
});
|
||||
await ctx.db.insert("skillDailyStats", {
|
||||
skillId,
|
||||
day: metricWindow.endDay,
|
||||
downloads: 100_000 - index,
|
||||
installs: 100_000 - index,
|
||||
bookmarks: 10_000 - index,
|
||||
rankingDatasetVersion: metricWindow.datasetVersion,
|
||||
rankingImportedAt: metricWindow.importedAt,
|
||||
updatedAt: metricWindow.importedAt,
|
||||
});
|
||||
}
|
||||
|
||||
const runId = await ctx.db.insert("skillsShMirrorRuns", {
|
||||
snapshotId: SOURCE_FIXTURE_ID,
|
||||
sourceView: "trending",
|
||||
sourceSnapshotHash: SOURCE_FIXTURE_ID,
|
||||
sourceCaptureWrites: 0,
|
||||
status: "completed",
|
||||
sourceTotal: EXTERNAL_COUNT,
|
||||
sourcePageSize: EXTERNAL_COUNT,
|
||||
sourceMeasuredAt: new Date(now).toISOString(),
|
||||
sourceDurationMs: 0,
|
||||
page: 1,
|
||||
offset: EXTERNAL_COUNT,
|
||||
counts: {
|
||||
observed: EXTERNAL_COUNT,
|
||||
inserted: EXTERNAL_COUNT,
|
||||
updated: 0,
|
||||
unchanged: 0,
|
||||
rejected: 0,
|
||||
quarantined: 0,
|
||||
quarantinedPreserved: 0,
|
||||
conflicts: 0,
|
||||
detailsInserted: 0,
|
||||
detailsUpdated: 0,
|
||||
detailsUnchanged: 0,
|
||||
detailsMissing: EXTERNAL_COUNT,
|
||||
detailsTruncated: 0,
|
||||
tombstoned: 0,
|
||||
reactivated: 0,
|
||||
scansPlanned: 0,
|
||||
scansAdmitted: 0,
|
||||
},
|
||||
operations: {
|
||||
functionCalls: 1,
|
||||
dbReads: 0,
|
||||
dbWrites: EXTERNAL_COUNT + 1,
|
||||
sourceRequests: 0,
|
||||
sourceBytes: 0,
|
||||
},
|
||||
actor: SOURCE_FIXTURE_ACTOR,
|
||||
reason: "Owned synthetic source corpus for CLAW-590 permanent-Test Trending proof.",
|
||||
startedAt: now,
|
||||
completedAt: now,
|
||||
updatedAt: now,
|
||||
});
|
||||
for (let index = 0; index < EXTERNAL_COUNT; index += 1) {
|
||||
const id = externalFixtureId(index);
|
||||
const slug = `trending-${fixtureOrdinal(index)}`;
|
||||
await ctx.db.insert("skillsShMirrorDigests", {
|
||||
externalId: id,
|
||||
sourceType: "github",
|
||||
upstreamSourceType: "github",
|
||||
owner: externalOwner(index),
|
||||
repo: "claw-590",
|
||||
slug,
|
||||
normalizedSlug: slug,
|
||||
normalizedSlugFirstToken: "trending",
|
||||
displayName: `CLAW-590 External ${fixtureOrdinal(index)}`,
|
||||
normalizedDisplayName: `claw-590 external ${fixtureOrdinal(index)}`,
|
||||
normalizedDisplayNameFirstToken: "claw",
|
||||
searchSummary: "Owned synthetic skills.sh candidate for permanent-Test Trending proof.",
|
||||
searchText: `claw 590 external trending ${fixtureOrdinal(index)}`,
|
||||
sourceUrl: `https://skills.sh/${id}`,
|
||||
canonicalRepoUrl: "https://github.com/clawhub-test/claw-590",
|
||||
githubPath: `skills/${slug}`,
|
||||
githubCommit: "0".repeat(40),
|
||||
sourceContentHash: "0".repeat(64),
|
||||
upstreamInstalls: 50_000 - index,
|
||||
trendingRank: index + 1,
|
||||
trendingLifetimeInstalls: 50_000 - index,
|
||||
trendingObservedAt: now,
|
||||
trendingSnapshotId: SOURCE_FIXTURE_ID,
|
||||
trendingObservedRunId: runId,
|
||||
upstreamScanners: {
|
||||
genAgentTrustHub: { status: "unavailable" },
|
||||
socket: { status: "unavailable" },
|
||||
snyk: { status: "unavailable" },
|
||||
},
|
||||
sourceFreshnessStatus: "observed-only",
|
||||
detailStatus: "missing",
|
||||
observationFingerprint: `${SOURCE_FIXTURE_ID}-${fixtureOrdinal(index)}`,
|
||||
sourceSnapshotId: SOURCE_FIXTURE_ID,
|
||||
lastObservedRunId: runId,
|
||||
active: true,
|
||||
publicVisible: true,
|
||||
installable: true,
|
||||
firstObservedAt: now,
|
||||
lastObservedAt: now,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
ok: true as const,
|
||||
created: true as const,
|
||||
nativeCount: NATIVE_COUNT,
|
||||
externalCount: EXTERNAL_COUNT,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
export const readCanonicalTrendingSourceFixture = internalQuery({
|
||||
args: confirmArgs,
|
||||
handler: async (ctx) => {
|
||||
assertTestSeedAllowed();
|
||||
const fixture = await readOwnedSourceFixture(ctx);
|
||||
if (!fixture) return { present: false as const };
|
||||
return {
|
||||
present: true as const,
|
||||
fixtureId: SOURCE_FIXTURE_ID,
|
||||
nativeCount: fixture.native.length,
|
||||
nativePublisherCount: fixture.users.length,
|
||||
externalCount: fixture.externalDigests.length,
|
||||
runId: fixture.run._id,
|
||||
scansPlanned: fixture.run.counts.scansPlanned,
|
||||
scansAdmitted: fixture.run.counts.scansAdmitted,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
export const cleanupCanonicalTrendingSourceFixture = internalMutation({
|
||||
args: confirmArgs,
|
||||
handler: async (ctx) => {
|
||||
assertTestSeedAllowed();
|
||||
const fixture = await readOwnedSourceFixture(ctx);
|
||||
if (!fixture) return { ok: true as const, removed: false as const };
|
||||
|
||||
for (const digest of fixture.externalDigests) {
|
||||
const [details, facets] = await Promise.all([
|
||||
ctx.db
|
||||
.query("skillsShMirrorDetails")
|
||||
.withIndex("by_digest_id", (q) => q.eq("digestId", digest._id))
|
||||
.take(1),
|
||||
ctx.db
|
||||
.query("skillsShMirrorFacets")
|
||||
.withIndex("by_digest_id_and_kind_and_term", (q) => q.eq("digestId", digest._id))
|
||||
.take(1),
|
||||
]);
|
||||
if (details.length > 0 || facets.length > 0) {
|
||||
throw new Error("CLAW-590 source fixture cleanup refused dependent mirror rows");
|
||||
}
|
||||
await ctx.db.delete(digest._id);
|
||||
}
|
||||
const conflicts = await ctx.db
|
||||
.query("skillsShMirrorConflicts")
|
||||
.withIndex("by_run_id", (q) => q.eq("runId", fixture.run._id))
|
||||
.take(1);
|
||||
if (conflicts.length > 0) {
|
||||
throw new Error("CLAW-590 source fixture cleanup refused dependent conflict rows");
|
||||
}
|
||||
|
||||
for (const row of fixture.native) {
|
||||
await ctx.db.delete(row.stat._id);
|
||||
await ctx.db.delete(row.digest._id);
|
||||
await ctx.db.delete(row.version._id);
|
||||
await ctx.db.delete(row.skill._id);
|
||||
}
|
||||
for (const user of fixture.users) await ctx.db.delete(user._id);
|
||||
await ctx.db.delete(fixture.run._id);
|
||||
return {
|
||||
ok: true as const,
|
||||
removed: true as const,
|
||||
nativeDeleted: fixture.native.length,
|
||||
externalDeleted: fixture.externalDigests.length,
|
||||
usersDeleted: fixture.users.length,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
export const readCanonicalTrendingProof = internalQuery({
|
||||
args: proofArgs,
|
||||
handler: async (ctx, args) => {
|
||||
assertTestSeedAllowed();
|
||||
assertProofSnapshotId(args.snapshotId);
|
||||
const snapshot = await ctx.db
|
||||
.query("canonicalTrendingSnapshots")
|
||||
.withIndex("by_snapshot_id", (q) => q.eq("snapshotId", args.snapshotId))
|
||||
.unique();
|
||||
const items = await ctx.db
|
||||
.query("canonicalTrendingItems")
|
||||
.withIndex("by_snapshot_id_and_position", (q) => q.eq("snapshotId", args.snapshotId))
|
||||
.take(SAMPLE_SIZE);
|
||||
if (!snapshot) {
|
||||
if (items.length > 0) throw new Error("CLAW-590 proof snapshot has orphaned items");
|
||||
return { present: false as const };
|
||||
}
|
||||
assertOwnedSnapshot(snapshot, args.snapshotId);
|
||||
const sample = await Promise.all(
|
||||
items.map(async (item) => {
|
||||
if (item.sourceRef.kind === "clawhub") {
|
||||
const skillId = item.sourceRef.skillId;
|
||||
const digest = await ctx.db
|
||||
.query("skillSearchDigest")
|
||||
.withIndex("by_skill", (q) => q.eq("skillId", skillId))
|
||||
.unique();
|
||||
if (!digest) throw new Error("CLAW-590 proof native source is missing");
|
||||
return {
|
||||
rank: item.position + 1,
|
||||
id: item.card.id,
|
||||
lane: item.lane,
|
||||
publisherKey: String(digest.ownerPublisherId ?? digest.ownerUserId),
|
||||
upstreamRank: null,
|
||||
metrics: item.card.metrics,
|
||||
};
|
||||
}
|
||||
const externalId = item.sourceRef.externalId;
|
||||
const digest = await ctx.db
|
||||
.query("skillsShMirrorDigests")
|
||||
.withIndex("by_external_id", (q) => q.eq("externalId", externalId))
|
||||
.unique();
|
||||
if (!digest) throw new Error("CLAW-590 proof external source is missing");
|
||||
return {
|
||||
rank: item.position + 1,
|
||||
id: item.card.id,
|
||||
lane: item.lane,
|
||||
publisherKey: digest.owner ?? digest.sourceHost ?? digest.externalId,
|
||||
upstreamRank: digest.trendingRank ?? null,
|
||||
metrics: item.card.metrics,
|
||||
};
|
||||
}),
|
||||
);
|
||||
return {
|
||||
present: true as const,
|
||||
snapshotId: snapshot.snapshotId,
|
||||
status: snapshot.status,
|
||||
generatedAt: snapshot.generatedAt,
|
||||
completedAt: snapshot.completedAt ?? null,
|
||||
totalItems: snapshot.totalItems ?? null,
|
||||
writtenItems: snapshot.writtenItems,
|
||||
sourceCounts: snapshot.sourceCounts ?? null,
|
||||
operations: snapshot.operations ?? null,
|
||||
sample,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
export const cleanupCanonicalTrendingProofBatch = internalMutation({
|
||||
args: proofArgs,
|
||||
handler: async (ctx, args) => {
|
||||
assertTestSeedAllowed();
|
||||
assertProofSnapshotId(args.snapshotId);
|
||||
const snapshot = await ctx.db
|
||||
.query("canonicalTrendingSnapshots")
|
||||
.withIndex("by_snapshot_id", (q) => q.eq("snapshotId", args.snapshotId))
|
||||
.unique();
|
||||
const items = await ctx.db
|
||||
.query("canonicalTrendingItems")
|
||||
.withIndex("by_snapshot_id_and_position", (q) => q.eq("snapshotId", args.snapshotId))
|
||||
.take(CLEANUP_BATCH_SIZE);
|
||||
if (!snapshot) {
|
||||
if (items.length > 0) throw new Error("CLAW-590 cleanup found orphaned proof items");
|
||||
return { done: true as const, itemsDeleted: 0, snapshotDeleted: false };
|
||||
}
|
||||
assertOwnedSnapshot(snapshot, args.snapshotId);
|
||||
for (const item of items) await ctx.db.delete(item._id);
|
||||
const done = items.length < CLEANUP_BATCH_SIZE;
|
||||
if (done) await ctx.db.delete(snapshot._id);
|
||||
return { done, itemsDeleted: items.length, snapshotDeleted: done };
|
||||
},
|
||||
});
|
||||
|
||||
export const cleanupCanonicalTrendingProof = internalAction({
|
||||
args: proofArgs,
|
||||
handler: async (ctx, args) => {
|
||||
assertTestSeedAllowed();
|
||||
assertProofSnapshotId(args.snapshotId);
|
||||
let itemsDeleted = 0;
|
||||
for (let batch = 1; batch <= CLEANUP_MAX_BATCHES; batch += 1) {
|
||||
const result = (await ctx.runMutation(
|
||||
internalRefs.canonicalTrendingTestFixtures.cleanupCanonicalTrendingProofBatch as never,
|
||||
args as never,
|
||||
)) as { done: boolean; itemsDeleted: number; snapshotDeleted: boolean };
|
||||
itemsDeleted += result.itemsDeleted;
|
||||
if (result.done) {
|
||||
return {
|
||||
ok: true as const,
|
||||
itemsDeleted,
|
||||
snapshotDeleted: result.snapshotDeleted,
|
||||
batches: batch,
|
||||
};
|
||||
}
|
||||
}
|
||||
throw new Error("CLAW-590 proof cleanup exceeded its bounded batch limit");
|
||||
},
|
||||
});
|
||||
@@ -17,6 +17,8 @@ const mocks = vi.hoisted(() => {
|
||||
const authRefreshTokensPruneRef = Symbol("auth-refresh-tokens-prune");
|
||||
const publisherInvitesPruneRef = Symbol("publisher-invites-prune");
|
||||
const promotionsFeedPublishRef = Symbol("promotions-feed-publish");
|
||||
const canonicalTrendingMaterializeRef = Symbol("canonical-trending-materialize");
|
||||
const canonicalTrendingPruneRef = Symbol("canonical-trending-prune");
|
||||
const prepublicationQueueHealthRef = Symbol("prepublication-queue-health");
|
||||
const securityScanExpiredLeaseRecoveryRef = Symbol("security-scan-expired-lease-recovery");
|
||||
const securityScanDispatchWatchdogRef = Symbol("security-scan-dispatch-watchdog");
|
||||
@@ -36,6 +38,8 @@ const mocks = vi.hoisted(() => {
|
||||
authRefreshTokensPruneRef,
|
||||
publisherInvitesPruneRef,
|
||||
promotionsFeedPublishRef,
|
||||
canonicalTrendingMaterializeRef,
|
||||
canonicalTrendingPruneRef,
|
||||
prepublicationQueueHealthRef,
|
||||
securityScanExpiredLeaseRecoveryRef,
|
||||
securityScanDispatchWatchdogRef,
|
||||
@@ -50,6 +54,10 @@ vi.mock("convex/server", () => ({
|
||||
|
||||
vi.mock("./_generated/api", () => ({
|
||||
internal: {
|
||||
canonicalTrending: {
|
||||
materializeInternal: mocks.canonicalTrendingMaterializeRef,
|
||||
pruneExpiredActionInternal: mocks.canonicalTrendingPruneRef,
|
||||
},
|
||||
githubSkillSyncNode: { syncGitHubSkillSourcesInternal: mocks.githubSkillSyncRef },
|
||||
leaderboards: { rebuildTrendingLeaderboardAction: Symbol("trending-leaderboard") },
|
||||
packageLeaderboards: {
|
||||
@@ -164,6 +172,28 @@ describe("crons", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("materializes the canonical Trending snapshot hourly", async () => {
|
||||
await import("./crons");
|
||||
|
||||
expect(mocks.interval).toHaveBeenCalledWith(
|
||||
"canonical-trending-snapshot",
|
||||
{ hours: 1 },
|
||||
mocks.canonicalTrendingMaterializeRef,
|
||||
{},
|
||||
);
|
||||
});
|
||||
|
||||
it("prunes canonical Trending snapshots independently each hour", async () => {
|
||||
await import("./crons");
|
||||
|
||||
expect(mocks.interval).toHaveBeenCalledWith(
|
||||
"canonical-trending-prune",
|
||||
{ hours: 1 },
|
||||
mocks.canonicalTrendingPruneRef,
|
||||
{},
|
||||
);
|
||||
});
|
||||
|
||||
it("prunes expired skill scan requests in bounded continuation batches", async () => {
|
||||
await import("./crons");
|
||||
|
||||
|
||||
@@ -26,6 +26,20 @@ if (process.env.CLAWHUB_DISABLE_CRONS !== "1" && process.env.CLAWHUB_PREVIEW !==
|
||||
{ limit: 200 },
|
||||
);
|
||||
|
||||
crons.interval(
|
||||
"canonical-trending-snapshot",
|
||||
{ hours: 1 },
|
||||
internal.canonicalTrending.materializeInternal,
|
||||
{},
|
||||
);
|
||||
|
||||
crons.interval(
|
||||
"canonical-trending-prune",
|
||||
{ hours: 1 },
|
||||
internal.canonicalTrending.pruneExpiredActionInternal,
|
||||
{},
|
||||
);
|
||||
|
||||
crons.interval(
|
||||
"package-trending-leaderboard",
|
||||
{ minutes: 60 },
|
||||
|
||||
+23
-1
@@ -1,6 +1,6 @@
|
||||
import { ApiRoutes, LegacyApiRoutes } from "clawhub-schema";
|
||||
/* @vitest-environment node */
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { vi } from "vitest";
|
||||
import type { ActionCtx } from "./_generated/server";
|
||||
import http from "./http";
|
||||
@@ -93,6 +93,28 @@ async function expectRouteUsesIpBucket({
|
||||
}
|
||||
|
||||
describe("HTTP route rate limit defaults", () => {
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
it("keeps the dark canonical Trending route ahead of rate limiting", async () => {
|
||||
vi.stubEnv("CLAWHUB_ENV", "production");
|
||||
vi.stubEnv("CLAWHUB_SKILLS_SH_ROLLOUT_MODE", "off");
|
||||
const route = http.lookup(ApiRoutes.trending, "GET");
|
||||
if (!route) throw new Error("Expected canonical Trending route");
|
||||
const [action] = route;
|
||||
const { ctx, runMutation } = makeDeniedRateLimitCtx();
|
||||
|
||||
const response = await (action as unknown as WrappedHttpAction)._handler(
|
||||
ctx,
|
||||
new Request(`https://example.com${ApiRoutes.trending}`),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(404);
|
||||
expect(response.headers.get("cache-control")).toBe("no-store");
|
||||
expect(runMutation).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("registers package version downloads behind the download limit", async () => {
|
||||
await expectRouteUsesIpBucket({
|
||||
path: "/api/v1/packages/demo/versions/1.0.0/download",
|
||||
|
||||
@@ -24,6 +24,7 @@ import {
|
||||
listPackagesV1Http,
|
||||
listPluginsV1Http,
|
||||
listSkillsV1Http,
|
||||
trendingV1Http,
|
||||
mintPublishTokenV1Http,
|
||||
npmMirrorGetHttp,
|
||||
packagesDeleteRouterV1Http,
|
||||
@@ -120,6 +121,12 @@ http.route({
|
||||
handler: listSkillsV1Http,
|
||||
});
|
||||
|
||||
http.route({
|
||||
path: ApiRoutes.trending,
|
||||
method: "GET",
|
||||
handler: trendingV1Http,
|
||||
});
|
||||
|
||||
http.route({
|
||||
pathPrefix: `${ApiRoutes.skillScans}/`,
|
||||
method: "GET",
|
||||
|
||||
@@ -2451,6 +2451,46 @@ describe("httpApiV1 handlers", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("deprecates native-only Trending with the canonical successor link", async () => {
|
||||
vi.stubEnv("CLAWHUB_ENV", "test");
|
||||
vi.stubEnv("CLAWHUB_SKILLS_SH_ROLLOUT_MODE", "test");
|
||||
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
|
||||
if ("limit" in args) return { items: [], nextCursor: null };
|
||||
return null;
|
||||
});
|
||||
const runMutation = vi.fn().mockResolvedValue(okRate());
|
||||
|
||||
const response = await __handlers.listSkillsV1Handler(
|
||||
makeCtx({ runQuery, runMutation }),
|
||||
new Request("https://example.com/api/v1/skills?sort=trending"),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get("deprecation")).toBe("true");
|
||||
expect(response.headers.get("link")).toBe(
|
||||
'</api/v1/trending?kind=skills>; rel="successor-version"',
|
||||
);
|
||||
});
|
||||
|
||||
it("does not advertise the dark canonical Trending successor", async () => {
|
||||
vi.stubEnv("CLAWHUB_ENV", "production");
|
||||
vi.stubEnv("CLAWHUB_SKILLS_SH_ROLLOUT_MODE", "off");
|
||||
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
|
||||
if ("limit" in args) return { items: [], nextCursor: null };
|
||||
return null;
|
||||
});
|
||||
const runMutation = vi.fn().mockResolvedValue(okRate());
|
||||
|
||||
const response = await __handlers.listSkillsV1Handler(
|
||||
makeCtx({ runQuery, runMutation }),
|
||||
new Request("https://example.com/api/v1/skills?sort=trending"),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get("deprecation")).toBeNull();
|
||||
expect(response.headers.get("link")).toBeNull();
|
||||
});
|
||||
|
||||
it("lists skills rejects invalid sort", async () => {
|
||||
const runQuery = vi.fn();
|
||||
const runMutation = vi.fn().mockResolvedValue(okRate());
|
||||
|
||||
@@ -49,6 +49,7 @@ import {
|
||||
} from "./httpApiV1/skillsV1";
|
||||
import { starsDeleteRouterV1Handler, starsPostRouterV1Handler } from "./httpApiV1/starsV1";
|
||||
import { transfersGetRouterV1Handler } from "./httpApiV1/transfersV1";
|
||||
import { trendingV1Handler } from "./httpApiV1/trendingV1";
|
||||
import {
|
||||
banAppealContextV1Handler,
|
||||
usersGetRouterV1Handler,
|
||||
@@ -82,6 +83,7 @@ export const promotionsFeedV1Http = httpAction(promotionsFeedV1Handler);
|
||||
export const searchSkillsV1Http = httpAction(searchSkillsV1Handler);
|
||||
export const resolveSkillVersionV1Http = httpAction(resolveSkillVersionV1Handler);
|
||||
export const listSkillsV1Http = httpAction(listSkillsV1Handler);
|
||||
export const trendingV1Http = httpAction(trendingV1Handler);
|
||||
export const skillsGetRouterV1Http = httpAction(skillsGetRouterV1Handler);
|
||||
export const publishSkillV1Http = httpAction(publishSkillV1Handler);
|
||||
export const skillSecurityVerdictsV1Http = httpAction(skillSecurityVerdictsV1Handler);
|
||||
@@ -132,6 +134,7 @@ export const __handlers = {
|
||||
searchSkillsV1Handler,
|
||||
resolveSkillVersionV1Handler,
|
||||
listSkillsV1Handler,
|
||||
trendingV1Handler,
|
||||
skillsGetRouterV1Handler,
|
||||
publishSkillV1Handler,
|
||||
skillSecurityVerdictsV1Handler,
|
||||
|
||||
@@ -1517,7 +1517,14 @@ export async function listSkillsV1Handler(ctx: ActionCtx, request: Request) {
|
||||
: null,
|
||||
}));
|
||||
|
||||
return json({ items, nextCursor: result.nextCursor ?? null }, 200, rate.headers);
|
||||
const responseHeaders =
|
||||
sort === "trending" && getRuntimeRolloutCapabilities().skillsSh.runtimeEnabled
|
||||
? mergeHeaders(rate.headers, {
|
||||
Deprecation: "true",
|
||||
Link: `<${ApiRoutes.trending}?kind=skills>; rel="successor-version"`,
|
||||
})
|
||||
: rate.headers;
|
||||
return json({ items, nextCursor: result.nextCursor ?? null }, 200, responseHeaders);
|
||||
}
|
||||
|
||||
async function describeOwnerVisibleSkillState(
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
/* @vitest-environment node */
|
||||
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("../lib/httpRateLimit", () => ({
|
||||
applyRateLimit: vi.fn(async () => ({ ok: true, headers: { "x-rate-limit": "ok" } })),
|
||||
}));
|
||||
|
||||
const { applyRateLimit } = await import("../lib/httpRateLimit");
|
||||
const { trendingV1Handler } = await import("./trendingV1");
|
||||
|
||||
beforeEach(() => {
|
||||
vi.stubEnv("CLAWHUB_ENV", "test");
|
||||
vi.stubEnv("CLAWHUB_SKILLS_SH_ROLLOUT_MODE", "test");
|
||||
vi.mocked(applyRateLimit).mockClear();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
describe("canonical Trending HTTP API", () => {
|
||||
it("stays dark before rate limiting while the skills.sh rollout is disabled", async () => {
|
||||
vi.stubEnv("CLAWHUB_SKILLS_SH_ROLLOUT_MODE", "off");
|
||||
const runQuery = vi.fn();
|
||||
|
||||
const response = await trendingV1Handler(
|
||||
{ runQuery } as never,
|
||||
new Request("https://clawhub.ai/api/v1/trending?kind=skills"),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(404);
|
||||
expect(response.headers.get("cache-control")).toBe("no-store");
|
||||
expect(applyRateLimit).not.toHaveBeenCalled();
|
||||
expect(runQuery).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns the materialized snapshot envelope without reordering cards", async () => {
|
||||
const page = {
|
||||
kind: "skills",
|
||||
snapshotId: "snap_123",
|
||||
snapshotCursor: "snapshot-cursor",
|
||||
generatedAt: "2026-07-25T16:00:00.000Z",
|
||||
windowHours: 24,
|
||||
rankingVersion: "skills-trending-v1",
|
||||
items: [
|
||||
{ id: "clawhub:skills:1", source: "clawhub", displayName: "Native" },
|
||||
{ id: "skills-sh:owner/repo/skill", source: "skills-sh", displayName: "External" },
|
||||
],
|
||||
nextCursor: "next-cursor",
|
||||
};
|
||||
const runQuery = vi.fn(async () => ({ status: "ok", page }));
|
||||
|
||||
const response = await trendingV1Handler(
|
||||
{ runQuery } as never,
|
||||
new Request(
|
||||
"https://academic-chihuahua-392.convex.site/api/v1/trending?kind=skills&limit=2&cursor=current-cursor",
|
||||
),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get("x-rate-limit")).toBe("ok");
|
||||
expect(runQuery).toHaveBeenCalledWith(expect.anything(), {
|
||||
cursor: "current-cursor",
|
||||
limit: 2,
|
||||
});
|
||||
expect(await response.json()).toEqual(page);
|
||||
});
|
||||
|
||||
it("rejects unsupported kinds and invalid limits before querying a snapshot", async () => {
|
||||
const runQuery = vi.fn();
|
||||
|
||||
const unsupported = await trendingV1Handler(
|
||||
{ runQuery } as never,
|
||||
new Request("https://clawhub.ai/api/v1/trending?kind=plugins"),
|
||||
);
|
||||
const invalidLimit = await trendingV1Handler(
|
||||
{ runQuery } as never,
|
||||
new Request("https://clawhub.ai/api/v1/trending?kind=skills&limit=0"),
|
||||
);
|
||||
|
||||
expect(unsupported.status).toBe(400);
|
||||
expect(invalidLimit.status).toBe(400);
|
||||
expect(runQuery).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns 503 until the first ready snapshot exists", async () => {
|
||||
const response = await trendingV1Handler(
|
||||
{ runQuery: vi.fn(async () => ({ status: "unavailable" })) } as never,
|
||||
new Request("https://clawhub.ai/api/v1/trending?kind=skills"),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(503);
|
||||
});
|
||||
|
||||
it("returns 410 when a stable cursor references a pruned snapshot", async () => {
|
||||
const response = await trendingV1Handler(
|
||||
{
|
||||
runQuery: vi.fn(async () => ({ status: "expired" })),
|
||||
} as never,
|
||||
new Request("https://clawhub.ai/api/v1/trending?cursor=expired"),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(410);
|
||||
});
|
||||
|
||||
it("returns 400 when the page query rejects a malformed cursor", async () => {
|
||||
const response = await trendingV1Handler(
|
||||
{ runQuery: vi.fn(async () => ({ status: "invalid-cursor" })) } as never,
|
||||
new Request("https://clawhub.ai/api/v1/trending?cursor=malformed"),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,54 @@
|
||||
import { internal } from "../_generated/api";
|
||||
import type { ActionCtx } from "../_generated/server";
|
||||
import { applyRateLimit } from "../lib/httpRateLimit";
|
||||
import { getRuntimeRolloutCapabilities } from "../lib/rolloutCapabilities";
|
||||
import { json, text } from "./shared";
|
||||
|
||||
const internalRefs = internal as unknown as {
|
||||
canonicalTrending: {
|
||||
getPageInternal: unknown;
|
||||
};
|
||||
};
|
||||
|
||||
const DEFAULT_TRENDING_LIMIT = 20;
|
||||
const MAX_TRENDING_LIMIT = 100;
|
||||
|
||||
type TrendingPageQueryResult =
|
||||
| { status: "ok"; page: unknown }
|
||||
| { status: "unavailable" }
|
||||
| { status: "invalid-cursor" }
|
||||
| { status: "expired" };
|
||||
|
||||
function parseLimit(value: string | null) {
|
||||
if (value === null) return DEFAULT_TRENDING_LIMIT;
|
||||
const parsed = Number(value);
|
||||
if (!Number.isSafeInteger(parsed) || parsed < 1 || parsed > MAX_TRENDING_LIMIT) return null;
|
||||
return parsed;
|
||||
}
|
||||
|
||||
export async function trendingV1Handler(ctx: ActionCtx, request: Request) {
|
||||
if (!getRuntimeRolloutCapabilities().skillsSh.runtimeEnabled) {
|
||||
return text("Not found", 404, { "cache-control": "no-store" });
|
||||
}
|
||||
|
||||
const rate = await applyRateLimit(ctx, request, "read");
|
||||
if (!rate.ok) return rate.response;
|
||||
|
||||
const url = new URL(request.url);
|
||||
const kind = url.searchParams.get("kind")?.trim() || "skills";
|
||||
if (kind !== "skills") return text("Unsupported trending kind", 400, rate.headers);
|
||||
const limit = parseLimit(url.searchParams.get("limit"));
|
||||
if (limit === null) return text("Invalid limit", 400, rate.headers);
|
||||
const cursor = url.searchParams.get("cursor")?.trim() || null;
|
||||
|
||||
const result = (await ctx.runQuery(
|
||||
internalRefs.canonicalTrending.getPageInternal as never,
|
||||
{ cursor, limit } as never,
|
||||
)) as TrendingPageQueryResult;
|
||||
if (result.status === "unavailable") {
|
||||
return text("Trending snapshot unavailable", 503, rate.headers);
|
||||
}
|
||||
if (result.status === "invalid-cursor") return text("Invalid cursor format", 400, rate.headers);
|
||||
if (result.status === "expired") return text("Trending snapshot expired", 410, rate.headers);
|
||||
return json(result.page, 200, rate.headers);
|
||||
}
|
||||
@@ -0,0 +1,308 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
blendCanonicalTrendingPools,
|
||||
buildExternalCanonicalTrendingCandidate,
|
||||
buildNativeCanonicalTrendingCandidate,
|
||||
decodeCanonicalTrendingCursor,
|
||||
encodeCanonicalTrendingCursor,
|
||||
sortCanonicalTrendingPools,
|
||||
type CanonicalTrendingCandidate,
|
||||
} from "./canonicalTrending";
|
||||
|
||||
function candidate(
|
||||
identity: string,
|
||||
lane: CanonicalTrendingCandidate["lane"],
|
||||
overrides: Partial<CanonicalTrendingCandidate> = {},
|
||||
): CanonicalTrendingCandidate {
|
||||
return {
|
||||
identity,
|
||||
lane,
|
||||
publisherKey: identity,
|
||||
installs24h: 0,
|
||||
bookmarks24h: 0,
|
||||
createdAt: 0,
|
||||
updatedAt: 0,
|
||||
upstreamRank: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("canonical Trending ordering", () => {
|
||||
it("interleaves complete pools as a continuous 40/20/40 feed", () => {
|
||||
const result = blendCanonicalTrendingPools({
|
||||
clawhubTrending: Array.from({ length: 4 }, (_, index) =>
|
||||
candidate(`c-${index + 1}`, "clawhub-trending"),
|
||||
),
|
||||
clawhubRising: Array.from({ length: 2 }, (_, index) =>
|
||||
candidate(`r-${index + 1}`, "clawhub-rising"),
|
||||
),
|
||||
skillsShTrending: Array.from({ length: 4 }, (_, index) =>
|
||||
candidate(`s-${index + 1}`, "skills-sh-trending"),
|
||||
),
|
||||
});
|
||||
|
||||
expect(result.map((entry) => entry.identity)).toEqual([
|
||||
"c-1",
|
||||
"s-1",
|
||||
"r-1",
|
||||
"c-2",
|
||||
"s-2",
|
||||
"c-3",
|
||||
"s-3",
|
||||
"r-2",
|
||||
"c-4",
|
||||
"s-4",
|
||||
]);
|
||||
});
|
||||
|
||||
it("backfills exhausted Rising capacity from ClawHub Trending", () => {
|
||||
const result = blendCanonicalTrendingPools({
|
||||
clawhubTrending: Array.from({ length: 6 }, (_, index) =>
|
||||
candidate(`c-${index + 1}`, "clawhub-trending"),
|
||||
),
|
||||
clawhubRising: [],
|
||||
skillsShTrending: Array.from({ length: 4 }, (_, index) =>
|
||||
candidate(`s-${index + 1}`, "skills-sh-trending"),
|
||||
),
|
||||
});
|
||||
|
||||
expect(result.slice(0, 5).map((entry) => entry.identity)).toEqual([
|
||||
"c-1",
|
||||
"s-1",
|
||||
"c-2",
|
||||
"c-3",
|
||||
"s-2",
|
||||
]);
|
||||
expect(result).toHaveLength(10);
|
||||
});
|
||||
|
||||
it("deduplicates native overlap and defers capped publishers beyond the first 20", () => {
|
||||
const clawhubTrending = [
|
||||
candidate("shared", "clawhub-trending", { publisherKey: "alpha", installs24h: 30 }),
|
||||
candidate("alpha-2", "clawhub-trending", { publisherKey: "alpha", installs24h: 29 }),
|
||||
candidate("alpha-3", "clawhub-trending", { publisherKey: "alpha", installs24h: 28 }),
|
||||
...Array.from({ length: 18 }, (_, index) =>
|
||||
candidate(`c-${index + 1}`, "clawhub-trending", {
|
||||
publisherKey: `c-${index + 1}`,
|
||||
installs24h: 27 - index,
|
||||
}),
|
||||
),
|
||||
];
|
||||
const clawhubRising = [
|
||||
candidate("shared", "clawhub-rising", { publisherKey: "alpha" }),
|
||||
...Array.from({ length: 10 }, (_, index) =>
|
||||
candidate(`r-${index + 1}`, "clawhub-rising", { publisherKey: `r-${index + 1}` }),
|
||||
),
|
||||
];
|
||||
const skillsShTrending = Array.from({ length: 20 }, (_, index) =>
|
||||
candidate(`s-${index + 1}`, "skills-sh-trending", { publisherKey: `s-${index + 1}` }),
|
||||
);
|
||||
|
||||
const result = blendCanonicalTrendingPools({
|
||||
clawhubTrending,
|
||||
clawhubRising,
|
||||
skillsShTrending,
|
||||
});
|
||||
|
||||
expect(result.filter((entry) => entry.identity === "shared")).toHaveLength(1);
|
||||
expect(result.slice(0, 20).filter((entry) => entry.publisherKey === "alpha")).toHaveLength(2);
|
||||
expect(result.findIndex((entry) => entry.identity === "alpha-3")).toBeGreaterThanOrEqual(20);
|
||||
expect(result).toHaveLength(51);
|
||||
});
|
||||
|
||||
it("completes an undersized feed after cap-compliant alternatives are exhausted", () => {
|
||||
const result = blendCanonicalTrendingPools({
|
||||
clawhubTrending: [
|
||||
candidate("alpha-1", "clawhub-trending", { publisherKey: "alpha" }),
|
||||
candidate("alpha-2", "clawhub-trending", { publisherKey: "alpha" }),
|
||||
candidate("alpha-3", "clawhub-trending", { publisherKey: "alpha" }),
|
||||
],
|
||||
clawhubRising: [],
|
||||
skillsShTrending: [],
|
||||
});
|
||||
|
||||
expect(result.map((entry) => entry.identity)).toEqual(["alpha-1", "alpha-2", "alpha-3"]);
|
||||
});
|
||||
|
||||
it("sorts native metrics and preserves exact skills.sh upstream rank", () => {
|
||||
const pools = sortCanonicalTrendingPools({
|
||||
clawhubTrending: [
|
||||
candidate("c-low", "clawhub-trending", { installs24h: 2, bookmarks24h: 9 }),
|
||||
candidate("c-high-bookmarks", "clawhub-trending", {
|
||||
installs24h: 3,
|
||||
bookmarks24h: 5,
|
||||
}),
|
||||
candidate("c-high", "clawhub-trending", { installs24h: 3, bookmarks24h: 1 }),
|
||||
],
|
||||
clawhubRising: [
|
||||
candidate("r-old", "clawhub-rising", {
|
||||
installs24h: 1,
|
||||
bookmarks24h: 1,
|
||||
createdAt: 10,
|
||||
}),
|
||||
candidate("r-new", "clawhub-rising", {
|
||||
installs24h: 1,
|
||||
bookmarks24h: 1,
|
||||
createdAt: 20,
|
||||
}),
|
||||
],
|
||||
skillsShTrending: [
|
||||
candidate("s-3", "skills-sh-trending", { upstreamRank: 3 }),
|
||||
candidate("s-1", "skills-sh-trending", { upstreamRank: 1 }),
|
||||
candidate("s-2", "skills-sh-trending", { upstreamRank: 2 }),
|
||||
],
|
||||
});
|
||||
|
||||
expect(pools.clawhubTrending.map((entry) => entry.identity)).toEqual([
|
||||
"c-high-bookmarks",
|
||||
"c-high",
|
||||
"c-low",
|
||||
]);
|
||||
expect(pools.clawhubRising.map((entry) => entry.identity)).toEqual(["r-new", "r-old"]);
|
||||
expect(pools.skillsShTrending.map((entry) => entry.identity)).toEqual(["s-1", "s-2", "s-3"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("canonical Trending cursors", () => {
|
||||
it("round-trips a stable snapshot position and rejects malformed cursors", () => {
|
||||
const cursor = encodeCanonicalTrendingCursor({ snapshotId: "snapshot-123", offset: 40 });
|
||||
|
||||
expect(decodeCanonicalTrendingCursor(cursor)).toEqual({
|
||||
snapshotId: "snapshot-123",
|
||||
offset: 40,
|
||||
});
|
||||
expect(() => decodeCanonicalTrendingCursor("not-a-cursor")).toThrow("Invalid cursor format");
|
||||
});
|
||||
});
|
||||
|
||||
describe("canonical Trending cards", () => {
|
||||
it("keeps native 24-hour metrics separate from lifetime installs", () => {
|
||||
const result = buildNativeCanonicalTrendingCandidate(
|
||||
{
|
||||
skillId: "skills:native" as never,
|
||||
slug: "native",
|
||||
displayName: "Native",
|
||||
summary: "Native summary",
|
||||
ownerUserId: "users:patrick" as never,
|
||||
ownerPublisherId: undefined,
|
||||
ownerHandle: "patrick",
|
||||
ownerKind: "user",
|
||||
ownerName: "patrick",
|
||||
ownerDisplayName: "Patrick",
|
||||
ownerImage: undefined,
|
||||
badges: undefined,
|
||||
installKind: undefined,
|
||||
githubScanStatus: undefined,
|
||||
moderationVerdict: "clean",
|
||||
statsInstallsAllTime: 900,
|
||||
stats: { downloads: 1_000, stars: 20, versions: 1, comments: 0 },
|
||||
createdAt: 100,
|
||||
updatedAt: 200,
|
||||
},
|
||||
{ installs: 12, bookmarks: 4, updatedAt: 300 },
|
||||
);
|
||||
|
||||
expect(result?.card.metrics).toEqual({
|
||||
trending24hInstalls: 12,
|
||||
trending24hBookmarks: 4,
|
||||
lifetimeInstalls: 900,
|
||||
lifetimeInstallsPeriod: "lifetime",
|
||||
updatedAt: 300,
|
||||
});
|
||||
});
|
||||
|
||||
it("excludes native rows without a routable public owner handle", () => {
|
||||
const result = buildNativeCanonicalTrendingCandidate(
|
||||
{
|
||||
skillId: "skills:native" as never,
|
||||
slug: "native",
|
||||
displayName: "Native",
|
||||
summary: undefined,
|
||||
ownerUserId: "users:patrick" as never,
|
||||
ownerPublisherId: undefined,
|
||||
ownerHandle: undefined,
|
||||
ownerKind: "user",
|
||||
ownerName: "patrick",
|
||||
ownerDisplayName: "Patrick",
|
||||
ownerImage: undefined,
|
||||
badges: undefined,
|
||||
installKind: undefined,
|
||||
githubScanStatus: undefined,
|
||||
moderationVerdict: "clean",
|
||||
statsInstallsAllTime: 0,
|
||||
stats: { downloads: 0, stars: 0, versions: 1, comments: 0 },
|
||||
createdAt: 100,
|
||||
updatedAt: 200,
|
||||
},
|
||||
{ installs: 1, bookmarks: 0, updatedAt: 300 },
|
||||
);
|
||||
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
it("preserves an unknown native lifetime install count as null", () => {
|
||||
const result = buildNativeCanonicalTrendingCandidate(
|
||||
{
|
||||
skillId: "skills:native" as never,
|
||||
slug: "native",
|
||||
displayName: "Native",
|
||||
summary: undefined,
|
||||
ownerUserId: "users:patrick" as never,
|
||||
ownerPublisherId: undefined,
|
||||
ownerHandle: "patrick",
|
||||
ownerKind: "user",
|
||||
ownerName: "patrick",
|
||||
ownerDisplayName: "Patrick",
|
||||
ownerImage: undefined,
|
||||
badges: undefined,
|
||||
installKind: undefined,
|
||||
githubScanStatus: undefined,
|
||||
moderationVerdict: "clean",
|
||||
statsInstallsAllTime: undefined,
|
||||
stats: { downloads: 0, stars: 0, versions: 1, comments: 0 },
|
||||
createdAt: 100,
|
||||
updatedAt: 200,
|
||||
},
|
||||
{ installs: 1, bookmarks: 0, updatedAt: 300 },
|
||||
);
|
||||
|
||||
expect(result?.card.metrics).toMatchObject({
|
||||
trending24hInstalls: 1,
|
||||
lifetimeInstalls: null,
|
||||
updatedAt: 300,
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves skills.sh rank while leaving unavailable 24-hour metrics null", () => {
|
||||
const result = buildExternalCanonicalTrendingCandidate({
|
||||
externalId: "patrick/repo/external",
|
||||
owner: "patrick",
|
||||
repo: "repo",
|
||||
sourceHost: undefined,
|
||||
slug: "external",
|
||||
displayName: "External",
|
||||
searchSummary: "External summary",
|
||||
sourceUrl: "https://skills.sh/patrick/repo/external",
|
||||
upstreamInstalls: 4_000,
|
||||
trendingRank: 7,
|
||||
trendingLifetimeInstalls: 4_200,
|
||||
trendingObservedAt: 300,
|
||||
upstreamScanners: {
|
||||
genAgentTrustHub: { status: "pass" },
|
||||
socket: { status: "pass" },
|
||||
snyk: { status: "pass" },
|
||||
},
|
||||
firstObservedAt: 100,
|
||||
lastObservedAt: 250,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ upstreamRank: 7 });
|
||||
expect(result?.card.metrics).toEqual({
|
||||
trending24hInstalls: null,
|
||||
trending24hBookmarks: null,
|
||||
lifetimeInstalls: 4_200,
|
||||
lifetimeInstallsPeriod: "lifetime",
|
||||
updatedAt: 300,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,444 @@
|
||||
import { type Infer, v } from "convex/values";
|
||||
import type { Doc } from "../_generated/dataModel";
|
||||
|
||||
export const CANONICAL_TRENDING_RANKING_VERSION = "skills-trending-v1";
|
||||
export const CANONICAL_TRENDING_WINDOW_HOURS = 24;
|
||||
export const CANONICAL_TRENDING_FIRST_PAGE_SIZE = 20;
|
||||
export const CANONICAL_TRENDING_PUBLISHER_CAP = 2;
|
||||
|
||||
const canonicalTrendingUpstreamScannerValidator = v.object({
|
||||
status: v.string(),
|
||||
sourceCheckedAt: v.optional(v.string()),
|
||||
sourceUrl: v.optional(v.string()),
|
||||
});
|
||||
|
||||
export const canonicalTrendingCardValidator = v.object({
|
||||
id: v.string(),
|
||||
source: v.union(v.literal("clawhub"), v.literal("skills-sh")),
|
||||
slug: v.string(),
|
||||
displayName: v.string(),
|
||||
summary: v.union(v.string(), v.null()),
|
||||
canonicalUrl: v.string(),
|
||||
links: v.object({
|
||||
canonical: v.string(),
|
||||
source: v.union(v.string(), v.null()),
|
||||
}),
|
||||
publisher: v.union(
|
||||
v.object({
|
||||
kind: v.union(v.literal("user"), v.literal("org")),
|
||||
handle: v.union(v.string(), v.null()),
|
||||
displayName: v.union(v.string(), v.null()),
|
||||
image: v.union(v.string(), v.null()),
|
||||
official: v.boolean(),
|
||||
}),
|
||||
v.null(),
|
||||
),
|
||||
official: v.boolean(),
|
||||
featured: v.boolean(),
|
||||
install: v.object({
|
||||
kind: v.union(v.literal("clawhub"), v.literal("github"), v.literal("skills-sh")),
|
||||
reference: v.string(),
|
||||
sourceUrl: v.union(v.string(), v.null()),
|
||||
}),
|
||||
sourceIdentity: v.object({
|
||||
id: v.string(),
|
||||
owner: v.union(v.string(), v.null()),
|
||||
repo: v.union(v.string(), v.null()),
|
||||
host: v.union(v.string(), v.null()),
|
||||
lifetimeInstalls: v.union(v.number(), v.null()),
|
||||
}),
|
||||
trust: v.object({
|
||||
visibility: v.literal("public"),
|
||||
installability: v.literal("installable"),
|
||||
clawHubVerdict: v.union(v.string(), v.null()),
|
||||
upstreamScanners: v.union(
|
||||
v.object({
|
||||
genAgentTrustHub: canonicalTrendingUpstreamScannerValidator,
|
||||
socket: canonicalTrendingUpstreamScannerValidator,
|
||||
snyk: canonicalTrendingUpstreamScannerValidator,
|
||||
}),
|
||||
v.null(),
|
||||
),
|
||||
sourceFreshness: v.union(v.literal("native"), v.literal("observed-only")),
|
||||
}),
|
||||
metrics: v.object({
|
||||
trending24hInstalls: v.union(v.number(), v.null()),
|
||||
trending24hBookmarks: v.union(v.number(), v.null()),
|
||||
lifetimeInstalls: v.union(v.number(), v.null()),
|
||||
lifetimeInstallsPeriod: v.literal("lifetime"),
|
||||
updatedAt: v.number(),
|
||||
}),
|
||||
});
|
||||
|
||||
export const canonicalTrendingSourceRefValidator = v.union(
|
||||
v.object({ kind: v.literal("clawhub"), skillId: v.id("skills") }),
|
||||
v.object({ kind: v.literal("skills-sh"), externalId: v.string() }),
|
||||
);
|
||||
|
||||
export type CanonicalTrendingCard = Infer<typeof canonicalTrendingCardValidator>;
|
||||
|
||||
export type CanonicalTrendingMaterializationCandidate = CanonicalTrendingCandidate & {
|
||||
card: CanonicalTrendingCard;
|
||||
sourceRef: Infer<typeof canonicalTrendingSourceRefValidator>;
|
||||
};
|
||||
|
||||
type NativeTrendingDigest = Pick<
|
||||
Doc<"skillSearchDigest">,
|
||||
| "skillId"
|
||||
| "slug"
|
||||
| "displayName"
|
||||
| "summary"
|
||||
| "ownerUserId"
|
||||
| "ownerPublisherId"
|
||||
| "ownerHandle"
|
||||
| "ownerKind"
|
||||
| "ownerName"
|
||||
| "ownerDisplayName"
|
||||
| "ownerImage"
|
||||
| "badges"
|
||||
| "installKind"
|
||||
| "githubScanStatus"
|
||||
| "moderationVerdict"
|
||||
| "statsInstallsAllTime"
|
||||
| "stats"
|
||||
| "createdAt"
|
||||
| "updatedAt"
|
||||
>;
|
||||
|
||||
type ExternalTrendingDigest = Pick<
|
||||
Doc<"skillsShMirrorDigests">,
|
||||
| "externalId"
|
||||
| "owner"
|
||||
| "repo"
|
||||
| "sourceHost"
|
||||
| "slug"
|
||||
| "displayName"
|
||||
| "searchSummary"
|
||||
| "sourceUrl"
|
||||
| "upstreamInstalls"
|
||||
| "trendingRank"
|
||||
| "trendingLifetimeInstalls"
|
||||
| "trendingObservedAt"
|
||||
| "upstreamScanners"
|
||||
| "firstObservedAt"
|
||||
| "lastObservedAt"
|
||||
>;
|
||||
|
||||
export function buildNativeCanonicalTrendingCandidate(
|
||||
digest: NativeTrendingDigest,
|
||||
usage: { installs: number; bookmarks: number; updatedAt: number },
|
||||
): CanonicalTrendingMaterializationCandidate | null {
|
||||
const ownerHandle = digest.ownerHandle?.trim();
|
||||
if (!ownerHandle) return null;
|
||||
const official = Boolean(digest.badges?.official);
|
||||
const canonicalUrl = `/${encodeURIComponent(ownerHandle)}/skills/${encodeURIComponent(digest.slug)}`;
|
||||
const identity = `clawhub:${String(digest.skillId)}`;
|
||||
const lifetimeInstalls = digest.statsInstallsAllTime ?? digest.stats.installsAllTime ?? null;
|
||||
return {
|
||||
identity,
|
||||
lane: "clawhub-trending",
|
||||
publisherKey: String(digest.ownerPublisherId ?? digest.ownerUserId),
|
||||
installs24h: Math.max(0, usage.installs),
|
||||
bookmarks24h: Math.max(0, usage.bookmarks),
|
||||
createdAt: digest.createdAt,
|
||||
updatedAt: digest.updatedAt,
|
||||
upstreamRank: null,
|
||||
sourceRef: { kind: "clawhub", skillId: digest.skillId },
|
||||
card: {
|
||||
id: identity,
|
||||
source: "clawhub",
|
||||
slug: digest.slug,
|
||||
displayName: digest.displayName,
|
||||
summary: digest.summary ?? null,
|
||||
canonicalUrl,
|
||||
links: { canonical: canonicalUrl, source: null },
|
||||
publisher: {
|
||||
kind: digest.ownerKind ?? "user",
|
||||
handle: ownerHandle,
|
||||
displayName: digest.ownerDisplayName ?? digest.ownerName ?? ownerHandle,
|
||||
image: digest.ownerImage ?? null,
|
||||
official,
|
||||
},
|
||||
official,
|
||||
featured: Boolean(digest.badges?.highlighted),
|
||||
install: {
|
||||
kind: digest.installKind === "github" ? "github" : "clawhub",
|
||||
reference: `${ownerHandle}/${digest.slug}`,
|
||||
sourceUrl: null,
|
||||
},
|
||||
sourceIdentity: {
|
||||
id: String(digest.skillId),
|
||||
owner: ownerHandle,
|
||||
repo: null,
|
||||
host: null,
|
||||
lifetimeInstalls,
|
||||
},
|
||||
trust: {
|
||||
visibility: "public",
|
||||
installability: "installable",
|
||||
clawHubVerdict: digest.githubScanStatus ?? digest.moderationVerdict ?? null,
|
||||
upstreamScanners: null,
|
||||
sourceFreshness: "native",
|
||||
},
|
||||
metrics: {
|
||||
trending24hInstalls: Math.max(0, usage.installs),
|
||||
trending24hBookmarks: Math.max(0, usage.bookmarks),
|
||||
lifetimeInstalls,
|
||||
lifetimeInstallsPeriod: "lifetime",
|
||||
updatedAt: usage.updatedAt,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function buildExternalCanonicalTrendingCandidate(
|
||||
digest: ExternalTrendingDigest,
|
||||
): CanonicalTrendingMaterializationCandidate | null {
|
||||
if (!Number.isSafeInteger(digest.trendingRank) || (digest.trendingRank ?? 0) < 1) return null;
|
||||
const encodedIdentity = digest.externalId
|
||||
.split("/")
|
||||
.map((segment) => encodeURIComponent(segment))
|
||||
.join("/");
|
||||
const canonicalUrl = `/skills-sh/${encodedIdentity}`;
|
||||
const identity = `skills-sh:${digest.externalId}`;
|
||||
const lifetimeInstalls = digest.trendingLifetimeInstalls ?? digest.upstreamInstalls;
|
||||
return {
|
||||
identity,
|
||||
lane: "skills-sh-trending",
|
||||
publisherKey: digest.owner ?? digest.sourceHost ?? digest.externalId,
|
||||
installs24h: 0,
|
||||
bookmarks24h: 0,
|
||||
createdAt: digest.firstObservedAt,
|
||||
updatedAt: digest.trendingObservedAt ?? digest.lastObservedAt,
|
||||
upstreamRank: digest.trendingRank ?? null,
|
||||
sourceRef: { kind: "skills-sh", externalId: digest.externalId },
|
||||
card: {
|
||||
id: identity,
|
||||
source: "skills-sh",
|
||||
slug: digest.slug,
|
||||
displayName: digest.displayName,
|
||||
summary: digest.searchSummary ?? null,
|
||||
canonicalUrl,
|
||||
links: { canonical: canonicalUrl, source: digest.sourceUrl },
|
||||
publisher: null,
|
||||
official: false,
|
||||
featured: false,
|
||||
install: {
|
||||
kind: "skills-sh",
|
||||
reference: identity,
|
||||
sourceUrl: digest.sourceUrl,
|
||||
},
|
||||
sourceIdentity: {
|
||||
id: digest.externalId,
|
||||
owner: digest.owner ?? null,
|
||||
repo: digest.repo ?? null,
|
||||
host: digest.sourceHost ?? null,
|
||||
lifetimeInstalls,
|
||||
},
|
||||
trust: {
|
||||
visibility: "public",
|
||||
installability: "installable",
|
||||
clawHubVerdict: null,
|
||||
upstreamScanners: digest.upstreamScanners,
|
||||
sourceFreshness: "observed-only",
|
||||
},
|
||||
metrics: {
|
||||
trending24hInstalls: null,
|
||||
trending24hBookmarks: null,
|
||||
lifetimeInstalls,
|
||||
lifetimeInstallsPeriod: "lifetime",
|
||||
updatedAt: digest.trendingObservedAt ?? digest.lastObservedAt,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export type CanonicalTrendingLane = "clawhub-trending" | "clawhub-rising" | "skills-sh-trending";
|
||||
|
||||
export type CanonicalTrendingCandidate = {
|
||||
identity: string;
|
||||
lane: CanonicalTrendingLane;
|
||||
publisherKey: string;
|
||||
installs24h: number;
|
||||
bookmarks24h: number;
|
||||
createdAt: number;
|
||||
updatedAt: number;
|
||||
upstreamRank: number | null;
|
||||
};
|
||||
|
||||
export type CanonicalTrendingPools<T extends CanonicalTrendingCandidate> = {
|
||||
clawhubTrending: T[];
|
||||
clawhubRising: T[];
|
||||
skillsShTrending: T[];
|
||||
};
|
||||
|
||||
const WEIGHTED_CYCLE: CanonicalTrendingLane[] = [
|
||||
"clawhub-trending",
|
||||
"skills-sh-trending",
|
||||
"clawhub-rising",
|
||||
"clawhub-trending",
|
||||
"skills-sh-trending",
|
||||
];
|
||||
|
||||
const SLOT_FALLBACKS: Record<CanonicalTrendingLane, CanonicalTrendingLane[]> = {
|
||||
"clawhub-trending": ["clawhub-trending", "skills-sh-trending", "clawhub-rising"],
|
||||
"clawhub-rising": ["clawhub-rising", "clawhub-trending", "skills-sh-trending"],
|
||||
"skills-sh-trending": ["skills-sh-trending", "clawhub-trending", "clawhub-rising"],
|
||||
};
|
||||
|
||||
type PoolState<T> = {
|
||||
queue: T[];
|
||||
deferred: T[];
|
||||
};
|
||||
|
||||
function compareNumberDesc(left: number, right: number) {
|
||||
return right - left;
|
||||
}
|
||||
|
||||
function compareIdentity(left: CanonicalTrendingCandidate, right: CanonicalTrendingCandidate) {
|
||||
return left.identity.localeCompare(right.identity);
|
||||
}
|
||||
|
||||
export function sortCanonicalTrendingPools<T extends CanonicalTrendingCandidate>(
|
||||
pools: CanonicalTrendingPools<T>,
|
||||
): CanonicalTrendingPools<T> {
|
||||
return {
|
||||
clawhubTrending: [...pools.clawhubTrending].sort(
|
||||
(left, right) =>
|
||||
compareNumberDesc(left.installs24h, right.installs24h) ||
|
||||
compareNumberDesc(left.bookmarks24h, right.bookmarks24h) ||
|
||||
compareNumberDesc(left.updatedAt, right.updatedAt) ||
|
||||
compareIdentity(left, right),
|
||||
),
|
||||
clawhubRising: [...pools.clawhubRising].sort(
|
||||
(left, right) =>
|
||||
compareNumberDesc(left.installs24h, right.installs24h) ||
|
||||
compareNumberDesc(left.bookmarks24h, right.bookmarks24h) ||
|
||||
compareNumberDesc(left.createdAt, right.createdAt) ||
|
||||
compareIdentity(left, right),
|
||||
),
|
||||
skillsShTrending: [...pools.skillsShTrending].sort(
|
||||
(left, right) =>
|
||||
(left.upstreamRank ?? Number.MAX_SAFE_INTEGER) -
|
||||
(right.upstreamRank ?? Number.MAX_SAFE_INTEGER) || compareIdentity(left, right),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
export function blendCanonicalTrendingPools<T extends CanonicalTrendingCandidate>(
|
||||
input: CanonicalTrendingPools<T>,
|
||||
options: {
|
||||
firstPageSize?: number;
|
||||
publisherCap?: number;
|
||||
} = {},
|
||||
) {
|
||||
const firstPageSize = options.firstPageSize ?? CANONICAL_TRENDING_FIRST_PAGE_SIZE;
|
||||
const publisherCap = options.publisherCap ?? CANONICAL_TRENDING_PUBLISHER_CAP;
|
||||
const sorted = sortCanonicalTrendingPools(input);
|
||||
const pools: Record<CanonicalTrendingLane, PoolState<T>> = {
|
||||
"clawhub-trending": { queue: sorted.clawhubTrending, deferred: [] },
|
||||
"clawhub-rising": { queue: sorted.clawhubRising, deferred: [] },
|
||||
"skills-sh-trending": { queue: sorted.skillsShTrending, deferred: [] },
|
||||
};
|
||||
const publisherCounts = new Map<string, number>();
|
||||
const seen = new Set<string>();
|
||||
const result: T[] = [];
|
||||
let cycleIndex = 0;
|
||||
let capReleased = firstPageSize === 0;
|
||||
|
||||
const releaseDeferred = () => {
|
||||
if (capReleased) return;
|
||||
capReleased = true;
|
||||
for (const pool of Object.values(pools)) {
|
||||
pool.queue = [...pool.deferred, ...pool.queue];
|
||||
pool.deferred = [];
|
||||
}
|
||||
};
|
||||
|
||||
const takeFromLane = (lane: CanonicalTrendingLane) => {
|
||||
const pool = pools[lane];
|
||||
while (pool.queue.length > 0) {
|
||||
const next = pool.queue.shift()!;
|
||||
if (seen.has(next.identity)) continue;
|
||||
if (!capReleased && (publisherCounts.get(next.publisherKey) ?? 0) >= publisherCap) {
|
||||
pool.deferred.push(next);
|
||||
continue;
|
||||
}
|
||||
return next;
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
while (true) {
|
||||
if (!capReleased && result.length >= firstPageSize) releaseDeferred();
|
||||
|
||||
const preferredLane = WEIGHTED_CYCLE[cycleIndex % WEIGHTED_CYCLE.length];
|
||||
cycleIndex += 1;
|
||||
let next: T | null = null;
|
||||
for (const lane of SLOT_FALLBACKS[preferredLane]) {
|
||||
next = takeFromLane(lane);
|
||||
if (next) break;
|
||||
}
|
||||
|
||||
if (!next) {
|
||||
const deferredCount = Object.values(pools).reduce(
|
||||
(total, pool) => total + pool.deferred.length,
|
||||
0,
|
||||
);
|
||||
if (!capReleased && deferredCount > 0) {
|
||||
// A complete feed and a strict first-page cap cannot both be satisfied
|
||||
// when every remaining row belongs to an already-capped publisher.
|
||||
// Preserve the cap while alternatives exist, then release deterministically.
|
||||
releaseDeferred();
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
seen.add(next.identity);
|
||||
publisherCounts.set(next.publisherKey, (publisherCounts.get(next.publisherKey) ?? 0) + 1);
|
||||
result.push(next);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
export type CanonicalTrendingCursor = {
|
||||
snapshotId: string;
|
||||
offset: number;
|
||||
};
|
||||
|
||||
export function encodeCanonicalTrendingCursor(cursor: CanonicalTrendingCursor) {
|
||||
if (!cursor.snapshotId || !/^[A-Za-z0-9:_-]+$/.test(cursor.snapshotId)) {
|
||||
throw new Error("Invalid snapshot ID");
|
||||
}
|
||||
if (!Number.isSafeInteger(cursor.offset) || cursor.offset < 0) {
|
||||
throw new Error("Invalid cursor offset");
|
||||
}
|
||||
return btoa(JSON.stringify({ v: 1, s: cursor.snapshotId, o: cursor.offset }))
|
||||
.replace(/\+/g, "-")
|
||||
.replace(/\//g, "_")
|
||||
.replace(/=+$/g, "");
|
||||
}
|
||||
|
||||
export function decodeCanonicalTrendingCursor(value: string): CanonicalTrendingCursor {
|
||||
try {
|
||||
const padded = value
|
||||
.replace(/-/g, "+")
|
||||
.replace(/_/g, "/")
|
||||
.padEnd(Math.ceil(value.length / 4) * 4, "=");
|
||||
const parsed = JSON.parse(atob(padded)) as { v?: unknown; s?: unknown; o?: unknown };
|
||||
if (
|
||||
parsed.v !== 1 ||
|
||||
typeof parsed.s !== "string" ||
|
||||
!/^[A-Za-z0-9:_-]+$/.test(parsed.s) ||
|
||||
!Number.isSafeInteger(parsed.o) ||
|
||||
(parsed.o as number) < 0
|
||||
) {
|
||||
throw new Error("invalid");
|
||||
}
|
||||
return { snapshotId: parsed.s, offset: parsed.o as number };
|
||||
} catch {
|
||||
throw new Error("Invalid cursor format");
|
||||
}
|
||||
}
|
||||
@@ -50,6 +50,9 @@ function resolveDefaultRouteRateLimit(spec: RouteSpec, request: Request): RouteR
|
||||
|
||||
const routedPath = getRoutedPath(spec);
|
||||
if (authMetadataPaths.has(routedPath)) return { kind: "none" };
|
||||
// The Trending handler owns its rollout gate before rate limiting so the
|
||||
// dark route remains indistinguishable from an absent endpoint.
|
||||
if (routedPath === ApiRoutes.trending) return { kind: "none" };
|
||||
|
||||
if (routedPath === ApiRoutes.publishTokenMint) return { kind: "trustedPublish" };
|
||||
if (routedPath === ApiRoutes.skillsExport || routedPath === ApiRoutes.pluginsExport) {
|
||||
|
||||
@@ -196,6 +196,24 @@ export const RETENTION_POLICIES = {
|
||||
rankingMetricImports: permanent(
|
||||
"Versioned Test ranking import provenance is retained until explicit cleanup.",
|
||||
),
|
||||
canonicalTrendingSnapshots: ephemeral(
|
||||
"Canonical Trending headers remain briefly available for stable cursor pagination.",
|
||||
{
|
||||
expirationField: "expiresAt",
|
||||
expirationIndex: "by_expires_at",
|
||||
prune: "canonicalTrending.pruneExpiredActionInternal",
|
||||
retention: "Forty-eight hours after snapshot generation.",
|
||||
},
|
||||
),
|
||||
canonicalTrendingItems: ephemeral(
|
||||
"Materialized Trending cards expire with their snapshot header.",
|
||||
{
|
||||
expirationField: "expiresAt",
|
||||
expirationIndex: "by_expires_at",
|
||||
prune: "canonicalTrending.pruneExpiredActionInternal",
|
||||
retention: "Forty-eight hours after snapshot generation.",
|
||||
},
|
||||
),
|
||||
skillStatEvents: ephemeral(
|
||||
"Skill stat event log is retained only after both consumers pass it.",
|
||||
{
|
||||
|
||||
@@ -2,6 +2,10 @@ import { authTables } from "@convex-dev/auth/server";
|
||||
import { createClawManifestSummarySchema } from "clawhub-schema";
|
||||
import { defineSchema, defineTable } from "convex/server";
|
||||
import { type GenericValidator, v } from "convex/values";
|
||||
import {
|
||||
canonicalTrendingCardValidator,
|
||||
canonicalTrendingSourceRefValidator,
|
||||
} from "./lib/canonicalTrending";
|
||||
import { EMBEDDING_DIMENSIONS } from "./lib/embeddings";
|
||||
|
||||
const PLATFORM_SKILL_LICENSE = "MIT-0" as const;
|
||||
@@ -2623,6 +2627,54 @@ const rankingMetricImports = defineTable({
|
||||
.index("by_dataset_version", ["datasetVersion"])
|
||||
.index("by_imported_at", ["importedAt"]);
|
||||
|
||||
const canonicalTrendingSnapshots = defineTable({
|
||||
snapshotId: v.string(),
|
||||
kind: v.literal("skills"),
|
||||
status: v.union(v.literal("building"), v.literal("ready"), v.literal("failed")),
|
||||
rankingVersion: v.string(),
|
||||
generatedAt: v.number(),
|
||||
completedAt: v.optional(v.number()),
|
||||
expiresAt: v.number(),
|
||||
windowHours: v.number(),
|
||||
windowStartDay: v.number(),
|
||||
windowEndDay: v.number(),
|
||||
writtenItems: v.number(),
|
||||
totalItems: v.optional(v.number()),
|
||||
sourceCounts: v.optional(
|
||||
v.object({
|
||||
clawhubTrending: v.number(),
|
||||
clawhubRising: v.number(),
|
||||
skillsShTrending: v.number(),
|
||||
}),
|
||||
),
|
||||
operations: v.optional(
|
||||
v.object({
|
||||
documentsRead: v.number(),
|
||||
documentsWritten: v.number(),
|
||||
functionCalls: v.number(),
|
||||
}),
|
||||
),
|
||||
error: v.optional(v.string()),
|
||||
})
|
||||
.index("by_snapshot_id", ["snapshotId"])
|
||||
.index("by_kind_and_status_and_expires_at", ["kind", "status", "expiresAt"])
|
||||
.index("by_expires_at", ["expiresAt"]);
|
||||
|
||||
const canonicalTrendingItems = defineTable({
|
||||
snapshotId: v.string(),
|
||||
position: v.number(),
|
||||
lane: v.union(
|
||||
v.literal("clawhub-trending"),
|
||||
v.literal("clawhub-rising"),
|
||||
v.literal("skills-sh-trending"),
|
||||
),
|
||||
sourceRef: canonicalTrendingSourceRefValidator,
|
||||
card: canonicalTrendingCardValidator,
|
||||
expiresAt: v.number(),
|
||||
})
|
||||
.index("by_snapshot_id_and_position", ["snapshotId", "position"])
|
||||
.index("by_expires_at", ["expiresAt"]);
|
||||
|
||||
const skillStatEvents = defineTable({
|
||||
skillId: v.id("skills"),
|
||||
kind: v.union(
|
||||
@@ -4096,6 +4148,8 @@ export default defineSchema({
|
||||
skillStatBackfillState,
|
||||
globalStats,
|
||||
rankingMetricImports,
|
||||
canonicalTrendingSnapshots,
|
||||
canonicalTrendingItems,
|
||||
skillStatEvents,
|
||||
skillStatUpdateCursors,
|
||||
skillStatDocSyncLeases,
|
||||
|
||||
@@ -88,6 +88,7 @@
|
||||
"test:pw:publish-lifecycle": "bun run test:pw:local-auth -- --project=chromium e2e/local-auth/publish-skill-lifecycle.pw.test.ts",
|
||||
"test:ui-contract": "vitest run src/__tests__/ui-design-contract.test.ts src/__tests__/header.test.tsx src/__tests__/home-route.test.tsx src/components/Footer.test.tsx src/lib/theme.test.tsx src/routes/-settings.test.tsx",
|
||||
"test:watch": "vitest",
|
||||
"trending:prove-test": "bun scripts/canonical-trending/prove-test.ts",
|
||||
"validate:public-corpus": "bun scripts/public-corpus/validate-public-corpus.ts",
|
||||
"verify:convex-contract": "bun scripts/verify-convex-contract.ts"
|
||||
},
|
||||
|
||||
@@ -17,6 +17,7 @@ export const ApiRoutes = {
|
||||
download: "/api/v1/download",
|
||||
publishTokenMint: "/api/v1/publish/token/mint",
|
||||
skills: "/api/v1/skills",
|
||||
trending: "/api/v1/trending",
|
||||
skillsSh: "/api/v1/skills-sh",
|
||||
skillScans: "/api/v1/skills/-/scan",
|
||||
packages: "/api/v1/packages",
|
||||
|
||||
Vendored
+1
@@ -16,6 +16,7 @@ export declare const ApiRoutes: {
|
||||
readonly download: "/api/v1/download";
|
||||
readonly publishTokenMint: "/api/v1/publish/token/mint";
|
||||
readonly skills: "/api/v1/skills";
|
||||
readonly trending: "/api/v1/trending";
|
||||
readonly skillsSh: "/api/v1/skills-sh";
|
||||
readonly skillScans: "/api/v1/skills/-/scan";
|
||||
readonly plugins: "/api/v1/plugins";
|
||||
|
||||
Vendored
+1
@@ -16,6 +16,7 @@ export const ApiRoutes = {
|
||||
download: "/api/v1/download",
|
||||
publishTokenMint: "/api/v1/publish/token/mint",
|
||||
skills: "/api/v1/skills",
|
||||
trending: "/api/v1/trending",
|
||||
skillsSh: "/api/v1/skills-sh",
|
||||
skillScans: "/api/v1/skills/-/scan",
|
||||
plugins: "/api/v1/plugins",
|
||||
|
||||
Vendored
+1
-1
@@ -1 +1 @@
|
||||
{"version":3,"file":"routes.js","sourceRoot":"","sources":["../src/routes.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,QAAQ,EAAE,eAAe;IACzB,MAAM,EAAE,aAAa;IACrB,KAAK,EAAE,YAAY;IACnB,YAAY,EAAE,oBAAoB;IAClC,SAAS,EAAE,iBAAiB;IAC5B,YAAY,EAAE,qBAAqB;IACnC,UAAU,EAAE,kBAAkB;IAC9B,mBAAmB,EAAE,4BAA4B;IACjD,cAAc,EAAE,uBAAuB;IACvC,gBAAgB,EAAE,yBAAyB;CACnC,CAAC;AAEX,MAAM,CAAC,MAAM,SAAS,GAAG;IACvB,MAAM,EAAE,gBAAgB;IACxB,OAAO,EAAE,iBAAiB;IAC1B,QAAQ,EAAE,kBAAkB;IAC5B,gBAAgB,EAAE,4BAA4B;IAC9C,MAAM,EAAE,gBAAgB;IACxB,QAAQ,EAAE,mBAAmB;IAC7B,UAAU,EAAE,uBAAuB;IACnC,OAAO,EAAE,iBAAiB;IAC1B,aAAa,EAAE,wBAAwB;IACvC,QAAQ,EAAE,kBAAkB;IAC5B,WAAW,EAAE,sBAAsB;IACnC,aAAa,EAAE,wBAAwB;IACvC,UAAU,EAAE,oBAAoB;IAChC,WAAW,EAAE,uBAAuB;IACpC,iBAAiB,EAAE,sBAAsB;IACzC,gBAAgB,EAAE,qBAAqB;IACvC,cAAc,EAAE,0BAA0B;IAC1C,KAAK,EAAE,eAAe;IACtB,SAAS,EAAE,mBAAmB;IAC9B,UAAU,EAAE,oBAAoB;IAChC,KAAK,EAAE,eAAe;IACtB,aAAa,EAAE,wBAAwB;IACvC,MAAM,EAAE,gBAAgB;IACxB,YAAY,EAAE,uBAAuB;CAC7B,CAAC"}
|
||||
{"version":3,"file":"routes.js","sourceRoot":"","sources":["../src/routes.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,QAAQ,EAAE,eAAe;IACzB,MAAM,EAAE,aAAa;IACrB,KAAK,EAAE,YAAY;IACnB,YAAY,EAAE,oBAAoB;IAClC,SAAS,EAAE,iBAAiB;IAC5B,YAAY,EAAE,qBAAqB;IACnC,UAAU,EAAE,kBAAkB;IAC9B,mBAAmB,EAAE,4BAA4B;IACjD,cAAc,EAAE,uBAAuB;IACvC,gBAAgB,EAAE,yBAAyB;CACnC,CAAC;AAEX,MAAM,CAAC,MAAM,SAAS,GAAG;IACvB,MAAM,EAAE,gBAAgB;IACxB,OAAO,EAAE,iBAAiB;IAC1B,QAAQ,EAAE,kBAAkB;IAC5B,gBAAgB,EAAE,4BAA4B;IAC9C,MAAM,EAAE,gBAAgB;IACxB,QAAQ,EAAE,kBAAkB;IAC5B,QAAQ,EAAE,mBAAmB;IAC7B,UAAU,EAAE,uBAAuB;IACnC,OAAO,EAAE,iBAAiB;IAC1B,aAAa,EAAE,wBAAwB;IACvC,QAAQ,EAAE,kBAAkB;IAC5B,WAAW,EAAE,sBAAsB;IACnC,aAAa,EAAE,wBAAwB;IACvC,UAAU,EAAE,oBAAoB;IAChC,WAAW,EAAE,uBAAuB;IACpC,iBAAiB,EAAE,sBAAsB;IACzC,gBAAgB,EAAE,qBAAqB;IACvC,cAAc,EAAE,0BAA0B;IAC1C,KAAK,EAAE,eAAe;IACtB,SAAS,EAAE,mBAAmB;IAC9B,UAAU,EAAE,oBAAoB;IAChC,KAAK,EAAE,eAAe;IACtB,aAAa,EAAE,wBAAwB;IACvC,MAAM,EAAE,gBAAgB;IACxB,YAAY,EAAE,uBAAuB;CAC7B,CAAC"}
|
||||
@@ -17,6 +17,7 @@ export const ApiRoutes = {
|
||||
download: "/api/v1/download",
|
||||
publishTokenMint: "/api/v1/publish/token/mint",
|
||||
skills: "/api/v1/skills",
|
||||
trending: "/api/v1/trending",
|
||||
skillsSh: "/api/v1/skills-sh",
|
||||
skillScans: "/api/v1/skills/-/scan",
|
||||
plugins: "/api/v1/plugins",
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
assertFirstPageContract,
|
||||
EXPECTED_FIRST_20_LANES,
|
||||
latencySummary,
|
||||
type ProofSampleRow,
|
||||
} from "./proof-contract";
|
||||
|
||||
function sampleRow(index: number): ProofSampleRow {
|
||||
const lane = EXPECTED_FIRST_20_LANES[index]!;
|
||||
const external = lane === "skills-sh-trending";
|
||||
return {
|
||||
rank: index + 1,
|
||||
id: `row-${index + 1}`,
|
||||
lane,
|
||||
publisherKey: `publisher-${Math.floor(index / 2)}`,
|
||||
upstreamRank: external ? index + 1 : null,
|
||||
metrics: {
|
||||
trending24hInstalls: external ? null : 20 - index,
|
||||
trending24hBookmarks: external ? null : index,
|
||||
lifetimeInstalls: 100 + index,
|
||||
lifetimeInstallsPeriod: "lifetime",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("CLAW-590 permanent Test proof contract", () => {
|
||||
it("accepts the continuous blend, cap, order, and metric provenance", () => {
|
||||
expect(assertFirstPageContract(Array.from({ length: 20 }, (_, index) => sampleRow(index))))
|
||||
.toMatchInlineSnapshot(`
|
||||
{
|
||||
"laneCounts": {
|
||||
"clawhub-rising": 4,
|
||||
"clawhub-trending": 8,
|
||||
"skills-sh-trending": 8,
|
||||
},
|
||||
"maximumPublisherCount": 2,
|
||||
"skillsShUpstreamRanks": [
|
||||
2,
|
||||
5,
|
||||
7,
|
||||
10,
|
||||
12,
|
||||
15,
|
||||
17,
|
||||
20,
|
||||
],
|
||||
}
|
||||
`);
|
||||
});
|
||||
|
||||
it("rejects publisher-cap and external metric provenance violations", () => {
|
||||
const publisherViolation = Array.from({ length: 20 }, (_, index) => sampleRow(index));
|
||||
publisherViolation[4]!.publisherKey = publisherViolation[0]!.publisherKey;
|
||||
expect(() => assertFirstPageContract(publisherViolation)).toThrow("Publisher cap exceeded");
|
||||
|
||||
const metricViolation = Array.from({ length: 20 }, (_, index) => sampleRow(index));
|
||||
metricViolation[1]!.metrics.trending24hInstalls = 12;
|
||||
expect(() => assertFirstPageContract(metricViolation)).toThrow(
|
||||
"skills.sh 24-hour metrics were inferred",
|
||||
);
|
||||
});
|
||||
|
||||
it("summarizes observed latency without imposing an unstated SLO", () => {
|
||||
expect(latencySummary([3, 1, 2])).toEqual({
|
||||
samplesMs: [3, 1, 2],
|
||||
medianMs: 2,
|
||||
p95Ms: 3,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,99 @@
|
||||
export const EXPECTED_FIRST_20_LANES = [
|
||||
"clawhub-trending",
|
||||
"skills-sh-trending",
|
||||
"clawhub-rising",
|
||||
"clawhub-trending",
|
||||
"skills-sh-trending",
|
||||
"clawhub-trending",
|
||||
"skills-sh-trending",
|
||||
"clawhub-rising",
|
||||
"clawhub-trending",
|
||||
"skills-sh-trending",
|
||||
"clawhub-trending",
|
||||
"skills-sh-trending",
|
||||
"clawhub-rising",
|
||||
"clawhub-trending",
|
||||
"skills-sh-trending",
|
||||
"clawhub-trending",
|
||||
"skills-sh-trending",
|
||||
"clawhub-rising",
|
||||
"clawhub-trending",
|
||||
"skills-sh-trending",
|
||||
] as const;
|
||||
|
||||
type ProofMetrics = {
|
||||
trending24hInstalls: number | null;
|
||||
trending24hBookmarks: number | null;
|
||||
lifetimeInstalls: number | null;
|
||||
lifetimeInstallsPeriod: string;
|
||||
};
|
||||
|
||||
export type ProofSampleRow = {
|
||||
rank: number;
|
||||
id: string;
|
||||
lane: string;
|
||||
publisherKey: string;
|
||||
upstreamRank: number | null;
|
||||
metrics: ProofMetrics;
|
||||
};
|
||||
|
||||
export function assertFirstPageContract(sample: ProofSampleRow[]) {
|
||||
if (sample.length !== EXPECTED_FIRST_20_LANES.length) {
|
||||
throw new Error(`Expected a 20-row Trending sample, received ${sample.length}`);
|
||||
}
|
||||
const publisherCounts = new Map<string, number>();
|
||||
const upstreamRanks: number[] = [];
|
||||
for (const [index, row] of sample.entries()) {
|
||||
if (row.rank !== index + 1) throw new Error(`Unexpected rank at sample offset ${index}`);
|
||||
if (row.lane !== EXPECTED_FIRST_20_LANES[index]) {
|
||||
throw new Error(`40/20/40 lane mismatch at rank ${row.rank}`);
|
||||
}
|
||||
const publisherCount = (publisherCounts.get(row.publisherKey) ?? 0) + 1;
|
||||
publisherCounts.set(row.publisherKey, publisherCount);
|
||||
if (publisherCount > 2) throw new Error(`Publisher cap exceeded by ${row.publisherKey}`);
|
||||
if (row.metrics.lifetimeInstallsPeriod !== "lifetime") {
|
||||
throw new Error(`Lifetime metric period is mislabeled for ${row.id}`);
|
||||
}
|
||||
if (row.lane === "skills-sh-trending") {
|
||||
if (row.metrics.trending24hInstalls !== null || row.metrics.trending24hBookmarks !== null) {
|
||||
throw new Error(`skills.sh 24-hour metrics were inferred for ${row.id}`);
|
||||
}
|
||||
if (typeof row.metrics.lifetimeInstalls !== "number") {
|
||||
throw new Error(`skills.sh lifetime installs are missing for ${row.id}`);
|
||||
}
|
||||
if (!Number.isSafeInteger(row.upstreamRank) || (row.upstreamRank ?? 0) < 1) {
|
||||
throw new Error(`skills.sh upstream rank is missing for ${row.id}`);
|
||||
}
|
||||
upstreamRanks.push(row.upstreamRank as number);
|
||||
} else if (
|
||||
typeof row.metrics.trending24hInstalls !== "number" ||
|
||||
typeof row.metrics.trending24hBookmarks !== "number"
|
||||
) {
|
||||
throw new Error(`Native 24-hour metrics are missing for ${row.id}`);
|
||||
}
|
||||
}
|
||||
if (upstreamRanks.some((rank, index) => index > 0 && rank <= upstreamRanks[index - 1]!)) {
|
||||
throw new Error("skills.sh upstream order was not preserved");
|
||||
}
|
||||
return {
|
||||
laneCounts: Object.fromEntries(
|
||||
[...new Set(EXPECTED_FIRST_20_LANES)].map((lane) => [
|
||||
lane,
|
||||
sample.filter((row) => row.lane === lane).length,
|
||||
]),
|
||||
),
|
||||
maximumPublisherCount: Math.max(...publisherCounts.values()),
|
||||
skillsShUpstreamRanks: upstreamRanks,
|
||||
};
|
||||
}
|
||||
|
||||
export function latencySummary(samples: number[]) {
|
||||
if (samples.length === 0) throw new Error("Latency samples are required");
|
||||
const sorted = [...samples].sort((a, b) => a - b);
|
||||
const percentile = (ratio: number) => sorted[Math.ceil(sorted.length * ratio) - 1] ?? sorted[0]!;
|
||||
return {
|
||||
samplesMs: samples.map((value) => Number(value.toFixed(2))),
|
||||
medianMs: Number(percentile(0.5).toFixed(2)),
|
||||
p95Ms: Number(percentile(0.95).toFixed(2)),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
#!/usr/bin/env bun
|
||||
|
||||
import { execFile } from "node:child_process";
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { assertFirstPageContract, latencySummary, type ProofSampleRow } from "./proof-contract";
|
||||
|
||||
const OUTPUT_PATH = resolve("proof/claw-590/canonical-trending-test-proof.json");
|
||||
const ACTIVE_PATH = resolve("proof/claw-590/active-snapshot.json");
|
||||
const SAMPLE_COUNT = 3;
|
||||
const PAGE_LIMIT = 20;
|
||||
const MAX_PAGES = 1_000;
|
||||
const CONFIRM = "manage-claw-590-canonical-trending-test-proof";
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
function requireEnv(name: string) {
|
||||
const value = process.env[name]?.trim();
|
||||
if (!value) throw new Error(`${name} is required`);
|
||||
return value;
|
||||
}
|
||||
|
||||
const deploySha = requireEnv("DEPLOY_SHA");
|
||||
if (!/^[0-9a-f]{40}$/.test(deploySha)) throw new Error("DEPLOY_SHA must be a full commit SHA");
|
||||
const siteUrl = requireEnv("TEST_SITE_URL").replace(/\/$/, "");
|
||||
const bypass = process.env.VERCEL_AUTOMATION_BYPASS_SECRET?.trim();
|
||||
const snapshotId = `claw-590-proof-${deploySha}`;
|
||||
|
||||
type ApiItem = {
|
||||
id: string;
|
||||
rank: number;
|
||||
lane: string;
|
||||
metrics: {
|
||||
trending24hInstalls: number | null;
|
||||
trending24hBookmarks: number | null;
|
||||
lifetimeInstalls: number | null;
|
||||
lifetimeInstallsPeriod: string;
|
||||
};
|
||||
};
|
||||
|
||||
type ApiPage = {
|
||||
kind: string;
|
||||
snapshotId: string;
|
||||
snapshotCursor: string;
|
||||
generatedAt: string;
|
||||
windowHours: number;
|
||||
rankingVersion: string;
|
||||
totalItems: number;
|
||||
items: ApiItem[];
|
||||
nextCursor: string | null;
|
||||
};
|
||||
|
||||
async function convexRun(functionName: string, args: Record<string, unknown>) {
|
||||
const { stdout } = await execFileAsync(
|
||||
"bunx",
|
||||
["convex", "run", "--no-push", functionName, JSON.stringify(args)],
|
||||
{ env: process.env, maxBuffer: 32 * 1024 * 1024 },
|
||||
);
|
||||
return JSON.parse(stdout.trim()) as Record<string, unknown>;
|
||||
}
|
||||
|
||||
async function fetchPage(cursor: string | null) {
|
||||
const url = new URL("/api/v1/trending", siteUrl);
|
||||
url.searchParams.set("kind", "skills");
|
||||
url.searchParams.set("limit", String(PAGE_LIMIT));
|
||||
if (cursor) url.searchParams.set("cursor", cursor);
|
||||
const startedAt = performance.now();
|
||||
const response = await fetch(url, {
|
||||
headers: bypass ? { "x-vercel-protection-bypass": bypass } : undefined,
|
||||
});
|
||||
const elapsedMs = performance.now() - startedAt;
|
||||
const text = await response.text();
|
||||
if (!response.ok) throw new Error(`Trending API returned HTTP ${response.status}: ${text}`);
|
||||
return { elapsedMs, page: JSON.parse(text) as ApiPage };
|
||||
}
|
||||
|
||||
function orderedIds(page: ApiPage) {
|
||||
return page.items.map((item) => item.id);
|
||||
}
|
||||
|
||||
await mkdir(dirname(ACTIVE_PATH), { recursive: true });
|
||||
await writeFile(
|
||||
ACTIVE_PATH,
|
||||
`${JSON.stringify({ armed: true, snapshotId, deploySha }, null, 2)}\n`,
|
||||
);
|
||||
|
||||
const materialization = await convexRun("canonicalTrending:materializeInternal", {
|
||||
proofSnapshotId: snapshotId,
|
||||
});
|
||||
if (materialization.status !== "ready" || materialization.snapshotId !== snapshotId) {
|
||||
throw new Error(`Unexpected materialization result: ${JSON.stringify(materialization)}`);
|
||||
}
|
||||
|
||||
const firstSamples = [];
|
||||
for (let index = 0; index < SAMPLE_COUNT; index += 1) firstSamples.push(await fetchPage(null));
|
||||
const firstPage = firstSamples[0]!.page;
|
||||
for (const sample of firstSamples) {
|
||||
if (
|
||||
sample.page.snapshotId !== snapshotId ||
|
||||
sample.page.snapshotCursor !== firstPage.snapshotCursor ||
|
||||
sample.page.nextCursor !== firstPage.nextCursor ||
|
||||
JSON.stringify(orderedIds(sample.page)) !== JSON.stringify(orderedIds(firstPage))
|
||||
) {
|
||||
throw new Error("Trending first-page cursor or order changed between samples");
|
||||
}
|
||||
}
|
||||
|
||||
const allItems = [...firstPage.items];
|
||||
let cursor = firstPage.nextCursor;
|
||||
let pages = 1;
|
||||
while (cursor) {
|
||||
if (pages >= MAX_PAGES) throw new Error("Trending pagination exceeded its proof bound");
|
||||
const result = await fetchPage(cursor);
|
||||
if (result.page.snapshotId !== snapshotId)
|
||||
throw new Error("Trending pagination changed snapshot");
|
||||
allItems.push(...result.page.items);
|
||||
cursor = result.page.nextCursor;
|
||||
pages += 1;
|
||||
}
|
||||
if (allItems.length !== firstPage.totalItems) {
|
||||
throw new Error(`Trending pagination returned ${allItems.length}/${firstPage.totalItems} items`);
|
||||
}
|
||||
if (new Set(allItems.map((item) => item.id)).size !== allItems.length) {
|
||||
throw new Error("Trending pagination returned duplicate identities");
|
||||
}
|
||||
if (allItems.some((item, index) => item.rank !== index + 1)) {
|
||||
throw new Error("Trending pagination returned unstable or discontinuous ranks");
|
||||
}
|
||||
|
||||
const readback = await convexRun("canonicalTrendingTestFixtures:readCanonicalTrendingProof", {
|
||||
confirm: CONFIRM,
|
||||
snapshotId,
|
||||
});
|
||||
if (readback.present !== true || readback.status !== "ready") {
|
||||
throw new Error(`Owned snapshot readback failed: ${JSON.stringify(readback)}`);
|
||||
}
|
||||
const sample = readback.sample as ProofSampleRow[];
|
||||
const publicSample = firstPage.items.map((item, index): ProofSampleRow => {
|
||||
const internalRow = sample[index];
|
||||
if (!internalRow) throw new Error(`Public API returned an unexpected row at rank ${item.rank}`);
|
||||
return {
|
||||
rank: item.rank,
|
||||
id: item.id,
|
||||
lane: item.lane,
|
||||
publisherKey: internalRow.publisherKey,
|
||||
upstreamRank: internalRow.upstreamRank,
|
||||
metrics: item.metrics,
|
||||
};
|
||||
});
|
||||
if (
|
||||
JSON.stringify(
|
||||
publicSample.map(({ rank, id, lane, metrics }) => ({ rank, id, lane, metrics })),
|
||||
) !== JSON.stringify(sample.map(({ rank, id, lane, metrics }) => ({ rank, id, lane, metrics })))
|
||||
) {
|
||||
throw new Error("Public API first-page fields differ from the owned snapshot readback");
|
||||
}
|
||||
const assertions = assertFirstPageContract(publicSample);
|
||||
const materializedSample = materialization.sample as Array<{ id: string }>;
|
||||
if (
|
||||
JSON.stringify(materializedSample.map((row) => row.id)) !== JSON.stringify(orderedIds(firstPage))
|
||||
) {
|
||||
throw new Error("Public API first page differs from the regenerated materialization sample");
|
||||
}
|
||||
|
||||
const proof = {
|
||||
generatedAt: new Date().toISOString(),
|
||||
target: {
|
||||
environment: "permanent Test",
|
||||
deploySha,
|
||||
siteUrl,
|
||||
convexDeployment: "academic-chihuahua-392",
|
||||
},
|
||||
materialization: {
|
||||
snapshotId,
|
||||
durationMs: materialization.durationMs,
|
||||
totalItems: materialization.totalItems,
|
||||
sourceCounts: materialization.sourceCounts,
|
||||
operations: materialization.operations,
|
||||
regeneratedFirst20: materializedSample,
|
||||
},
|
||||
publicApi: {
|
||||
samples: SAMPLE_COUNT,
|
||||
latency: latencySummary(firstSamples.map((sampleResult) => sampleResult.elapsedMs)),
|
||||
snapshotCursor: firstPage.snapshotCursor,
|
||||
generatedAt: firstPage.generatedAt,
|
||||
windowHours: firstPage.windowHours,
|
||||
rankingVersion: firstPage.rankingVersion,
|
||||
totalItems: firstPage.totalItems,
|
||||
pages,
|
||||
first20: firstPage.items,
|
||||
},
|
||||
assertions: {
|
||||
...assertions,
|
||||
stableSnapshot: true,
|
||||
stableCursor: true,
|
||||
stableOrder: true,
|
||||
completePagination: true,
|
||||
duplicateIdentities: 0,
|
||||
lifetimeMetricsLabeled: true,
|
||||
skillsShTrending24hInstalls: assertions.skillsShUpstreamRanks.every(
|
||||
(_, index) =>
|
||||
publicSample.filter((row) => row.lane === "skills-sh-trending")[index]?.metrics
|
||||
.trending24hInstalls === null,
|
||||
)
|
||||
? null
|
||||
: "unexpected-non-null",
|
||||
},
|
||||
readback,
|
||||
};
|
||||
|
||||
await mkdir(dirname(OUTPUT_PATH), { recursive: true });
|
||||
await writeFile(OUTPUT_PATH, `${JSON.stringify(proof, null, 2)}\n`);
|
||||
console.log(JSON.stringify({ ok: true, outputPath: OUTPUT_PATH, snapshotId }));
|
||||
@@ -111,6 +111,8 @@ describe("Test deploy workflow", () => {
|
||||
expect(revision).toContain("deploy-claw-577-to-permanent-test");
|
||||
expect(revision).toContain("refs/heads/pe/claw-583-mirrored-search-journey");
|
||||
expect(revision).toContain("deploy-claw-583-to-permanent-test");
|
||||
expect(revision).toContain("refs/heads/pe/claw-590-trending-snapshot");
|
||||
expect(revision).toContain("deploy-claw-590-to-permanent-test");
|
||||
expect(revision).toContain("${{ inputs.expected_sha }}");
|
||||
expect(revision).toContain("${{ github.event.pull_request.head.sha }}");
|
||||
expect(revision).toContain("${{ github.event.pull_request.head.repo.full_name }}");
|
||||
@@ -307,4 +309,44 @@ describe("Test deploy workflow", () => {
|
||||
expect(upload?.with?.path).toContain("proof/claw-583/external-detail.png");
|
||||
expect(upload?.with?.path).toContain("claw583-cleanup-readback.json");
|
||||
});
|
||||
|
||||
it("runs the CLAW-590 materialization proof only for its exact guarded branch", async () => {
|
||||
const workflow = await readWorkflow();
|
||||
const job = workflow.jobs?.["claw590-canonical-trending-proof"];
|
||||
const proofStep = job?.steps?.find(
|
||||
(candidate) =>
|
||||
candidate.name === "Prove canonical Trending materialization and API in permanent Test",
|
||||
);
|
||||
const upload = job?.steps?.find(
|
||||
(candidate) => candidate.name === "Upload permanent Test canonical Trending proof",
|
||||
);
|
||||
const run = proofStep?.run ?? "";
|
||||
|
||||
expect(job?.needs).toBe("deploy-test");
|
||||
expect(job?.if).toContain("github.ref == 'refs/heads/pe/claw-590-trending-snapshot'");
|
||||
expect(job?.if).toContain("inputs.branch_test_confirm == 'deploy-claw-590-to-permanent-test'");
|
||||
expect(job?.if).toContain("inputs.expected_sha == needs.deploy-test.outputs.deploy_sha");
|
||||
expect(job?.environment?.name).toBe("Test");
|
||||
expect(job?.steps?.[0]?.with?.ref).toBe("${{ inputs.expected_sha }}");
|
||||
expect(proofStep?.env?.DEPLOY_SHA).toBe("${{ needs.deploy-test.outputs.deploy_sha }}");
|
||||
expect(run).toContain("appMeta:getDeploymentInfo");
|
||||
expect(run).toContain("bun run trending:prove-test");
|
||||
expect(run).toContain("trap cleanup EXIT");
|
||||
expect(run).toContain("canonicalTrendingTestFixtures:cleanupCanonicalTrendingProof");
|
||||
expect(run).toContain("canonicalTrendingTestFixtures:readCanonicalTrendingProof");
|
||||
expect(run).toContain("canonicalTrendingTestFixtures:seedCanonicalTrendingSourceFixture");
|
||||
expect(run).toContain("canonicalTrendingTestFixtures:cleanupCanonicalTrendingSourceFixture");
|
||||
expect(run).toContain("canonicalTrendingTestFixtures:readCanonicalTrendingSourceFixture");
|
||||
expect(run).toContain("claw590-recovery-readback.json");
|
||||
expect(run).toContain("claw590-source-cleanup-readback.json");
|
||||
expect(run).toContain("snapshot cleanup incomplete; source retained");
|
||||
expect(run).toContain("documentsRead > 0");
|
||||
expect(run).toContain('laneCounts["skills-sh-trending"] == 8');
|
||||
expect(upload?.uses).toBe("actions/upload-artifact@v7");
|
||||
expect(upload?.with?.name).toBe("claw590-canonical-trending-proof");
|
||||
expect(upload?.with?.["if-no-files-found"]).toBe("error");
|
||||
expect(upload?.with?.path).toContain("proof/claw-590/canonical-trending-test-proof.json");
|
||||
expect(upload?.with?.path).toContain("claw590-cleanup-readback.json");
|
||||
expect(upload?.with?.path).toContain("claw590-source-cleanup-readback.json");
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user