mirror of
https://github.com/openclaw/clawhub.git
synced 2026-08-14 00:47:57 +00:00
Seed local dev with public corpus (#2226)
* feat: seed local dev with public corpus * fix: preserve local seed owner helpers after merge
This commit is contained in:
Patrick Erichsen
+7
-11
@@ -82,18 +82,14 @@ Change the port if 3000 is already in use, and update `SITE_URL` in both `.env.l
|
||||
|
||||
### Seed the database
|
||||
|
||||
Populate shared `@local` sample skills, plugins, and scanner fixtures so the UI is not empty:
|
||||
Populate local QA fixtures and the committed public corpus so the UI isn't empty:
|
||||
|
||||
```bash
|
||||
bun run seed:dev
|
||||
```
|
||||
# local moderation/security fixtures
|
||||
bunx convex run --no-push devSeed:seedLocalFixtures
|
||||
|
||||
The script waits for the local Convex deployment, runs the fixture seed, and refreshes global stats.
|
||||
If you need to run the pieces manually:
|
||||
|
||||
```bash
|
||||
# Skills, plugins, and moderation/scanner fixtures
|
||||
bunx convex run --no-push devSeed:seedNixSkills
|
||||
# real-ish public corpus rows under deterministic dummy accounts
|
||||
bun run seed:public-corpus
|
||||
|
||||
# 50 extra skills for pagination testing (optional)
|
||||
bunx convex run --no-push devSeedExtra:seedExtraSkillsInternal
|
||||
@@ -105,8 +101,8 @@ bunx convex run --no-push statsMaintenance:updateGlobalStatsAction
|
||||
To reset and re-seed:
|
||||
|
||||
```bash
|
||||
bunx convex run --no-push devSeed:seedNixSkills '{"reset": true}'
|
||||
bunx convex run --no-push statsMaintenance:updateGlobalStatsAction
|
||||
bunx convex run --no-push devSeed:seedLocalFixtures '{"reset": true}'
|
||||
bun run seed:public-corpus -- --reset
|
||||
```
|
||||
|
||||
### Optional environment variables
|
||||
|
||||
@@ -108,7 +108,7 @@ bun run dev
|
||||
# detached/Codex worktree preview
|
||||
bun run dev:worktree
|
||||
|
||||
# optional: seed local skills, plugins, and scanner fixtures
|
||||
# seed local QA fixtures and the public corpus
|
||||
bun run seed:dev
|
||||
```
|
||||
|
||||
|
||||
@@ -53,8 +53,9 @@
|
||||
"zod": "4.4.3",
|
||||
},
|
||||
"devDependencies": {
|
||||
"@playwright/test": "1.60.0",
|
||||
"@tailwindcss/vite": "4.3.0",
|
||||
"@faker-js/faker": "^10.4.0",
|
||||
"@playwright/test": "^1.60.0",
|
||||
"@tailwindcss/vite": "^4.3.0",
|
||||
"@tanstack/devtools-vite": "0.6.0",
|
||||
"@testing-library/dom": "10.4.1",
|
||||
"@testing-library/react": "16.3.2",
|
||||
@@ -278,6 +279,8 @@
|
||||
|
||||
"@exodus/bytes": ["@exodus/bytes@1.15.0", "", { "peerDependencies": { "@noble/hashes": "^1.8.0 || ^2.0.0" }, "optionalPeers": ["@noble/hashes"] }, "sha512-UY0nlA+feH81UGSHv92sLEPLCeZFjXOuHhrIo0HQydScuQc8s0A7kL/UdgwgDq8g8ilksmuoF35YVTNphV2aBQ=="],
|
||||
|
||||
"@faker-js/faker": ["@faker-js/faker@10.4.0", "", {}, "sha512-sDBWI3yLy8EcDzgobvJTWq1MJYzAkQdpjXuPukga9wXonhpMRvd1Izuo2Qgwey2OiEoRIBr35RMU9HJRoOHzpw=="],
|
||||
|
||||
"@floating-ui/core": ["@floating-ui/core@1.7.5", "", { "dependencies": { "@floating-ui/utils": "^0.2.11" } }, "sha512-1Ih4WTWyw0+lKyFMcBHGbb5U5FtuHJuujoyyr5zTaWS5EYMeT6Jb2AuDeftsCsEuchO+mM2ij5+q9crhydzLhQ=="],
|
||||
|
||||
"@floating-ui/dom": ["@floating-ui/dom@1.7.6", "", { "dependencies": { "@floating-ui/core": "^1.7.5", "@floating-ui/utils": "^0.2.11" } }, "sha512-9gZSAI5XM36880PPMm//9dfiEngYoC6Am2izES1FF406YFsjvyBMmeJ2g4SAju3xWwtuynNRFL2s9hgxpLI5SQ=="],
|
||||
|
||||
+480
-428
@@ -4,10 +4,10 @@ import type { Id } from "./_generated/dataModel";
|
||||
import type { ActionCtx, MutationCtx } from "./_generated/server";
|
||||
import { internalMutation as rawInternalMutation } from "./_generated/server";
|
||||
import { internalAction, internalMutation } from "./functions";
|
||||
import { EMBEDDING_DIMENSIONS } from "./lib/embeddings";
|
||||
import { EMBEDDING_DIMENSIONS, generateEmbedding } from "./lib/embeddings";
|
||||
import { normalizePackageName } from "./lib/packageRegistry";
|
||||
import { ensurePersonalPublisherForUser } from "./lib/publishers";
|
||||
import { parseClawdisMetadata, parseFrontmatter } from "./lib/skills";
|
||||
import { buildEmbeddingText, parseClawdisMetadata, parseFrontmatter } from "./lib/skills";
|
||||
import { generateToken, hashToken } from "./lib/tokens";
|
||||
|
||||
type SeedSkillSpec = {
|
||||
@@ -19,19 +19,6 @@ type SeedSkillSpec = {
|
||||
rawSkillMd: string;
|
||||
};
|
||||
|
||||
type SeedPluginSpec = {
|
||||
name: string;
|
||||
displayName: string;
|
||||
summary: string;
|
||||
version: string;
|
||||
runtimeId: string;
|
||||
sourceRepo: string;
|
||||
isOfficial: boolean;
|
||||
capabilityTags: string[];
|
||||
stats: { downloads: number; installs: number; stars: number; versions: number };
|
||||
readme: string;
|
||||
};
|
||||
|
||||
type SeedActionArgs = {
|
||||
reset?: boolean;
|
||||
ownerUserId?: Id<"users">;
|
||||
@@ -44,9 +31,95 @@ type SeedActionResult = {
|
||||
|
||||
type SeedMutationResult = Record<string, unknown>;
|
||||
|
||||
type PublicCorpusDummyOwner = {
|
||||
handle: string;
|
||||
displayName: string;
|
||||
image: string;
|
||||
};
|
||||
|
||||
const publicCorpusDummyOwnerValidator = v.object({
|
||||
handle: v.string(),
|
||||
displayName: v.string(),
|
||||
image: v.string(),
|
||||
});
|
||||
|
||||
const publicCorpusSkillRowValidator = v.object({
|
||||
kind: v.literal("skill"),
|
||||
slug: v.string(),
|
||||
displayName: v.string(),
|
||||
version: v.string(),
|
||||
skillMd: v.string(),
|
||||
summary: v.optional(v.string()),
|
||||
capabilityTags: v.optional(v.array(v.string())),
|
||||
createdAt: v.optional(v.number()),
|
||||
dummyOwner: publicCorpusDummyOwnerValidator,
|
||||
});
|
||||
|
||||
const publicCorpusPluginRowValidator = v.object({
|
||||
kind: v.literal("plugin"),
|
||||
name: v.string(),
|
||||
displayName: v.string(),
|
||||
version: v.string(),
|
||||
readme: v.string(),
|
||||
summary: v.optional(v.string()),
|
||||
capabilityTags: v.optional(v.array(v.string())),
|
||||
family: v.optional(
|
||||
v.union(v.literal("skill"), v.literal("code-plugin"), v.literal("bundle-plugin")),
|
||||
),
|
||||
channel: v.optional(v.union(v.literal("official"), v.literal("community"), v.literal("private"))),
|
||||
executesCode: v.optional(v.boolean()),
|
||||
sourceRepoHost: v.optional(v.union(v.string(), v.null())),
|
||||
createdAt: v.optional(v.number()),
|
||||
dummyOwner: publicCorpusDummyOwnerValidator,
|
||||
});
|
||||
|
||||
const publicCorpusSeedRowValidator = v.union(
|
||||
publicCorpusSkillRowValidator,
|
||||
publicCorpusPluginRowValidator,
|
||||
);
|
||||
|
||||
const publicCorpusPreparedSkillRowValidator = v.object({
|
||||
kind: v.literal("skill"),
|
||||
slug: v.string(),
|
||||
displayName: v.string(),
|
||||
version: v.string(),
|
||||
skillMd: v.string(),
|
||||
summary: v.optional(v.string()),
|
||||
capabilityTags: v.optional(v.array(v.string())),
|
||||
createdAt: v.optional(v.number()),
|
||||
dummyOwner: publicCorpusDummyOwnerValidator,
|
||||
storageId: v.id("_storage"),
|
||||
embedding: v.array(v.number()),
|
||||
});
|
||||
|
||||
const publicCorpusPreparedPluginRowValidator = v.object({
|
||||
kind: v.literal("plugin"),
|
||||
name: v.string(),
|
||||
displayName: v.string(),
|
||||
version: v.string(),
|
||||
readme: v.string(),
|
||||
summary: v.optional(v.string()),
|
||||
capabilityTags: v.optional(v.array(v.string())),
|
||||
family: v.optional(
|
||||
v.union(v.literal("skill"), v.literal("code-plugin"), v.literal("bundle-plugin")),
|
||||
),
|
||||
channel: v.optional(v.union(v.literal("official"), v.literal("community"), v.literal("private"))),
|
||||
executesCode: v.optional(v.boolean()),
|
||||
sourceRepoHost: v.optional(v.union(v.string(), v.null())),
|
||||
createdAt: v.optional(v.number()),
|
||||
dummyOwner: publicCorpusDummyOwnerValidator,
|
||||
storageId: v.id("_storage"),
|
||||
});
|
||||
|
||||
const publicCorpusPreparedRowValidator = v.union(
|
||||
publicCorpusPreparedSkillRowValidator,
|
||||
publicCorpusPreparedPluginRowValidator,
|
||||
);
|
||||
|
||||
const LOCAL_SEED_HANDLE = "local";
|
||||
const LOCAL_SEED_GITHUB_CREATED_AT = Date.parse("2020-01-01T00:00:00.000Z");
|
||||
const CURRENT_USER_SEED_PREFIX = "dev";
|
||||
const PUBLIC_CORPUS_BATCH = "public-corpus-v1";
|
||||
const FLAGGED_SKILL_SLUG = "local-flagged-wallet-sync";
|
||||
const SCANNED_SKILL_SLUG = "local-agentic-risk-demo";
|
||||
const FLAGGED_PLUGIN_NAME = "local-flagged-runtime-plugin";
|
||||
@@ -171,243 +244,6 @@ This seeded plugin is public and intentionally has completed scan results so loc
|
||||
preview plugin scanner detail pages without owner-only visibility.
|
||||
`;
|
||||
|
||||
function isProductionDeployment() {
|
||||
const deployment = process.env.CONVEX_DEPLOYMENT?.trim() ?? "";
|
||||
return deployment.startsWith("prod:") || deployment.includes("production");
|
||||
}
|
||||
|
||||
function assertDevSeedAllowed() {
|
||||
if (isProductionDeployment()) {
|
||||
throw new Error("Dev fixture seeding is disabled on production deployments.");
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeSeedPart(value: string) {
|
||||
return value
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, "-")
|
||||
.replace(/^-+|-+$/g, "")
|
||||
.replace(/-{2,}/g, "-");
|
||||
}
|
||||
|
||||
function currentUserSeedKey(userId: Id<"users">) {
|
||||
const raw = String(userId).split(":").pop() ?? String(userId);
|
||||
const normalized = normalizeSeedPart(raw).replace(/-/g, "");
|
||||
return (normalized || "user").slice(-8);
|
||||
}
|
||||
|
||||
export function currentUserSeedSkillSlug(userId: Id<"users">, baseSlug: string) {
|
||||
return `${CURRENT_USER_SEED_PREFIX}-${currentUserSeedKey(userId)}-${baseSlug}`;
|
||||
}
|
||||
|
||||
export function currentUserSeedPackageName(userId: Id<"users">, baseName: string) {
|
||||
const normalized = normalizePackageName(baseName).replace(/^@/, "").replace("/", "-");
|
||||
return `${CURRENT_USER_SEED_PREFIX}-${currentUserSeedKey(userId)}-${normalized}`;
|
||||
}
|
||||
|
||||
function withFrontmatterName(rawSkillMd: string, name: string) {
|
||||
const frontmatterEnd = rawSkillMd.indexOf("\n---", 3);
|
||||
if (frontmatterEnd === -1) return rawSkillMd;
|
||||
const frontmatter = rawSkillMd.slice(0, frontmatterEnd);
|
||||
const body = rawSkillMd.slice(frontmatterEnd);
|
||||
if (/^name:\s*.+$/m.test(frontmatter)) {
|
||||
return `${frontmatter.replace(/^name:\s*.+$/m, `name: ${name}`)}${body}`;
|
||||
}
|
||||
return `${frontmatter}\nname: ${name}${body}`;
|
||||
}
|
||||
|
||||
const FEATURED_PLUGIN_SEEDS: SeedPluginSpec[] = [
|
||||
{
|
||||
name: "@apify/apify-openclaw-plugin",
|
||||
displayName: "Apify",
|
||||
summary:
|
||||
"Scrape websites through Apify actors and make structured web data available to agents.",
|
||||
version: "1.0.0",
|
||||
runtimeId: "apify",
|
||||
sourceRepo: "apify/apify-openclaw-plugin",
|
||||
isOfficial: false,
|
||||
capabilityTags: ["web", "scraping", "automation"],
|
||||
stats: { downloads: 1200, installs: 320, stars: 45, versions: 1 },
|
||||
readme: "# Apify\n\nScrape websites through Apify actors from OpenClaw.",
|
||||
},
|
||||
{
|
||||
name: "openclaw-codex-app-server",
|
||||
displayName: "Codex App Server Bridge",
|
||||
summary: "Bind OpenClaw chats to Codex App Server conversations and control threads from chat.",
|
||||
version: "1.0.0",
|
||||
runtimeId: "codex-app-server",
|
||||
sourceRepo: "pwrdrvr/openclaw-codex-app-server",
|
||||
isOfficial: false,
|
||||
capabilityTags: ["codex", "chat", "bridge"],
|
||||
stats: { downloads: 980, installs: 280, stars: 37, versions: 1 },
|
||||
readme: "# Codex App Server Bridge\n\nBridge OpenClaw chat sessions to Codex App Server.",
|
||||
},
|
||||
{
|
||||
name: "@largezhou/ddingtalk",
|
||||
displayName: "DingTalk",
|
||||
summary: "Connect OpenClaw to DingTalk enterprise robots with text, image, and file messages.",
|
||||
version: "1.0.0",
|
||||
runtimeId: "dingtalk",
|
||||
sourceRepo: "largezhou/openclaw-dingtalk",
|
||||
isOfficial: false,
|
||||
capabilityTags: ["channel", "dingtalk", "enterprise"],
|
||||
stats: { downloads: 930, installs: 250, stars: 32, versions: 1 },
|
||||
readme: "# DingTalk\n\nDingTalk enterprise robot plugin for OpenClaw.",
|
||||
},
|
||||
{
|
||||
name: "kudosity-openclaw-sms",
|
||||
displayName: "Kudosity SMS",
|
||||
summary: "Send and receive SMS through Kudosity as an OpenClaw plugin.",
|
||||
version: "1.0.0",
|
||||
runtimeId: "kudosity-sms",
|
||||
sourceRepo: "kudosity/openclaw-sms",
|
||||
isOfficial: false,
|
||||
capabilityTags: ["channel", "sms", "kudosity"],
|
||||
stats: { downloads: 860, installs: 210, stars: 29, versions: 1 },
|
||||
readme: "# Kudosity SMS\n\nKudosity SMS channel plugin for OpenClaw.",
|
||||
},
|
||||
{
|
||||
name: "@martian-engineering/lossless-claw",
|
||||
displayName: "Lossless Claw",
|
||||
summary:
|
||||
"Preserve conversation context with DAG-based summarization and incremental compaction.",
|
||||
version: "1.0.0",
|
||||
runtimeId: "lossless-claw",
|
||||
sourceRepo: "Martian-Engineering/lossless-claw",
|
||||
isOfficial: false,
|
||||
capabilityTags: ["memory", "context", "summarization"],
|
||||
stats: { downloads: 820, installs: 190, stars: 28, versions: 1 },
|
||||
readme: "# Lossless Claw\n\nLossless context management plugin for OpenClaw.",
|
||||
},
|
||||
{
|
||||
name: "@opik/opik-openclaw",
|
||||
displayName: "Opik",
|
||||
summary: "Export OpenClaw traces to Opik for monitoring, costs, token usage, and debugging.",
|
||||
version: "1.0.0",
|
||||
runtimeId: "opik",
|
||||
sourceRepo: "comet-ml/opik-openclaw",
|
||||
isOfficial: true,
|
||||
capabilityTags: ["observability", "tracing", "monitoring"],
|
||||
stats: { downloads: 760, installs: 180, stars: 25, versions: 1 },
|
||||
readme: "# Opik\n\nTrace OpenClaw agents with Opik.",
|
||||
},
|
||||
{
|
||||
name: "@prometheusavatar/openclaw-plugin",
|
||||
displayName: "Prometheus Avatar",
|
||||
summary: "Give OpenClaw agents a Live2D avatar with lip-sync, expressions, and speech.",
|
||||
version: "1.0.0",
|
||||
runtimeId: "prometheus-avatar",
|
||||
sourceRepo: "myths-labs/prometheus-avatar",
|
||||
isOfficial: false,
|
||||
capabilityTags: ["avatar", "tts", "live2d"],
|
||||
stats: { downloads: 690, installs: 150, stars: 22, versions: 1 },
|
||||
readme: "# Prometheus Avatar\n\nLive2D avatar plugin for OpenClaw.",
|
||||
},
|
||||
{
|
||||
name: "@tencent-connect/openclaw-qqbot",
|
||||
displayName: "QQbot",
|
||||
summary:
|
||||
"Connect OpenClaw to QQ private chats, group mentions, channel messages, and rich media.",
|
||||
version: "1.0.0",
|
||||
runtimeId: "qqbot",
|
||||
sourceRepo: "tencent-connect/openclaw-qqbot",
|
||||
isOfficial: true,
|
||||
capabilityTags: ["channel", "qq", "messaging"],
|
||||
stats: { downloads: 640, installs: 140, stars: 20, versions: 1 },
|
||||
readme: "# QQbot\n\nQQ Bot plugin for OpenClaw.",
|
||||
},
|
||||
{
|
||||
name: "@wecom/wecom-openclaw-plugin",
|
||||
displayName: "wecom",
|
||||
summary:
|
||||
"Use WeCom Bot WebSocket connections for direct messages, group chats, and proactive messaging.",
|
||||
version: "1.0.0",
|
||||
runtimeId: "wecom",
|
||||
sourceRepo: "WecomTeam/wecom-openclaw-plugin",
|
||||
isOfficial: true,
|
||||
capabilityTags: ["channel", "wecom", "enterprise"],
|
||||
stats: { downloads: 610, installs: 130, stars: 18, versions: 1 },
|
||||
readme: "# wecom\n\nWeCom channel plugin for OpenClaw.",
|
||||
},
|
||||
{
|
||||
name: "openclaw-plugin-yuanbao",
|
||||
displayName: "Yuanbao",
|
||||
summary:
|
||||
"Connect OpenClaw to Yuanbao with direct messages, group chats, media, and slash commands.",
|
||||
version: "1.0.0",
|
||||
runtimeId: "yuanbao",
|
||||
sourceRepo: "yb-claw/openclaw-plugin-yuanbao",
|
||||
isOfficial: false,
|
||||
capabilityTags: ["channel", "yuanbao", "messaging"],
|
||||
stats: { downloads: 580, installs: 125, stars: 17, versions: 1 },
|
||||
readme: "# Yuanbao\n\nYuanbao channel plugin for OpenClaw.",
|
||||
},
|
||||
];
|
||||
|
||||
const LOCAL_OWNER_PLUGIN_SEEDS: SeedPluginSpec[] = [
|
||||
{
|
||||
name: "local-merge-notes-plugin",
|
||||
displayName: "Local Merge Notes",
|
||||
summary: "Local owner fixture for validating plugin inventory and skill merge settings.",
|
||||
version: "0.1.0",
|
||||
runtimeId: "local.merge.notes",
|
||||
sourceRepo: "openclaw/local-merge-notes-plugin",
|
||||
isOfficial: false,
|
||||
capabilityTags: ["notes", "local-dev", "merge-fixture"],
|
||||
stats: { downloads: 18, installs: 6, stars: 2, versions: 1 },
|
||||
readme: "# Local Merge Notes\n\nLocal dev plugin fixture for owner inventory screens.",
|
||||
},
|
||||
{
|
||||
name: "local-merge-browser-plugin",
|
||||
displayName: "Local Merge Browser",
|
||||
summary: "Browser automation fixture owned by the local dev account.",
|
||||
version: "0.1.0",
|
||||
runtimeId: "local.merge.browser",
|
||||
sourceRepo: "openclaw/local-merge-browser-plugin",
|
||||
isOfficial: false,
|
||||
capabilityTags: ["browser", "automation", "merge-fixture"],
|
||||
stats: { downloads: 16, installs: 5, stars: 2, versions: 1 },
|
||||
readme: "# Local Merge Browser\n\nLocal dev browser plugin fixture.",
|
||||
},
|
||||
{
|
||||
name: "local-merge-terminal-plugin",
|
||||
displayName: "Local Merge Terminal",
|
||||
summary: "Terminal command fixture owned by the local dev account.",
|
||||
version: "0.1.0",
|
||||
runtimeId: "local.merge.terminal",
|
||||
sourceRepo: "openclaw/local-merge-terminal-plugin",
|
||||
isOfficial: false,
|
||||
capabilityTags: ["terminal", "commands", "merge-fixture"],
|
||||
stats: { downloads: 14, installs: 5, stars: 1, versions: 1 },
|
||||
readme: "# Local Merge Terminal\n\nLocal dev terminal plugin fixture.",
|
||||
},
|
||||
{
|
||||
name: "local-merge-calendar-plugin",
|
||||
displayName: "Local Merge Calendar",
|
||||
summary: "Calendar workflow fixture owned by the local dev account.",
|
||||
version: "0.1.0",
|
||||
runtimeId: "local.merge.calendar",
|
||||
sourceRepo: "openclaw/local-merge-calendar-plugin",
|
||||
isOfficial: false,
|
||||
capabilityTags: ["calendar", "scheduling", "merge-fixture"],
|
||||
stats: { downloads: 12, installs: 4, stars: 1, versions: 1 },
|
||||
readme: "# Local Merge Calendar\n\nLocal dev calendar plugin fixture.",
|
||||
},
|
||||
{
|
||||
name: "local-merge-git-plugin",
|
||||
displayName: "Local Merge Git",
|
||||
summary: "Git workflow fixture owned by the local dev account.",
|
||||
version: "0.1.0",
|
||||
runtimeId: "local.merge.git",
|
||||
sourceRepo: "openclaw/local-merge-git-plugin",
|
||||
isOfficial: false,
|
||||
capabilityTags: ["git", "workflow", "merge-fixture"],
|
||||
stats: { downloads: 10, installs: 3, stars: 1, versions: 1 },
|
||||
readme: "# Local Merge Git\n\nLocal dev git workflow plugin fixture.",
|
||||
},
|
||||
];
|
||||
|
||||
type RoleHelpFixtureUser = {
|
||||
handle: string;
|
||||
displayName: string;
|
||||
@@ -728,6 +564,20 @@ redirect behavior.
|
||||
},
|
||||
];
|
||||
|
||||
function currentUserSeedKey(userId: Id<"users">) {
|
||||
const normalized = String(userId).replace(/[^a-zA-Z0-9]/g, "");
|
||||
return (normalized || "user").slice(-8);
|
||||
}
|
||||
|
||||
export function currentUserSeedSkillSlug(userId: Id<"users">, baseSlug: string) {
|
||||
return `${CURRENT_USER_SEED_PREFIX}-${currentUserSeedKey(userId)}-${baseSlug}`;
|
||||
}
|
||||
|
||||
export function currentUserSeedPackageName(userId: Id<"users">, baseName: string) {
|
||||
const normalized = normalizePackageName(baseName).replace(/^@/, "").replace("/", "-");
|
||||
return `${CURRENT_USER_SEED_PREFIX}-${currentUserSeedKey(userId)}-${normalized}`;
|
||||
}
|
||||
|
||||
function injectMetadata(rawSkillMd: string, metadata: Record<string, unknown>) {
|
||||
const frontmatterEnd = rawSkillMd.indexOf("\n---", 3);
|
||||
if (frontmatterEnd === -1) return rawSkillMd;
|
||||
@@ -736,63 +586,10 @@ function injectMetadata(rawSkillMd: string, metadata: Record<string, unknown>) {
|
||||
)}${rawSkillMd.slice(frontmatterEnd)}`;
|
||||
}
|
||||
|
||||
async function seedPluginPackageBatch(
|
||||
ctx: ActionCtx,
|
||||
args: SeedActionArgs,
|
||||
specs: SeedPluginSpec[],
|
||||
): Promise<SeedMutationResult> {
|
||||
const storageIds = await Promise.all(
|
||||
specs.map(async (spec) =>
|
||||
ctx.storage.store(new Blob([spec.readme], { type: "text/markdown" })),
|
||||
),
|
||||
);
|
||||
return (await ctx.runMutation(internal.devSeed.seedFeaturedPluginPackagesMutation, {
|
||||
reset: args.reset,
|
||||
ownerUserId: args.ownerUserId,
|
||||
packages: specs.map((spec, index) => ({
|
||||
name: spec.name,
|
||||
displayName: spec.displayName,
|
||||
summary: spec.summary,
|
||||
version: spec.version,
|
||||
runtimeId: spec.runtimeId,
|
||||
sourceRepo: spec.sourceRepo,
|
||||
isOfficial: spec.isOfficial,
|
||||
capabilityTags: spec.capabilityTags,
|
||||
stats: spec.stats,
|
||||
storageId: storageIds[index],
|
||||
readmeSize: spec.readme.length,
|
||||
})),
|
||||
})) as SeedMutationResult;
|
||||
}
|
||||
|
||||
async function seedNixSkillsHandler(
|
||||
async function seedLocalFixturesHandler(
|
||||
ctx: ActionCtx,
|
||||
args: SeedActionArgs,
|
||||
): Promise<SeedActionResult> {
|
||||
const results: Array<Record<string, unknown> & { slug: string }> = [];
|
||||
|
||||
for (const spec of SEED_SKILLS) {
|
||||
const skillMd = injectMetadata(spec.rawSkillMd, spec.metadata);
|
||||
const frontmatter = parseFrontmatter(skillMd);
|
||||
const clawdis = parseClawdisMetadata(frontmatter);
|
||||
const storageId = await ctx.storage.store(new Blob([skillMd], { type: "text/markdown" }));
|
||||
|
||||
const result: SeedMutationResult = await ctx.runMutation(internal.devSeed.seedSkillMutation, {
|
||||
reset: args.reset,
|
||||
storageId,
|
||||
metadata: spec.metadata,
|
||||
frontmatter,
|
||||
clawdis,
|
||||
skillMd,
|
||||
slug: spec.slug,
|
||||
displayName: spec.displayName,
|
||||
summary: spec.summary,
|
||||
version: spec.version,
|
||||
});
|
||||
|
||||
results.push({ slug: spec.slug, ...result });
|
||||
}
|
||||
|
||||
const [
|
||||
flaggedSkillStorageId,
|
||||
scannedSkillStorageId,
|
||||
@@ -804,6 +601,7 @@ async function seedNixSkillsHandler(
|
||||
ctx.storage.store(new Blob([FLAGGED_PLUGIN_README], { type: "text/markdown" })),
|
||||
ctx.storage.store(new Blob([SCANNED_PLUGIN_README], { type: "text/markdown" })),
|
||||
]);
|
||||
|
||||
const fixtureResult: SeedMutationResult = await ctx.runMutation(
|
||||
internal.devSeed.seedLocalModerationFixturesMutation,
|
||||
{
|
||||
@@ -818,145 +616,320 @@ async function seedNixSkillsHandler(
|
||||
scannedPluginReadme: SCANNED_PLUGIN_README,
|
||||
},
|
||||
);
|
||||
results.push({ slug: FLAGGED_SKILL_SLUG, ...fixtureResult });
|
||||
|
||||
const featuredResult = await seedPluginPackageBatch(ctx, args, FEATURED_PLUGIN_SEEDS);
|
||||
results.push({ slug: "featured-plugins", ...featuredResult });
|
||||
const ownerPluginResult = await seedPluginPackageBatch(ctx, args, LOCAL_OWNER_PLUGIN_SEEDS);
|
||||
results.push({ slug: "local-owner-plugins", ...ownerPluginResult });
|
||||
|
||||
return { ok: true, results };
|
||||
return { ok: true, results: [{ slug: "local-moderation-fixtures", ...fixtureResult }] };
|
||||
}
|
||||
|
||||
export const seedNixSkills: ReturnType<typeof internalAction> = internalAction({
|
||||
export const seedLocalFixtures: ReturnType<typeof internalAction> = internalAction({
|
||||
args: {
|
||||
reset: v.optional(v.boolean()),
|
||||
},
|
||||
handler: seedNixSkillsHandler,
|
||||
handler: seedLocalFixturesHandler,
|
||||
});
|
||||
|
||||
type SeedCurrentUserFixturesArgs = {
|
||||
reset?: boolean;
|
||||
ownerUserId: Id<"users">;
|
||||
};
|
||||
export const seedPublicCorpusBatch: ReturnType<typeof internalAction> = internalAction({
|
||||
args: {
|
||||
reset: v.optional(v.boolean()),
|
||||
resetOwnerHandles: v.optional(v.array(v.string())),
|
||||
rows: v.array(publicCorpusSeedRowValidator),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const preparedRows = await Promise.all(
|
||||
args.rows.map(async (row) => {
|
||||
if (row.kind === "skill") {
|
||||
const storageId = await ctx.storage.store(
|
||||
new Blob([row.skillMd], { type: "text/markdown" }),
|
||||
);
|
||||
const frontmatter = parseFrontmatter(row.skillMd);
|
||||
const embeddingText = buildEmbeddingText({
|
||||
frontmatter,
|
||||
readme: row.skillMd,
|
||||
otherFiles: [],
|
||||
});
|
||||
const embedding = await generateEmbedding(embeddingText);
|
||||
return { ...row, storageId, embedding };
|
||||
}
|
||||
const storageId = await ctx.storage.store(
|
||||
new Blob([row.readme], { type: "text/markdown" }),
|
||||
);
|
||||
return { ...row, storageId };
|
||||
}),
|
||||
);
|
||||
|
||||
async function seedCurrentUserFixturesHandler(ctx: ActionCtx, args: SeedCurrentUserFixturesArgs) {
|
||||
assertDevSeedAllowed();
|
||||
const userId = args.ownerUserId;
|
||||
const skillSlugs: string[] = [];
|
||||
const packageNames: string[] = [];
|
||||
const results: Array<Record<string, unknown> & { slug: string }> = [];
|
||||
|
||||
for (const spec of SEED_SKILLS) {
|
||||
const slug = currentUserSeedSkillSlug(userId, spec.slug);
|
||||
const skillMd = withFrontmatterName(injectMetadata(spec.rawSkillMd, spec.metadata), slug);
|
||||
const frontmatter = parseFrontmatter(skillMd);
|
||||
const clawdis = parseClawdisMetadata(frontmatter);
|
||||
const storageId = await ctx.storage.store(new Blob([skillMd], { type: "text/markdown" }));
|
||||
|
||||
const result: SeedMutationResult = await ctx.runMutation(internal.devSeed.seedSkillMutation, {
|
||||
return await ctx.runMutation(internal.devSeed.seedPublicCorpusBatchMutation, {
|
||||
reset: args.reset,
|
||||
ownerUserId: userId,
|
||||
storageId,
|
||||
metadata: spec.metadata,
|
||||
frontmatter,
|
||||
clawdis,
|
||||
skillMd,
|
||||
slug,
|
||||
displayName: spec.displayName,
|
||||
summary: spec.summary,
|
||||
version: spec.version,
|
||||
resetOwnerHandles: args.resetOwnerHandles,
|
||||
rows: preparedRows,
|
||||
});
|
||||
skillSlugs.push(slug);
|
||||
results.push({ slug, ...result });
|
||||
},
|
||||
});
|
||||
|
||||
export const seedPublicCorpusBatchMutation = internalMutation({
|
||||
args: {
|
||||
reset: v.optional(v.boolean()),
|
||||
resetOwnerHandles: v.optional(v.array(v.string())),
|
||||
rows: v.array(publicCorpusPreparedRowValidator),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const now = Date.now();
|
||||
if (args.reset) await resetPublicCorpusRows(ctx, args.resetOwnerHandles ?? []);
|
||||
|
||||
const seeded: string[] = [];
|
||||
const skipped: string[] = [];
|
||||
|
||||
for (const row of args.rows) {
|
||||
const { userId, publisherId } = await ensurePublicCorpusOwner(ctx, row.dummyOwner);
|
||||
if (row.kind === "skill") {
|
||||
const existing = await ctx.db
|
||||
.query("skills")
|
||||
.withIndex("by_slug", (q) => q.eq("slug", row.slug))
|
||||
.unique();
|
||||
if (existing) {
|
||||
skipped.push(`skill:${row.slug}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const frontmatter = parseFrontmatter(row.skillMd);
|
||||
const clawdis = parseClawdisMetadata(frontmatter);
|
||||
const metadata =
|
||||
frontmatter.metadata && typeof frontmatter.metadata === "object"
|
||||
? (frontmatter.metadata as Record<string, unknown>)
|
||||
: {};
|
||||
const summary = row.summary ?? publicCorpusSummaryFromFrontmatter(frontmatter);
|
||||
const createdAt = row.createdAt ?? now;
|
||||
const stats = publicCorpusSkillStats(row.slug);
|
||||
const skillId = await ctx.db.insert("skills", {
|
||||
slug: row.slug,
|
||||
displayName: row.displayName,
|
||||
summary,
|
||||
ownerUserId: userId,
|
||||
ownerPublisherId: publisherId,
|
||||
latestVersionId: undefined,
|
||||
latestVersionSummary: undefined,
|
||||
tags: {},
|
||||
capabilityTags: row.capabilityTags ?? [],
|
||||
badges: { highlighted: undefined, redactionApproved: undefined },
|
||||
batch: PUBLIC_CORPUS_BATCH,
|
||||
statsDownloads: stats.downloads,
|
||||
statsStars: stats.stars,
|
||||
statsInstallsCurrent: stats.installsCurrent,
|
||||
statsInstallsAllTime: stats.installsAllTime,
|
||||
stats: {
|
||||
downloads: stats.downloads,
|
||||
installsCurrent: stats.installsCurrent,
|
||||
installsAllTime: stats.installsAllTime,
|
||||
stars: stats.stars,
|
||||
versions: 0,
|
||||
comments: 0,
|
||||
},
|
||||
createdAt,
|
||||
updatedAt: now,
|
||||
});
|
||||
const versionId = await ctx.db.insert("skillVersions", {
|
||||
skillId,
|
||||
version: row.version,
|
||||
changelog: "Seeded from the public corpus fixture.",
|
||||
changelogSource: "user",
|
||||
files: [
|
||||
{
|
||||
path: "SKILL.md",
|
||||
size: row.skillMd.length,
|
||||
storageId: row.storageId,
|
||||
sha256: `public-corpus-${row.slug}`,
|
||||
contentType: "text/markdown",
|
||||
},
|
||||
],
|
||||
parsed: {
|
||||
frontmatter,
|
||||
metadata,
|
||||
clawdis,
|
||||
},
|
||||
createdBy: userId,
|
||||
createdAt,
|
||||
softDeletedAt: undefined,
|
||||
});
|
||||
const embeddingId = await ctx.db.insert("skillEmbeddings", {
|
||||
skillId,
|
||||
versionId,
|
||||
ownerId: userId,
|
||||
ownerPublisherId: publisherId,
|
||||
embedding: row.embedding,
|
||||
isLatest: true,
|
||||
isApproved: true,
|
||||
visibility: "latest-approved",
|
||||
updatedAt: now,
|
||||
});
|
||||
await ctx.db.insert("embeddingSkillMap", { embeddingId, skillId });
|
||||
await ctx.db.patch(skillId, {
|
||||
latestVersionId: versionId,
|
||||
latestVersionSummary: {
|
||||
version: row.version,
|
||||
createdAt,
|
||||
changelog: "Seeded from the public corpus fixture.",
|
||||
changelogSource: "user",
|
||||
clawdis,
|
||||
},
|
||||
tags: { latest: versionId },
|
||||
stats: {
|
||||
downloads: stats.downloads,
|
||||
installsCurrent: stats.installsCurrent,
|
||||
installsAllTime: stats.installsAllTime,
|
||||
stars: stats.stars,
|
||||
versions: 1,
|
||||
comments: 0,
|
||||
},
|
||||
updatedAt: now,
|
||||
});
|
||||
seeded.push(`skill:${row.slug}`);
|
||||
} else {
|
||||
const normalizedName = normalizePackageName(row.name);
|
||||
const existing = await ctx.db
|
||||
.query("packages")
|
||||
.withIndex("by_name", (q) => q.eq("normalizedName", normalizedName))
|
||||
.unique();
|
||||
if (existing) {
|
||||
skipped.push(`plugin:${row.name}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const createdAt = row.createdAt ?? now;
|
||||
const stats = publicCorpusPackageStats(row.name);
|
||||
const capabilityTags = row.capabilityTags ?? [];
|
||||
const compatibility = { pluginApiRange: ">=0.1.0" };
|
||||
const capabilities = {
|
||||
executesCode: row.executesCode ?? true,
|
||||
runtimeId: normalizedName,
|
||||
pluginKind: "runtime",
|
||||
capabilityTags,
|
||||
};
|
||||
const verification = {
|
||||
tier: "structural" as const,
|
||||
scope: "artifact-only" as const,
|
||||
summary: "Seeded from the public corpus fixture.",
|
||||
scanStatus: "clean" as const,
|
||||
};
|
||||
const packageId = await ctx.db.insert("packages", {
|
||||
name: row.name,
|
||||
normalizedName,
|
||||
displayName: row.displayName,
|
||||
summary: row.summary ?? `${row.displayName} public corpus plugin fixture.`,
|
||||
ownerUserId: userId,
|
||||
ownerPublisherId: publisherId,
|
||||
family: row.family ?? "code-plugin",
|
||||
channel: row.channel ?? "community",
|
||||
isOfficial: row.channel === "official",
|
||||
runtimeId: normalizedName,
|
||||
latestReleaseId: undefined,
|
||||
latestVersionSummary: undefined,
|
||||
tags: {},
|
||||
capabilityTags,
|
||||
executesCode: row.executesCode ?? true,
|
||||
compatibility,
|
||||
capabilities,
|
||||
verification,
|
||||
scanStatus: "clean",
|
||||
stats: { ...stats, versions: 0 },
|
||||
softDeletedAt: undefined,
|
||||
createdAt,
|
||||
updatedAt: now,
|
||||
});
|
||||
const releaseId = await ctx.db.insert("packageReleases", {
|
||||
packageId,
|
||||
version: row.version,
|
||||
changelog: "Seeded from the public corpus fixture.",
|
||||
summary: row.summary ?? `${row.displayName} public corpus plugin fixture.`,
|
||||
distTags: ["latest"],
|
||||
files: [
|
||||
{
|
||||
path: "README.md",
|
||||
size: row.readme.length,
|
||||
storageId: row.storageId,
|
||||
sha256: `public-corpus-${normalizedName}`,
|
||||
contentType: "text/markdown",
|
||||
},
|
||||
],
|
||||
integritySha256: `public-corpus-integrity-${normalizedName}`,
|
||||
extractedPackageJson: {
|
||||
name: row.name,
|
||||
version: row.version,
|
||||
description: row.summary ?? `${row.displayName} public corpus plugin fixture.`,
|
||||
},
|
||||
compatibility,
|
||||
capabilities,
|
||||
verification,
|
||||
sha256hash: `public-corpus-hash-${normalizedName}`,
|
||||
source: row.sourceRepoHost
|
||||
? { kind: "github", repo: row.sourceRepoHost, path: "." }
|
||||
: undefined,
|
||||
createdBy: userId,
|
||||
publishActor: { kind: "user", userId },
|
||||
createdAt,
|
||||
softDeletedAt: undefined,
|
||||
});
|
||||
await ctx.db.patch(packageId, {
|
||||
latestReleaseId: releaseId,
|
||||
latestVersionSummary: {
|
||||
version: row.version,
|
||||
createdAt,
|
||||
changelog: "Seeded from the public corpus fixture.",
|
||||
compatibility,
|
||||
capabilities,
|
||||
verification,
|
||||
},
|
||||
tags: { latest: releaseId },
|
||||
stats: { ...stats, versions: 1 },
|
||||
updatedAt: now,
|
||||
});
|
||||
seeded.push(`plugin:${row.name}`);
|
||||
}
|
||||
}
|
||||
|
||||
return { ok: true, seeded, skipped };
|
||||
},
|
||||
});
|
||||
|
||||
function publicCorpusSummaryFromFrontmatter(frontmatter: Record<string, unknown>) {
|
||||
if (typeof frontmatter.description === "string" && frontmatter.description.trim()) {
|
||||
return frontmatter.description.trim();
|
||||
}
|
||||
const metadata = frontmatter.metadata;
|
||||
if (
|
||||
metadata &&
|
||||
typeof metadata === "object" &&
|
||||
!Array.isArray(metadata) &&
|
||||
typeof (metadata as Record<string, unknown>).description === "string"
|
||||
) {
|
||||
return ((metadata as Record<string, unknown>).description as string).trim();
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const flaggedSkillSlug = currentUserSeedSkillSlug(userId, FLAGGED_SKILL_SLUG);
|
||||
const scannedSkillSlug = currentUserSeedSkillSlug(userId, SCANNED_SKILL_SLUG);
|
||||
const flaggedPluginName = currentUserSeedPackageName(userId, FLAGGED_PLUGIN_NAME);
|
||||
const scannedPluginName = currentUserSeedPackageName(userId, SCANNED_PLUGIN_NAME);
|
||||
const flaggedSkillMd = withFrontmatterName(FLAGGED_SKILL_MD, flaggedSkillSlug);
|
||||
const scannedSkillMd = withFrontmatterName(SCANNED_SKILL_MD, scannedSkillSlug);
|
||||
const flaggedPluginReadme = FLAGGED_PLUGIN_README.replace(
|
||||
"# Local Flagged Runtime Plugin",
|
||||
`# ${flaggedPluginName}`,
|
||||
);
|
||||
const scannedPluginReadme = SCANNED_PLUGIN_README.replace(
|
||||
"# Local Scanned Runtime Plugin",
|
||||
`# ${scannedPluginName}`,
|
||||
);
|
||||
const [
|
||||
flaggedSkillStorageId,
|
||||
scannedSkillStorageId,
|
||||
flaggedPluginStorageId,
|
||||
scannedPluginStorageId,
|
||||
] = await Promise.all([
|
||||
ctx.storage.store(new Blob([flaggedSkillMd], { type: "text/markdown" })),
|
||||
ctx.storage.store(new Blob([scannedSkillMd], { type: "text/markdown" })),
|
||||
ctx.storage.store(new Blob([flaggedPluginReadme], { type: "text/markdown" })),
|
||||
ctx.storage.store(new Blob([scannedPluginReadme], { type: "text/markdown" })),
|
||||
]);
|
||||
|
||||
const fixtureResult: SeedMutationResult = await ctx.runMutation(
|
||||
internal.devSeed.seedLocalModerationFixturesMutation,
|
||||
{
|
||||
reset: args.reset,
|
||||
ownerUserId: userId,
|
||||
flaggedSkillSlug,
|
||||
scannedSkillSlug,
|
||||
flaggedPluginName,
|
||||
scannedPluginName,
|
||||
flaggedSkillStorageId,
|
||||
flaggedSkillMd,
|
||||
scannedSkillStorageId,
|
||||
scannedSkillMd,
|
||||
flaggedPluginStorageId,
|
||||
flaggedPluginReadme,
|
||||
scannedPluginStorageId,
|
||||
scannedPluginReadme,
|
||||
},
|
||||
);
|
||||
skillSlugs.push(flaggedSkillSlug, scannedSkillSlug);
|
||||
packageNames.push(flaggedPluginName, scannedPluginName);
|
||||
results.push({ slug: flaggedSkillSlug, ...fixtureResult });
|
||||
|
||||
const ownerPluginSpecs = LOCAL_OWNER_PLUGIN_SEEDS.map((spec) => ({
|
||||
...spec,
|
||||
name: currentUserSeedPackageName(userId, spec.name),
|
||||
runtimeId: `${CURRENT_USER_SEED_PREFIX}.${currentUserSeedKey(userId)}.${spec.runtimeId}`,
|
||||
}));
|
||||
const ownerPluginResult = await seedPluginPackageBatch(
|
||||
ctx,
|
||||
{ reset: args.reset, ownerUserId: userId },
|
||||
ownerPluginSpecs,
|
||||
);
|
||||
packageNames.push(...ownerPluginSpecs.map((spec) => spec.name));
|
||||
results.push({ slug: "current-user-local-plugins", ...ownerPluginResult });
|
||||
|
||||
const statsResult = (await ctx.runAction(
|
||||
internal.statsMaintenance.updateGlobalStatsAction,
|
||||
{},
|
||||
)) as { count: number } | null;
|
||||
|
||||
function publicCorpusSkillStats(slug: string) {
|
||||
const score = publicCorpusStableNumber(slug);
|
||||
return {
|
||||
ok: true,
|
||||
ownerUserId: userId,
|
||||
skillCount: skillSlugs.length,
|
||||
pluginCount: packageNames.length,
|
||||
skillSlugs,
|
||||
packageNames,
|
||||
statsCount: statsResult?.count ?? null,
|
||||
results,
|
||||
downloads: score % 400,
|
||||
stars: score % 40,
|
||||
installsCurrent: score % 25,
|
||||
installsAllTime: score % 120,
|
||||
};
|
||||
}
|
||||
|
||||
export const seedCurrentUserFixtures: ReturnType<typeof internalAction> = internalAction({
|
||||
args: {
|
||||
reset: v.optional(v.boolean()),
|
||||
ownerUserId: v.id("users"),
|
||||
},
|
||||
handler: seedCurrentUserFixturesHandler,
|
||||
});
|
||||
function publicCorpusPackageStats(name: string) {
|
||||
const score = publicCorpusStableNumber(name);
|
||||
return {
|
||||
downloads: score % 600,
|
||||
installs: score % 80,
|
||||
stars: score % 60,
|
||||
};
|
||||
}
|
||||
|
||||
function publicCorpusStableNumber(value: string) {
|
||||
let hash = 0;
|
||||
for (let index = 0; index < value.length; index += 1) {
|
||||
hash = (hash * 31 + value.charCodeAt(index)) >>> 0;
|
||||
}
|
||||
return hash;
|
||||
}
|
||||
|
||||
async function seedPadelSkillHandler(
|
||||
ctx: ActionCtx,
|
||||
@@ -1034,6 +1007,40 @@ async function ensureSeedOwner(ctx: MutationCtx, ownerUserId?: Id<"users">) {
|
||||
return { userId: user._id, publisherId: publisher._id };
|
||||
}
|
||||
|
||||
async function ensurePublicCorpusOwner(ctx: MutationCtx, owner: PublicCorpusDummyOwner) {
|
||||
const now = Date.now();
|
||||
const existingUsers = await ctx.db
|
||||
.query("users")
|
||||
.withIndex("handle", (q) => q.eq("handle", owner.handle))
|
||||
.collect();
|
||||
const userId =
|
||||
existingUsers[0]?._id ??
|
||||
(await ctx.db.insert("users", {
|
||||
handle: owner.handle,
|
||||
displayName: owner.displayName,
|
||||
name: owner.displayName,
|
||||
image: owner.image,
|
||||
role: "user",
|
||||
githubCreatedAt: LOCAL_SEED_GITHUB_CREATED_AT,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
}));
|
||||
if (existingUsers[0]) {
|
||||
await ctx.db.patch(userId, {
|
||||
displayName: owner.displayName,
|
||||
name: owner.displayName,
|
||||
image: owner.image,
|
||||
githubCreatedAt: LOCAL_SEED_GITHUB_CREATED_AT,
|
||||
updatedAt: now,
|
||||
});
|
||||
}
|
||||
const user = await ctx.db.get(userId);
|
||||
if (!user) throw new Error(`Public corpus owner was not created: ${owner.handle}`);
|
||||
const publisher = await ensurePersonalPublisherForUser(ctx, user);
|
||||
if (!publisher) throw new Error(`Public corpus publisher was not created: ${owner.handle}`);
|
||||
return { userId, publisherId: publisher._id };
|
||||
}
|
||||
|
||||
async function deleteSkillEmbeddingsForSkill(ctx: MutationCtx, skillId: Id<"skills">) {
|
||||
const embeddings = await ctx.db
|
||||
.query("skillEmbeddings")
|
||||
@@ -1049,6 +1056,51 @@ async function deleteSkillEmbeddingsForSkill(ctx: MutationCtx, skillId: Id<"skil
|
||||
}
|
||||
}
|
||||
|
||||
async function deleteSkillAndVersions(ctx: MutationCtx, skillId: Id<"skills">) {
|
||||
const versions = await ctx.db
|
||||
.query("skillVersions")
|
||||
.withIndex("by_skill", (q) => q.eq("skillId", skillId))
|
||||
.collect();
|
||||
for (const version of versions) await ctx.db.delete(version._id);
|
||||
await deleteSkillEmbeddingsForSkill(ctx, skillId);
|
||||
await deleteSkillBadgesForSkill(ctx, skillId);
|
||||
await ctx.db.delete(skillId);
|
||||
}
|
||||
|
||||
async function deletePackageAndReleases(ctx: MutationCtx, packageId: Id<"packages">) {
|
||||
const releases = await ctx.db
|
||||
.query("packageReleases")
|
||||
.withIndex("by_package", (q) => q.eq("packageId", packageId))
|
||||
.collect();
|
||||
await deletePackageBadgesForPackage(ctx, packageId);
|
||||
await ctx.db.delete(packageId);
|
||||
for (const release of releases) await ctx.db.delete(release._id);
|
||||
}
|
||||
|
||||
async function resetPublicCorpusRows(ctx: MutationCtx, ownerHandles: string[]) {
|
||||
for (const handle of ownerHandles) {
|
||||
const owners = await ctx.db
|
||||
.query("users")
|
||||
.withIndex("handle", (q) => q.eq("handle", handle))
|
||||
.collect();
|
||||
for (const owner of owners) {
|
||||
const skills = await ctx.db
|
||||
.query("skills")
|
||||
.withIndex("by_owner", (q) => q.eq("ownerUserId", owner._id))
|
||||
.collect();
|
||||
for (const skill of skills) {
|
||||
if (skill.batch === PUBLIC_CORPUS_BATCH) await deleteSkillAndVersions(ctx, skill._id);
|
||||
}
|
||||
|
||||
const packages = await ctx.db
|
||||
.query("packages")
|
||||
.withIndex("by_owner", (q) => q.eq("ownerUserId", owner._id))
|
||||
.collect();
|
||||
for (const pkg of packages) await deletePackageAndReleases(ctx, pkg._id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function deleteSkillBadgesForSkill(ctx: MutationCtx, skillId: Id<"skills">) {
|
||||
const badges = await ctx.db
|
||||
.query("skillBadges")
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"fixture_schema_version": 1,
|
||||
"created_at": "2026-05-14T18:16:13.492Z",
|
||||
"source_snapshot": "clawhub-wry-manatee-359-20260513T165423Z-a1666bb1",
|
||||
"redaction_policy_version": "public-signals-v2",
|
||||
"row_counts": {
|
||||
"total": 1250,
|
||||
"skills": 1000,
|
||||
"plugins": 250
|
||||
},
|
||||
"fields_dropped": [
|
||||
"source_doc_id_hash",
|
||||
"parent_doc_id_hash",
|
||||
"owner identity",
|
||||
"raw Convex ids",
|
||||
"scan findings",
|
||||
"labels"
|
||||
],
|
||||
"ownership_policy": "Importer assigns deterministic faker-generated dummy local accounts at seed time.",
|
||||
"refresh_policy": "Maintainer-prepared fixture; ordinary developers do not need production or private export data access."
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
+5
-2
@@ -52,6 +52,7 @@
|
||||
"proof:ui": "node scripts/ui-proof.mjs",
|
||||
"release:clawhub:cli:npm:check": "node scripts/clawhub-cli-npm-release-check.mjs",
|
||||
"seed:dev": "bun run setup:worktree -- --quiet && bun scripts/dev-worktree.ts --seed-only",
|
||||
"seed:public-corpus": "bun run setup:worktree -- --quiet && bunx convex dev --once --typecheck=disable && bun scripts/public-corpus/seed-public-corpus.ts",
|
||||
"setup:worktree": "bun scripts/setup-worktree.ts",
|
||||
"test": "vitest run",
|
||||
"test:e2e": "vitest run -c vitest.e2e.config.ts",
|
||||
@@ -62,6 +63,7 @@
|
||||
"test:pw:publish-lifecycle": "bun run test:pw:local-auth -- --project=chromium e2e/local-auth/publish-skill-lifecycle.pw.test.ts",
|
||||
"test:ui-contract": "vitest run src/__tests__/ui-design-contract.test.ts src/__tests__/header.test.tsx src/__tests__/home-route.test.tsx src/components/Footer.test.tsx src/lib/theme.test.tsx src/routes/-settings.test.tsx",
|
||||
"test:watch": "vitest",
|
||||
"validate:public-corpus": "bun scripts/public-corpus/validate-public-corpus.ts",
|
||||
"verify:convex-contract": "bun scripts/verify-convex-contract.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
@@ -114,8 +116,9 @@
|
||||
"zod": "4.4.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@playwright/test": "1.60.0",
|
||||
"@tailwindcss/vite": "4.3.0",
|
||||
"@faker-js/faker": "^10.4.0",
|
||||
"@playwright/test": "^1.60.0",
|
||||
"@tailwindcss/vite": "^4.3.0",
|
||||
"@tanstack/devtools-vite": "0.6.0",
|
||||
"@testing-library/dom": "10.4.1",
|
||||
"@testing-library/react": "16.3.2",
|
||||
|
||||
@@ -76,8 +76,8 @@ branch refs/heads/feature
|
||||
it("recognizes Convex functions that are not queryable yet", () => {
|
||||
expect(
|
||||
isConvexFunctionUnavailableOutput(`
|
||||
Failed to run function "devSeed:seedNixSkills":
|
||||
Could not find function for 'devSeed:seedNixSkills'. Did you forget to run \`npx convex dev\`?
|
||||
Failed to run function "devSeed:seedLocalFixtures":
|
||||
Could not find function for 'devSeed:seedLocalFixtures'. Did you forget to run \`npx convex dev\`?
|
||||
No functions found.
|
||||
`),
|
||||
).toBe(true);
|
||||
|
||||
@@ -359,15 +359,18 @@ async function main() {
|
||||
await ensureConvex(convexUrl);
|
||||
|
||||
if (options.seed) {
|
||||
console.log("Seeding sample skills...");
|
||||
console.log("Seeding local fixtures and public corpus...");
|
||||
const seedStatus = await runConvexFunctionWhenReady([
|
||||
"convex",
|
||||
"run",
|
||||
"--no-push",
|
||||
"devSeed:seedNixSkills",
|
||||
"devSeed:seedLocalFixtures",
|
||||
]);
|
||||
if (seedStatus !== 0) process.exit(seedStatus);
|
||||
|
||||
const publicCorpusStatus = runSync("bun", ["scripts/public-corpus/seed-public-corpus.ts"], {});
|
||||
if (publicCorpusStatus !== 0) process.exit(publicCorpusStatus);
|
||||
|
||||
const statsStatus = await runConvexFunctionWhenReady([
|
||||
"convex",
|
||||
"run",
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildDummyOwnerPool, ownerForCorpusKey } from "./dummyOwners";
|
||||
|
||||
describe("public corpus dummy owners", () => {
|
||||
it("generates deterministic dummy local accounts", () => {
|
||||
expect(buildDummyOwnerPool(3)).toEqual([
|
||||
{
|
||||
handle: "local-corpus-electa-pfeffer",
|
||||
displayName: "Electa Pfeffer",
|
||||
image: expect.stringContaining("api.dicebear.com"),
|
||||
},
|
||||
{
|
||||
handle: "local-corpus-angelo-marvin",
|
||||
displayName: "Angelo Marvin",
|
||||
image: expect.stringContaining("api.dicebear.com"),
|
||||
},
|
||||
{
|
||||
handle: "local-corpus-cedrick-rowe",
|
||||
displayName: "Cedrick Rowe",
|
||||
image: expect.stringContaining("api.dicebear.com"),
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("maps the same corpus key to the same dummy account", () => {
|
||||
const pool = buildDummyOwnerPool(8);
|
||||
|
||||
expect(ownerForCorpusKey("skill:demo-skill", pool)).toEqual(
|
||||
ownerForCorpusKey("skill:demo-skill", pool),
|
||||
);
|
||||
expect(ownerForCorpusKey("plugin:@demo/plugin", pool).handle).toMatch(/^local-corpus-/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,57 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { faker } from "@faker-js/faker";
|
||||
|
||||
export type DummyCorpusOwner = {
|
||||
handle: string;
|
||||
displayName: string;
|
||||
image: string;
|
||||
};
|
||||
|
||||
const DEFAULT_OWNER_SEED = 20260513;
|
||||
const DEFAULT_OWNER_COUNT = 24;
|
||||
|
||||
export function buildDummyOwnerPool(count = DEFAULT_OWNER_COUNT): DummyCorpusOwner[] {
|
||||
faker.seed(DEFAULT_OWNER_SEED);
|
||||
const owners: DummyCorpusOwner[] = [];
|
||||
const handles = new Set<string>();
|
||||
|
||||
while (owners.length < count) {
|
||||
const firstName = faker.person.firstName();
|
||||
const lastName = faker.person.lastName();
|
||||
const displayName = `${firstName} ${lastName}`;
|
||||
const baseHandle = slugify(displayName);
|
||||
const handle = uniqueHandle(baseHandle, handles);
|
||||
owners.push({
|
||||
handle,
|
||||
displayName,
|
||||
image: `https://api.dicebear.com/9.x/shapes/svg?seed=${encodeURIComponent(handle)}`,
|
||||
});
|
||||
}
|
||||
|
||||
return owners;
|
||||
}
|
||||
|
||||
export function ownerForCorpusKey(key: string, owners = buildDummyOwnerPool()): DummyCorpusOwner {
|
||||
if (owners.length === 0) throw new Error("Dummy owner pool must not be empty.");
|
||||
const digest = createHash("sha256").update(key).digest("hex");
|
||||
const index = Number.parseInt(digest.slice(0, 8), 16) % owners.length;
|
||||
return owners[index]!;
|
||||
}
|
||||
|
||||
function uniqueHandle(baseHandle: string, handles: Set<string>) {
|
||||
let handle = `local-corpus-${baseHandle}`;
|
||||
let suffix = 2;
|
||||
while (handles.has(handle)) {
|
||||
handle = `local-corpus-${baseHandle}-${suffix}`;
|
||||
suffix += 1;
|
||||
}
|
||||
handles.add(handle);
|
||||
return handle;
|
||||
}
|
||||
|
||||
function slugify(value: string) {
|
||||
return value
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, "-")
|
||||
.replace(/^-+|-+$/g, "");
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
#!/usr/bin/env bun
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { buildDummyOwnerPool, ownerForCorpusKey, type DummyCorpusOwner } from "./dummyOwners";
|
||||
import {
|
||||
DEFAULT_PUBLIC_CORPUS_FIXTURE,
|
||||
parseCorpusJsonl,
|
||||
validateCorpusRows,
|
||||
type PublicCorpusRow,
|
||||
} from "./validate";
|
||||
|
||||
type Options = {
|
||||
fixture: string;
|
||||
reset: boolean;
|
||||
limit: number | null;
|
||||
batchBytes: number;
|
||||
};
|
||||
|
||||
type SeedCorpusRow = PublicCorpusRow & {
|
||||
dummyOwner: DummyCorpusOwner;
|
||||
};
|
||||
|
||||
const DEFAULT_BATCH_BYTES = 96_000;
|
||||
|
||||
async function main() {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
const text = await readFile(options.fixture, "utf8");
|
||||
const rows = parseCorpusJsonl(text).slice(0, options.limit ?? undefined);
|
||||
const validation = validateCorpusRows(rows);
|
||||
if (!validation.ok) {
|
||||
console.error(
|
||||
JSON.stringify({ ok: false, findings: validation.findings.slice(0, 100) }, null, 2),
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const owners = buildDummyOwnerPool();
|
||||
const seedRows = rows.map((row) => ({
|
||||
...row,
|
||||
dummyOwner: ownerForCorpusKey(corpusKey(row), owners),
|
||||
}));
|
||||
const batches = chunkRowsByOwner(seedRows, options.batchBytes);
|
||||
|
||||
for (let index = 0; index < batches.length; index += 1) {
|
||||
const batch = batches[index]!;
|
||||
const args = {
|
||||
reset: options.reset && index === 0,
|
||||
resetOwnerHandles: options.reset ? owners.map((owner) => owner.handle) : [],
|
||||
rows: batch,
|
||||
};
|
||||
const status = runConvexSeedBatch(args);
|
||||
if (status !== 0) process.exit(status);
|
||||
console.log(
|
||||
`Seeded public corpus batch ${index + 1}/${batches.length} (${batch.length} rows).`,
|
||||
);
|
||||
}
|
||||
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{
|
||||
ok: true,
|
||||
fixture: options.fixture,
|
||||
seededRows: seedRows.length,
|
||||
batches: batches.length,
|
||||
reset: options.reset,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
function parseArgs(args: string[]): Options {
|
||||
const options: Options = {
|
||||
fixture: DEFAULT_PUBLIC_CORPUS_FIXTURE,
|
||||
reset: false,
|
||||
limit: null,
|
||||
batchBytes: DEFAULT_BATCH_BYTES,
|
||||
};
|
||||
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const arg = args[index];
|
||||
if (arg === "--reset") {
|
||||
options.reset = true;
|
||||
} else if (arg === "--fixture") {
|
||||
options.fixture = readValue(args, ++index, arg);
|
||||
} else if (arg === "--limit") {
|
||||
options.limit = readPositiveInt(readValue(args, ++index, arg), arg);
|
||||
} else if (arg === "--batch-bytes") {
|
||||
options.batchBytes = readPositiveInt(readValue(args, ++index, arg), arg);
|
||||
} else {
|
||||
throw new Error(`Unknown argument: ${arg}`);
|
||||
}
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
function chunkRowsByOwner(rows: SeedCorpusRow[], maxBytes: number) {
|
||||
const byOwner = new Map<string, SeedCorpusRow[]>();
|
||||
for (const row of rows) {
|
||||
const ownerRows = byOwner.get(row.dummyOwner.handle) ?? [];
|
||||
ownerRows.push(row);
|
||||
byOwner.set(row.dummyOwner.handle, ownerRows);
|
||||
}
|
||||
|
||||
return Array.from(byOwner.keys())
|
||||
.sort((left, right) => left.localeCompare(right))
|
||||
.flatMap((handle) => chunkRows(byOwner.get(handle) ?? [], maxBytes));
|
||||
}
|
||||
|
||||
function chunkRows(rows: SeedCorpusRow[], maxBytes: number) {
|
||||
const batches: SeedCorpusRow[][] = [];
|
||||
let current: SeedCorpusRow[] = [];
|
||||
let currentBytes = 0;
|
||||
|
||||
for (const row of rows) {
|
||||
const rowBytes = JSON.stringify(row).length + 2;
|
||||
if (current.length > 0 && currentBytes + rowBytes > maxBytes) {
|
||||
batches.push(current);
|
||||
current = [];
|
||||
currentBytes = 0;
|
||||
}
|
||||
current.push(row);
|
||||
currentBytes += rowBytes;
|
||||
}
|
||||
|
||||
if (current.length > 0) batches.push(current);
|
||||
return batches;
|
||||
}
|
||||
|
||||
function corpusKey(row: PublicCorpusRow) {
|
||||
return row.kind === "skill" ? `skill:${row.slug}` : `plugin:${row.name}`;
|
||||
}
|
||||
|
||||
function runConvexSeedBatch(args: unknown) {
|
||||
const result = spawnSync(
|
||||
"bunx",
|
||||
["convex", "run", "--no-push", "devSeed:seedPublicCorpusBatch", JSON.stringify(args)],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
encoding: "utf8",
|
||||
stdio: "inherit",
|
||||
},
|
||||
);
|
||||
return result.status ?? 1;
|
||||
}
|
||||
|
||||
function readValue(args: string[], index: number, flag: string) {
|
||||
const value = args[index];
|
||||
if (!value) throw new Error(`Missing value for ${flag}`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function readPositiveInt(value: string, flag: string) {
|
||||
const parsed = Number.parseInt(value, 10);
|
||||
if (!Number.isFinite(parsed) || parsed <= 0)
|
||||
throw new Error(`Expected positive integer for ${flag}`);
|
||||
return parsed;
|
||||
}
|
||||
|
||||
main().catch((error: unknown) => {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env bun
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { DEFAULT_PUBLIC_CORPUS_FIXTURE, validateCorpusJsonl } from "./validate";
|
||||
|
||||
async function main() {
|
||||
const fixture = process.argv[2] ?? DEFAULT_PUBLIC_CORPUS_FIXTURE;
|
||||
const text = await readFile(fixture, "utf8");
|
||||
const result = validateCorpusJsonl(text);
|
||||
|
||||
if (result.ok) {
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{
|
||||
ok: true,
|
||||
fixture,
|
||||
rowCount: result.rowCount,
|
||||
skillCount: result.skillCount,
|
||||
pluginCount: result.pluginCount,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
console.error(
|
||||
JSON.stringify(
|
||||
{
|
||||
ok: false,
|
||||
fixture,
|
||||
rowCount: result.rowCount,
|
||||
skillCount: result.skillCount,
|
||||
pluginCount: result.pluginCount,
|
||||
findings: result.findings.slice(0, 100),
|
||||
findingCount: result.findings.length,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
main().catch((error: unknown) => {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,110 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
DEFAULT_PUBLIC_CORPUS_FIXTURE,
|
||||
parseCorpusJsonl,
|
||||
validateCorpusRows,
|
||||
type PublicCorpusRow,
|
||||
} from "./validate";
|
||||
|
||||
const skillRow: PublicCorpusRow = {
|
||||
kind: "skill",
|
||||
slug: "demo-skill",
|
||||
displayName: "Demo Skill",
|
||||
version: "1.0.0",
|
||||
skillMd: "---\nname: demo-skill\ndescription: Demo skill\n---\n# Demo Skill",
|
||||
capabilityTags: ["automation"],
|
||||
createdAt: 1770000000000,
|
||||
};
|
||||
|
||||
const pluginRow: PublicCorpusRow = {
|
||||
kind: "plugin",
|
||||
name: "@demo/plugin",
|
||||
displayName: "Demo Plugin",
|
||||
version: "1.0.0",
|
||||
readme: "# Demo Plugin\n\nRuntime plugin for local fixture testing.",
|
||||
capabilityTags: ["executes-code"],
|
||||
family: "code-plugin",
|
||||
channel: "community",
|
||||
executesCode: true,
|
||||
createdAt: 1770000000000,
|
||||
};
|
||||
|
||||
describe("public corpus fixture validation", () => {
|
||||
it("accepts clean skill and plugin rows", () => {
|
||||
expect(validateCorpusRows([skillRow, pluginRow])).toEqual({
|
||||
ok: true,
|
||||
rowCount: 2,
|
||||
skillCount: 1,
|
||||
pluginCount: 1,
|
||||
findings: [],
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects owner identity fields and raw Convex ids", () => {
|
||||
const row = {
|
||||
...skillRow,
|
||||
ownerHandle: "real-user",
|
||||
sourceDocId: "skillVersions:abc123",
|
||||
} as PublicCorpusRow;
|
||||
|
||||
const result = validateCorpusRows([row]);
|
||||
|
||||
expect(result.ok).toBe(false);
|
||||
expect(result.findings.map((finding) => finding.reason)).toContain("disallowed_field");
|
||||
expect(result.findings.map((finding) => finding.reason)).toContain("raw_convex_id");
|
||||
});
|
||||
|
||||
it("rejects duplicate slugs within each artifact kind", () => {
|
||||
const result = validateCorpusRows([skillRow, { ...skillRow, displayName: "Duplicate" }]);
|
||||
|
||||
expect(result.ok).toBe(false);
|
||||
expect(result.findings).toContainEqual(
|
||||
expect.objectContaining({ reason: "duplicate_slug", value: "skill:demo-skill" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects empty content and secret-like text", () => {
|
||||
const result = validateCorpusRows([
|
||||
{ ...skillRow, slug: "empty", skillMd: "" },
|
||||
{
|
||||
...skillRow,
|
||||
slug: "secret",
|
||||
skillMd: `# Secret\nOPENAI_API_KEY=${"sk"}-${"proj"}-abc12345678901234567890`,
|
||||
},
|
||||
]);
|
||||
|
||||
expect(result.ok).toBe(false);
|
||||
expect(result.findings.map((finding) => finding.reason)).toContain("empty_skill_text");
|
||||
expect(result.findings.map((finding) => finding.reason)).toContain("secret_like_text");
|
||||
});
|
||||
|
||||
it("rejects raw local absolute paths but allows redacted path placeholders", () => {
|
||||
const rawLocalPath = ["", "Users", "alice", "project", "secret.txt"].join("/");
|
||||
const result = validateCorpusRows([
|
||||
{ ...skillRow, slug: "local-path", skillMd: `# Path\n${rawLocalPath}` },
|
||||
{
|
||||
...skillRow,
|
||||
slug: "redacted-path",
|
||||
skillMd: "# Path\n/Users/[REDACTED_USER]/project/example.txt",
|
||||
},
|
||||
]);
|
||||
|
||||
expect(result.ok).toBe(false);
|
||||
expect(result.findings).toContainEqual(
|
||||
expect.objectContaining({ reason: "local_path", field: "skillMd" }),
|
||||
);
|
||||
expect(result.findings).not.toContainEqual(
|
||||
expect.objectContaining({ line: 2, reason: "local_path" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("parses newline-delimited corpus rows", () => {
|
||||
expect(
|
||||
parseCorpusJsonl(`${JSON.stringify(skillRow)}\n\n${JSON.stringify(pluginRow)}\n`),
|
||||
).toEqual([skillRow, pluginRow]);
|
||||
});
|
||||
|
||||
it("documents the committed fixture path", () => {
|
||||
expect(DEFAULT_PUBLIC_CORPUS_FIXTURE).toBe("fixtures/public-corpus/corpus.jsonl");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,232 @@
|
||||
export const DEFAULT_PUBLIC_CORPUS_FIXTURE = "fixtures/public-corpus/corpus.jsonl";
|
||||
|
||||
export type PublicCorpusSkillRow = {
|
||||
kind: "skill";
|
||||
slug: string;
|
||||
displayName: string;
|
||||
version: string;
|
||||
skillMd: string;
|
||||
summary?: string;
|
||||
capabilityTags?: string[];
|
||||
createdAt?: number;
|
||||
};
|
||||
|
||||
export type PublicCorpusPluginRow = {
|
||||
kind: "plugin";
|
||||
name: string;
|
||||
displayName: string;
|
||||
version: string;
|
||||
readme: string;
|
||||
summary?: string;
|
||||
capabilityTags?: string[];
|
||||
family?: "skill" | "code-plugin" | "bundle-plugin";
|
||||
channel?: "official" | "community" | "private";
|
||||
executesCode?: boolean;
|
||||
sourceRepoHost?: string | null;
|
||||
createdAt?: number;
|
||||
};
|
||||
|
||||
export type PublicCorpusRow = PublicCorpusSkillRow | PublicCorpusPluginRow;
|
||||
|
||||
export type CorpusValidationFinding = {
|
||||
line: number;
|
||||
reason:
|
||||
| "invalid_json"
|
||||
| "invalid_kind"
|
||||
| "missing_required_field"
|
||||
| "empty_skill_text"
|
||||
| "empty_plugin_text"
|
||||
| "disallowed_field"
|
||||
| "raw_convex_id"
|
||||
| "duplicate_slug"
|
||||
| "local_path"
|
||||
| "secret_like_text";
|
||||
field?: string;
|
||||
value?: string;
|
||||
};
|
||||
|
||||
export type CorpusValidationResult = {
|
||||
ok: boolean;
|
||||
rowCount: number;
|
||||
skillCount: number;
|
||||
pluginCount: number;
|
||||
findings: CorpusValidationFinding[];
|
||||
};
|
||||
|
||||
const DISALLOWED_FIELD_PATTERNS = [
|
||||
/^owner/i,
|
||||
/^publisher/i,
|
||||
/^user/i,
|
||||
/^email$/i,
|
||||
/^auth/i,
|
||||
/^token/i,
|
||||
];
|
||||
|
||||
const RAW_CONVEX_ID_PATTERN =
|
||||
/\b(?:users|publishers|skills|skillVersions|packages|packageReleases):[A-Za-z0-9_-]+\b/;
|
||||
const RAW_LOCAL_PATH_PATTERN =
|
||||
/(?:\/Users\/(?!\[REDACTED_USER\])|\/private\/tmp\/(?!\[REDACTED_PATH\])|\/var\/folders\/(?!\[REDACTED_PATH\])|\/mnt\/c\/Users\/(?!\[REDACTED_USER\])|C:\\Users\\(?!\[REDACTED_USER\]))/;
|
||||
|
||||
const SECRET_PATTERNS = [
|
||||
/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/i,
|
||||
/\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9_]{20,}\b/,
|
||||
/\bgithub_pat_[A-Za-z0-9_]{20,}\b/,
|
||||
/\bsk-(?:proj-)?[A-Za-z0-9_-]{20,}\b/,
|
||||
/\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/,
|
||||
/-----BEGIN [A-Z ]*PRIVATE KEY-----/,
|
||||
/\b(?:authorization|x-api-key)\s*[:=]\s*["']?(?:bearer|basic)?\s+[A-Za-z0-9._~+/=-]{12,}/i,
|
||||
];
|
||||
|
||||
export function parseCorpusJsonl(text: string): PublicCorpusRow[] {
|
||||
return text
|
||||
.split(/\r?\n/)
|
||||
.filter((line) => line.trim().length > 0)
|
||||
.map((line) => JSON.parse(line) as PublicCorpusRow);
|
||||
}
|
||||
|
||||
export function validateCorpusJsonl(text: string): CorpusValidationResult {
|
||||
const rows: PublicCorpusRow[] = [];
|
||||
const findings: CorpusValidationFinding[] = [];
|
||||
|
||||
text.split(/\r?\n/).forEach((line, index) => {
|
||||
if (line.trim().length === 0) return;
|
||||
try {
|
||||
rows.push(JSON.parse(line) as PublicCorpusRow);
|
||||
} catch {
|
||||
findings.push({ line: index + 1, reason: "invalid_json" });
|
||||
}
|
||||
});
|
||||
|
||||
return mergeValidationResults(validateCorpusRows(rows), findings);
|
||||
}
|
||||
|
||||
export function validateCorpusRows(rows: PublicCorpusRow[]): CorpusValidationResult {
|
||||
const findings: CorpusValidationFinding[] = [];
|
||||
const seenKeys = new Set<string>();
|
||||
let skillCount = 0;
|
||||
let pluginCount = 0;
|
||||
|
||||
rows.forEach((row, index) => {
|
||||
const line = index + 1;
|
||||
if (!isRecord(row) || (row.kind !== "skill" && row.kind !== "plugin")) {
|
||||
findings.push({ line, reason: "invalid_kind" });
|
||||
return;
|
||||
}
|
||||
|
||||
collectDisallowedFieldFindings(row, line, findings);
|
||||
collectTextFindings(row, line, findings);
|
||||
|
||||
if (row.kind === "skill") {
|
||||
skillCount += 1;
|
||||
requireString(row.slug, "slug", line, findings);
|
||||
requireString(row.displayName, "displayName", line, findings);
|
||||
requireString(row.version, "version", line, findings);
|
||||
if (!row.skillMd?.trim())
|
||||
findings.push({ line, reason: "empty_skill_text", field: "skillMd" });
|
||||
collectDuplicate("skill", row.slug, line, seenKeys, findings);
|
||||
} else {
|
||||
pluginCount += 1;
|
||||
requireString(row.name, "name", line, findings);
|
||||
requireString(row.displayName, "displayName", line, findings);
|
||||
requireString(row.version, "version", line, findings);
|
||||
if (!row.readme?.trim())
|
||||
findings.push({ line, reason: "empty_plugin_text", field: "readme" });
|
||||
collectDuplicate("plugin", row.name, line, seenKeys, findings);
|
||||
}
|
||||
});
|
||||
|
||||
return {
|
||||
ok: findings.length === 0,
|
||||
rowCount: rows.length,
|
||||
skillCount,
|
||||
pluginCount,
|
||||
findings,
|
||||
};
|
||||
}
|
||||
|
||||
function mergeValidationResults(
|
||||
result: CorpusValidationResult,
|
||||
findings: CorpusValidationFinding[],
|
||||
): CorpusValidationResult {
|
||||
return {
|
||||
...result,
|
||||
ok: result.ok && findings.length === 0,
|
||||
findings: [...findings, ...result.findings],
|
||||
};
|
||||
}
|
||||
|
||||
function collectDuplicate(
|
||||
kind: "skill" | "plugin",
|
||||
slug: unknown,
|
||||
line: number,
|
||||
seenKeys: Set<string>,
|
||||
findings: CorpusValidationFinding[],
|
||||
) {
|
||||
if (typeof slug !== "string" || !slug) return;
|
||||
const key = `${kind}:${slug}`;
|
||||
if (seenKeys.has(key)) findings.push({ line, reason: "duplicate_slug", value: key });
|
||||
seenKeys.add(key);
|
||||
}
|
||||
|
||||
function requireString(
|
||||
value: unknown,
|
||||
field: string,
|
||||
line: number,
|
||||
findings: CorpusValidationFinding[],
|
||||
) {
|
||||
if (typeof value === "string" && value.trim()) return;
|
||||
findings.push({ line, reason: "missing_required_field", field });
|
||||
}
|
||||
|
||||
function collectDisallowedFieldFindings(
|
||||
value: unknown,
|
||||
line: number,
|
||||
findings: CorpusValidationFinding[],
|
||||
path = "",
|
||||
) {
|
||||
if (!isRecord(value)) return;
|
||||
for (const [key, nested] of Object.entries(value)) {
|
||||
const field = path ? `${path}.${key}` : key;
|
||||
if (DISALLOWED_FIELD_PATTERNS.some((pattern) => pattern.test(key))) {
|
||||
findings.push({ line, reason: "disallowed_field", field });
|
||||
}
|
||||
collectDisallowedFieldFindings(nested, line, findings, field);
|
||||
}
|
||||
}
|
||||
|
||||
function collectTextFindings(
|
||||
value: unknown,
|
||||
line: number,
|
||||
findings: CorpusValidationFinding[],
|
||||
path = "",
|
||||
) {
|
||||
if (typeof value === "string") {
|
||||
if (RAW_CONVEX_ID_PATTERN.test(value)) {
|
||||
findings.push({ line, reason: "raw_convex_id", field: path, value: preview(value) });
|
||||
}
|
||||
if (RAW_LOCAL_PATH_PATTERN.test(value)) {
|
||||
findings.push({ line, reason: "local_path", field: path, value: preview(value) });
|
||||
}
|
||||
if (SECRET_PATTERNS.some((pattern) => pattern.test(value))) {
|
||||
findings.push({ line, reason: "secret_like_text", field: path, value: preview(value) });
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
value.forEach((item, index) => collectTextFindings(item, line, findings, `${path}[${index}]`));
|
||||
return;
|
||||
}
|
||||
if (isRecord(value)) {
|
||||
for (const [key, nested] of Object.entries(value)) {
|
||||
collectTextFindings(nested, line, findings, path ? `${path}.${key}` : key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function preview(value: string) {
|
||||
return value.slice(0, 120);
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
@@ -1,6 +1,9 @@
|
||||
# Local Moderation Fixtures
|
||||
|
||||
This note records the intended local-only QA fixtures created by `bun run seed:dev`.
|
||||
The real-ish catalog density comes from the committed public corpus fixture; these
|
||||
hand-authored fixtures remain for security and moderation states that need stable
|
||||
local reproduction.
|
||||
|
||||
The fixtures exist so developers can exercise ClawHub moderation, scan, publisher-note, and artifact UI states without hand-editing Convex data. They are not production behavior and should not introduce appeal-specific flows.
|
||||
|
||||
|
||||
+1
-1
@@ -156,7 +156,7 @@ From SKILL.md frontmatter + AgentSkills + Clawdis extensions:
|
||||
5. Server stores files + metadata, sets `latest` tag, updates stats.
|
||||
|
||||
Soul upload flow: same as skills (including GitHub account age checks), but only `SOUL.md` is allowed.
|
||||
Seed data lives in `convex/seed.ts` for local dev.
|
||||
Local fixture data lives in `convex/devSeed.ts` and `fixtures/public-corpus/`.
|
||||
|
||||
## Versioning + tags
|
||||
|
||||
|
||||
Reference in New Issue
Block a user