fix(seed): populate public corpus plugin metadata

This commit is contained in:
Patrick Erichsen
2026-06-23 13:14:30 -07:00
parent 071c43da43
commit dfde400fcf
4 changed files with 703 additions and 1 deletions
+358
View File
@@ -507,8 +507,345 @@ describe("devSeed local fixtures", () => {
).toBe(installs);
});
it("populates public corpus plugin catalog metadata, digests, and validation findings", async () => {
const { db, tables } = createDb();
await seedPublicCorpusBatchHandler(
createMutationCtx(db) as never,
{
rows: [
{
kind: "plugin",
name: "gmail-agent-plugin",
displayName: "Gmail Agent Plugin",
version: "0.1.0",
readme: "# Gmail Agent Plugin\n\nWatches Gmail and notifies an OpenClaw channel.",
storageId: "storage:gmail-agent-plugin",
categories: ["channels", "tools"],
topics: ["Gmail", "Notifications"],
dummyOwner: {
handle: "corpus-owner",
displayName: "Corpus Owner",
image: "https://example.invalid/avatar.png",
},
},
],
} as never,
);
const pkg = tables.packages?.find((candidate) => candidate.name === "gmail-agent-plugin");
const release = tables.packageReleases?.find((candidate) => candidate.packageId === pkg?._id);
expect(pkg).toEqual(
expect.objectContaining({
categories: ["channels", "tools"],
topics: ["Gmail", "Notifications"],
}),
);
expect(tables.packageSearchDigest?.[0]).toEqual(
expect.objectContaining({
packageId: pkg?._id,
categories: ["channels", "tools"],
topics: ["Gmail", "Notifications"],
pluginCategoryTags: ["channels", "tools"],
}),
);
expect(
tables.packagePluginCategorySearchDigest
?.map((row) => String(row.pluginCategory))
.sort((left, right) => left.localeCompare(right)),
).toEqual(["channels", "tools"]);
expect(
tables.packageTopicSearchDigest
?.map((row) => String(row.topic))
.sort((left, right) => left.localeCompare(right)),
).toEqual(["gmail", "notifications"]);
expect(tables.packageInspectorWarnings).toEqual([
expect.objectContaining({
packageId: pkg?._id,
releaseId: release?._id,
packageName: "gmail-agent-plugin",
version: "0.1.0",
findingKind: "warning",
code: "package-min-host-version-drift",
authorRemediation: expect.objectContaining({
docsUrl:
"https://docs.openclaw.ai/clawhub/plugin-validation-fixes#package-min-host-version-drift",
}),
}),
]);
});
it("backfills catalog metadata and validation findings for existing public corpus packages", async () => {
const { db, tables } = createDb();
const userId = (await db.insert("users", {
handle: "corpus-owner",
displayName: "Corpus Owner",
role: "user",
createdAt: 1,
updatedAt: 1,
})) as Id<"users">;
const publisherId = (await db.insert("publishers", {
kind: "user",
handle: "corpus-owner",
displayName: "Corpus Owner",
linkedUserId: userId,
createdAt: 1,
updatedAt: 1,
})) as Id<"publishers">;
const packageId = (await db.insert("packages", {
name: "gmail-agent-plugin",
normalizedName: "gmail-agent-plugin",
displayName: "Gmail Agent Plugin",
summary: "Existing public corpus plugin fixture.",
ownerUserId: userId,
ownerPublisherId: publisherId,
family: "code-plugin",
channel: "community",
isOfficial: false,
runtimeId: "gmail-agent-plugin",
latestReleaseId: undefined,
latestVersionSummary: undefined,
tags: {},
compatibility: { pluginApiRange: ">=0.1.0" },
verification: {
tier: "structural",
scope: "artifact-only",
summary: "Seeded from the public corpus fixture.",
scanStatus: "clean",
},
scanStatus: "clean",
stats: { downloads: 57, installs: 13, stars: 2, versions: 1 },
createdAt: 1,
updatedAt: 1,
})) as Id<"packages">;
const releaseId = (await db.insert("packageReleases", {
packageId,
version: "0.1.0",
changelog: "Existing public corpus fixture.",
distTags: ["latest"],
files: [],
integritySha256: "existing-integrity",
compatibility: { pluginApiRange: ">=0.1.0" },
verification: {
tier: "structural",
scope: "artifact-only",
summary: "Seeded from the public corpus fixture.",
scanStatus: "clean",
},
createdBy: userId,
publishActor: { kind: "user", userId },
createdAt: 1,
})) as Id<"packageReleases">;
await db.patch(packageId, {
latestReleaseId: releaseId,
latestVersionSummary: {
version: "0.1.0",
createdAt: 1,
changelog: "Existing public corpus fixture.",
compatibility: { pluginApiRange: ">=0.1.0" },
verification: {
tier: "structural",
scope: "artifact-only",
summary: "Seeded from the public corpus fixture.",
scanStatus: "clean",
},
},
});
await seedPublicCorpusBatchHandler(
createMutationCtx(db) as never,
{
rows: [
{
kind: "plugin",
name: "gmail-agent-plugin",
displayName: "Gmail Agent Plugin",
version: "0.1.0",
readme: "# Gmail Agent Plugin\n\nWatches Gmail and notifies an OpenClaw channel.",
storageId: "storage:gmail-agent-plugin",
categories: ["channels", "tools"],
topics: ["Gmail", "Notifications"],
dummyOwner: {
handle: "corpus-owner",
displayName: "Corpus Owner",
image: "https://example.invalid/avatar.png",
},
},
],
} as never,
);
expect(tables.packages?.[0]).toEqual(
expect.objectContaining({
categories: ["channels", "tools"],
topics: ["Gmail", "Notifications"],
}),
);
expect(tables.packageSearchDigest?.[0]).toEqual(
expect.objectContaining({
packageId,
pluginCategoryTags: ["channels", "tools"],
}),
);
expect(tables.packageInspectorWarnings).toHaveLength(1);
});
it("does not backfill catalog metadata onto non-corpus package name collisions", async () => {
const { db, tables } = createDb();
const userId = (await db.insert("users", {
handle: "real-owner",
displayName: "Real Owner",
role: "user",
createdAt: 1,
updatedAt: 1,
})) as Id<"users">;
const publisherId = (await db.insert("publishers", {
kind: "user",
handle: "real-owner",
displayName: "Real Owner",
linkedUserId: userId,
createdAt: 1,
updatedAt: 1,
})) as Id<"publishers">;
const packageId = (await db.insert("packages", {
name: "gmail-agent-plugin",
normalizedName: "gmail-agent-plugin",
displayName: "Gmail Agent Plugin",
summary: "A real package that happens to collide with the corpus fixture.",
ownerUserId: userId,
ownerPublisherId: publisherId,
family: "code-plugin",
channel: "community",
isOfficial: false,
runtimeId: "gmail-agent-plugin",
latestReleaseId: undefined,
latestVersionSummary: undefined,
tags: {},
categories: ["models"],
topics: ["Original Topic"],
compatibility: { pluginApiRange: ">=0.1.0" },
verification: {
tier: "structural",
scope: "artifact-only",
summary: "Real package verification.",
scanStatus: "clean",
},
scanStatus: "clean",
stats: { downloads: 57, installs: 13, stars: 2, versions: 1 },
createdAt: 1,
updatedAt: 1,
})) as Id<"packages">;
const releaseId = (await db.insert("packageReleases", {
packageId,
version: "0.1.0",
changelog: "Real package release.",
distTags: ["latest"],
files: [],
integritySha256: "existing-integrity",
compatibility: { pluginApiRange: ">=0.1.0" },
verification: {
tier: "structural",
scope: "artifact-only",
summary: "Real package verification.",
scanStatus: "clean",
},
createdBy: userId,
publishActor: { kind: "user", userId },
createdAt: 1,
})) as Id<"packageReleases">;
await db.patch(packageId, {
latestReleaseId: releaseId,
latestVersionSummary: {
version: "0.1.0",
createdAt: 1,
changelog: "Real package release.",
compatibility: { pluginApiRange: ">=0.1.0" },
verification: {
tier: "structural",
scope: "artifact-only",
summary: "Real package verification.",
scanStatus: "clean",
},
},
});
await backfillExistingPublicCorpusBatchRowsHandler(
createMutationCtx(db) as never,
{
rows: [
{
kind: "plugin",
name: "gmail-agent-plugin",
displayName: "Gmail Agent Plugin",
version: "0.1.0",
readme: "# Gmail Agent Plugin\n\nWatches Gmail and notifies an OpenClaw channel.",
storageId: "storage:gmail-agent-plugin",
categories: ["channels", "tools"],
topics: ["Gmail", "Notifications"],
dummyOwner: {
handle: "corpus-owner",
displayName: "Corpus Owner",
image: "https://example.invalid/avatar.png",
},
},
],
} as never,
);
expect(tables.packages?.[0]).toEqual(
expect.objectContaining({
categories: ["models"],
topics: ["Original Topic"],
}),
);
expect(tables.packageSearchDigest).toBeUndefined();
expect(tables.packageDailyStats).toBeUndefined();
expect(tables.packageInspectorWarnings).toBeUndefined();
});
it("caps inferred public corpus plugin categories at the catalog limit", async () => {
const { db, tables } = createDb();
await seedPublicCorpusBatchHandler(
createMutationCtx(db) as never,
{
rows: [
{
kind: "plugin",
name: "context-security-openclaw-email-guard-plugin",
displayName: "Context Security Email Guard Plugin",
version: "0.1.0",
readme:
"# Context Security Email Guard Plugin\n\nA runtime plugin for Gmail, model providers, memory, context, web search, GitHub tools, gateway operations, and OAuth policy checks.",
storageId: "storage:context-security-openclaw-email-guard-plugin",
dummyOwner: {
handle: "corpus-owner",
displayName: "Corpus Owner",
image: "https://example.invalid/avatar.png",
},
},
],
} as never,
);
const pkg = tables.packages?.find(
(candidate) => candidate.name === "context-security-openclaw-email-guard-plugin",
);
const categories = Array.isArray(pkg?.categories) ? pkg.categories : [];
expect(categories.length).toBeLessThanOrEqual(3);
expect(categories.length).toBeGreaterThan(0);
});
it("removes public corpus daily activity rows during reset", async () => {
const { db, tables } = createDb();
await db.insert("globalStats", {
key: "default",
activeSkillsCount: 0,
activePluginsCount: 0,
updatedAt: 1,
});
const rows = [
{
kind: "skill",
@@ -544,6 +881,11 @@ describe("devSeed local fixtures", () => {
const firstPackageId = tables.packages?.[0]?._id;
const firstSkillDailyRows = tables.skillDailyStats?.length ?? 0;
const firstPackageDailyRows = tables.packageDailyStats?.length ?? 0;
const firstPackageDigestRows = tables.packageSearchDigest?.length ?? 0;
const firstPackageCategoryDigestRows = tables.packagePluginCategorySearchDigest?.length ?? 0;
const firstPackageTopicDigestRows = tables.packageTopicSearchDigest?.length ?? 0;
const firstPackageInspectorWarningRows = tables.packageInspectorWarnings?.length ?? 0;
const firstActivePluginsCount = tables.globalStats?.[0]?.activePluginsCount;
await seedPublicCorpusBatchHandler(
createMutationCtx(db) as never,
@@ -556,6 +898,22 @@ describe("devSeed local fixtures", () => {
expect(tables.packageDailyStats).toHaveLength(firstPackageDailyRows);
expect(tables.skillDailyStats?.some((row) => row.skillId === firstSkillId)).toBe(false);
expect(tables.packageDailyStats?.some((row) => row.packageId === firstPackageId)).toBe(false);
expect(firstPackageDigestRows).toBeGreaterThan(0);
expect(firstPackageCategoryDigestRows).toBeGreaterThan(0);
expect(firstPackageTopicDigestRows).toBeGreaterThan(0);
expect(firstPackageInspectorWarningRows).toBeGreaterThan(0);
expect(tables.packageSearchDigest?.some((row) => row.packageId === firstPackageId)).toBe(false);
expect(
tables.packagePluginCategorySearchDigest?.some((row) => row.packageId === firstPackageId),
).toBe(false);
expect(tables.packageTopicSearchDigest?.some((row) => row.packageId === firstPackageId)).toBe(
false,
);
expect(tables.packageInspectorWarnings?.some((row) => row.packageId === firstPackageId)).toBe(
false,
);
expect(firstActivePluginsCount).toBe(1);
expect(tables.globalStats?.[0]?.activePluginsCount).toBe(1);
});
it("seeds a GitHub-backed source and skills without creating mirrored versions", async () => {
+283 -1
View File
@@ -1,6 +1,14 @@
import {
CATALOG_CATEGORY_LIMIT,
PLUGIN_CATEGORY_DEFINITIONS,
normalizeCatalogTopic,
normalizeCatalogTopics,
normalizePluginCategories,
resolvePluginCategories,
} from "clawhub-schema";
import { v } from "convex/values";
import { internal } from "./_generated/api";
import type { Id } from "./_generated/dataModel";
import type { Doc, Id } from "./_generated/dataModel";
import type { ActionCtx, MutationCtx } from "./_generated/server";
import { internalMutation as rawInternalMutation } from "./_generated/server";
import { internalAction, internalMutation } from "./functions";
@@ -9,6 +17,11 @@ import { EMBEDDING_DIMENSIONS, generateEmbedding } from "./lib/embeddings";
import { deleteGitHubSkillScansForSkill } from "./lib/githubSkillScans";
import { toDayKey } from "./lib/leaderboards";
import { normalizePackageName } from "./lib/packageRegistry";
import {
deletePackageSearchDigests,
extractPackageDigestFields,
upsertPackageSearchDigest,
} from "./lib/packageSearchDigest";
import { ensurePersonalPublisherForUser } from "./lib/publishers";
import {
computeRecommendationScore,
@@ -159,6 +172,8 @@ const publicCorpusPluginRowValidator = v.object({
version: v.string(),
readme: v.string(),
summary: v.optional(v.string()),
categories: v.optional(v.array(v.string())),
topics: v.optional(v.array(v.string())),
family: v.optional(
v.union(v.literal("skill"), v.literal("code-plugin"), v.literal("bundle-plugin")),
),
@@ -193,6 +208,8 @@ const publicCorpusPreparedPluginRowValidator = v.object({
version: v.string(),
readme: v.string(),
summary: v.optional(v.string()),
categories: v.optional(v.array(v.string())),
topics: v.optional(v.array(v.string())),
family: v.optional(
v.union(v.literal("skill"), v.literal("code-plugin"), v.literal("bundle-plugin")),
),
@@ -866,6 +883,32 @@ export const backfillExistingPublicCorpusBatchRows = internalMutation({
missingKeys.push(publicCorpusSeedRowKey(row));
continue;
}
if (!(await packageBelongsToPublicCorpusOwner(ctx, existing, row.dummyOwner))) {
skipped.push(`plugin:${row.name}`);
continue;
}
const catalogMetadata = publicCorpusPluginCatalogMetadata(row);
await ctx.db.patch(existing._id, {
categories: catalogMetadata.categories,
topics: catalogMetadata.topics,
updatedAt: now,
});
if (existing.latestReleaseId) {
await ensurePublicCorpusPackageValidationWarning(ctx, {
packageId: existing._id,
releaseId: existing.latestReleaseId,
ownerUserId: existing.ownerUserId,
ownerPublisherId: existing.ownerPublisherId,
packageName: existing.name,
normalizedName: existing.normalizedName,
version: row.version,
createdAt: now,
});
}
const updatedPackage = await ctx.db.get(existing._id);
if (updatedPackage) {
await upsertPackageSearchDigest(ctx, extractPackageDigestFields(updatedPackage));
}
await ensurePublicCorpusPackageDailyStats(ctx, {
packageId: existing._id,
key: row.name,
@@ -945,6 +988,21 @@ function publicCorpusSeedRowKey(
return row.kind === "skill" ? `skill:${row.slug}` : `plugin:${row.name}`;
}
async function packageBelongsToPublicCorpusOwner(
ctx: Pick<MutationCtx, "db">,
pkg: Pick<Doc<"packages">, "ownerUserId">,
dummyOwner: { handle: string },
ownerUserId?: Id<"users">,
) {
if (ownerUserId) return pkg.ownerUserId === ownerUserId;
const owners = await ctx.db
.query("users")
.withIndex("handle", (q) => q.eq("handle", dummyOwner.handle))
.collect();
return owners.some((owner) => owner._id === pkg.ownerUserId);
}
export const seedPublicCorpusBatchMutation = internalMutation({
args: {
reset: v.optional(v.boolean()),
@@ -1083,6 +1141,32 @@ export const seedPublicCorpusBatchMutation = internalMutation({
.withIndex("by_name", (q) => q.eq("normalizedName", normalizedName))
.unique();
if (existing) {
if (!(await packageBelongsToPublicCorpusOwner(ctx, existing, row.dummyOwner, userId))) {
skipped.push(`plugin:${row.name}`);
continue;
}
const catalogMetadata = publicCorpusPluginCatalogMetadata(row);
await ctx.db.patch(existing._id, {
categories: catalogMetadata.categories,
topics: catalogMetadata.topics,
updatedAt: now,
});
if (existing.latestReleaseId) {
await ensurePublicCorpusPackageValidationWarning(ctx, {
packageId: existing._id,
releaseId: existing.latestReleaseId,
ownerUserId: existing.ownerUserId,
ownerPublisherId: existing.ownerPublisherId,
packageName: existing.name,
normalizedName: existing.normalizedName,
version: row.version,
createdAt: now,
});
}
const updatedPackage = await ctx.db.get(existing._id);
if (updatedPackage) {
await upsertPackageSearchDigest(ctx, extractPackageDigestFields(updatedPackage));
}
await ensurePublicCorpusPackageDailyStats(ctx, {
packageId: existing._id,
key: row.name,
@@ -1097,6 +1181,7 @@ export const seedPublicCorpusBatchMutation = internalMutation({
const createdAt = row.createdAt ?? now;
const stats = publicCorpusPackageStats(row.name);
const compatibility = { pluginApiRange: ">=0.1.0" };
const catalogMetadata = publicCorpusPluginCatalogMetadata(row);
const verification = {
tier: "structural" as const,
scope: "artifact-only" as const,
@@ -1117,6 +1202,8 @@ export const seedPublicCorpusBatchMutation = internalMutation({
latestReleaseId: undefined,
latestVersionSummary: undefined,
tags: {},
categories: catalogMetadata.categories,
topics: catalogMetadata.topics,
compatibility,
verification,
scanStatus: "clean",
@@ -1158,6 +1245,16 @@ export const seedPublicCorpusBatchMutation = internalMutation({
createdAt,
softDeletedAt: undefined,
});
await ensurePublicCorpusPackageValidationWarning(ctx, {
packageId,
releaseId,
ownerUserId: userId,
ownerPublisherId: publisherId,
packageName: row.name,
normalizedName,
version: row.version,
createdAt: now,
});
await ctx.db.patch(packageId, {
latestReleaseId: releaseId,
latestVersionSummary: {
@@ -1171,6 +1268,10 @@ export const seedPublicCorpusBatchMutation = internalMutation({
stats: { ...stats, versions: 1 },
updatedAt: now,
});
const packageDoc = await ctx.db.get(packageId);
if (packageDoc) {
await upsertPackageSearchDigest(ctx, extractPackageDigestFields(packageDoc));
}
await ensurePublicCorpusPackageDailyStats(ctx, {
packageId,
key: row.name,
@@ -1337,6 +1438,176 @@ function publicCorpusPackageStats(name: string) {
};
}
const PUBLIC_CORPUS_PLUGIN_FALLBACK_CATEGORIES = PLUGIN_CATEGORY_DEFINITIONS.map(
(category) => category.slug,
).filter((slug) => slug !== "other");
const PUBLIC_CORPUS_PLUGIN_CATEGORY_TOPICS: Record<string, string> = {
channels: "Messaging",
models: "Model Providers",
memory: "Memory",
context: "Context",
voice: "Voice",
media: "Media",
web: "Web Search",
tools: "Automation",
runtime: "Runtime",
gateway: "Gateway",
security: "Security",
other: "Utilities",
};
function publicCorpusPluginCatalogMetadata(row: {
name: string;
displayName: string;
summary?: string;
readme: string;
categories?: string[];
topics?: string[];
}) {
const declaredCategories =
row.categories === undefined ? undefined : normalizePluginCategories(row.categories);
const categories = resolvePluginCategories({
declared: declaredCategories,
inferred: declaredCategories === undefined ? inferPublicCorpusPluginCategories(row) : undefined,
});
const topics =
row.topics === undefined
? inferPublicCorpusPluginTopics(row, categories)
: normalizeCatalogTopics(row.topics);
return { categories, topics };
}
async function ensurePublicCorpusPackageValidationWarning(
ctx: Pick<MutationCtx, "db">,
params: {
packageId: Id<"packages">;
releaseId: Id<"packageReleases">;
ownerUserId: Id<"users">;
ownerPublisherId?: Id<"publishers">;
packageName: string;
normalizedName: string;
version: string;
createdAt: number;
},
) {
const existingWarnings = await ctx.db
.query("packageInspectorWarnings")
.withIndex("by_release", (q) => q.eq("releaseId", params.releaseId))
.collect();
if (
existingWarnings.some(
(warning) =>
warning.code === "package-min-host-version-drift" &&
warning.inspectorFindingId === `${params.normalizedName}:package-min-host-version-drift`,
)
) {
return;
}
await ctx.db.insert("packageInspectorWarnings", {
packageId: params.packageId,
releaseId: params.releaseId,
ownerUserId: params.ownerUserId,
ownerPublisherId: params.ownerPublisherId,
packageName: params.packageName,
version: params.version,
findingKind: "warning",
scanSource: "publish",
inspectorVersion: "0.3.15",
targetOpenClawVersion: "2026.6.9",
code: "package-min-host-version-drift",
severity: "P2",
level: "warning",
issueClass: "upstream-metadata",
compatStatus: "warning",
message: `${params.packageName}: OpenClaw package minimum host version drifts from build target`,
evidence: ["minHostVersion: >=2026.4.25", "buildOpenClawVersion: 2026.6.9"],
authorRemediation: {
summary:
"Set the package minimum host version to the OpenClaw version range the plugin was built and tested against.",
docsUrl:
"https://docs.openclaw.ai/clawhub/plugin-validation-fixes#package-min-host-version-drift",
},
fixture: "public-corpus",
decision: "seeded-warning",
inspectorFindingId: `${params.normalizedName}:package-min-host-version-drift`,
createdAt: params.createdAt,
});
}
function inferPublicCorpusPluginCategories(row: {
name: string;
displayName: string;
summary?: string;
readme: string;
}) {
const text = [row.name, row.displayName, row.summary, row.readme]
.filter(Boolean)
.join(" ")
.toLocaleLowerCase("en-US");
const categories: string[] = [];
const add = (category: string) => {
if (!categories.includes(category)) categories.push(category);
};
if (/\b(slack|discord|telegram|whatsapp|gmail|email|chat|message|messenger|sms)\b/.test(text)) {
add("channels");
}
if (/\b(model|models|provider|providers|gpt|llm|openai|claude|inference|modelark)\b/.test(text)) {
add("models");
}
if (/\b(memory|recall|embedding|embeddings|vector|session)\b/.test(text)) add("memory");
if (/\b(context|knowledge|document|docs|pdf)\b/.test(text)) add("context");
if (/\b(voice|speech|tts|transcription|audio)\b/.test(text)) add("voice");
if (/\b(image|media|video|youtube|twitter|x-|music|render)\b/.test(text)) add("media");
if (/\b(web|browser|search|reddit|fetch|crawl|url|http)\b/.test(text)) add("web");
if (/\b(tool|tools|workflow|automation|cli|command|shell|github|actions)\b/.test(text)) {
add("tools");
}
if (/\b(runtime|codex|developer|dev|test|deploy|openclaw)\b/.test(text)) add("runtime");
if (/\b(gateway|observability|worker|ops|operator)\b/.test(text)) add("gateway");
if (/\b(auth|oauth|security|secret|policy|permission|trust)\b/.test(text)) add("security");
if (categories.length > 0) return categories.slice(0, CATALOG_CATEGORY_LIMIT);
const fallbackIndex =
publicCorpusStableNumber(row.name) % PUBLIC_CORPUS_PLUGIN_FALLBACK_CATEGORIES.length;
return [PUBLIC_CORPUS_PLUGIN_FALLBACK_CATEGORIES[fallbackIndex] ?? "other"];
}
function inferPublicCorpusPluginTopics(
row: { name: string; displayName: string },
categories: readonly string[],
) {
const topics: string[] = [];
const seenSlugs = new Set<string>();
const add = (candidate: string | undefined) => {
if (!candidate) return;
const slug = normalizeCatalogTopic(candidate);
if (!slug || seenSlugs.has(slug)) return;
try {
const [topic] = normalizeCatalogTopics([candidate]);
if (!topic) return;
seenSlugs.add(slug);
topics.push(topic);
} catch {
return;
}
};
add(PUBLIC_CORPUS_PLUGIN_CATEGORY_TOPICS[categories[0] ?? "other"]);
for (const rawToken of `${row.displayName} ${row.name}`.split(/[^\p{L}\p{N}]+/u)) {
const token = rawToken.trim();
if (token.length < 3) continue;
if (/^(plugin|openclaw|clawhub|agent|agents|the)$/i.test(token)) continue;
add(token.slice(0, 1).toLocaleUpperCase("en-US") + token.slice(1));
if (topics.length >= 5) break;
}
return topics.length > 0 ? topics : ["Utilities"];
}
function publicCorpusStableNumber(value: string) {
let hash = 0;
for (let index = 0; index < value.length; index += 1) {
@@ -1580,6 +1851,7 @@ async function deletePackageAndReleases(ctx: MutationCtx, packageId: Id<"package
.collect();
await deletePackageBadgesForPackage(ctx, packageId);
await deletePackageDailyStatsForPackage(ctx, packageId);
await deletePackageDerivedSearchData(ctx, packageId);
await ctx.db.delete(packageId);
for (const release of releases) await ctx.db.delete(release._id);
}
@@ -1640,6 +1912,16 @@ async function deletePackageDailyStatsForPackage(ctx: MutationCtx, packageId: Id
for (const row of rows) await ctx.db.delete(row._id);
}
async function deletePackageDerivedSearchData(ctx: MutationCtx, packageId: Id<"packages">) {
await deletePackageSearchDigests(ctx, packageId);
const inspectorWarnings = await ctx.db
.query("packageInspectorWarnings")
.withIndex("by_package_created", (q) => q.eq("packageId", packageId))
.collect();
for (const row of inspectorWarnings) await ctx.db.delete(row._id);
}
async function deleteSeedSkillFixture(ctx: MutationCtx, slug = FLAGGED_SKILL_SLUG) {
const existing = await findSeedSkillFixture(ctx, slug);
if (!existing) return;
+24
View File
@@ -60,6 +60,30 @@ describe("public corpus fixture validation", () => {
);
});
it("rejects invalid plugin categories and topics", () => {
const result = validateCorpusRows([
{
...pluginRow,
categories: ["not-a-category"],
topics: ["Official"],
},
]);
expect(result.ok).toBe(false);
expect(result.findings).toContainEqual(
expect.objectContaining({
reason: "invalid_catalog_metadata",
field: "categories",
}),
);
expect(result.findings).toContainEqual(
expect.objectContaining({
reason: "invalid_catalog_metadata",
field: "topics",
}),
);
});
it("rejects empty content and secret-like text", () => {
const result = validateCorpusRows([
{ ...skillRow, slug: "empty", skillMd: "" },
+38
View File
@@ -1,3 +1,5 @@
import { normalizeCatalogTopics, normalizePluginCategories } from "clawhub-schema";
export const DEFAULT_PUBLIC_CORPUS_FIXTURE = "fixtures/public-corpus/corpus.jsonl";
export type PublicCorpusSkillRow = {
@@ -17,6 +19,8 @@ export type PublicCorpusPluginRow = {
version: string;
readme: string;
summary?: string;
categories?: string[];
topics?: string[];
family?: "skill" | "code-plugin" | "bundle-plugin";
channel?: "official" | "community" | "private";
sourceRepoHost?: string | null;
@@ -36,6 +40,7 @@ export type CorpusValidationFinding = {
| "disallowed_field"
| "raw_convex_id"
| "duplicate_slug"
| "invalid_catalog_metadata"
| "local_path"
| "secret_like_text";
field?: string;
@@ -128,6 +133,7 @@ export function validateCorpusRows(rows: PublicCorpusRow[]): CorpusValidationRes
requireString(row.version, "version", line, findings);
if (!row.readme?.trim())
findings.push({ line, reason: "empty_plugin_text", field: "readme" });
collectPluginCatalogMetadataFindings(row, line, findings);
collectDuplicate("plugin", row.name, line, seenKeys, findings);
}
});
@@ -175,6 +181,38 @@ function requireString(
findings.push({ line, reason: "missing_required_field", field });
}
function collectPluginCatalogMetadataFindings(
row: PublicCorpusPluginRow,
line: number,
findings: CorpusValidationFinding[],
) {
if (row.categories !== undefined) {
try {
normalizePluginCategories(row.categories);
} catch (error) {
findings.push({
line,
reason: "invalid_catalog_metadata",
field: "categories",
value: error instanceof Error ? error.message : undefined,
});
}
}
if (row.topics !== undefined) {
try {
normalizeCatalogTopics(row.topics);
} catch (error) {
findings.push({
line,
reason: "invalid_catalog_metadata",
field: "topics",
value: error instanceof Error ? error.message : undefined,
});
}
}
}
function collectDisallowedFieldFindings(
value: unknown,
line: number,