mirror of
https://github.com/openclaw/clawhub.git
synced 2026-08-14 00:47:57 +00:00
fix(seed): populate public corpus plugin metadata
This commit is contained in:
@@ -507,8 +507,345 @@ describe("devSeed local fixtures", () => {
|
||||
).toBe(installs);
|
||||
});
|
||||
|
||||
it("populates public corpus plugin catalog metadata, digests, and validation findings", async () => {
|
||||
const { db, tables } = createDb();
|
||||
|
||||
await seedPublicCorpusBatchHandler(
|
||||
createMutationCtx(db) as never,
|
||||
{
|
||||
rows: [
|
||||
{
|
||||
kind: "plugin",
|
||||
name: "gmail-agent-plugin",
|
||||
displayName: "Gmail Agent Plugin",
|
||||
version: "0.1.0",
|
||||
readme: "# Gmail Agent Plugin\n\nWatches Gmail and notifies an OpenClaw channel.",
|
||||
storageId: "storage:gmail-agent-plugin",
|
||||
categories: ["channels", "tools"],
|
||||
topics: ["Gmail", "Notifications"],
|
||||
dummyOwner: {
|
||||
handle: "corpus-owner",
|
||||
displayName: "Corpus Owner",
|
||||
image: "https://example.invalid/avatar.png",
|
||||
},
|
||||
},
|
||||
],
|
||||
} as never,
|
||||
);
|
||||
|
||||
const pkg = tables.packages?.find((candidate) => candidate.name === "gmail-agent-plugin");
|
||||
const release = tables.packageReleases?.find((candidate) => candidate.packageId === pkg?._id);
|
||||
|
||||
expect(pkg).toEqual(
|
||||
expect.objectContaining({
|
||||
categories: ["channels", "tools"],
|
||||
topics: ["Gmail", "Notifications"],
|
||||
}),
|
||||
);
|
||||
expect(tables.packageSearchDigest?.[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
packageId: pkg?._id,
|
||||
categories: ["channels", "tools"],
|
||||
topics: ["Gmail", "Notifications"],
|
||||
pluginCategoryTags: ["channels", "tools"],
|
||||
}),
|
||||
);
|
||||
expect(
|
||||
tables.packagePluginCategorySearchDigest
|
||||
?.map((row) => String(row.pluginCategory))
|
||||
.sort((left, right) => left.localeCompare(right)),
|
||||
).toEqual(["channels", "tools"]);
|
||||
expect(
|
||||
tables.packageTopicSearchDigest
|
||||
?.map((row) => String(row.topic))
|
||||
.sort((left, right) => left.localeCompare(right)),
|
||||
).toEqual(["gmail", "notifications"]);
|
||||
expect(tables.packageInspectorWarnings).toEqual([
|
||||
expect.objectContaining({
|
||||
packageId: pkg?._id,
|
||||
releaseId: release?._id,
|
||||
packageName: "gmail-agent-plugin",
|
||||
version: "0.1.0",
|
||||
findingKind: "warning",
|
||||
code: "package-min-host-version-drift",
|
||||
authorRemediation: expect.objectContaining({
|
||||
docsUrl:
|
||||
"https://docs.openclaw.ai/clawhub/plugin-validation-fixes#package-min-host-version-drift",
|
||||
}),
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("backfills catalog metadata and validation findings for existing public corpus packages", async () => {
|
||||
const { db, tables } = createDb();
|
||||
const userId = (await db.insert("users", {
|
||||
handle: "corpus-owner",
|
||||
displayName: "Corpus Owner",
|
||||
role: "user",
|
||||
createdAt: 1,
|
||||
updatedAt: 1,
|
||||
})) as Id<"users">;
|
||||
const publisherId = (await db.insert("publishers", {
|
||||
kind: "user",
|
||||
handle: "corpus-owner",
|
||||
displayName: "Corpus Owner",
|
||||
linkedUserId: userId,
|
||||
createdAt: 1,
|
||||
updatedAt: 1,
|
||||
})) as Id<"publishers">;
|
||||
const packageId = (await db.insert("packages", {
|
||||
name: "gmail-agent-plugin",
|
||||
normalizedName: "gmail-agent-plugin",
|
||||
displayName: "Gmail Agent Plugin",
|
||||
summary: "Existing public corpus plugin fixture.",
|
||||
ownerUserId: userId,
|
||||
ownerPublisherId: publisherId,
|
||||
family: "code-plugin",
|
||||
channel: "community",
|
||||
isOfficial: false,
|
||||
runtimeId: "gmail-agent-plugin",
|
||||
latestReleaseId: undefined,
|
||||
latestVersionSummary: undefined,
|
||||
tags: {},
|
||||
compatibility: { pluginApiRange: ">=0.1.0" },
|
||||
verification: {
|
||||
tier: "structural",
|
||||
scope: "artifact-only",
|
||||
summary: "Seeded from the public corpus fixture.",
|
||||
scanStatus: "clean",
|
||||
},
|
||||
scanStatus: "clean",
|
||||
stats: { downloads: 57, installs: 13, stars: 2, versions: 1 },
|
||||
createdAt: 1,
|
||||
updatedAt: 1,
|
||||
})) as Id<"packages">;
|
||||
const releaseId = (await db.insert("packageReleases", {
|
||||
packageId,
|
||||
version: "0.1.0",
|
||||
changelog: "Existing public corpus fixture.",
|
||||
distTags: ["latest"],
|
||||
files: [],
|
||||
integritySha256: "existing-integrity",
|
||||
compatibility: { pluginApiRange: ">=0.1.0" },
|
||||
verification: {
|
||||
tier: "structural",
|
||||
scope: "artifact-only",
|
||||
summary: "Seeded from the public corpus fixture.",
|
||||
scanStatus: "clean",
|
||||
},
|
||||
createdBy: userId,
|
||||
publishActor: { kind: "user", userId },
|
||||
createdAt: 1,
|
||||
})) as Id<"packageReleases">;
|
||||
await db.patch(packageId, {
|
||||
latestReleaseId: releaseId,
|
||||
latestVersionSummary: {
|
||||
version: "0.1.0",
|
||||
createdAt: 1,
|
||||
changelog: "Existing public corpus fixture.",
|
||||
compatibility: { pluginApiRange: ">=0.1.0" },
|
||||
verification: {
|
||||
tier: "structural",
|
||||
scope: "artifact-only",
|
||||
summary: "Seeded from the public corpus fixture.",
|
||||
scanStatus: "clean",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await seedPublicCorpusBatchHandler(
|
||||
createMutationCtx(db) as never,
|
||||
{
|
||||
rows: [
|
||||
{
|
||||
kind: "plugin",
|
||||
name: "gmail-agent-plugin",
|
||||
displayName: "Gmail Agent Plugin",
|
||||
version: "0.1.0",
|
||||
readme: "# Gmail Agent Plugin\n\nWatches Gmail and notifies an OpenClaw channel.",
|
||||
storageId: "storage:gmail-agent-plugin",
|
||||
categories: ["channels", "tools"],
|
||||
topics: ["Gmail", "Notifications"],
|
||||
dummyOwner: {
|
||||
handle: "corpus-owner",
|
||||
displayName: "Corpus Owner",
|
||||
image: "https://example.invalid/avatar.png",
|
||||
},
|
||||
},
|
||||
],
|
||||
} as never,
|
||||
);
|
||||
|
||||
expect(tables.packages?.[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
categories: ["channels", "tools"],
|
||||
topics: ["Gmail", "Notifications"],
|
||||
}),
|
||||
);
|
||||
expect(tables.packageSearchDigest?.[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
packageId,
|
||||
pluginCategoryTags: ["channels", "tools"],
|
||||
}),
|
||||
);
|
||||
expect(tables.packageInspectorWarnings).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("does not backfill catalog metadata onto non-corpus package name collisions", async () => {
|
||||
const { db, tables } = createDb();
|
||||
const userId = (await db.insert("users", {
|
||||
handle: "real-owner",
|
||||
displayName: "Real Owner",
|
||||
role: "user",
|
||||
createdAt: 1,
|
||||
updatedAt: 1,
|
||||
})) as Id<"users">;
|
||||
const publisherId = (await db.insert("publishers", {
|
||||
kind: "user",
|
||||
handle: "real-owner",
|
||||
displayName: "Real Owner",
|
||||
linkedUserId: userId,
|
||||
createdAt: 1,
|
||||
updatedAt: 1,
|
||||
})) as Id<"publishers">;
|
||||
const packageId = (await db.insert("packages", {
|
||||
name: "gmail-agent-plugin",
|
||||
normalizedName: "gmail-agent-plugin",
|
||||
displayName: "Gmail Agent Plugin",
|
||||
summary: "A real package that happens to collide with the corpus fixture.",
|
||||
ownerUserId: userId,
|
||||
ownerPublisherId: publisherId,
|
||||
family: "code-plugin",
|
||||
channel: "community",
|
||||
isOfficial: false,
|
||||
runtimeId: "gmail-agent-plugin",
|
||||
latestReleaseId: undefined,
|
||||
latestVersionSummary: undefined,
|
||||
tags: {},
|
||||
categories: ["models"],
|
||||
topics: ["Original Topic"],
|
||||
compatibility: { pluginApiRange: ">=0.1.0" },
|
||||
verification: {
|
||||
tier: "structural",
|
||||
scope: "artifact-only",
|
||||
summary: "Real package verification.",
|
||||
scanStatus: "clean",
|
||||
},
|
||||
scanStatus: "clean",
|
||||
stats: { downloads: 57, installs: 13, stars: 2, versions: 1 },
|
||||
createdAt: 1,
|
||||
updatedAt: 1,
|
||||
})) as Id<"packages">;
|
||||
const releaseId = (await db.insert("packageReleases", {
|
||||
packageId,
|
||||
version: "0.1.0",
|
||||
changelog: "Real package release.",
|
||||
distTags: ["latest"],
|
||||
files: [],
|
||||
integritySha256: "existing-integrity",
|
||||
compatibility: { pluginApiRange: ">=0.1.0" },
|
||||
verification: {
|
||||
tier: "structural",
|
||||
scope: "artifact-only",
|
||||
summary: "Real package verification.",
|
||||
scanStatus: "clean",
|
||||
},
|
||||
createdBy: userId,
|
||||
publishActor: { kind: "user", userId },
|
||||
createdAt: 1,
|
||||
})) as Id<"packageReleases">;
|
||||
await db.patch(packageId, {
|
||||
latestReleaseId: releaseId,
|
||||
latestVersionSummary: {
|
||||
version: "0.1.0",
|
||||
createdAt: 1,
|
||||
changelog: "Real package release.",
|
||||
compatibility: { pluginApiRange: ">=0.1.0" },
|
||||
verification: {
|
||||
tier: "structural",
|
||||
scope: "artifact-only",
|
||||
summary: "Real package verification.",
|
||||
scanStatus: "clean",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await backfillExistingPublicCorpusBatchRowsHandler(
|
||||
createMutationCtx(db) as never,
|
||||
{
|
||||
rows: [
|
||||
{
|
||||
kind: "plugin",
|
||||
name: "gmail-agent-plugin",
|
||||
displayName: "Gmail Agent Plugin",
|
||||
version: "0.1.0",
|
||||
readme: "# Gmail Agent Plugin\n\nWatches Gmail and notifies an OpenClaw channel.",
|
||||
storageId: "storage:gmail-agent-plugin",
|
||||
categories: ["channels", "tools"],
|
||||
topics: ["Gmail", "Notifications"],
|
||||
dummyOwner: {
|
||||
handle: "corpus-owner",
|
||||
displayName: "Corpus Owner",
|
||||
image: "https://example.invalid/avatar.png",
|
||||
},
|
||||
},
|
||||
],
|
||||
} as never,
|
||||
);
|
||||
|
||||
expect(tables.packages?.[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
categories: ["models"],
|
||||
topics: ["Original Topic"],
|
||||
}),
|
||||
);
|
||||
expect(tables.packageSearchDigest).toBeUndefined();
|
||||
expect(tables.packageDailyStats).toBeUndefined();
|
||||
expect(tables.packageInspectorWarnings).toBeUndefined();
|
||||
});
|
||||
|
||||
it("caps inferred public corpus plugin categories at the catalog limit", async () => {
|
||||
const { db, tables } = createDb();
|
||||
|
||||
await seedPublicCorpusBatchHandler(
|
||||
createMutationCtx(db) as never,
|
||||
{
|
||||
rows: [
|
||||
{
|
||||
kind: "plugin",
|
||||
name: "context-security-openclaw-email-guard-plugin",
|
||||
displayName: "Context Security Email Guard Plugin",
|
||||
version: "0.1.0",
|
||||
readme:
|
||||
"# Context Security Email Guard Plugin\n\nA runtime plugin for Gmail, model providers, memory, context, web search, GitHub tools, gateway operations, and OAuth policy checks.",
|
||||
storageId: "storage:context-security-openclaw-email-guard-plugin",
|
||||
dummyOwner: {
|
||||
handle: "corpus-owner",
|
||||
displayName: "Corpus Owner",
|
||||
image: "https://example.invalid/avatar.png",
|
||||
},
|
||||
},
|
||||
],
|
||||
} as never,
|
||||
);
|
||||
|
||||
const pkg = tables.packages?.find(
|
||||
(candidate) => candidate.name === "context-security-openclaw-email-guard-plugin",
|
||||
);
|
||||
const categories = Array.isArray(pkg?.categories) ? pkg.categories : [];
|
||||
|
||||
expect(categories.length).toBeLessThanOrEqual(3);
|
||||
expect(categories.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("removes public corpus daily activity rows during reset", async () => {
|
||||
const { db, tables } = createDb();
|
||||
await db.insert("globalStats", {
|
||||
key: "default",
|
||||
activeSkillsCount: 0,
|
||||
activePluginsCount: 0,
|
||||
updatedAt: 1,
|
||||
});
|
||||
const rows = [
|
||||
{
|
||||
kind: "skill",
|
||||
@@ -544,6 +881,11 @@ describe("devSeed local fixtures", () => {
|
||||
const firstPackageId = tables.packages?.[0]?._id;
|
||||
const firstSkillDailyRows = tables.skillDailyStats?.length ?? 0;
|
||||
const firstPackageDailyRows = tables.packageDailyStats?.length ?? 0;
|
||||
const firstPackageDigestRows = tables.packageSearchDigest?.length ?? 0;
|
||||
const firstPackageCategoryDigestRows = tables.packagePluginCategorySearchDigest?.length ?? 0;
|
||||
const firstPackageTopicDigestRows = tables.packageTopicSearchDigest?.length ?? 0;
|
||||
const firstPackageInspectorWarningRows = tables.packageInspectorWarnings?.length ?? 0;
|
||||
const firstActivePluginsCount = tables.globalStats?.[0]?.activePluginsCount;
|
||||
|
||||
await seedPublicCorpusBatchHandler(
|
||||
createMutationCtx(db) as never,
|
||||
@@ -556,6 +898,22 @@ describe("devSeed local fixtures", () => {
|
||||
expect(tables.packageDailyStats).toHaveLength(firstPackageDailyRows);
|
||||
expect(tables.skillDailyStats?.some((row) => row.skillId === firstSkillId)).toBe(false);
|
||||
expect(tables.packageDailyStats?.some((row) => row.packageId === firstPackageId)).toBe(false);
|
||||
expect(firstPackageDigestRows).toBeGreaterThan(0);
|
||||
expect(firstPackageCategoryDigestRows).toBeGreaterThan(0);
|
||||
expect(firstPackageTopicDigestRows).toBeGreaterThan(0);
|
||||
expect(firstPackageInspectorWarningRows).toBeGreaterThan(0);
|
||||
expect(tables.packageSearchDigest?.some((row) => row.packageId === firstPackageId)).toBe(false);
|
||||
expect(
|
||||
tables.packagePluginCategorySearchDigest?.some((row) => row.packageId === firstPackageId),
|
||||
).toBe(false);
|
||||
expect(tables.packageTopicSearchDigest?.some((row) => row.packageId === firstPackageId)).toBe(
|
||||
false,
|
||||
);
|
||||
expect(tables.packageInspectorWarnings?.some((row) => row.packageId === firstPackageId)).toBe(
|
||||
false,
|
||||
);
|
||||
expect(firstActivePluginsCount).toBe(1);
|
||||
expect(tables.globalStats?.[0]?.activePluginsCount).toBe(1);
|
||||
});
|
||||
|
||||
it("seeds a GitHub-backed source and skills without creating mirrored versions", async () => {
|
||||
|
||||
+283
-1
@@ -1,6 +1,14 @@
|
||||
import {
|
||||
CATALOG_CATEGORY_LIMIT,
|
||||
PLUGIN_CATEGORY_DEFINITIONS,
|
||||
normalizeCatalogTopic,
|
||||
normalizeCatalogTopics,
|
||||
normalizePluginCategories,
|
||||
resolvePluginCategories,
|
||||
} from "clawhub-schema";
|
||||
import { v } from "convex/values";
|
||||
import { internal } from "./_generated/api";
|
||||
import type { Id } from "./_generated/dataModel";
|
||||
import type { Doc, Id } from "./_generated/dataModel";
|
||||
import type { ActionCtx, MutationCtx } from "./_generated/server";
|
||||
import { internalMutation as rawInternalMutation } from "./_generated/server";
|
||||
import { internalAction, internalMutation } from "./functions";
|
||||
@@ -9,6 +17,11 @@ import { EMBEDDING_DIMENSIONS, generateEmbedding } from "./lib/embeddings";
|
||||
import { deleteGitHubSkillScansForSkill } from "./lib/githubSkillScans";
|
||||
import { toDayKey } from "./lib/leaderboards";
|
||||
import { normalizePackageName } from "./lib/packageRegistry";
|
||||
import {
|
||||
deletePackageSearchDigests,
|
||||
extractPackageDigestFields,
|
||||
upsertPackageSearchDigest,
|
||||
} from "./lib/packageSearchDigest";
|
||||
import { ensurePersonalPublisherForUser } from "./lib/publishers";
|
||||
import {
|
||||
computeRecommendationScore,
|
||||
@@ -159,6 +172,8 @@ const publicCorpusPluginRowValidator = v.object({
|
||||
version: v.string(),
|
||||
readme: v.string(),
|
||||
summary: v.optional(v.string()),
|
||||
categories: v.optional(v.array(v.string())),
|
||||
topics: v.optional(v.array(v.string())),
|
||||
family: v.optional(
|
||||
v.union(v.literal("skill"), v.literal("code-plugin"), v.literal("bundle-plugin")),
|
||||
),
|
||||
@@ -193,6 +208,8 @@ const publicCorpusPreparedPluginRowValidator = v.object({
|
||||
version: v.string(),
|
||||
readme: v.string(),
|
||||
summary: v.optional(v.string()),
|
||||
categories: v.optional(v.array(v.string())),
|
||||
topics: v.optional(v.array(v.string())),
|
||||
family: v.optional(
|
||||
v.union(v.literal("skill"), v.literal("code-plugin"), v.literal("bundle-plugin")),
|
||||
),
|
||||
@@ -866,6 +883,32 @@ export const backfillExistingPublicCorpusBatchRows = internalMutation({
|
||||
missingKeys.push(publicCorpusSeedRowKey(row));
|
||||
continue;
|
||||
}
|
||||
if (!(await packageBelongsToPublicCorpusOwner(ctx, existing, row.dummyOwner))) {
|
||||
skipped.push(`plugin:${row.name}`);
|
||||
continue;
|
||||
}
|
||||
const catalogMetadata = publicCorpusPluginCatalogMetadata(row);
|
||||
await ctx.db.patch(existing._id, {
|
||||
categories: catalogMetadata.categories,
|
||||
topics: catalogMetadata.topics,
|
||||
updatedAt: now,
|
||||
});
|
||||
if (existing.latestReleaseId) {
|
||||
await ensurePublicCorpusPackageValidationWarning(ctx, {
|
||||
packageId: existing._id,
|
||||
releaseId: existing.latestReleaseId,
|
||||
ownerUserId: existing.ownerUserId,
|
||||
ownerPublisherId: existing.ownerPublisherId,
|
||||
packageName: existing.name,
|
||||
normalizedName: existing.normalizedName,
|
||||
version: row.version,
|
||||
createdAt: now,
|
||||
});
|
||||
}
|
||||
const updatedPackage = await ctx.db.get(existing._id);
|
||||
if (updatedPackage) {
|
||||
await upsertPackageSearchDigest(ctx, extractPackageDigestFields(updatedPackage));
|
||||
}
|
||||
await ensurePublicCorpusPackageDailyStats(ctx, {
|
||||
packageId: existing._id,
|
||||
key: row.name,
|
||||
@@ -945,6 +988,21 @@ function publicCorpusSeedRowKey(
|
||||
return row.kind === "skill" ? `skill:${row.slug}` : `plugin:${row.name}`;
|
||||
}
|
||||
|
||||
async function packageBelongsToPublicCorpusOwner(
|
||||
ctx: Pick<MutationCtx, "db">,
|
||||
pkg: Pick<Doc<"packages">, "ownerUserId">,
|
||||
dummyOwner: { handle: string },
|
||||
ownerUserId?: Id<"users">,
|
||||
) {
|
||||
if (ownerUserId) return pkg.ownerUserId === ownerUserId;
|
||||
|
||||
const owners = await ctx.db
|
||||
.query("users")
|
||||
.withIndex("handle", (q) => q.eq("handle", dummyOwner.handle))
|
||||
.collect();
|
||||
return owners.some((owner) => owner._id === pkg.ownerUserId);
|
||||
}
|
||||
|
||||
export const seedPublicCorpusBatchMutation = internalMutation({
|
||||
args: {
|
||||
reset: v.optional(v.boolean()),
|
||||
@@ -1083,6 +1141,32 @@ export const seedPublicCorpusBatchMutation = internalMutation({
|
||||
.withIndex("by_name", (q) => q.eq("normalizedName", normalizedName))
|
||||
.unique();
|
||||
if (existing) {
|
||||
if (!(await packageBelongsToPublicCorpusOwner(ctx, existing, row.dummyOwner, userId))) {
|
||||
skipped.push(`plugin:${row.name}`);
|
||||
continue;
|
||||
}
|
||||
const catalogMetadata = publicCorpusPluginCatalogMetadata(row);
|
||||
await ctx.db.patch(existing._id, {
|
||||
categories: catalogMetadata.categories,
|
||||
topics: catalogMetadata.topics,
|
||||
updatedAt: now,
|
||||
});
|
||||
if (existing.latestReleaseId) {
|
||||
await ensurePublicCorpusPackageValidationWarning(ctx, {
|
||||
packageId: existing._id,
|
||||
releaseId: existing.latestReleaseId,
|
||||
ownerUserId: existing.ownerUserId,
|
||||
ownerPublisherId: existing.ownerPublisherId,
|
||||
packageName: existing.name,
|
||||
normalizedName: existing.normalizedName,
|
||||
version: row.version,
|
||||
createdAt: now,
|
||||
});
|
||||
}
|
||||
const updatedPackage = await ctx.db.get(existing._id);
|
||||
if (updatedPackage) {
|
||||
await upsertPackageSearchDigest(ctx, extractPackageDigestFields(updatedPackage));
|
||||
}
|
||||
await ensurePublicCorpusPackageDailyStats(ctx, {
|
||||
packageId: existing._id,
|
||||
key: row.name,
|
||||
@@ -1097,6 +1181,7 @@ export const seedPublicCorpusBatchMutation = internalMutation({
|
||||
const createdAt = row.createdAt ?? now;
|
||||
const stats = publicCorpusPackageStats(row.name);
|
||||
const compatibility = { pluginApiRange: ">=0.1.0" };
|
||||
const catalogMetadata = publicCorpusPluginCatalogMetadata(row);
|
||||
const verification = {
|
||||
tier: "structural" as const,
|
||||
scope: "artifact-only" as const,
|
||||
@@ -1117,6 +1202,8 @@ export const seedPublicCorpusBatchMutation = internalMutation({
|
||||
latestReleaseId: undefined,
|
||||
latestVersionSummary: undefined,
|
||||
tags: {},
|
||||
categories: catalogMetadata.categories,
|
||||
topics: catalogMetadata.topics,
|
||||
compatibility,
|
||||
verification,
|
||||
scanStatus: "clean",
|
||||
@@ -1158,6 +1245,16 @@ export const seedPublicCorpusBatchMutation = internalMutation({
|
||||
createdAt,
|
||||
softDeletedAt: undefined,
|
||||
});
|
||||
await ensurePublicCorpusPackageValidationWarning(ctx, {
|
||||
packageId,
|
||||
releaseId,
|
||||
ownerUserId: userId,
|
||||
ownerPublisherId: publisherId,
|
||||
packageName: row.name,
|
||||
normalizedName,
|
||||
version: row.version,
|
||||
createdAt: now,
|
||||
});
|
||||
await ctx.db.patch(packageId, {
|
||||
latestReleaseId: releaseId,
|
||||
latestVersionSummary: {
|
||||
@@ -1171,6 +1268,10 @@ export const seedPublicCorpusBatchMutation = internalMutation({
|
||||
stats: { ...stats, versions: 1 },
|
||||
updatedAt: now,
|
||||
});
|
||||
const packageDoc = await ctx.db.get(packageId);
|
||||
if (packageDoc) {
|
||||
await upsertPackageSearchDigest(ctx, extractPackageDigestFields(packageDoc));
|
||||
}
|
||||
await ensurePublicCorpusPackageDailyStats(ctx, {
|
||||
packageId,
|
||||
key: row.name,
|
||||
@@ -1337,6 +1438,176 @@ function publicCorpusPackageStats(name: string) {
|
||||
};
|
||||
}
|
||||
|
||||
const PUBLIC_CORPUS_PLUGIN_FALLBACK_CATEGORIES = PLUGIN_CATEGORY_DEFINITIONS.map(
|
||||
(category) => category.slug,
|
||||
).filter((slug) => slug !== "other");
|
||||
|
||||
const PUBLIC_CORPUS_PLUGIN_CATEGORY_TOPICS: Record<string, string> = {
|
||||
channels: "Messaging",
|
||||
models: "Model Providers",
|
||||
memory: "Memory",
|
||||
context: "Context",
|
||||
voice: "Voice",
|
||||
media: "Media",
|
||||
web: "Web Search",
|
||||
tools: "Automation",
|
||||
runtime: "Runtime",
|
||||
gateway: "Gateway",
|
||||
security: "Security",
|
||||
other: "Utilities",
|
||||
};
|
||||
|
||||
function publicCorpusPluginCatalogMetadata(row: {
|
||||
name: string;
|
||||
displayName: string;
|
||||
summary?: string;
|
||||
readme: string;
|
||||
categories?: string[];
|
||||
topics?: string[];
|
||||
}) {
|
||||
const declaredCategories =
|
||||
row.categories === undefined ? undefined : normalizePluginCategories(row.categories);
|
||||
const categories = resolvePluginCategories({
|
||||
declared: declaredCategories,
|
||||
inferred: declaredCategories === undefined ? inferPublicCorpusPluginCategories(row) : undefined,
|
||||
});
|
||||
const topics =
|
||||
row.topics === undefined
|
||||
? inferPublicCorpusPluginTopics(row, categories)
|
||||
: normalizeCatalogTopics(row.topics);
|
||||
|
||||
return { categories, topics };
|
||||
}
|
||||
|
||||
async function ensurePublicCorpusPackageValidationWarning(
|
||||
ctx: Pick<MutationCtx, "db">,
|
||||
params: {
|
||||
packageId: Id<"packages">;
|
||||
releaseId: Id<"packageReleases">;
|
||||
ownerUserId: Id<"users">;
|
||||
ownerPublisherId?: Id<"publishers">;
|
||||
packageName: string;
|
||||
normalizedName: string;
|
||||
version: string;
|
||||
createdAt: number;
|
||||
},
|
||||
) {
|
||||
const existingWarnings = await ctx.db
|
||||
.query("packageInspectorWarnings")
|
||||
.withIndex("by_release", (q) => q.eq("releaseId", params.releaseId))
|
||||
.collect();
|
||||
if (
|
||||
existingWarnings.some(
|
||||
(warning) =>
|
||||
warning.code === "package-min-host-version-drift" &&
|
||||
warning.inspectorFindingId === `${params.normalizedName}:package-min-host-version-drift`,
|
||||
)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
await ctx.db.insert("packageInspectorWarnings", {
|
||||
packageId: params.packageId,
|
||||
releaseId: params.releaseId,
|
||||
ownerUserId: params.ownerUserId,
|
||||
ownerPublisherId: params.ownerPublisherId,
|
||||
packageName: params.packageName,
|
||||
version: params.version,
|
||||
findingKind: "warning",
|
||||
scanSource: "publish",
|
||||
inspectorVersion: "0.3.15",
|
||||
targetOpenClawVersion: "2026.6.9",
|
||||
code: "package-min-host-version-drift",
|
||||
severity: "P2",
|
||||
level: "warning",
|
||||
issueClass: "upstream-metadata",
|
||||
compatStatus: "warning",
|
||||
message: `${params.packageName}: OpenClaw package minimum host version drifts from build target`,
|
||||
evidence: ["minHostVersion: >=2026.4.25", "buildOpenClawVersion: 2026.6.9"],
|
||||
authorRemediation: {
|
||||
summary:
|
||||
"Set the package minimum host version to the OpenClaw version range the plugin was built and tested against.",
|
||||
docsUrl:
|
||||
"https://docs.openclaw.ai/clawhub/plugin-validation-fixes#package-min-host-version-drift",
|
||||
},
|
||||
fixture: "public-corpus",
|
||||
decision: "seeded-warning",
|
||||
inspectorFindingId: `${params.normalizedName}:package-min-host-version-drift`,
|
||||
createdAt: params.createdAt,
|
||||
});
|
||||
}
|
||||
|
||||
function inferPublicCorpusPluginCategories(row: {
|
||||
name: string;
|
||||
displayName: string;
|
||||
summary?: string;
|
||||
readme: string;
|
||||
}) {
|
||||
const text = [row.name, row.displayName, row.summary, row.readme]
|
||||
.filter(Boolean)
|
||||
.join(" ")
|
||||
.toLocaleLowerCase("en-US");
|
||||
const categories: string[] = [];
|
||||
const add = (category: string) => {
|
||||
if (!categories.includes(category)) categories.push(category);
|
||||
};
|
||||
|
||||
if (/\b(slack|discord|telegram|whatsapp|gmail|email|chat|message|messenger|sms)\b/.test(text)) {
|
||||
add("channels");
|
||||
}
|
||||
if (/\b(model|models|provider|providers|gpt|llm|openai|claude|inference|modelark)\b/.test(text)) {
|
||||
add("models");
|
||||
}
|
||||
if (/\b(memory|recall|embedding|embeddings|vector|session)\b/.test(text)) add("memory");
|
||||
if (/\b(context|knowledge|document|docs|pdf)\b/.test(text)) add("context");
|
||||
if (/\b(voice|speech|tts|transcription|audio)\b/.test(text)) add("voice");
|
||||
if (/\b(image|media|video|youtube|twitter|x-|music|render)\b/.test(text)) add("media");
|
||||
if (/\b(web|browser|search|reddit|fetch|crawl|url|http)\b/.test(text)) add("web");
|
||||
if (/\b(tool|tools|workflow|automation|cli|command|shell|github|actions)\b/.test(text)) {
|
||||
add("tools");
|
||||
}
|
||||
if (/\b(runtime|codex|developer|dev|test|deploy|openclaw)\b/.test(text)) add("runtime");
|
||||
if (/\b(gateway|observability|worker|ops|operator)\b/.test(text)) add("gateway");
|
||||
if (/\b(auth|oauth|security|secret|policy|permission|trust)\b/.test(text)) add("security");
|
||||
|
||||
if (categories.length > 0) return categories.slice(0, CATALOG_CATEGORY_LIMIT);
|
||||
const fallbackIndex =
|
||||
publicCorpusStableNumber(row.name) % PUBLIC_CORPUS_PLUGIN_FALLBACK_CATEGORIES.length;
|
||||
return [PUBLIC_CORPUS_PLUGIN_FALLBACK_CATEGORIES[fallbackIndex] ?? "other"];
|
||||
}
|
||||
|
||||
function inferPublicCorpusPluginTopics(
|
||||
row: { name: string; displayName: string },
|
||||
categories: readonly string[],
|
||||
) {
|
||||
const topics: string[] = [];
|
||||
const seenSlugs = new Set<string>();
|
||||
const add = (candidate: string | undefined) => {
|
||||
if (!candidate) return;
|
||||
const slug = normalizeCatalogTopic(candidate);
|
||||
if (!slug || seenSlugs.has(slug)) return;
|
||||
try {
|
||||
const [topic] = normalizeCatalogTopics([candidate]);
|
||||
if (!topic) return;
|
||||
seenSlugs.add(slug);
|
||||
topics.push(topic);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
add(PUBLIC_CORPUS_PLUGIN_CATEGORY_TOPICS[categories[0] ?? "other"]);
|
||||
for (const rawToken of `${row.displayName} ${row.name}`.split(/[^\p{L}\p{N}]+/u)) {
|
||||
const token = rawToken.trim();
|
||||
if (token.length < 3) continue;
|
||||
if (/^(plugin|openclaw|clawhub|agent|agents|the)$/i.test(token)) continue;
|
||||
add(token.slice(0, 1).toLocaleUpperCase("en-US") + token.slice(1));
|
||||
if (topics.length >= 5) break;
|
||||
}
|
||||
|
||||
return topics.length > 0 ? topics : ["Utilities"];
|
||||
}
|
||||
|
||||
function publicCorpusStableNumber(value: string) {
|
||||
let hash = 0;
|
||||
for (let index = 0; index < value.length; index += 1) {
|
||||
@@ -1580,6 +1851,7 @@ async function deletePackageAndReleases(ctx: MutationCtx, packageId: Id<"package
|
||||
.collect();
|
||||
await deletePackageBadgesForPackage(ctx, packageId);
|
||||
await deletePackageDailyStatsForPackage(ctx, packageId);
|
||||
await deletePackageDerivedSearchData(ctx, packageId);
|
||||
await ctx.db.delete(packageId);
|
||||
for (const release of releases) await ctx.db.delete(release._id);
|
||||
}
|
||||
@@ -1640,6 +1912,16 @@ async function deletePackageDailyStatsForPackage(ctx: MutationCtx, packageId: Id
|
||||
for (const row of rows) await ctx.db.delete(row._id);
|
||||
}
|
||||
|
||||
async function deletePackageDerivedSearchData(ctx: MutationCtx, packageId: Id<"packages">) {
|
||||
await deletePackageSearchDigests(ctx, packageId);
|
||||
|
||||
const inspectorWarnings = await ctx.db
|
||||
.query("packageInspectorWarnings")
|
||||
.withIndex("by_package_created", (q) => q.eq("packageId", packageId))
|
||||
.collect();
|
||||
for (const row of inspectorWarnings) await ctx.db.delete(row._id);
|
||||
}
|
||||
|
||||
async function deleteSeedSkillFixture(ctx: MutationCtx, slug = FLAGGED_SKILL_SLUG) {
|
||||
const existing = await findSeedSkillFixture(ctx, slug);
|
||||
if (!existing) return;
|
||||
|
||||
@@ -60,6 +60,30 @@ describe("public corpus fixture validation", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects invalid plugin categories and topics", () => {
|
||||
const result = validateCorpusRows([
|
||||
{
|
||||
...pluginRow,
|
||||
categories: ["not-a-category"],
|
||||
topics: ["Official"],
|
||||
},
|
||||
]);
|
||||
|
||||
expect(result.ok).toBe(false);
|
||||
expect(result.findings).toContainEqual(
|
||||
expect.objectContaining({
|
||||
reason: "invalid_catalog_metadata",
|
||||
field: "categories",
|
||||
}),
|
||||
);
|
||||
expect(result.findings).toContainEqual(
|
||||
expect.objectContaining({
|
||||
reason: "invalid_catalog_metadata",
|
||||
field: "topics",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects empty content and secret-like text", () => {
|
||||
const result = validateCorpusRows([
|
||||
{ ...skillRow, slug: "empty", skillMd: "" },
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { normalizeCatalogTopics, normalizePluginCategories } from "clawhub-schema";
|
||||
|
||||
export const DEFAULT_PUBLIC_CORPUS_FIXTURE = "fixtures/public-corpus/corpus.jsonl";
|
||||
|
||||
export type PublicCorpusSkillRow = {
|
||||
@@ -17,6 +19,8 @@ export type PublicCorpusPluginRow = {
|
||||
version: string;
|
||||
readme: string;
|
||||
summary?: string;
|
||||
categories?: string[];
|
||||
topics?: string[];
|
||||
family?: "skill" | "code-plugin" | "bundle-plugin";
|
||||
channel?: "official" | "community" | "private";
|
||||
sourceRepoHost?: string | null;
|
||||
@@ -36,6 +40,7 @@ export type CorpusValidationFinding = {
|
||||
| "disallowed_field"
|
||||
| "raw_convex_id"
|
||||
| "duplicate_slug"
|
||||
| "invalid_catalog_metadata"
|
||||
| "local_path"
|
||||
| "secret_like_text";
|
||||
field?: string;
|
||||
@@ -128,6 +133,7 @@ export function validateCorpusRows(rows: PublicCorpusRow[]): CorpusValidationRes
|
||||
requireString(row.version, "version", line, findings);
|
||||
if (!row.readme?.trim())
|
||||
findings.push({ line, reason: "empty_plugin_text", field: "readme" });
|
||||
collectPluginCatalogMetadataFindings(row, line, findings);
|
||||
collectDuplicate("plugin", row.name, line, seenKeys, findings);
|
||||
}
|
||||
});
|
||||
@@ -175,6 +181,38 @@ function requireString(
|
||||
findings.push({ line, reason: "missing_required_field", field });
|
||||
}
|
||||
|
||||
function collectPluginCatalogMetadataFindings(
|
||||
row: PublicCorpusPluginRow,
|
||||
line: number,
|
||||
findings: CorpusValidationFinding[],
|
||||
) {
|
||||
if (row.categories !== undefined) {
|
||||
try {
|
||||
normalizePluginCategories(row.categories);
|
||||
} catch (error) {
|
||||
findings.push({
|
||||
line,
|
||||
reason: "invalid_catalog_metadata",
|
||||
field: "categories",
|
||||
value: error instanceof Error ? error.message : undefined,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (row.topics !== undefined) {
|
||||
try {
|
||||
normalizeCatalogTopics(row.topics);
|
||||
} catch (error) {
|
||||
findings.push({
|
||||
line,
|
||||
reason: "invalid_catalog_metadata",
|
||||
field: "topics",
|
||||
value: error instanceof Error ? error.message : undefined,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function collectDisallowedFieldFindings(
|
||||
value: unknown,
|
||||
line: number,
|
||||
|
||||
Reference in New Issue
Block a user