mirror of
https://github.com/openclaw/clawhub.git
synced 2026-08-14 00:47:57 +00:00
chore: remove catalog classification rollout migrations (#2735)
* chore: remove catalog classification rollout migrations * chore: keep static audit gate green
This commit is contained in:
@@ -307,19 +307,6 @@ export const classifyCatalog: ReturnType<typeof action> = action({
|
||||
},
|
||||
});
|
||||
|
||||
export const applyCatalogClassifications: ReturnType<typeof action> = action({
|
||||
args: {
|
||||
dryRun: v.optional(v.boolean()),
|
||||
minimumConfidence: v.union(v.literal("high"), v.literal("medium")),
|
||||
confirm: v.optional(v.string()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const { user } = await requireUserFromAction(ctx);
|
||||
assertRole(user, ["admin"]);
|
||||
return ctx.runAction(internal.migrations.runCatalogClassificationApply, args);
|
||||
},
|
||||
});
|
||||
|
||||
export type StoredCatalogClassificationInput = ReturnType<
|
||||
typeof import("./lib/catalogClassification").prepareCatalogClassificationResult
|
||||
> & {
|
||||
|
||||
@@ -1,59 +0,0 @@
|
||||
/* @vitest-environment node */
|
||||
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { internal } from "./_generated/api";
|
||||
import { runCatalogClassificationApply, runCatalogTaxonomyPrerequisites } from "./migrations";
|
||||
|
||||
type WrappedHandler = {
|
||||
_handler: (ctx: unknown, args: { dryRun?: boolean }) => Promise<unknown>;
|
||||
};
|
||||
|
||||
type ClassificationApplyWrappedHandler = {
|
||||
_handler: (
|
||||
ctx: unknown,
|
||||
args: { dryRun?: boolean; minimumConfidence: "high" | "medium"; confirm?: string },
|
||||
) => Promise<unknown>;
|
||||
};
|
||||
|
||||
describe("catalog taxonomy migrations", () => {
|
||||
it("dry-runs both tracked digest migrations", async () => {
|
||||
const runMutation = vi.fn().mockResolvedValue({});
|
||||
const handler = (runCatalogTaxonomyPrerequisites as unknown as WrappedHandler)._handler;
|
||||
|
||||
await handler({ runMutation }, { dryRun: true });
|
||||
|
||||
expect(runMutation).toHaveBeenNthCalledWith(1, internal.migrations.run, {
|
||||
fn: "migrations:rebuildCatalogTaxonomyPackageDigests",
|
||||
dryRun: true,
|
||||
reset: true,
|
||||
});
|
||||
expect(runMutation).toHaveBeenNthCalledWith(2, internal.migrations.run, {
|
||||
fn: "migrations:rebuildCatalogTaxonomySkillDigests",
|
||||
dryRun: true,
|
||||
reset: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("dry-runs the selected classification apply migration", async () => {
|
||||
const runMutation = vi.fn().mockResolvedValue({});
|
||||
const handler = (runCatalogClassificationApply as unknown as ClassificationApplyWrappedHandler)
|
||||
._handler;
|
||||
|
||||
await handler({ runMutation }, { minimumConfidence: "medium" });
|
||||
|
||||
expect(runMutation).toHaveBeenCalledWith(internal.migrations.run, {
|
||||
fn: "migrations:applyMediumConfidenceCatalogClassifications",
|
||||
dryRun: true,
|
||||
reset: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("requires an explicit confidence-specific confirmation before applying", async () => {
|
||||
const handler = (runCatalogClassificationApply as unknown as ClassificationApplyWrappedHandler)
|
||||
._handler;
|
||||
|
||||
await expect(
|
||||
handler({ runMutation: vi.fn() }, { dryRun: false, minimumConfidence: "high" }),
|
||||
).rejects.toThrow('Pass confirm="apply-high-confidence-catalog-classifications" to apply.');
|
||||
});
|
||||
});
|
||||
+2
-249
@@ -1,257 +1,10 @@
|
||||
import { Migrations, runToCompletion } from "@convex-dev/migrations";
|
||||
import {
|
||||
normalizeInferredCatalogTopics,
|
||||
normalizePluginCategories,
|
||||
normalizeSkillCategories,
|
||||
} from "clawhub-schema";
|
||||
import { ConvexError, v } from "convex/values";
|
||||
import { components, internal } from "./_generated/api";
|
||||
import type { Doc } from "./_generated/dataModel";
|
||||
import type { MutationCtx } from "./_generated/server";
|
||||
import { internalAction } from "./_generated/server";
|
||||
import { syncPackageSearchDigestForPackageId } from "./functions";
|
||||
import {
|
||||
selectCatalogInference,
|
||||
type CatalogClassificationConfidence,
|
||||
} from "./lib/catalogClassification";
|
||||
import { syncSkillSearchDigestForSkill } from "./lib/skillSearchDigest";
|
||||
import { Migrations } from "@convex-dev/migrations";
|
||||
import { components } from "./_generated/api";
|
||||
import schema from "./schema";
|
||||
|
||||
const APPLY_HIGH_CONFIDENCE_CONFIRM = "apply-high-confidence-catalog-classifications";
|
||||
const APPLY_MEDIUM_CONFIDENCE_CONFIRM = "apply-medium-confidence-catalog-classifications";
|
||||
|
||||
export const migrations = new Migrations(components.migrations, {
|
||||
schema,
|
||||
defaultBatchSize: 25,
|
||||
});
|
||||
|
||||
export const rebuildCatalogTaxonomyPackageDigests = migrations.define({
|
||||
table: "packages",
|
||||
migrateOne: async (ctx, pkg) => {
|
||||
await syncPackageSearchDigestForPackageId(ctx, pkg._id);
|
||||
},
|
||||
});
|
||||
|
||||
export const rebuildCatalogTaxonomySkillDigests = migrations.define({
|
||||
table: "skills",
|
||||
migrateOne: async (ctx, skill) => {
|
||||
await syncSkillSearchDigestForSkill(ctx, skill);
|
||||
},
|
||||
});
|
||||
|
||||
async function applyCatalogClassification(
|
||||
ctx: Pick<MutationCtx, "db">,
|
||||
result: Doc<"catalogClassificationResults">,
|
||||
minimumConfidence: CatalogClassificationConfidence,
|
||||
) {
|
||||
const now = Date.now();
|
||||
if (result.targetKind === "skill" && result.skillId) {
|
||||
const skill = await ctx.db.get(result.skillId);
|
||||
if (!skill) {
|
||||
await ctx.db.patch(result._id, {
|
||||
applyStatus: "stale",
|
||||
error: "Skill no longer exists",
|
||||
appliedAt: undefined,
|
||||
});
|
||||
return;
|
||||
}
|
||||
const selection = selectCatalogInference({
|
||||
currentSourceId: skill.latestVersionId,
|
||||
resultSourceId: result.skillVersionId,
|
||||
authorCategories: skill.categories,
|
||||
authorTopics: skill.topics,
|
||||
result: {
|
||||
categories: result.categories,
|
||||
topics: result.topics,
|
||||
confidence: result.categoryConfidence,
|
||||
topicConfidence: result.topicConfidence,
|
||||
},
|
||||
minimumConfidence,
|
||||
});
|
||||
if (selection.status !== "applied") {
|
||||
await ctx.db.patch(result._id, {
|
||||
applyStatus: selection.status,
|
||||
error: undefined,
|
||||
appliedAt: undefined,
|
||||
});
|
||||
return;
|
||||
}
|
||||
const patch = {
|
||||
inferredCategories: selection.categories
|
||||
? normalizeSkillCategories(selection.categories)
|
||||
: undefined,
|
||||
inferredTopics: selection.topics
|
||||
? normalizeInferredCatalogTopics(selection.topics)
|
||||
: undefined,
|
||||
inferredFromVersionId: result.skillVersionId,
|
||||
inferredCategoryConfidence: selection.categories ? result.categoryConfidence : undefined,
|
||||
inferredTopicConfidence: selection.topics ? result.topicConfidence : undefined,
|
||||
inferredClassifierVersion: result.classifierVersion,
|
||||
inferredTopicClassifierVersion: result.topicClassifierVersion,
|
||||
inferredInputHash: result.inputHash,
|
||||
inferredTopicInputHash: result.topicInputHash,
|
||||
inferredAt: now,
|
||||
};
|
||||
await ctx.db.patch(skill._id, patch);
|
||||
await syncSkillSearchDigestForSkill(ctx, { ...skill, ...patch });
|
||||
await ctx.db.patch(result._id, {
|
||||
applyStatus: "applied",
|
||||
error: undefined,
|
||||
appliedAt: now,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (result.targetKind === "plugin" && result.packageId) {
|
||||
const pkg = await ctx.db.get(result.packageId);
|
||||
if (!pkg) {
|
||||
await ctx.db.patch(result._id, {
|
||||
applyStatus: "stale",
|
||||
error: "Package no longer exists",
|
||||
appliedAt: undefined,
|
||||
});
|
||||
return;
|
||||
}
|
||||
const selection = selectCatalogInference({
|
||||
currentSourceId: pkg.latestReleaseId,
|
||||
resultSourceId: result.packageReleaseId,
|
||||
authorCategories: pkg.categories,
|
||||
authorTopics: pkg.topics,
|
||||
result: {
|
||||
categories: result.categories,
|
||||
topics: result.topics,
|
||||
confidence: result.categoryConfidence,
|
||||
topicConfidence: result.topicConfidence,
|
||||
},
|
||||
minimumConfidence,
|
||||
});
|
||||
if (selection.status !== "applied") {
|
||||
await ctx.db.patch(result._id, {
|
||||
applyStatus: selection.status,
|
||||
error: undefined,
|
||||
appliedAt: undefined,
|
||||
});
|
||||
return;
|
||||
}
|
||||
await ctx.db.patch(pkg._id, {
|
||||
inferredCategories: selection.categories
|
||||
? normalizePluginCategories(selection.categories)
|
||||
: undefined,
|
||||
inferredTopics: selection.topics
|
||||
? normalizeInferredCatalogTopics(selection.topics)
|
||||
: undefined,
|
||||
inferredFromReleaseId: result.packageReleaseId,
|
||||
inferredCategoryConfidence: selection.categories ? result.categoryConfidence : undefined,
|
||||
inferredTopicConfidence: selection.topics ? result.topicConfidence : undefined,
|
||||
inferredClassifierVersion: result.classifierVersion,
|
||||
inferredTopicClassifierVersion: result.topicClassifierVersion,
|
||||
inferredInputHash: result.inputHash,
|
||||
inferredTopicInputHash: result.topicInputHash,
|
||||
inferredAt: now,
|
||||
});
|
||||
await syncPackageSearchDigestForPackageId(ctx, pkg._id);
|
||||
await ctx.db.patch(result._id, {
|
||||
applyStatus: "applied",
|
||||
error: undefined,
|
||||
appliedAt: now,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
await ctx.db.patch(result._id, {
|
||||
applyStatus: "error",
|
||||
error: "Classification target is inconsistent",
|
||||
appliedAt: undefined,
|
||||
});
|
||||
}
|
||||
|
||||
export const applyHighConfidenceCatalogClassifications = migrations.define({
|
||||
table: "catalogClassificationResults",
|
||||
batchSize: 10,
|
||||
migrateOne: (ctx, result) => applyCatalogClassification(ctx, result, "high"),
|
||||
});
|
||||
|
||||
export const applyMediumConfidenceCatalogClassifications = migrations.define({
|
||||
table: "catalogClassificationResults",
|
||||
batchSize: 10,
|
||||
migrateOne: (ctx, result) => applyCatalogClassification(ctx, result, "medium"),
|
||||
});
|
||||
|
||||
export const run = migrations.runner();
|
||||
|
||||
export const runCatalogClassificationApply = internalAction({
|
||||
args: {
|
||||
dryRun: v.optional(v.boolean()),
|
||||
minimumConfidence: v.union(v.literal("high"), v.literal("medium")),
|
||||
confirm: v.optional(v.string()),
|
||||
},
|
||||
returns: v.object({
|
||||
ok: v.literal(true),
|
||||
dryRun: v.boolean(),
|
||||
minimumConfidence: v.union(v.literal("high"), v.literal("medium")),
|
||||
confirmRequired: v.optional(v.string()),
|
||||
}),
|
||||
handler: async (ctx, args) => {
|
||||
const dryRun = args.dryRun !== false;
|
||||
const confirmRequired =
|
||||
args.minimumConfidence === "high"
|
||||
? APPLY_HIGH_CONFIDENCE_CONFIRM
|
||||
: APPLY_MEDIUM_CONFIDENCE_CONFIRM;
|
||||
if (!dryRun && args.confirm !== confirmRequired) {
|
||||
throw new ConvexError(`Pass confirm="${confirmRequired}" to apply.`);
|
||||
}
|
||||
const migration =
|
||||
args.minimumConfidence === "high"
|
||||
? internal.migrations.applyHighConfidenceCatalogClassifications
|
||||
: internal.migrations.applyMediumConfidenceCatalogClassifications;
|
||||
if (dryRun) {
|
||||
await ctx.runMutation(internal.migrations.run, {
|
||||
fn:
|
||||
args.minimumConfidence === "high"
|
||||
? "migrations:applyHighConfidenceCatalogClassifications"
|
||||
: "migrations:applyMediumConfidenceCatalogClassifications",
|
||||
dryRun: true,
|
||||
reset: true,
|
||||
});
|
||||
} else {
|
||||
await runToCompletion(ctx, components.migrations, migration);
|
||||
}
|
||||
return {
|
||||
ok: true as const,
|
||||
dryRun,
|
||||
minimumConfidence: args.minimumConfidence,
|
||||
confirmRequired: dryRun ? confirmRequired : undefined,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
export const runCatalogTaxonomyPrerequisites = internalAction({
|
||||
args: { dryRun: v.optional(v.boolean()) },
|
||||
returns: v.null(),
|
||||
handler: async (ctx, args): Promise<null> => {
|
||||
if (args.dryRun) {
|
||||
for (const fn of [
|
||||
"migrations:rebuildCatalogTaxonomyPackageDigests",
|
||||
"migrations:rebuildCatalogTaxonomySkillDigests",
|
||||
]) {
|
||||
await ctx.runMutation(internal.migrations.run, {
|
||||
fn,
|
||||
dryRun: true,
|
||||
reset: true,
|
||||
});
|
||||
}
|
||||
return null;
|
||||
}
|
||||
await runToCompletion(
|
||||
ctx,
|
||||
components.migrations,
|
||||
internal.migrations.rebuildCatalogTaxonomyPackageDigests,
|
||||
);
|
||||
await runToCompletion(
|
||||
ctx,
|
||||
components.migrations,
|
||||
internal.migrations.rebuildCatalogTaxonomySkillDigests,
|
||||
);
|
||||
return null;
|
||||
},
|
||||
});
|
||||
|
||||
@@ -22,9 +22,8 @@
|
||||
- Capability tags are not taxonomy inputs.
|
||||
- Existing items without valid stored categories remain in `other` until an author or operator
|
||||
explicitly accepts generated or manually selected categories.
|
||||
- Operators run the tracked package and skill catalog digest migrations after backend deployment
|
||||
and verify completion before considering the taxonomy rollout complete. The deploy workflow does
|
||||
not invoke them automatically.
|
||||
- The one-time package and skill catalog digest migrations completed during the production rollout.
|
||||
No catalog taxonomy migration remains in the backend deploy checklist.
|
||||
|
||||
## Topics
|
||||
|
||||
@@ -67,19 +66,5 @@ Corpus classification is a separate operator-run phase from digest projection:
|
||||
- The preview runner is cursor-batched and resumable. It uses an action instead of
|
||||
`@convex-dev/migrations` because it must read immutable storage blobs; the source-changing apply
|
||||
phase uses the migrations component.
|
||||
- Apply defaults to a dry run and requires a confidence-specific confirmation string. High
|
||||
confidence is the default production lane. Medium confidence requires an explicit operator
|
||||
decision after reviewing the generated corpus report.
|
||||
|
||||
Operator sequence:
|
||||
|
||||
```bash
|
||||
bunx convex run catalogClassificationNode:classifyCatalogInternal \
|
||||
'{"targetKind":"skill","maxBatches":20,"continueOnIncomplete":true}' --prod
|
||||
bunx convex run catalogClassificationNode:classifyCatalogInternal \
|
||||
'{"targetKind":"plugin","maxBatches":20,"continueOnIncomplete":true}' --prod
|
||||
bunx convex run migrations:runCatalogClassificationApply \
|
||||
'{"minimumConfidence":"high","dryRun":true}' --prod
|
||||
bunx convex run migrations:runCatalogClassificationApply \
|
||||
'{"minimumConfidence":"high","dryRun":false,"confirm":"apply-high-confidence-catalog-classifications"}' --prod
|
||||
```
|
||||
- High- and medium-confidence rollout apply was a one-time production migration. The temporary
|
||||
apply migrations and operator wrappers were removed after the verified rollout completed.
|
||||
|
||||
+2
-9
@@ -32,15 +32,8 @@ Production deploy notes:
|
||||
|
||||
- `deploy.yml` is manual-only (`workflow_dispatch`). Merging to `main` does not deploy.
|
||||
- The workflow must be started from `main`.
|
||||
- Backend deploys do not run the tracked catalog-taxonomy digest migrations automatically. For the
|
||||
taxonomy rollout, an operator must run
|
||||
`bunx convex run migrations:runCatalogTaxonomyPrerequisites --prod` after Convex deploy and verify
|
||||
completion before considering the deployment complete. The migration is idempotent, resumable,
|
||||
and a no-op after it completes.
|
||||
- Catalog classification/backfill is also operator-run. Generate skill and plugin preview rows with
|
||||
`catalogClassificationNode:classifyCatalogInternal`, review the resulting confidence lanes, then
|
||||
dry-run and explicitly confirm `migrations:runCatalogClassificationApply`. See
|
||||
`specs/catalog-taxonomy.md` for the exact commands and confidence-specific confirmation strings.
|
||||
- The catalog-taxonomy digest and high-/medium-confidence classification rollout migrations were
|
||||
one-time production operations and are no longer part of the deploy checklist.
|
||||
- Deploy targets:
|
||||
- `full`: deploy Convex, verify contract, wait for the matching Vercel production deploy, then run smoke tests
|
||||
- `backend`: deploy Convex, verify contract, then run smoke tests against current production
|
||||
|
||||
Reference in New Issue
Block a user