diff --git a/convex/lib/searchText.test.ts b/convex/lib/searchText.test.ts index de8fdc9c..799f1831 100644 --- a/convex/lib/searchText.test.ts +++ b/convex/lib/searchText.test.ts @@ -94,8 +94,39 @@ describe("searchText", () => { }); it("handles Japanese text", () => { - const tokens = tokenize("こんにちは世界"); - expect(tokens.length).toBeGreaterThan(0); + expect(tokenize("こんにちは世界")).toEqual(["こんにちは", "世界"]); + }); + + it("keeps katakana words that contain a prolonged sound mark intact", () => { + expect(tokenize("データベース")).toEqual(["データベース"]); + expect(tokenize("データベース管理")).toEqual(["データベース", "管理"]); + expect(tokenize("ユーザーインターフェース")).toEqual(["ユーザー", "インターフェース"]); + }); + + it("keeps the iteration mark attached to the character it repeats", () => { + expect(tokenize("人々")).toEqual(["人々"]); + expect(tokenize("時々")).toEqual(["時々"]); + }); + + it("keeps the marks attached when Intl.Segmenter is unavailable", () => { + // segmentCJKByChar is the no-Segmenter fallback. Emitting ー or 々 on their own + // would leave one-character tokens that exploratory matching discards. + expect(__test.segmentCJKByChar("データベース")).toEqual(["デー", "タ", "ベー", "ス"]); + expect(__test.segmentCJKByChar("人々")).toEqual(["人々"]); + expect(__test.segmentCJKByChar("時々の記録")).toEqual(["時々", "の", "記", "録"]); + }); + + it("matches Japanese query tokens against Japanese skill names", () => { + const queryTokens = tokenize("データベース"); + expect(matchesExactTokens(queryTokens, ["データベース管理ツール"])).toBe(true); + }); + + it("lets katakana queries reach the exploratory match tiers", () => { + // Exploratory tiers require every query token to clear a three-character floor. + const queryTokens = tokenize("データベース"); + expect(matchesExploratoryTokenPrefixes(queryTokens, ["データベース管理ツール"], 3)).toBe( + true, + ); }); it("handles Korean text", () => { diff --git a/convex/lib/searchText.ts b/convex/lib/searchText.ts index 1dbb35f5..0ebfa6fd 100644 --- a/convex/lib/searchText.ts +++ b/convex/lib/searchText.ts @@ -1,4 +1,10 @@ -const CJK_RE = /[\u4e00-\u9fff\u3400-\u4dbf\u3041-\u3096\u30a1-\u30fa\uac00-\ud7af]/; +// U+30FC (ー) and U+3005 (々) extend the word they follow, so the pre-split in tokenize() +// must keep them with that word instead of treating them as separators. +const CJK_RE = /[\u4e00-\u9fff\u3400-\u4dbf\u3041-\u3096\u30a1-\u30fa\u30fc\u3005\uac00-\ud7af]/; + +// The same two marks the class above admits: they extend the preceding character rather +// than standing on their own, so the per-character fallback must not emit them alone. +const CJK_EXTENDER_RE = /[\u30fc\u3005]/; const hasSegmenter = typeof Intl !== "undefined" && "Segmenter" in Intl; @@ -34,9 +40,12 @@ function getKoSegmenter(): Intl.Segmenter { function segmentCJKByChar(text: string): string[] { const tokens: string[] = []; for (const ch of text) { - if (CJK_RE.test(ch)) { - tokens.push(ch); + if (!CJK_RE.test(ch)) continue; + if (CJK_EXTENDER_RE.test(ch) && tokens.length > 0) { + tokens[tokens.length - 1] += ch; + continue; } + tokens.push(ch); } return tokens; } @@ -116,7 +125,7 @@ export function tokenize(value: string): string[] { const tokens: string[] = []; const parts = normalized.split( - /([^\u4e00-\u9fff\u3400-\u4dbf\u3041-\u3096\u30a1-\u30fa\uac00-\ud7af]+)/g, + /([^\u4e00-\u9fff\u3400-\u4dbf\u3041-\u3096\u30a1-\u30fa\u30fc\u3005\uac00-\ud7af]+)/g, ); for (const part of parts) { diff --git a/convex/lib/skillSearchDigest.ts b/convex/lib/skillSearchDigest.ts index eba87c0a..6ea26952 100644 --- a/convex/lib/skillSearchDigest.ts +++ b/convex/lib/skillSearchDigest.ts @@ -146,6 +146,12 @@ export function getFirstSearchToken(value: string) { return tokenize(value)[0]; } +// The skills.sh mirror persists the same tokenizer-derived first tokens, but its columns +// are required, so it falls back to the normalized text when the tokenizer yields nothing. +export function getMirrorFirstSearchToken(value: string) { + return getFirstSearchToken(value) ?? normalizeSkillSearchText(value); +} + /** * Map a digest row to the HydratableSkill shape expected by toPublicSkill / * isPublicSkillDoc / isSkillSuspicious. Fully type-checked: if diff --git a/convex/maintenance.test.ts b/convex/maintenance.test.ts index 2a7323c6..252f372f 100644 --- a/convex/maintenance.test.ts +++ b/convex/maintenance.test.ts @@ -33,6 +33,12 @@ vi.mock("./_generated/api", () => ({ backfillSkillSearchDigestModerationVerdictsInternal: Symbol( "backfillSkillSearchDigestModerationVerdictsInternal", ), + backfillSkillSearchDigestFirstTokensInternal: Symbol( + "backfillSkillSearchDigestFirstTokensInternal", + ), + backfillSkillsShMirrorDigestFirstTokensInternal: Symbol( + "backfillSkillsShMirrorDigestFirstTokensInternal", + ), getEmptySkillCleanupPageInternal: Symbol("getEmptySkillCleanupPageInternal"), applyEmptySkillCleanupInternal: Symbol("applyEmptySkillCleanupInternal"), nominateUserForEmptySkillSpamInternal: Symbol("nominateUserForEmptySkillSpamInternal"), @@ -79,6 +85,8 @@ vi.mock("./lib/skillSummary", () => ({ const { backfillLatestVersionSummaryInternal, backfillSkillSearchDigestModerationVerdictsInternal, + backfillSkillSearchDigestFirstTokensInternal, + backfillSkillsShMirrorDigestFirstTokensInternal, backfillPublisherStatsInternalHandler, backfillSkillFingerprintsInternalHandler, backfillSkillSummariesInternalHandler, @@ -2052,3 +2060,419 @@ describe("maintenance empty skill nominations", () => { ]); }); }); + +const SKILL_SEARCH_DIGEST_FIRST_TOKEN_BACKFILL_CONFIRM = + "backfill-skill-search-digest-first-tokens"; +const SKILLS_SH_MIRROR_DIGEST_FIRST_TOKEN_BACKFILL_CONFIRM = + "backfill-skills-sh-mirror-digest-first-tokens"; + +describe("backfillSkillSearchDigestFirstTokensInternal", () => { + it("repairs digest rows whose stored first tokens predate the tokenizer", async () => { + const paginate = vi.fn().mockResolvedValue({ + page: [ + { + _id: "skillSearchDigest:stale", + skillId: "skills:stale", + normalizedSlugFirstToken: "database", + normalizedDisplayNameFirstToken: "デ", + }, + { + _id: "skillSearchDigest:fresh", + skillId: "skills:fresh", + normalizedSlugFirstToken: "deploy", + normalizedDisplayNameFirstToken: "deploy", + }, + { + _id: "skillSearchDigest:orphan", + skillId: "skills:missing", + normalizedSlugFirstToken: "gone", + normalizedDisplayNameFirstToken: "gone", + }, + ], + continueCursor: "next-page", + isDone: false, + }); + const query = vi.fn().mockReturnValue({ paginate }); + const get = vi + .fn() + .mockResolvedValueOnce({ + _id: "skills:stale", + slug: "database", + displayName: "データベース管理", + }) + .mockResolvedValueOnce({ + _id: "skills:fresh", + slug: "deploy", + displayName: "Deploy helper", + }) + .mockResolvedValueOnce(null); + const patch = vi.fn().mockResolvedValue(undefined); + const runAfter = vi.fn().mockResolvedValue(undefined); + + const result = await ( + backfillSkillSearchDigestFirstTokensInternal as unknown as { _handler: Function } + )._handler( + { + db: { query, get, patch, normalizeId: vi.fn() }, + scheduler: { runAfter }, + } as never, + { + cursor: "start", + batchSize: 25, + dryRun: false, + confirm: SKILL_SEARCH_DIGEST_FIRST_TOKEN_BACKFILL_CONFIRM, + }, + ); + + expect(result).toEqual({ + scanned: 3, + patched: 1, + missingSkills: 1, + cursor: "next-page", + isDone: false, + dryRun: false, + confirmRequired: undefined, + }); + expect(query).toHaveBeenCalledWith("skillSearchDigest"); + expect(paginate).toHaveBeenCalledWith({ cursor: "start", numItems: 25 }); + expect(patch).toHaveBeenCalledTimes(1); + expect(patch).toHaveBeenCalledWith("skillSearchDigest:stale", { + normalizedSlugFirstToken: "database", + normalizedDisplayNameFirstToken: "データベース", + }); + // The scheduled page has to carry the token, otherwise the run stalls on its own guard. + expect(runAfter).toHaveBeenCalledWith( + 500, + internal.maintenance.backfillSkillSearchDigestFirstTokensInternal, + { + cursor: "next-page", + batchSize: 25, + delayMs: undefined, + dryRun: false, + confirm: SKILL_SEARCH_DIGEST_FIRST_TOKEN_BACKFILL_CONFIRM, + }, + ); + }); + + it("previews without writing or scheduling when arguments are omitted", async () => { + const paginate = vi.fn().mockResolvedValue({ + page: [ + { + _id: "skillSearchDigest:stale", + skillId: "skills:stale", + normalizedSlugFirstToken: "database", + normalizedDisplayNameFirstToken: "デ", + }, + ], + continueCursor: "next-page", + isDone: false, + }); + const query = vi.fn().mockReturnValue({ paginate }); + const get = vi.fn().mockResolvedValue({ + _id: "skills:stale", + slug: "database", + displayName: "データベース管理", + }); + const patch = vi.fn().mockResolvedValue(undefined); + const runAfter = vi.fn().mockResolvedValue(undefined); + + const result = await ( + backfillSkillSearchDigestFirstTokensInternal as unknown as { _handler: Function } + )._handler( + { + db: { query, get, patch, normalizeId: vi.fn() }, + scheduler: { runAfter }, + } as never, + {}, + ); + + expect(result.dryRun).toBe(true); + expect(result.patched).toBe(1); + expect(result.confirmRequired).toBe(SKILL_SEARCH_DIGEST_FIRST_TOKEN_BACKFILL_CONFIRM); + expect(patch).not.toHaveBeenCalled(); + expect(runAfter).not.toHaveBeenCalled(); + }); + + it("rejects an apply that omits the confirmation token", async () => { + const paginate = vi.fn(); + const query = vi.fn().mockReturnValue({ paginate }); + const patch = vi.fn(); + const runAfter = vi.fn(); + const ctx = { + db: { query, get: vi.fn(), patch, normalizeId: vi.fn() }, + scheduler: { runAfter }, + } as never; + const handler = ( + backfillSkillSearchDigestFirstTokensInternal as unknown as { _handler: Function } + )._handler; + + await expect(handler(ctx, { dryRun: false })).rejects.toThrow( + `Pass confirm="${SKILL_SEARCH_DIGEST_FIRST_TOKEN_BACKFILL_CONFIRM}" to apply.`, + ); + await expect(handler(ctx, { dryRun: false, confirm: "wrong-token" })).rejects.toThrow( + `Pass confirm="${SKILL_SEARCH_DIGEST_FIRST_TOKEN_BACKFILL_CONFIRM}" to apply.`, + ); + expect(paginate).not.toHaveBeenCalled(); + expect(patch).not.toHaveBeenCalled(); + expect(runAfter).not.toHaveBeenCalled(); + }); + + it("spaces the next batch by the requested delay and clamps it", async () => { + const paginate = vi.fn().mockResolvedValue({ + page: [ + { + _id: "skillSearchDigest:stale", + skillId: "skills:stale", + normalizedSlugFirstToken: "database", + normalizedDisplayNameFirstToken: "デ", + }, + ], + continueCursor: "next-page", + isDone: false, + }); + const query = vi.fn().mockReturnValue({ paginate }); + const get = vi.fn().mockResolvedValue({ + _id: "skills:stale", + slug: "database", + displayName: "データベース管理", + }); + const patch = vi.fn().mockResolvedValue(undefined); + const runAfter = vi.fn().mockResolvedValue(undefined); + const handler = ( + backfillSkillSearchDigestFirstTokensInternal as unknown as { _handler: Function } + )._handler; + const ctx = { + db: { query, get, patch, normalizeId: vi.fn() }, + scheduler: { runAfter }, + } as never; + + await handler(ctx, { + delayMs: 2_000, + dryRun: false, + confirm: SKILL_SEARCH_DIGEST_FIRST_TOKEN_BACKFILL_CONFIRM, + }); + expect(runAfter).toHaveBeenLastCalledWith( + 2_000, + internal.maintenance.backfillSkillSearchDigestFirstTokensInternal, + { + cursor: "next-page", + batchSize: undefined, + delayMs: 2_000, + dryRun: false, + confirm: SKILL_SEARCH_DIGEST_FIRST_TOKEN_BACKFILL_CONFIRM, + }, + ); + + await handler(ctx, { + delayMs: 600_000, + dryRun: false, + confirm: SKILL_SEARCH_DIGEST_FIRST_TOKEN_BACKFILL_CONFIRM, + }); + expect(runAfter).toHaveBeenLastCalledWith( + 60_000, + internal.maintenance.backfillSkillSearchDigestFirstTokensInternal, + { + cursor: "next-page", + batchSize: undefined, + delayMs: 600_000, + dryRun: false, + confirm: SKILL_SEARCH_DIGEST_FIRST_TOKEN_BACKFILL_CONFIRM, + }, + ); + }); +}); + +describe("backfillSkillsShMirrorDigestFirstTokensInternal", () => { + const mirrorPage = () => [ + { + _id: "skillsShMirrorDigests:stale", + slug: "database", + displayName: "データベース管理", + normalizedSlugFirstToken: "database", + normalizedDisplayNameFirstToken: "デ", + }, + { + _id: "skillsShMirrorDigests:fresh", + slug: "deploy", + displayName: "Deploy helper", + normalizedSlugFirstToken: "deploy", + normalizedDisplayNameFirstToken: "deploy", + }, + ]; + + it("repairs mirrored rows whose stored first tokens predate the tokenizer", async () => { + const paginate = vi.fn().mockResolvedValue({ + page: mirrorPage(), + continueCursor: "next-page", + isDone: false, + }); + const query = vi.fn().mockReturnValue({ paginate }); + const patch = vi.fn().mockResolvedValue(undefined); + const runAfter = vi.fn().mockResolvedValue(undefined); + + const result = await ( + backfillSkillsShMirrorDigestFirstTokensInternal as unknown as { _handler: Function } + )._handler( + { + db: { query, get: vi.fn(), patch, normalizeId: vi.fn() }, + scheduler: { runAfter }, + } as never, + { + cursor: "start", + batchSize: 25, + dryRun: false, + confirm: SKILLS_SH_MIRROR_DIGEST_FIRST_TOKEN_BACKFILL_CONFIRM, + }, + ); + + expect(result).toEqual({ + scanned: 2, + patched: 1, + cursor: "next-page", + isDone: false, + dryRun: false, + confirmRequired: undefined, + }); + expect(query).toHaveBeenCalledWith("skillsShMirrorDigests"); + expect(paginate).toHaveBeenCalledWith({ cursor: "start", numItems: 25 }); + expect(patch).toHaveBeenCalledTimes(1); + expect(patch).toHaveBeenCalledWith("skillsShMirrorDigests:stale", { + normalizedSlugFirstToken: "database", + normalizedDisplayNameFirstToken: "データベース", + }); + expect(runAfter).toHaveBeenCalledWith( + 500, + internal.maintenance.backfillSkillsShMirrorDigestFirstTokensInternal, + { + cursor: "next-page", + batchSize: 25, + delayMs: undefined, + dryRun: false, + confirm: SKILLS_SH_MIRROR_DIGEST_FIRST_TOKEN_BACKFILL_CONFIRM, + }, + ); + }); + + it("previews without writing or scheduling when arguments are omitted", async () => { + const paginate = vi.fn().mockResolvedValue({ + page: mirrorPage(), + continueCursor: "next-page", + isDone: false, + }); + const query = vi.fn().mockReturnValue({ paginate }); + const patch = vi.fn().mockResolvedValue(undefined); + const runAfter = vi.fn().mockResolvedValue(undefined); + + const result = await ( + backfillSkillsShMirrorDigestFirstTokensInternal as unknown as { _handler: Function } + )._handler( + { + db: { query, get: vi.fn(), patch, normalizeId: vi.fn() }, + scheduler: { runAfter }, + } as never, + {}, + ); + + expect(result.dryRun).toBe(true); + expect(result.patched).toBe(1); + expect(result.confirmRequired).toBe(SKILLS_SH_MIRROR_DIGEST_FIRST_TOKEN_BACKFILL_CONFIRM); + expect(patch).not.toHaveBeenCalled(); + expect(runAfter).not.toHaveBeenCalled(); + }); + + it("rejects an apply that omits the confirmation token", async () => { + const paginate = vi.fn(); + const query = vi.fn().mockReturnValue({ paginate }); + const patch = vi.fn(); + const runAfter = vi.fn(); + const ctx = { + db: { query, get: vi.fn(), patch, normalizeId: vi.fn() }, + scheduler: { runAfter }, + } as never; + const handler = ( + backfillSkillsShMirrorDigestFirstTokensInternal as unknown as { _handler: Function } + )._handler; + + await expect(handler(ctx, { dryRun: false })).rejects.toThrow( + `Pass confirm="${SKILLS_SH_MIRROR_DIGEST_FIRST_TOKEN_BACKFILL_CONFIRM}" to apply.`, + ); + await expect( + handler(ctx, { + dryRun: false, + // The native token must not unlock the mirror path. + confirm: SKILL_SEARCH_DIGEST_FIRST_TOKEN_BACKFILL_CONFIRM, + }), + ).rejects.toThrow( + `Pass confirm="${SKILLS_SH_MIRROR_DIGEST_FIRST_TOKEN_BACKFILL_CONFIRM}" to apply.`, + ); + expect(paginate).not.toHaveBeenCalled(); + expect(patch).not.toHaveBeenCalled(); + expect(runAfter).not.toHaveBeenCalled(); + }); + + it("reports would-be patches without writing or scheduling in dry run mode", async () => { + const paginate = vi.fn().mockResolvedValue({ + page: mirrorPage(), + continueCursor: "next-page", + isDone: false, + }); + const query = vi.fn().mockReturnValue({ paginate }); + const patch = vi.fn().mockResolvedValue(undefined); + const runAfter = vi.fn().mockResolvedValue(undefined); + + const result = await ( + backfillSkillsShMirrorDigestFirstTokensInternal as unknown as { _handler: Function } + )._handler( + { + db: { query, get: vi.fn(), patch, normalizeId: vi.fn() }, + scheduler: { runAfter }, + } as never, + { dryRun: true }, + ); + + expect(result.patched).toBe(1); + expect(result.dryRun).toBe(true); + expect(patch).not.toHaveBeenCalled(); + expect(runAfter).not.toHaveBeenCalled(); + }); +}); + +describe("backfillSkillSearchDigestFirstTokensInternal dry run", () => { + it("reports would-be patches without writing or scheduling in dry run mode", async () => { + const paginate = vi.fn().mockResolvedValue({ + page: [ + { + _id: "skillSearchDigest:stale", + skillId: "skills:stale", + normalizedSlugFirstToken: "database", + normalizedDisplayNameFirstToken: "デ", + }, + ], + continueCursor: "next-page", + isDone: false, + }); + const query = vi.fn().mockReturnValue({ paginate }); + const get = vi.fn().mockResolvedValue({ + _id: "skills:stale", + slug: "database", + displayName: "データベース管理", + }); + const patch = vi.fn().mockResolvedValue(undefined); + const runAfter = vi.fn().mockResolvedValue(undefined); + + const result = await ( + backfillSkillSearchDigestFirstTokensInternal as unknown as { _handler: Function } + )._handler( + { + db: { query, get, patch, normalizeId: vi.fn() }, + scheduler: { runAfter }, + } as never, + { dryRun: true }, + ); + + expect(result.patched).toBe(1); + expect(result.dryRun).toBe(true); + expect(patch).not.toHaveBeenCalled(); + expect(runAfter).not.toHaveBeenCalled(); + }); +}); diff --git a/convex/maintenance.ts b/convex/maintenance.ts index 08225b4c..7170099b 100644 --- a/convex/maintenance.ts +++ b/convex/maintenance.ts @@ -30,6 +30,7 @@ import { } from "./lib/skillQuality"; import { getFrontmatterValue, hashSkillFiles } from "./lib/skills"; import { computeIsSuspicious } from "./lib/skillSafety"; +import { getFirstSearchToken, getMirrorFirstSearchToken } from "./lib/skillSearchDigest"; import { generateSkillSummary } from "./lib/skillSummary"; const DEFAULT_BATCH_SIZE = 50; @@ -2749,6 +2750,208 @@ export const backfillSkillSearchDigestModerationVerdicts: ReturnType { + const batchSize = clampInt(args.batchSize ?? 100, 10, 200); + // Catalog search subscribes to skillSearchDigest, so batches are spaced out to keep the + // backfill from driving reactive re-reads back to back. + const delayMs = clampInt(args.delayMs ?? 500, 0, 60_000); + // Preview unless the caller opts into applying, matching the catalog-digest resync + // contract: an omitted argument must never start a table-wide write. + const dryRun = args.dryRun !== false; + if (!dryRun && args.confirm !== SKILL_SEARCH_DIGEST_FIRST_TOKEN_BACKFILL_CONFIRM) { + throw new ConvexError( + `Pass confirm="${SKILL_SEARCH_DIGEST_FIRST_TOKEN_BACKFILL_CONFIRM}" to apply.`, + ); + } + const { page, continueCursor, isDone } = await ctx.db + .query("skillSearchDigest") + .paginate({ cursor: args.cursor ?? null, numItems: batchSize }); + + let patched = 0; + let missingSkills = 0; + for (const digest of page) { + const skill = await ctx.db.get(digest.skillId); + if (!skill) { + missingSkills++; + continue; + } + + const normalizedSlugFirstToken = getFirstSearchToken(skill.slug); + const normalizedDisplayNameFirstToken = getFirstSearchToken(skill.displayName); + if ( + digest.normalizedSlugFirstToken === normalizedSlugFirstToken && + digest.normalizedDisplayNameFirstToken === normalizedDisplayNameFirstToken + ) { + continue; + } + + patched++; + if (!dryRun) { + await ctx.db.patch(digest._id, { + normalizedSlugFirstToken, + normalizedDisplayNameFirstToken, + }); + } + } + + if (!dryRun && !isDone) { + await ctx.scheduler.runAfter( + delayMs, + internal.maintenance.backfillSkillSearchDigestFirstTokensInternal, + { + cursor: continueCursor, + batchSize: args.batchSize, + delayMs: args.delayMs, + dryRun, + // Continuations re-enter the same guard, so the token has to travel with them. + confirm: args.confirm, + }, + ); + } + + return { + scanned: page.length, + patched, + missingSkills, + cursor: continueCursor, + isDone, + dryRun, + confirmRequired: dryRun ? SKILL_SEARCH_DIGEST_FIRST_TOKEN_BACKFILL_CONFIRM : undefined, + }; + }, +}); + +export const backfillSkillSearchDigestFirstTokens: ReturnType = action({ + args: { + cursor: v.optional(v.string()), + batchSize: v.optional(v.number()), + delayMs: v.optional(v.number()), + dryRun: v.optional(v.boolean()), + confirm: v.optional(v.string()), + }, + handler: async (ctx, args) => { + const { user } = await requireUserFromAction(ctx); + assertRole(user, ["admin"]); + return await ctx.runMutation( + internal.maintenance.backfillSkillSearchDigestFirstTokensInternal, + args, + ); + }, +}); + +// Recompute the stored first-token fields on skillsShMirrorDigests rows. The skills.sh +// mirror derives them through the same tokenizer as the native digest above, and external +// candidate search range-scans them, so a tokenizer change strands mirrored rows the same +// way. Run once after deploying such a change, preview first: +// npx convex run maintenance:backfillSkillsShMirrorDigestFirstTokens --prod +// npx convex run maintenance:backfillSkillsShMirrorDigestFirstTokens \ +// '{"dryRun": false, "confirm": "backfill-skills-sh-mirror-digest-first-tokens"}' --prod +export const backfillSkillsShMirrorDigestFirstTokensInternal = internalMutation({ + args: { + cursor: v.optional(v.string()), + batchSize: v.optional(v.number()), + delayMs: v.optional(v.number()), + dryRun: v.optional(v.boolean()), + confirm: v.optional(v.string()), + }, + handler: async (ctx, args) => { + const batchSize = clampInt(args.batchSize ?? 100, 10, 200); + // The catalog subscribes to mirrored rows too, so pages are spaced apart here as well. + const delayMs = clampInt(args.delayMs ?? 500, 0, 60_000); + // Same preview-then-confirm contract as the native backfill above. + const dryRun = args.dryRun !== false; + if (!dryRun && args.confirm !== SKILLS_SH_MIRROR_DIGEST_FIRST_TOKEN_BACKFILL_CONFIRM) { + throw new ConvexError( + `Pass confirm="${SKILLS_SH_MIRROR_DIGEST_FIRST_TOKEN_BACKFILL_CONFIRM}" to apply.`, + ); + } + const { page, continueCursor, isDone } = await ctx.db + .query("skillsShMirrorDigests") + .paginate({ cursor: args.cursor ?? null, numItems: batchSize }); + + let patched = 0; + for (const digest of page) { + const normalizedSlugFirstToken = getMirrorFirstSearchToken(digest.slug); + const normalizedDisplayNameFirstToken = getMirrorFirstSearchToken(digest.displayName); + if ( + digest.normalizedSlugFirstToken === normalizedSlugFirstToken && + digest.normalizedDisplayNameFirstToken === normalizedDisplayNameFirstToken + ) { + continue; + } + + patched++; + if (!dryRun) { + await ctx.db.patch(digest._id, { + normalizedSlugFirstToken, + normalizedDisplayNameFirstToken, + }); + } + } + + if (!dryRun && !isDone) { + await ctx.scheduler.runAfter( + delayMs, + internal.maintenance.backfillSkillsShMirrorDigestFirstTokensInternal, + { + cursor: continueCursor, + batchSize: args.batchSize, + delayMs: args.delayMs, + dryRun, + confirm: args.confirm, + }, + ); + } + + return { + scanned: page.length, + patched, + cursor: continueCursor, + isDone, + dryRun, + confirmRequired: dryRun ? SKILLS_SH_MIRROR_DIGEST_FIRST_TOKEN_BACKFILL_CONFIRM : undefined, + }; + }, +}); + +export const backfillSkillsShMirrorDigestFirstTokens: ReturnType = action({ + args: { + cursor: v.optional(v.string()), + batchSize: v.optional(v.number()), + delayMs: v.optional(v.number()), + dryRun: v.optional(v.boolean()), + confirm: v.optional(v.string()), + }, + handler: async (ctx, args) => { + const { user } = await requireUserFromAction(ctx); + assertRole(user, ["admin"]); + return await ctx.runMutation( + internal.maintenance.backfillSkillsShMirrorDigestFirstTokensInternal, + args, + ); + }, +}); + // Repair stale skill-level moderation that was sourced from a non-latest version. // Run once after deploying the latest-version moderation fix: // npx convex run maintenance:backfillLatestSkillModeration --prod diff --git a/convex/skillSearchDigestFirstTokens.runtime.test.ts b/convex/skillSearchDigestFirstTokens.runtime.test.ts new file mode 100644 index 00000000..28195570 --- /dev/null +++ b/convex/skillSearchDigestFirstTokens.runtime.test.ts @@ -0,0 +1,153 @@ +/// +/* @vitest-environment edge-runtime */ +import { convexTest } from "convex-test"; +import { describe, expect, it } from "vitest"; +import { internal } from "./_generated/api"; +import { getFirstSearchToken } from "./lib/skillSearchDigest"; +import schema from "./schema"; + +const modules = import.meta.glob("./**/*.ts"); + +// What a row written before ー joined the CJK class holds for this display name. +const STALE_FIRST_TOKEN = "デ"; +const DISPLAY_NAME = "データベース管理"; +const SLUG = "database-kanri"; +// The backfill previews unless an apply is confirmed, so the runtime cases that expect +// writes have to opt in the same way an operator does. +const APPLY = { + dryRun: false, + confirm: "backfill-skill-search-digest-first-tokens", +} as const; + +async function insertDigestWithStaleFirstToken(t: ReturnType) { + return await t.run(async (ctx) => { + const now = Date.now(); + const userId = await ctx.db.insert("users", { + handle: "patrick", + displayName: "Patrick", + createdAt: now, + updatedAt: now, + }); + const skillId = await ctx.db.insert("skills", { + slug: SLUG, + displayName: DISPLAY_NAME, + ownerUserId: userId, + tags: {}, + stats: { downloads: 0, stars: 0, versions: 1, comments: 0 }, + createdAt: now, + updatedAt: now, + }); + const versionId = await ctx.db.insert("skillVersions", { + skillId, + version: "1.0.0", + changelog: "Initial", + files: [], + parsed: { frontmatter: {} }, + createdBy: userId, + createdAt: now, + }); + const digestId = await ctx.db.insert("skillSearchDigest", { + skillId, + slug: SLUG, + displayName: DISPLAY_NAME, + normalizedSlug: SLUG, + normalizedSlugFirstToken: "database", + normalizedDisplayName: DISPLAY_NAME, + normalizedDisplayNameFirstToken: STALE_FIRST_TOKEN, + ownerUserId: userId, + ownerHandle: "patrick", + ownerKind: "user", + ownerName: "patrick", + ownerDisplayName: "Patrick", + latestVersionId: versionId, + latestVersionSkillId: skillId, + publicVersion: { status: "available", versionId }, + tags: {}, + stats: { downloads: 0, stars: 0, versions: 1, comments: 0 }, + createdAt: now, + updatedAt: now, + }); + return { digestId, skillId }; + }); +} + +describe("skillSearchDigest first-token resynchronization", () => { + it("repairs a pre-existing row the current tokenizer no longer agrees with", async () => { + const t = convexTest(schema, modules); + const { digestId } = await insertDigestWithStaleFirstToken(t); + + const expected = getFirstSearchToken(DISPLAY_NAME); + expect(expected).not.toBe(STALE_FIRST_TOKEN); + + const before = await t.run(async (ctx) => await ctx.db.get(digestId)); + expect(before?.normalizedDisplayNameFirstToken).toBe(STALE_FIRST_TOKEN); + + // An unconfirmed call reports the same repair without performing it. + const preview = await t.mutation( + internal.maintenance.backfillSkillSearchDigestFirstTokensInternal, + {}, + ); + expect(preview.patched).toBe(1); + expect(preview.dryRun).toBe(true); + const afterPreview = await t.run(async (ctx) => await ctx.db.get(digestId)); + expect(afterPreview?.normalizedDisplayNameFirstToken).toBe(STALE_FIRST_TOKEN); + + const result = await t.mutation( + internal.maintenance.backfillSkillSearchDigestFirstTokensInternal, + APPLY, + ); + expect(result.patched).toBe(1); + expect(result.missingSkills).toBe(0); + + const after = await t.run(async (ctx) => await ctx.db.get(digestId)); + expect(after?.normalizedDisplayNameFirstToken).toBe(expected); + expect(after?.normalizedSlugFirstToken).toBe(getFirstSearchToken(SLUG)); + }); + + it("makes the row reachable again through the index the search actually queries", async () => { + const t = convexTest(schema, modules); + await insertDigestWithStaleFirstToken(t); + + const token = getFirstSearchToken(DISPLAY_NAME) as string; + const upperBound = + token.slice(0, -1) + String.fromCharCode(token.charCodeAt(token.length - 1) + 1); + const recall = async () => + await t.run( + async (ctx) => + await ctx.db + .query("skillSearchDigest") + .withIndex("by_active_normalized_display_name_first_token", (q) => + q + .eq("softDeletedAt", undefined) + .gte("normalizedDisplayNameFirstToken", token) + .lt("normalizedDisplayNameFirstToken", upperBound), + ) + .collect(), + ); + + // The row is on disk and matches the query the user typed, but the stored token + // predates the tokenizer, so the range bound the search computes never reaches it. + expect(await recall()).toHaveLength(0); + + await t.mutation(internal.maintenance.backfillSkillSearchDigestFirstTokensInternal, APPLY); + + expect(await recall()).toHaveLength(1); + }); + + it("leaves an already-current row untouched", async () => { + const t = convexTest(schema, modules); + const { digestId } = await insertDigestWithStaleFirstToken(t); + + await t.mutation(internal.maintenance.backfillSkillSearchDigestFirstTokensInternal, APPLY); + const second = await t.mutation( + internal.maintenance.backfillSkillSearchDigestFirstTokensInternal, + APPLY, + ); + + expect(second.scanned).toBe(1); + expect(second.patched).toBe(0); + + const row = await t.run(async (ctx) => await ctx.db.get(digestId)); + expect(row?.normalizedDisplayNameFirstToken).toBe(getFirstSearchToken(DISPLAY_NAME)); + }); +}); diff --git a/convex/skillsShMirror.ts b/convex/skillsShMirror.ts index abdeeda5..43a16980 100644 --- a/convex/skillsShMirror.ts +++ b/convex/skillsShMirror.ts @@ -8,7 +8,7 @@ import { ConvexError, type Infer, v } from "convex/values"; import type { Doc, Id } from "./_generated/dataModel"; import type { MutationCtx, QueryCtx } from "./_generated/server"; import { internalMutation, internalQuery } from "./functions"; -import { tokenize } from "./lib/searchText"; +import { getMirrorFirstSearchToken } from "./lib/skillSearchDigest"; import { assertSkillsShMirrorEnvironmentAllowed } from "./lib/skillsShCatalogEnvironment"; import { skillsShMirrorFreshObservationFlags } from "./lib/skillsShPublicVisibility"; @@ -404,10 +404,6 @@ function normalizedTopicLabel(value: string) { return value.normalize("NFKC").trim().replace(/\s+/g, " "); } -function firstSearchToken(value: string) { - return tokenize(value)[0] ?? normalizedSearchText(value); -} - function requiredSearchValue(name: string, value: string) { const normalized = normalizedSearchText(value); if (!normalized) throw new ConvexError(`${name} is required`); @@ -433,9 +429,9 @@ function searchFields(row: MirrorRow) { .slice(0, 512); return { normalizedSlug, - normalizedSlugFirstToken: firstSearchToken(row.slug), + normalizedSlugFirstToken: getMirrorFirstSearchToken(row.slug), normalizedDisplayName, - normalizedDisplayNameFirstToken: firstSearchToken(row.displayName), + normalizedDisplayNameFirstToken: getMirrorFirstSearchToken(row.displayName), ...(searchSummary ? { searchSummary } : {}), searchText: [ row.displayName,