From 58260017954f702a8949ff512e661cffa0d8490c Mon Sep 17 00:00:00 2001 From: pqdong <40668796+pq-dong@users.noreply.github.com> Date: Fri, 10 Apr 2026 12:20:40 +0800 Subject: [PATCH] feat: Add support for Chinese Japanese and Korean(CJK) skills search (#1596) Merged via squash. Prepared head SHA: ab58f01be712cb7b9a6ce6cf52d61afab0022446 Co-authored-by: pq-dong <40668796+pq-dong@users.noreply.github.com> Co-authored-by: momothemage <35096042+momothemage@users.noreply.github.com> Reviewed-by: @momothemage --- CHANGELOG.md | 6 ++ convex/_generated/api.d.ts | 6 ++ convex/devSeed.ts | 68 +++++++++++++++++ convex/devSeedExtra.ts | 16 ++++ convex/lib/searchText.test.ts | 51 +++++++++++++ convex/lib/searchText.ts | 133 +++++++++++++++++++++++++++++++++- package.json | 1 + 7 files changed, 278 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48589a63..3975780b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## Unreleased + +### Changed + +- Search: add CJK tokenization support (Chinese/Japanese/Korean) with Intl.Segmenter plus fallback behavior to improve skill query matching (#1596) (thanks @pq-dong). + ## 0.10.0 - 2026-04-05 ### Added diff --git a/convex/_generated/api.d.ts b/convex/_generated/api.d.ts index 083510fa..d772fe59 100644 --- a/convex/_generated/api.d.ts +++ b/convex/_generated/api.d.ts @@ -48,6 +48,7 @@ import type * as lib_contentTypes from "../lib/contentTypes.js"; import type * as lib_embeddingVisibility from "../lib/embeddingVisibility.js"; import type * as lib_embeddings from "../lib/embeddings.js"; import type * as lib_githubAccount from "../lib/githubAccount.js"; +import type * as lib_githubActionsOidc from "../lib/githubActionsOidc.js"; import type * as lib_githubBackup from "../lib/githubBackup.js"; import type * as lib_githubIdentity from "../lib/githubIdentity.js"; import type * as lib_githubImport from "../lib/githubImport.js"; @@ -76,6 +77,7 @@ import type * as lib_reservedSlugs from "../lib/reservedSlugs.js"; import type * as lib_searchText from "../lib/searchText.js"; import type * as lib_securityPrompt from "../lib/securityPrompt.js"; import type * as lib_skillBackfill from "../lib/skillBackfill.js"; +import type * as lib_skillCapabilityTags from "../lib/skillCapabilityTags.js"; import type * as lib_skillPublish from "../lib/skillPublish.js"; import type * as lib_skillQuality from "../lib/skillQuality.js"; import type * as lib_skillSafety from "../lib/skillSafety.js"; @@ -92,6 +94,7 @@ import type * as lib_userSearch from "../lib/userSearch.js"; import type * as lib_webhooks from "../lib/webhooks.js"; import type * as llmEval from "../llmEval.js"; import type * as maintenance from "../maintenance.js"; +import type * as packagePublishTokens from "../packagePublishTokens.js"; import type * as packages from "../packages.js"; import type * as publishers from "../publishers.js"; import type * as rateLimits from "../rateLimits.js"; @@ -161,6 +164,7 @@ declare const fullApi: ApiFromModules<{ "lib/embeddingVisibility": typeof lib_embeddingVisibility; "lib/embeddings": typeof lib_embeddings; "lib/githubAccount": typeof lib_githubAccount; + "lib/githubActionsOidc": typeof lib_githubActionsOidc; "lib/githubBackup": typeof lib_githubBackup; "lib/githubIdentity": typeof lib_githubIdentity; "lib/githubImport": typeof lib_githubImport; @@ -189,6 +193,7 @@ declare const fullApi: ApiFromModules<{ "lib/searchText": typeof lib_searchText; "lib/securityPrompt": typeof lib_securityPrompt; "lib/skillBackfill": typeof lib_skillBackfill; + "lib/skillCapabilityTags": typeof lib_skillCapabilityTags; "lib/skillPublish": typeof lib_skillPublish; "lib/skillQuality": typeof lib_skillQuality; "lib/skillSafety": typeof lib_skillSafety; @@ -205,6 +210,7 @@ declare const fullApi: ApiFromModules<{ "lib/webhooks": typeof lib_webhooks; llmEval: typeof llmEval; maintenance: typeof maintenance; + packagePublishTokens: typeof packagePublishTokens; packages: typeof packages; publishers: typeof publishers; rateLimits: typeof rateLimits; diff --git a/convex/devSeed.ts b/convex/devSeed.ts index 2fbc56d9..2c5ad44c 100644 --- a/convex/devSeed.ts +++ b/convex/devSeed.ts @@ -237,6 +237,74 @@ xuezh snapshot --profile default xuezh review next --limit 10 xuezh audio process-voice --file ./utterance.wav \`\`\` +`, + }, + { + slug: "hanzi-helper", + displayName: "汉字助手", + summary: "汉字学习与分析工具，支持笔画查询、部首检索和组词生成。", + version: "0.1.0", + metadata: { + clawdbot: { + nix: { + plugin: "github:example/hanzi-helper", + systems: ["aarch64-darwin", "x86_64-linux"], + }, + config: { + requiredEnv: ["HANZI_DB_PATH"], + stateDirs: [".config/hanzi"], + example: + 'config = { env = { HANZI_DB_PATH = ".config/hanzi/db"; }; stateDirs = [ ".config/hanzi" ]; };', + }, + cliHelp: `汉字助手 - Chinese character learning and analysis + +Usage: + hanzi-helper [command] + +Available Commands: + lookup 查询汉字信息（笔画、部首、释义） + radical 按部首检索汉字 + stroke 按笔画数筛选汉字 + words 生成汉字组词 + practice 练习汉字书写 + quiz 汉字听写测试 + +Flags: + -h, --help help for hanzi-helper + --json Output JSON +`, + }, + }, + rawSkillMd: `--- +name: hanzi-helper +description: 汉字学习与分析工具，提供笔画查询、部首检索、组词生成和汉字听写练习功能。 +--- + +# 汉字助手 + +## 功能介绍 + +汉字助手是一个强大的中文汉字学习工具，帮助用户深入了解每个汉字的结构和含义。 + +## CLI + +\`\`\`bash +hanzi-helper lookup --char 学 +hanzi-helper radical --name 木 +hanzi-helper stroke --count 8 +hanzi-helper words --char 大 --limit 20 +\`\`\` + +## 使用场景 + +- **汉字查询**：输入任意汉字，查看笔画数、部首、繁体形式和基本释义 +- **部首检索**：按部首浏览相关汉字，了解汉字的分类规律 +- **组词生成**：输入一个汉字，自动生成常用词语和成语 +- **听写练习**：随机生成汉字听写测试，巩固学习效果 + +## 学习建议 + +建议每天学习五个新汉字，结合组词和例句加深记忆。坚持使用听写练习功能可以有效提高汉字识别能力。 `, }, ]; diff --git a/convex/devSeedExtra.ts b/convex/devSeedExtra.ts index e655b253..e8e798b1 100644 --- a/convex/devSeedExtra.ts +++ b/convex/devSeedExtra.ts @@ -448,6 +448,22 @@ const EXTRA_SEED_SKILLS: SeedSkillSpec[] = [ ["SSH_KEY_DIR"], ["generate", "rotate", "deploy", "list", "revoke"], ), + + // CJK Language Support (2) + makeSkill( + "nihongo-check", + "日本語チェッカー", + "日本語文章の文法チェックと翻訳支援ツール。Japanese grammar checker and translation assistant.", + ["NIHONGO_API_KEY"], + ["check", "translate", "kanji", "grammar", "vocabulary"], + ), + makeSkill( + "hangukgeo-helper", + "한국어 도우미", + "한국어 학습 보조 도구입니다. Korean language learning assistant with vocabulary and grammar support.", + ["HANGUL_API_KEY"], + ["learn", "quiz", "vocabulary", "grammar", "pronunciation"], + ), ]; function injectMetadata(rawSkillMd: string, metadata: Record) { diff --git a/convex/lib/searchText.test.ts b/convex/lib/searchText.test.ts index a28c6b55..d3621f9a 100644 --- a/convex/lib/searchText.test.ts +++ b/convex/lib/searchText.test.ts @@ -47,4 +47,55 @@ describe("searchText", () => { it("normalize uses lowercase", () => { expect(__test.normalize("AbC")).toBe("abc"); }); + + // CJK (Chinese, Japanese, Korean) support tests + describe("CJK tokenization", () => { + it("tokenizes Chinese text using Intl.Segmenter", () => { + const tokens = tokenize("中文搜索"); + expect(tokens.length).toBeGreaterThan(0); + expect(tokens).toContain("中文"); + expect(tokens).toContain("搜索"); + }); + + it("tokenizes mixed Chinese and English text", () => { + const tokens = tokenize("React 组件开发"); + expect(tokens).toContain("react"); + expect(tokens.some((t) => t.includes("组") || t.includes("件"))).toBe(true); + }); + + it("matches Chinese query tokens against Chinese skill names", () => { + const queryTokens = tokenize("翻译"); + const skillName = "AI翻译助手"; + expect(matchesExactTokens(queryTokens, [skillName])).toBe(true); + }); + + it("matches partial Chinese words", () => { + const queryTokens = tokenize("助手"); + const skillName = "AI翻译助手"; + expect(matchesExactTokens(queryTokens, [skillName])).toBe(true); + }); + + it("handles Japanese text", () => { + const tokens = tokenize("こんにちは世界"); + expect(tokens.length).toBeGreaterThan(0); + }); + + it("handles Korean text", () => { + const tokens = tokenize("안녕하세요"); + expect(tokens.length).toBeGreaterThan(0); + }); + + it("returns empty array for empty or whitespace-only input", () => { + expect(tokenize("")).toEqual([]); + expect(tokenize(" ")).toEqual([]); + expect(tokenize("!!!")).toEqual([]); + }); + + it("detects CJK language correctly", () => { + expect(__test.detectCJKLanguage("中文")).toBe("zh"); + expect(__test.detectCJKLanguage("こんにちは")).toBe("ja"); + expect(__test.detectCJKLanguage("안녕하세요")).toBe("ko"); + expect(__test.detectCJKLanguage("hello")).toBeNull(); + }); + }); }); diff --git a/convex/lib/searchText.ts b/convex/lib/searchText.ts index 8e8e0692..8f1874c9 100644 --- a/convex/lib/searchText.ts +++ b/convex/lib/searchText.ts @@ -1,12 +1,135 @@ -const WORD_RE = /[a-z0-9]+/g; +const CJK_RE = /[\u4e00-\u9fff\u3400-\u4dbf\u3041-\u3096\u30a1-\u30fa\uac00-\ud7af]/; + +const hasSegmenter = typeof Intl !== "undefined" && "Segmenter" in Intl; + +let zhSegmenter: Intl.Segmenter | null = null; +let jaSegmenter: Intl.Segmenter | null = null; +let koSegmenter: Intl.Segmenter | null = null; + +function getZhSegmenter(): Intl.Segmenter { + if (!zhSegmenter) { + zhSegmenter = new Intl.Segmenter("zh-CN", { granularity: "word" }); + } + return zhSegmenter; +} + +function getJaSegmenter(): Intl.Segmenter { + if (!jaSegmenter) { + jaSegmenter = new Intl.Segmenter("ja", { granularity: "word" }); + } + return jaSegmenter; +} + +function getKoSegmenter(): Intl.Segmenter { + if (!koSegmenter) { + koSegmenter = new Intl.Segmenter("ko", { granularity: "word" }); + } + return koSegmenter; +} + +/** + * Fallback: split CJK text into individual characters. + * Used when Intl.Segmenter is unavailable (e.g. stripped V8 runtime). + */ +function segmentCJKByChar(text: string): string[] { + const tokens: string[] = []; + for (const ch of text) { + if (CJK_RE.test(ch)) { + tokens.push(ch); + } + } + return tokens; +} function normalize(value: string) { return value.toLowerCase(); } +/** + * Detect the primary CJK language in a text + * Returns 'zh' for Chinese, 'ja' for Japanese, 'ko' for Korean, or null + */ +function detectCJKLanguage(text: string): "zh" | "ja" | "ko" | null { + const chineseCount = (text.match(/[\u4e00-\u9fff\u3400-\u4dbf]/g) || []).length; + const hiraganaCount = (text.match(/[\u3040-\u309f]/g) || []).length; + const katakanaCount = (text.match(/[\u30a0-\u30ff]/g) || []).length; + const hangulCount = (text.match(/[\uac00-\ud7af]/g) || []).length; + if (hiraganaCount + katakanaCount > 0) { + return "ja"; + } + if (hangulCount > 0) { + return "ko"; + } + if (chineseCount > 0) { + return "zh"; + } + return null; +} + +/** + * Segment CJK text using Intl.Segmenter, falling back to character-level + * tokenization when the API is unavailable. + */ +function segmentCJK(text: string): string[] { + if (!hasSegmenter) return segmentCJKByChar(text); + + const lang = detectCJKLanguage(text); + if (!lang) return []; + + let segmenter: Intl.Segmenter; + switch (lang) { + case "ja": + segmenter = getJaSegmenter(); + break; + case "ko": + segmenter = getKoSegmenter(); + break; + default: + segmenter = getZhSegmenter(); + } + + const segments: string[] = []; + for (const { segment, isWordLike } of segmenter.segment(text)) { + const trimmed = segment.trim(); + if (trimmed && isWordLike) { + segments.push(trimmed); + } + } + return segments; +} + +/** + * Tokenize text for search, supporting both English and CJK languages + * + * For English: uses word boundaries (whitespace, punctuation) + * For CJK: uses Intl.Segmenter for proper word segmentation + */ export function tokenize(value: string): string[] { if (!value) return []; - return normalize(value).match(WORD_RE) ?? []; + + const normalized = normalize(value); + + if (!CJK_RE.test(normalized)) { + return normalized.match(/[a-z0-9]+/g) ?? []; + } + + const tokens: string[] = []; + + const parts = normalized.split(/([^\u4e00-\u9fff\u3400-\u4dbf\u3041-\u3096\u30a1-\u30fa\uac00-\ud7af]+)/g); + + for (const part of parts) { + if (!part.trim()) continue; + + if (CJK_RE.test(part)) { + const cjkTokens = segmentCJK(part); + tokens.push(...cjkTokens); + } else { + const asciiTokens = part.match(/[a-z0-9]+/g) ?? []; + tokens.push(...asciiTokens); + } + } + + return tokens; } export function matchesExactTokens( @@ -24,4 +147,8 @@ export function matchesExactTokens( ); } -export const __test = { normalize, tokenize, matchesExactTokens }; +export const __test = { + normalize, + detectCJKLanguage, + segmentCJKByChar, +}; diff --git a/package.json b/package.json index e656b82e..a5fff501 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "type": "module", "scripts": { "build": "bun --bun vite build && bun scripts/copy-og-assets.ts", + "check": "bun run lint", "check:peers": "bun scripts/check-peer-deps.ts", "check:secrets": "bun scripts/check-staged-secrets.mjs", "convex:deploy": "bunx convex deploy --typecheck=disable --yes",