feat: Add support for Chinese Japanese and Korean(CJK) skills search (#1596)

Merged via squash.

Prepared head SHA: ab58f01be7
Co-authored-by: pq-dong <40668796+pq-dong@users.noreply.github.com>
Co-authored-by: momothemage <35096042+momothemage@users.noreply.github.com>
Reviewed-by: @momothemage

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## Unreleased
+
+### Changed
+
+- Search: add CJK tokenization support (Chinese/Japanese/Korean) with Intl.Segmenter plus fallback behavior to improve skill query matching (#1596) (thanks @pq-dong).
+
 ## 0.10.0 - 2026-04-05
 
 ### Added

convex/_generated/api.d.ts

@@ -48,6 +48,7 @@ import type * as lib_contentTypes from "../lib/contentTypes.js";
 import type * as lib_embeddingVisibility from "../lib/embeddingVisibility.js";
 import type * as lib_embeddings from "../lib/embeddings.js";
 import type * as lib_githubAccount from "../lib/githubAccount.js";
+import type * as lib_githubActionsOidc from "../lib/githubActionsOidc.js";
 import type * as lib_githubBackup from "../lib/githubBackup.js";
 import type * as lib_githubIdentity from "../lib/githubIdentity.js";
 import type * as lib_githubImport from "../lib/githubImport.js";
@@ -76,6 +77,7 @@ import type * as lib_reservedSlugs from "../lib/reservedSlugs.js";
 import type * as lib_searchText from "../lib/searchText.js";
 import type * as lib_securityPrompt from "../lib/securityPrompt.js";
 import type * as lib_skillBackfill from "../lib/skillBackfill.js";
+import type * as lib_skillCapabilityTags from "../lib/skillCapabilityTags.js";
 import type * as lib_skillPublish from "../lib/skillPublish.js";
 import type * as lib_skillQuality from "../lib/skillQuality.js";
 import type * as lib_skillSafety from "../lib/skillSafety.js";
@@ -92,6 +94,7 @@ import type * as lib_userSearch from "../lib/userSearch.js";
 import type * as lib_webhooks from "../lib/webhooks.js";
 import type * as llmEval from "../llmEval.js";
 import type * as maintenance from "../maintenance.js";
+import type * as packagePublishTokens from "../packagePublishTokens.js";
 import type * as packages from "../packages.js";
 import type * as publishers from "../publishers.js";
 import type * as rateLimits from "../rateLimits.js";
@@ -161,6 +164,7 @@ declare const fullApi: ApiFromModules<{
   "lib/embeddingVisibility": typeof lib_embeddingVisibility;
   "lib/embeddings": typeof lib_embeddings;
   "lib/githubAccount": typeof lib_githubAccount;
+  "lib/githubActionsOidc": typeof lib_githubActionsOidc;
   "lib/githubBackup": typeof lib_githubBackup;
   "lib/githubIdentity": typeof lib_githubIdentity;
   "lib/githubImport": typeof lib_githubImport;
@@ -189,6 +193,7 @@ declare const fullApi: ApiFromModules<{
   "lib/searchText": typeof lib_searchText;
   "lib/securityPrompt": typeof lib_securityPrompt;
   "lib/skillBackfill": typeof lib_skillBackfill;
+  "lib/skillCapabilityTags": typeof lib_skillCapabilityTags;
   "lib/skillPublish": typeof lib_skillPublish;
   "lib/skillQuality": typeof lib_skillQuality;
   "lib/skillSafety": typeof lib_skillSafety;
@@ -205,6 +210,7 @@ declare const fullApi: ApiFromModules<{
   "lib/webhooks": typeof lib_webhooks;
   llmEval: typeof llmEval;
   maintenance: typeof maintenance;
+  packagePublishTokens: typeof packagePublishTokens;
   packages: typeof packages;
   publishers: typeof publishers;
   rateLimits: typeof rateLimits;

convex/devSeed.ts

@@ -237,6 +237,74 @@ xuezh snapshot --profile default
 xuezh review next --limit 10
 xuezh audio process-voice --file ./utterance.wav
 \`\`\`
+`,
+  },
+  {
+    slug: "hanzi-helper",
+    displayName: "汉字助手",
+    summary: "汉字学习与分析工具，支持笔画查询、部首检索和组词生成。",
+    version: "0.1.0",
+    metadata: {
+      clawdbot: {
+        nix: {
+          plugin: "github:example/hanzi-helper",
+          systems: ["aarch64-darwin", "x86_64-linux"],
+        },
+        config: {
+          requiredEnv: ["HANZI_DB_PATH"],
+          stateDirs: [".config/hanzi"],
+          example:
+            'config = { env = { HANZI_DB_PATH = ".config/hanzi/db"; }; stateDirs = [ ".config/hanzi" ]; };',
+        },
+        cliHelp: `汉字助手 - Chinese character learning and analysis
+
+Usage:
+  hanzi-helper [command]
+
+Available Commands:
+  lookup      查询汉字信息（笔画、部首、释义）
+  radical     按部首检索汉字
+  stroke      按笔画数筛选汉字
+  words       生成汉字组词
+  practice    练习汉字书写
+  quiz        汉字听写测试
+
+Flags:
+  -h, --help   help for hanzi-helper
+  --json       Output JSON
+`,
+      },
+    },
+    rawSkillMd: `---
+name: hanzi-helper
+description: 汉字学习与分析工具，提供笔画查询、部首检索、组词生成和汉字听写练习功能。
+---
+
+# 汉字助手
+
+## 功能介绍
+
+汉字助手是一个强大的中文汉字学习工具，帮助用户深入了解每个汉字的结构和含义。
+
+## CLI
+
+\`\`\`bash
+hanzi-helper lookup --char 学
+hanzi-helper radical --name 木
+hanzi-helper stroke --count 8
+hanzi-helper words --char 大 --limit 20
+\`\`\`
+
+## 使用场景
+
+- **汉字查询**：输入任意汉字，查看笔画数、部首、繁体形式和基本释义
+- **部首检索**：按部首浏览相关汉字，了解汉字的分类规律
+- **组词生成**：输入一个汉字，自动生成常用词语和成语
+- **听写练习**：随机生成汉字听写测试，巩固学习效果
+
+## 学习建议
+
+建议每天学习五个新汉字，结合组词和例句加深记忆。坚持使用听写练习功能可以有效提高汉字识别能力。
 `,
   },
 ];

convex/devSeedExtra.ts

@@ -448,6 +448,22 @@ const EXTRA_SEED_SKILLS: SeedSkillSpec[] = [
     ["SSH_KEY_DIR"],
     ["generate", "rotate", "deploy", "list", "revoke"],
   ),
+
+  // CJK Language Support (2)
+  makeSkill(
+    "nihongo-check",
+    "日本語チェッカー",
+    "日本語文章の文法チェックと翻訳支援ツール。Japanese grammar checker and translation assistant.",
+    ["NIHONGO_API_KEY"],
+    ["check", "translate", "kanji", "grammar", "vocabulary"],
+  ),
+  makeSkill(
+    "hangukgeo-helper",
+    "한국어 도우미",
+    "한국어 학습 보조 도구입니다. Korean language learning assistant with vocabulary and grammar support.",
+    ["HANGUL_API_KEY"],
+    ["learn", "quiz", "vocabulary", "grammar", "pronunciation"],
+  ),
 ];
 
 function injectMetadata(rawSkillMd: string, metadata: Record<string, unknown>) {

convex/lib/searchText.test.ts

@@ -47,4 +47,55 @@ describe("searchText", () => {
   it("normalize uses lowercase", () => {
     expect(__test.normalize("AbC")).toBe("abc");
   });
+
+  // CJK (Chinese, Japanese, Korean) support tests
+  describe("CJK tokenization", () => {
+    it("tokenizes Chinese text using Intl.Segmenter", () => {
+      const tokens = tokenize("中文搜索");
+      expect(tokens.length).toBeGreaterThan(0);
+      expect(tokens).toContain("中文");
+      expect(tokens).toContain("搜索");
+    });
+
+    it("tokenizes mixed Chinese and English text", () => {
+      const tokens = tokenize("React 组件开发");
+      expect(tokens).toContain("react");
+      expect(tokens.some((t) => t.includes("组") || t.includes("件"))).toBe(true);
+    });
+
+    it("matches Chinese query tokens against Chinese skill names", () => {
+      const queryTokens = tokenize("翻译");
+      const skillName = "AI翻译助手";
+      expect(matchesExactTokens(queryTokens, [skillName])).toBe(true);
+    });
+
+    it("matches partial Chinese words", () => {
+      const queryTokens = tokenize("助手");
+      const skillName = "AI翻译助手";
+      expect(matchesExactTokens(queryTokens, [skillName])).toBe(true);
+    });
+
+    it("handles Japanese text", () => {
+      const tokens = tokenize("こんにちは世界");
+      expect(tokens.length).toBeGreaterThan(0);
+    });
+
+    it("handles Korean text", () => {
+      const tokens = tokenize("안녕하세요");
+      expect(tokens.length).toBeGreaterThan(0);
+    });
+
+    it("returns empty array for empty or whitespace-only input", () => {
+      expect(tokenize("")).toEqual([]);
+      expect(tokenize("   ")).toEqual([]);
+      expect(tokenize("!!!")).toEqual([]);
+    });
+
+    it("detects CJK language correctly", () => {
+      expect(__test.detectCJKLanguage("中文")).toBe("zh");
+      expect(__test.detectCJKLanguage("こんにちは")).toBe("ja");
+      expect(__test.detectCJKLanguage("안녕하세요")).toBe("ko");
+      expect(__test.detectCJKLanguage("hello")).toBeNull();
+    });
+  });
 });

convex/lib/searchText.ts

@@ -1,12 +1,135 @@
-const WORD_RE = /[a-z0-9]+/g;
+const CJK_RE = /[\u4e00-\u9fff\u3400-\u4dbf\u3041-\u3096\u30a1-\u30fa\uac00-\ud7af]/;
+
+const hasSegmenter = typeof Intl !== "undefined" && "Segmenter" in Intl;
+
+let zhSegmenter: Intl.Segmenter | null = null;
+let jaSegmenter: Intl.Segmenter | null = null;
+let koSegmenter: Intl.Segmenter | null = null;
+
+function getZhSegmenter(): Intl.Segmenter {
+  if (!zhSegmenter) {
+    zhSegmenter = new Intl.Segmenter("zh-CN", { granularity: "word" });
+  }
+  return zhSegmenter;
+}
+
+function getJaSegmenter(): Intl.Segmenter {
+  if (!jaSegmenter) {
+    jaSegmenter = new Intl.Segmenter("ja", { granularity: "word" });
+  }
+  return jaSegmenter;
+}
+
+function getKoSegmenter(): Intl.Segmenter {
+  if (!koSegmenter) {
+    koSegmenter = new Intl.Segmenter("ko", { granularity: "word" });
+  }
+  return koSegmenter;
+}
+
+/**
+ * Fallback: split CJK text into individual characters.
+ * Used when Intl.Segmenter is unavailable (e.g. stripped V8 runtime).
+ */
+function segmentCJKByChar(text: string): string[] {
+  const tokens: string[] = [];
+  for (const ch of text) {
+    if (CJK_RE.test(ch)) {
+      tokens.push(ch);
+    }
+  }
+  return tokens;
+}
 
 function normalize(value: string) {
   return value.toLowerCase();
 }
 
+/**
+ * Detect the primary CJK language in a text
+ * Returns 'zh' for Chinese, 'ja' for Japanese, 'ko' for Korean, or null
+ */
+function detectCJKLanguage(text: string): "zh" | "ja" | "ko" | null {
+  const chineseCount = (text.match(/[\u4e00-\u9fff\u3400-\u4dbf]/g) || []).length;
+  const hiraganaCount = (text.match(/[\u3040-\u309f]/g) || []).length;
+  const katakanaCount = (text.match(/[\u30a0-\u30ff]/g) || []).length;
+  const hangulCount = (text.match(/[\uac00-\ud7af]/g) || []).length;
+  if (hiraganaCount + katakanaCount > 0) {
+    return "ja";
+  }
+  if (hangulCount > 0) {
+    return "ko";
+  }
+  if (chineseCount > 0) {
+    return "zh";
+  }
+  return null;
+}
+
+/**
+ * Segment CJK text using Intl.Segmenter, falling back to character-level
+ * tokenization when the API is unavailable.
+ */
+function segmentCJK(text: string): string[] {
+  if (!hasSegmenter) return segmentCJKByChar(text);
+
+  const lang = detectCJKLanguage(text);
+  if (!lang) return [];
+
+  let segmenter: Intl.Segmenter;
+  switch (lang) {
+    case "ja":
+      segmenter = getJaSegmenter();
+      break;
+    case "ko":
+      segmenter = getKoSegmenter();
+      break;
+    default:
+      segmenter = getZhSegmenter();
+  }
+
+  const segments: string[] = [];
+  for (const { segment, isWordLike } of segmenter.segment(text)) {
+    const trimmed = segment.trim();
+    if (trimmed && isWordLike) {
+      segments.push(trimmed);
+    }
+  }
+  return segments;
+}
+
+/**
+ * Tokenize text for search, supporting both English and CJK languages
+ *
+ * For English: uses word boundaries (whitespace, punctuation)
+ * For CJK: uses Intl.Segmenter for proper word segmentation
+ */
 export function tokenize(value: string): string[] {
   if (!value) return [];
-  return normalize(value).match(WORD_RE) ?? [];
+
+  const normalized = normalize(value);
+
+  if (!CJK_RE.test(normalized)) {
+    return normalized.match(/[a-z0-9]+/g) ?? [];
+  }
+
+  const tokens: string[] = [];
+
+  const parts = normalized.split(/([^\u4e00-\u9fff\u3400-\u4dbf\u3041-\u3096\u30a1-\u30fa\uac00-\ud7af]+)/g);
+
+  for (const part of parts) {
+    if (!part.trim()) continue;
+
+    if (CJK_RE.test(part)) {
+      const cjkTokens = segmentCJK(part);
+      tokens.push(...cjkTokens);
+    } else {
+      const asciiTokens = part.match(/[a-z0-9]+/g) ?? [];
+      tokens.push(...asciiTokens);
+    }
+  }
+
+  return tokens;
 }
 
 export function matchesExactTokens(
@@ -24,4 +147,8 @@ export function matchesExactTokens(
   );
 }
 
-export const __test = { normalize, tokenize, matchesExactTokens };
+export const __test = {
+  normalize,
+  detectCJKLanguage,
+  segmentCJKByChar,
+};

package.json

@@ -7,6 +7,7 @@
   "type": "module",
   "scripts": {
     "build": "bun --bun vite build && bun scripts/copy-og-assets.ts",
+    "check": "bun run lint",
     "check:peers": "bun scripts/check-peer-deps.ts",
     "check:secrets": "bun scripts/check-staged-secrets.mjs",
     "convex:deploy": "bunx convex deploy --typecheck=disable --yes",