fix: catalog previews cut Chinese and Japanese summaries at the first Latin word (#3362)

* fix: catalog previews cut Chinese and Japanese summaries at the first Latin word

truncateText backtracks to the last space in the slice unconditionally. Scripts
that do not separate words with spaces usually carry a single Latin space near
the start of a summary, so that backtrack discards nearly the whole preview:
across fixtures/public-corpus/corpus.jsonl, 25 catalog entries render with a
handful of characters instead of their budget, one of them as just "|".

Honour the word boundary only when it keeps most of the slice. All 1362
space-separated previews in the same corpus are unchanged.

* fix: keep the word boundary for space-separated previews

The kept-ratio fallback was added for CJK summaries whose only Latin space
sits near the start, but it applied to every script. A space-separated
summary ending in a long token — a URL, a compound word — lost its word
boundary and was cut mid-token instead.

Gate the ratio on the discarded tail actually being non-spacing script,
reusing the character class the catalog search tokenizer already relies on
in convex/lib/searchText.ts.

src/lib/truncateText.test.ts

@@ -0,0 +1,84 @@
+import { describe, expect, it } from "vitest";
+import { PUBLIC_CATALOG_NAME_PREVIEW_LENGTH, truncateText } from "./truncateText";
+
+// Catalog summaries taken verbatim from fixtures/public-corpus/corpus.jsonl.
+const ZH_SUMMARY =
+  "5GC Web仪表自动化技能，支持AMF/UDM/AUSF/SMF/PGW-C/UPF/PGW-U/GNB/UE/PCF/NRF/QoS/TC/PCC/smpolicy的批量添加与编辑及PCF默认规则一键配置";
+const ZH_MIXED_SUMMARY =
+  "实施Claude Code架构的5阶段进化计划，将OpenClaw系统升级到生产级Agent架构。包括记忆系统升级、工具系统优化、多Agent协作增强、安全架构强化和Prompt优化。当用户需要：1) 将现有OpenClaw系统升级到Claude Code架构标准，2) 实施结构化记忆系统，3) 建立四层权限模型，4) 配置多Agent协作，5) 增强安全架构，6) 优化Prompt和上下文管理时使用此技能。";
+const JA_SUMMARY =
+  "Claude Code向けのワークフロー自動化スキルパックです。日々のリリース作業やレビュー依頼をまとめて処理し、開発チームの負担を減らします。設定ファイルは不要で、導入したその日から使えます。";
+
+describe("truncateText", () => {
+  it("returns short values unchanged", () => {
+    expect(truncateText("Deploy helper", 40)).toBe("Deploy helper");
+    expect(truncateText("  Deploy helper  ", 40)).toBe("Deploy helper");
+  });
+
+  it("collapses internal whitespace before measuring", () => {
+    expect(truncateText("Deploy\n\thelper   pack", 40)).toBe("Deploy helper pack");
+  });
+
+  it("cuts space-separated text at a word boundary", () => {
+    expect(truncateText("A skill that automates deployment pipelines", 40)).toBe(
+      "A skill that automates deployment…",
+    );
+    expect(truncateText("Generate release notes from your commit history", 30)).toBe(
+      "Generate release notes from…",
+    );
+    expect(truncateText("automation toolkit for release engineering teams", 30)).toBe(
+      "automation toolkit for…",
+    );
+  });
+
+  it("keeps the word boundary when a long Latin word ends the slice", () => {
+    // The only space sits before 60% of the slice, but the discarded tail is space-separated
+    // text, so the preview must still end on a word rather than mid-word.
+    expect(truncateText("Read this hyperextendedword", 20)).toBe("Read this…");
+    expect(truncateText("Ship internationalization", 22)).toBe("Ship…");
+  });
+
+  it("keeps the Latin word boundary when only a short CJK tail is discarded", () => {
+    // Here the boundary already keeps most of the slice, so backtracking loses nothing but a
+    // stray CJK character that reads as noise on its own.
+    expect(truncateText("Deploy helper for 中文文档管理", 20)).toBe("Deploy helper for…");
+    expect(truncateText("Release notes generator 日本語対応", 28)).toBe("Release notes generator…");
+  });
+
+  it("keeps the preview budget for Chinese summaries carrying an early Latin space", () => {
+    // The single space after "5GC" is the only space in 104 characters, so backtracking
+    // to it would leave a three-character preview.
+    expect(truncateText(ZH_SUMMARY, 80)).toBe(
+      "5GC Web仪表自动化技能，支持AMF/UDM/AUSF/SMF/PGW-C/UPF/PGW-U/GNB/UE/PCF/NRF/QoS/TC/PCC/smp…",
+    );
+    expect(truncateText(ZH_SUMMARY, 100)).toBe(
+      "5GC Web仪表自动化技能，支持AMF/UDM/AUSF/SMF/PGW-C/UPF/PGW-U/GNB/UE/PCF/NRF/QoS/TC/PCC/smpolicy的批量添加与编辑及PCF默认规…",
+    );
+  });
+
+  it("keeps the preview budget when a Latin product name opens a Chinese summary", () => {
+    expect(truncateText(ZH_MIXED_SUMMARY, 80)).toBe(
+      "实施Claude Code架构的5阶段进化计划，将OpenClaw系统升级到生产级Agent架构。包括记忆系统升级、工具系统优化、多Agent协作增强、安全架…",
+    );
+  });
+
+  it("keeps the preview budget for Japanese summaries carrying an early Latin space", () => {
+    expect(truncateText(JA_SUMMARY, 80)).toBe(
+      "Claude Code向けのワークフロー自動化スキルパックです。日々のリリース作業やレビュー依頼をまとめて処理し、開発チームの負担を減らします。設定ファイルは…",
+    );
+  });
+
+  it("cuts hard when the text contains no space at all", () => {
+    expect(truncateText("仪表自动化技能支持批量添加与编辑及默认规则一键配置", 10)).toBe(
+      "仪表自动化技能支持…",
+    );
+  });
+
+  it("never exceeds the requested budget", () => {
+    for (const sample of [ZH_SUMMARY, ZH_MIXED_SUMMARY, JA_SUMMARY]) {
+      for (const budget of [PUBLIC_CATALOG_NAME_PREVIEW_LENGTH, 80, 100]) {
+        expect(truncateText(sample, budget).length).toBeLessThanOrEqual(budget);
+      }
+    }
+  });
+});

src/lib/truncateText.ts

@@ -1,10 +1,22 @@
 export const PUBLIC_CATALOG_NAME_PREVIEW_LENGTH = 70;
 
+// Scripts that do not separate words with spaces (Chinese, Japanese, Korean) often carry a
+// single Latin space near the start of a summary. Backtracking to that space would drop
+// nearly the whole preview, so only honour a word boundary that keeps most of the slice.
+const WORD_BOUNDARY_MIN_KEPT_RATIO = 0.6;
+// Same character class as the catalog search tokenizer in convex/lib/searchText.ts.
+const CJK_RE = /[\u4e00-\u9fff\u3400-\u4dbf\u3041-\u3096\u30a1-\u30fa\uac00-\ud7af]/;
+
 export function truncateText(value: string, maxLength: number) {
   const normalized = value.trim().replace(/\s+/g, " ");
   if (normalized.length <= maxLength) return normalized;
   const truncated = normalized.slice(0, Math.max(0, maxLength - 1)).trimEnd();
   const wordBoundary = truncated.lastIndexOf(" ");
-  const text = wordBoundary > 0 ? truncated.slice(0, wordBoundary) : truncated;
+  // The ratio only guards non-spacing scripts. Space-separated text keeps its word boundary
+  // however long the trailing word is.
+  const discardsCJK = CJK_RE.test(truncated.slice(wordBoundary + 1));
+  const keepsMostOfSlice = wordBoundary >= truncated.length * WORD_BOUNDARY_MIN_KEPT_RATIO;
+  const honoursBoundary = wordBoundary > 0 && (keepsMostOfSlice || !discardsCJK);
+  const text = honoursBoundary ? truncated.slice(0, wordBoundary) : truncated;
   return `${text}…`;
 }