mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-14 00:48:18 +00:00
742 lines
33 KiB
TypeScript
742 lines
33 KiB
TypeScript
/**
|
||
* Unit tests for src/core/by-mention.ts.
|
||
*
|
||
* Pure-function coverage of `findMentionedEntities` + `buildGazetteer`.
|
||
* Hermetic via PGLite for buildGazetteer (needs engine); pure-fn cases
|
||
* for findMentionedEntities (no engine needed).
|
||
*
|
||
* Covers all 20 cases enumerated in the v0.42.0.0 plan:
|
||
* 1. Single-token title match
|
||
* 2. Multi-word phrase pass ("Acme Corp" matches "Acme Corp" not "Acme")
|
||
* 3. Case folding
|
||
* 4. Whole-word boundary
|
||
* 5. Possessive form
|
||
* 6. Code-block stripping
|
||
* 7. Min-length filter
|
||
* 8. Ignore-list at gazetteer build (Apple suppressed when no page)
|
||
* 9. Ignore-list inverse (Apple matches when page exists)
|
||
* 10. First-mention-only cap
|
||
* 11. Empty gazetteer
|
||
* 12. Empty text
|
||
* 13. All entity pages soft-deleted → empty gazetteer
|
||
* 14. Multi-word shared first token (longest-match wins)
|
||
* 15. Determinism across 10 calls
|
||
* 16. Self-link guard (D13)
|
||
* 17. Cross-source guard
|
||
* 18. Hardcoded type filter (meeting NOT in gazetteer)
|
||
* 19. Min-length + ignore-list interaction
|
||
* 20. Code-block + token interaction
|
||
*/
|
||
|
||
import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
|
||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||
import {
|
||
buildGazetteer,
|
||
findMentionedEntities,
|
||
tokenizeForScan,
|
||
tokenizeTitle,
|
||
LINKABLE_ENTITY_TYPES,
|
||
type Gazetteer,
|
||
type GazetteerEntry,
|
||
} from '../src/core/by-mention.ts';
|
||
|
||
let engine: PGLiteEngine;
|
||
|
||
beforeAll(async () => {
|
||
engine = new PGLiteEngine();
|
||
await engine.connect({});
|
||
await engine.initSchema();
|
||
}, 60_000);
|
||
|
||
afterAll(async () => {
|
||
await engine.disconnect();
|
||
});
|
||
|
||
beforeEach(async () => {
|
||
await engine.executeRaw('DELETE FROM links');
|
||
await engine.executeRaw('DELETE FROM pages');
|
||
});
|
||
|
||
// Tiny gazetteer builder for pure-fn cases that don't need engine.
|
||
//
|
||
// Deliberately calls the PRODUCTION `tokenizeTitle` rather than re-declaring
|
||
// the tokenizer. A duplicated copy makes every test here non-discriminating:
|
||
// reverting the source tokenizer would leave the fixture on the new one, so
|
||
// title and body would keep agreeing and the tests would pass either way.
|
||
function gazetteerFromEntries(entries: Omit<GazetteerEntry, 'tokens'>[]): Gazetteer {
|
||
const g: Gazetteer = new Map();
|
||
for (const raw of entries) {
|
||
const tokens = tokenizeTitle(raw.title);
|
||
if (tokens.length === 0) continue;
|
||
const key = tokens[0]!;
|
||
const entry: GazetteerEntry = { ...raw, tokens };
|
||
const bucket = g.get(key);
|
||
if (bucket) bucket.push(entry);
|
||
else g.set(key, [entry]);
|
||
}
|
||
for (const bucket of g.values()) bucket.sort((a, b) => b.tokens.length - a.tokens.length);
|
||
return g;
|
||
}
|
||
|
||
// ============================================================
|
||
// findMentionedEntities — pure unit tests
|
||
// ============================================================
|
||
|
||
describe('findMentionedEntities — pure cases', () => {
|
||
test('1. single-token title match', () => {
|
||
const g = gazetteerFromEntries([
|
||
{ slug: 'companies/acme', source_id: 'default', title: 'Acme' },
|
||
]);
|
||
const mentions = findMentionedEntities('Acme launched today.', g, {
|
||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||
});
|
||
expect(mentions).toHaveLength(1);
|
||
expect(mentions[0]!.slug).toBe('companies/acme');
|
||
expect(mentions[0]!.name).toBe('Acme');
|
||
expect(mentions[0]!.offset).toBe(0);
|
||
});
|
||
|
||
test('2. multi-word phrase pass — "Acme Corp" matches multi-word, not single', () => {
|
||
const g = gazetteerFromEntries([
|
||
{ slug: 'companies/acme', source_id: 'default', title: 'Acme' },
|
||
{ slug: 'companies/acme-corp', source_id: 'default', title: 'Acme Corp' },
|
||
]);
|
||
const mentions = findMentionedEntities('We met with Acme Corp last week.', g, {
|
||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||
});
|
||
// longest-match wins → only the multi-word target
|
||
expect(mentions).toHaveLength(1);
|
||
expect(mentions[0]!.slug).toBe('companies/acme-corp');
|
||
});
|
||
|
||
test('3. case folding — "iOS Engineer" title matches "ios engineer" in body', () => {
|
||
const g = gazetteerFromEntries([
|
||
{ slug: 'people/ios-engineer', source_id: 'default', title: 'iOS Engineer' },
|
||
]);
|
||
const mentions = findMentionedEntities('Looking to hire an ios engineer.', g, {
|
||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||
});
|
||
expect(mentions).toHaveLength(1);
|
||
expect(mentions[0]!.slug).toBe('people/ios-engineer');
|
||
});
|
||
|
||
test('4. whole-word boundary — "Acme" matches "Acme." but NOT "Acmecorp"', () => {
|
||
const g = gazetteerFromEntries([
|
||
{ slug: 'companies/acme', source_id: 'default', title: 'Acme' },
|
||
]);
|
||
// Should match (sentence-ending dot is a token break)
|
||
const m1 = findMentionedEntities('We bought Acme.', g, {
|
||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||
});
|
||
expect(m1).toHaveLength(1);
|
||
// Should NOT match — "Acmecorp" tokenizes as single token "acmecorp"
|
||
const m2 = findMentionedEntities('Acmecorp is unrelated.', g, {
|
||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||
});
|
||
expect(m2).toHaveLength(0);
|
||
});
|
||
|
||
test('5. possessive form — "Acme\'s growth" → Acme matches', () => {
|
||
const g = gazetteerFromEntries([
|
||
{ slug: 'companies/acme', source_id: 'default', title: 'Acme' },
|
||
]);
|
||
const mentions = findMentionedEntities("Acme's growth is impressive.", g, {
|
||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||
});
|
||
expect(mentions).toHaveLength(1);
|
||
expect(mentions[0]!.slug).toBe('companies/acme');
|
||
});
|
||
|
||
test('6. code-block stripping — mentions inside ``` blocks ignored', () => {
|
||
const g = gazetteerFromEntries([
|
||
{ slug: 'companies/acme', source_id: 'default', title: 'Acme' },
|
||
]);
|
||
const body = '```\nAcme code\n```\nNothing here.';
|
||
const mentions = findMentionedEntities(body, g, {
|
||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||
});
|
||
expect(mentions).toHaveLength(0);
|
||
});
|
||
|
||
test('10. first-mention-only cap — 5 body mentions of same entity → 1 link', () => {
|
||
const g = gazetteerFromEntries([
|
||
{ slug: 'companies/acme', source_id: 'default', title: 'Acme' },
|
||
]);
|
||
const body = 'Acme one. Acme two. Acme three. Acme four. Acme five.';
|
||
const mentions = findMentionedEntities(body, g, {
|
||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||
});
|
||
expect(mentions).toHaveLength(1);
|
||
});
|
||
|
||
test('11. empty gazetteer → empty result', () => {
|
||
const g: Gazetteer = new Map();
|
||
const mentions = findMentionedEntities('Anything goes here.', g, {
|
||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||
});
|
||
expect(mentions).toEqual([]);
|
||
});
|
||
|
||
test('12. empty text → empty result', () => {
|
||
const g = gazetteerFromEntries([
|
||
{ slug: 'companies/acme', source_id: 'default', title: 'Acme' },
|
||
]);
|
||
const mentions = findMentionedEntities('', g, {
|
||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||
});
|
||
expect(mentions).toEqual([]);
|
||
});
|
||
|
||
test('14. multi-word shared first token — longest-match wins', () => {
|
||
const g = gazetteerFromEntries([
|
||
{ slug: 'companies/acme', source_id: 'default', title: 'Acme' },
|
||
{ slug: 'companies/acme-corp', source_id: 'default', title: 'Acme Corp' },
|
||
{ slug: 'companies/acme-foundation', source_id: 'default', title: 'Acme Foundation' },
|
||
]);
|
||
const body = 'Acme Foundation announced. Then Acme Corp. Then plain Acme.';
|
||
const mentions = findMentionedEntities(body, g, {
|
||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||
});
|
||
// First offset: "Acme Foundation" — longest match wins.
|
||
// Second occurrence of "Acme": multi-word "Acme Corp" matches → multi-word wins.
|
||
// Third: plain "Acme" alone — single-word match.
|
||
const slugs = mentions.map(m => m.slug);
|
||
expect(slugs).toContain('companies/acme-foundation');
|
||
expect(slugs).toContain('companies/acme-corp');
|
||
expect(slugs).toContain('companies/acme');
|
||
});
|
||
|
||
test('15. determinism — same body + same gazetteer → identical output across 10 calls', () => {
|
||
const g = gazetteerFromEntries([
|
||
{ slug: 'companies/acme', source_id: 'default', title: 'Acme' },
|
||
{ slug: 'companies/acme-corp', source_id: 'default', title: 'Acme Corp' },
|
||
{ slug: 'people/alice', source_id: 'default', title: 'Alice Smith' },
|
||
]);
|
||
const body = 'Acme Corp and Alice Smith and Acme met. Then Alice Smith again.';
|
||
const refs = new Set<string>();
|
||
for (let i = 0; i < 10; i++) {
|
||
const mentions = findMentionedEntities(body, g, {
|
||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||
});
|
||
refs.add(JSON.stringify(mentions));
|
||
}
|
||
expect(refs.size).toBe(1);
|
||
});
|
||
|
||
test('16. self-link guard (D13) — entity page mentioning own title skips', () => {
|
||
const g = gazetteerFromEntries([
|
||
{ slug: 'companies/acme', source_id: 'default', title: 'Acme' },
|
||
]);
|
||
// Page IS the Acme page; body mentions "Acme" → self-link guard skips.
|
||
const mentions = findMentionedEntities('Acme has 500 customers.', g, {
|
||
fromSlug: 'companies/acme', fromSourceId: 'default',
|
||
});
|
||
expect(mentions).toEqual([]);
|
||
});
|
||
|
||
test('17. cross-source guard — page in source A mentions entity in source B → no link', () => {
|
||
const g = gazetteerFromEntries([
|
||
{ slug: 'companies/acme', source_id: 'team-b', title: 'Acme' },
|
||
]);
|
||
const mentions = findMentionedEntities('We met Acme today.', g, {
|
||
fromSlug: 'writing/post-1', fromSourceId: 'team-a', // different source
|
||
});
|
||
expect(mentions).toEqual([]);
|
||
});
|
||
|
||
test('20. code-block + token interaction — body text outside block linked, inside skipped', () => {
|
||
const g = gazetteerFromEntries([
|
||
{ slug: 'companies/acme', source_id: 'default', title: 'Acme' },
|
||
]);
|
||
// Single backtick inline-code blocks the inner mention; outer mention fires.
|
||
const body = 'Outside: Acme works. Inline `Acme inside` should skip. After.';
|
||
const mentions = findMentionedEntities(body, g, {
|
||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||
});
|
||
expect(mentions).toHaveLength(1); // first-mention-only cap
|
||
expect(mentions[0]!.slug).toBe('companies/acme');
|
||
});
|
||
});
|
||
|
||
// ============================================================
|
||
// CJK — entity extraction tests
|
||
// ============================================================
|
||
|
||
describe('findMentionedEntities — CJK cases', () => {
|
||
test('CJK single-name match — "纳瓦尔" in body → matched', () => {
|
||
const g = gazetteerFromEntries([
|
||
{ slug: 'people/naval', source_id: 'default', title: '纳瓦尔' },
|
||
]);
|
||
const mentions = findMentionedEntities('我最近读了纳瓦尔的书。', g, {
|
||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||
});
|
||
expect(mentions).toHaveLength(1);
|
||
expect(mentions[0]!.slug).toBe('people/naval');
|
||
expect(mentions[0]!.name).toBe('纳瓦尔');
|
||
});
|
||
|
||
test('CJK multi-name — two different CJK entities in one body', () => {
|
||
const g = gazetteerFromEntries([
|
||
{ slug: 'people/naval', source_id: 'default', title: '纳瓦尔' },
|
||
{ slug: 'people/shuang-xuetao', source_id: 'default', title: '双雪涛' },
|
||
]);
|
||
const mentions = findMentionedEntities('纳瓦尔和双雪涛都是作家。', g, {
|
||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||
});
|
||
expect(mentions).toHaveLength(2);
|
||
const slugs = mentions.map(m => m.slug);
|
||
expect(slugs).toContain('people/naval');
|
||
expect(slugs).toContain('people/shuang-xuetao');
|
||
});
|
||
|
||
test('CJK first-mention-only — repeated name → single link', () => {
|
||
const g = gazetteerFromEntries([
|
||
{ slug: 'people/naval', source_id: 'default', title: '纳瓦尔' },
|
||
]);
|
||
const mentions = findMentionedEntities('纳瓦尔说过。然后纳瓦尔又说过。', g, {
|
||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||
});
|
||
expect(mentions).toHaveLength(1);
|
||
});
|
||
|
||
test('CJK self-link guard — entity page mentioning itself is skipped', () => {
|
||
const g = gazetteerFromEntries([
|
||
{ slug: 'people/naval', source_id: 'default', title: '纳瓦尔' },
|
||
]);
|
||
const mentions = findMentionedEntities('纳瓦尔是一位投资人。', g, {
|
||
fromSlug: 'people/naval', fromSourceId: 'default',
|
||
});
|
||
expect(mentions).toEqual([]);
|
||
});
|
||
|
||
test('CJK cross-source guard — entity in different source skipped', () => {
|
||
const g = gazetteerFromEntries([
|
||
{ slug: 'people/naval', source_id: 'team-b', title: '纳瓦尔' },
|
||
]);
|
||
const mentions = findMentionedEntities('纳瓦尔写了这本书。', g, {
|
||
fromSlug: 'writing/post-1', fromSourceId: 'team-a',
|
||
});
|
||
expect(mentions).toEqual([]);
|
||
});
|
||
|
||
test('CJK code-block stripping — CJK name inside ``` is skipped, outside matched', () => {
|
||
const g = gazetteerFromEntries([
|
||
{ slug: 'people/naval', source_id: 'default', title: '纳瓦尔' },
|
||
]);
|
||
// "纳瓦尔" only appears inside code block → should be skipped.
|
||
const body = '```\n纳瓦尔\n```\n只有代码块里面有。';
|
||
const mentions = findMentionedEntities(body, g, {
|
||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||
});
|
||
expect(mentions).toHaveLength(0);
|
||
});
|
||
|
||
test('CJK determinism — same output across 10 calls', () => {
|
||
const g = gazetteerFromEntries([
|
||
{ slug: 'people/naval', source_id: 'default', title: '纳瓦尔' },
|
||
{ slug: 'people/shuang-xuetao', source_id: 'default', title: '双雪涛' },
|
||
]);
|
||
const body = '纳瓦尔和双雪涛。纳瓦尔再说一次。';
|
||
const refs = new Set<string>();
|
||
for (let i = 0; i < 10; i++) {
|
||
const mentions = findMentionedEntities(body, g, {
|
||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||
});
|
||
refs.add(JSON.stringify(mentions));
|
||
}
|
||
expect(refs.size).toBe(1);
|
||
});
|
||
|
||
test('CJK mixed body — CJK entity matched in body with ASCII around it', () => {
|
||
const g = gazetteerFromEntries([
|
||
{ slug: 'people/naval', source_id: 'default', title: '纳瓦尔' },
|
||
{ slug: 'companies/acme', source_id: 'default', title: 'Acme' },
|
||
]);
|
||
const mentions = findMentionedEntities('Acme was founded by 纳瓦尔 in 2020.', g, {
|
||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||
});
|
||
expect(mentions).toHaveLength(2);
|
||
const slugs = mentions.map(m => m.slug);
|
||
expect(slugs).toContain('people/naval');
|
||
expect(slugs).toContain('companies/acme');
|
||
});
|
||
|
||
test('CJK empty gazetteer — no false positives', () => {
|
||
const g: Gazetteer = new Map();
|
||
const mentions = findMentionedEntities('纳瓦尔和双雪涛。', g, {
|
||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||
});
|
||
expect(mentions).toEqual([]);
|
||
});
|
||
|
||
test('CJK empty text → empty result', () => {
|
||
const g = gazetteerFromEntries([
|
||
{ slug: 'people/naval', source_id: 'default', title: '纳瓦尔' },
|
||
]);
|
||
const mentions = findMentionedEntities('', g, {
|
||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||
});
|
||
expect(mentions).toEqual([]);
|
||
});
|
||
});
|
||
|
||
// ============================================================
|
||
// Vietnamese (diacritic Latin) — entity extraction tests
|
||
// ============================================================
|
||
|
||
// Fictional Vietnamese names only (privacy rule: no real people in fixtures).
|
||
// "Đà Nẵng" is a public city, not a person, and is the canonical đ-diacritic case.
|
||
//
|
||
// Every case below asserts TOKENIZATION, not just the resolved mention. A
|
||
// mention-only assertion does not discriminate: the previous ASCII tokenizer
|
||
// fragmented the gazetteer title and the body symmetrically, so a 5-fragment
|
||
// entry still matched a 5-fragment body run, and `Mention.name` is copied from
|
||
// the untouched `title` column rather than derived from tokens.
|
||
describe('findMentionedEntities — Vietnamese cases', () => {
|
||
test('VN multi-syllable name matches as a WHOLE (regression: no diacritic fragmentation)', () => {
|
||
// Discriminating assertion: the ASCII tokenizer produced
|
||
// ['nguy','n','v','n','c'] for this title.
|
||
expect(tokenizeTitle('Nguyễn Văn Đức')).toEqual(['nguyễn', 'văn', 'đức']);
|
||
const body = 'Hôm nay mình học bài của thầy Nguyễn Văn Đức.';
|
||
expect(tokenizeForScan(body).map(t => t.text)).toContain('nguyễn');
|
||
|
||
const g = gazetteerFromEntries([
|
||
{ slug: 'people/nguyen-van-duc', source_id: 'default', title: 'Nguyễn Văn Đức' },
|
||
]);
|
||
const mentions = findMentionedEntities(body, g, {
|
||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||
});
|
||
expect(mentions).toHaveLength(1);
|
||
expect(mentions[0]!.slug).toBe('people/nguyen-van-duc');
|
||
expect(mentions[0]!.name).toBe('Nguyễn Văn Đức');
|
||
});
|
||
|
||
test('VN place name with đ/diacritics — "Đà Nẵng" matched', () => {
|
||
// Discriminating assertion: the ASCII tokenizer produced ['n','ng'],
|
||
// which is what made this entity match 820 pages instead of 440.
|
||
expect(tokenizeTitle('Đà Nẵng')).toEqual(['đà', 'nẵng']);
|
||
const body = 'Gia đình mình chuyển tới Đà Nẵng năm ngoái.';
|
||
expect(tokenizeForScan(body).map(t => t.text)).toContain('nẵng');
|
||
|
||
const g = gazetteerFromEntries([
|
||
{ slug: 'places/da-nang', source_id: 'default', title: 'Đà Nẵng' },
|
||
]);
|
||
const mentions = findMentionedEntities(body, g, {
|
||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||
});
|
||
expect(mentions).toHaveLength(1);
|
||
expect(mentions[0]!.slug).toBe('places/da-nang');
|
||
expect(mentions[0]!.name).toBe('Đà Nẵng');
|
||
});
|
||
|
||
test('VN NFD body matches an NFC gazetteer title (and the reverse)', () => {
|
||
const nfc = 'Nguyễn Văn';
|
||
const nfd = nfc.normalize('NFD');
|
||
expect(nfd).not.toBe(nfc); // fixture really is decomposed
|
||
expect(tokenizeTitle(nfd)).toEqual(tokenizeTitle(nfc));
|
||
expect(tokenizeTitle(nfd)).toEqual(['nguyễn', 'văn']);
|
||
|
||
const opts = { fromSlug: 'writing/post-1', fromSourceId: 'default' };
|
||
const gNfc = gazetteerFromEntries([{ slug: 'people/nvd', source_id: 'default', title: nfc }]);
|
||
expect(findMentionedEntities(`Thầy ${nfd} nói.`, gNfc, opts)).toHaveLength(1);
|
||
|
||
const gNfd = gazetteerFromEntries([{ slug: 'people/nvd', source_id: 'default', title: nfd }]);
|
||
expect(findMentionedEntities(`Thầy ${nfc} nói.`, gNfd, opts)).toHaveLength(1);
|
||
});
|
||
|
||
test('VN diacritics are significant — "Hồng" title does NOT match diacritic-free "Hong"', () => {
|
||
// The title must survive tokenization intact for this to mean anything:
|
||
// under the ASCII tokenizer it became ['l','th','h','ng'] and missed for
|
||
// the wrong reason.
|
||
expect(tokenizeTitle('Lê Thị Hồng')).toEqual(['lê', 'thị', 'hồng']);
|
||
const g = gazetteerFromEntries([
|
||
{ slug: 'people/le-thi-hong', source_id: 'default', title: 'Lê Thị Hồng' },
|
||
]);
|
||
// Body uses the ASCII-typed variant "Le Thi Hong" — tokens differ, no false match.
|
||
const mentions = findMentionedEntities('Gặp Le Thi Hong hôm qua.', g, {
|
||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||
});
|
||
expect(mentions).toEqual([]);
|
||
});
|
||
|
||
test('VN mixed with ASCII — Vietnamese name + ASCII company in one body', () => {
|
||
const body = 'Phạm Quốc Bảo hợp tác với Acme.';
|
||
// Discriminating: the ASCII tokenizer emitted ['ph','m','qu','c','b','o',
|
||
// 'h','p','t','c','v','i','acme'] here — only the ASCII control survived.
|
||
expect(tokenizeForScan(body).map(t => t.text))
|
||
.toEqual(['phạm', 'quốc', 'bảo', 'hợp', 'tác', 'với', 'acme']);
|
||
|
||
const g = gazetteerFromEntries([
|
||
{ slug: 'people/pham-quoc-bao', source_id: 'default', title: 'Phạm Quốc Bảo' },
|
||
{ slug: 'companies/acme', source_id: 'default', title: 'Acme' },
|
||
]);
|
||
const mentions = findMentionedEntities(body, g, {
|
||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||
});
|
||
expect(mentions).toHaveLength(2);
|
||
const slugs = mentions.map(m => m.slug);
|
||
expect(slugs).toContain('people/pham-quoc-bao');
|
||
expect(slugs).toContain('companies/acme');
|
||
});
|
||
|
||
test('VN longest-match wins — "Nguyễn Văn Đức" beats a shorter "Nguyễn Văn" entry', () => {
|
||
// Both entries must share a real first token for maximal-munch to be
|
||
// exercised at all; under the ASCII tokenizer both keyed on 'nguy'.
|
||
expect(tokenizeTitle('Nguyễn Văn')).toEqual(['nguyễn', 'văn']);
|
||
expect(tokenizeTitle('Nguyễn Văn Đức')[0]).toBe('nguyễn');
|
||
const g = gazetteerFromEntries([
|
||
{ slug: 'people/nguyen-van-duc', source_id: 'default', title: 'Nguyễn Văn Đức' },
|
||
{ slug: 'people/nguyen-van', source_id: 'default', title: 'Nguyễn Văn' },
|
||
]);
|
||
const mentions = findMentionedEntities('Bài giảng của Nguyễn Văn Đức rất hay.', g, {
|
||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||
});
|
||
expect(mentions).toHaveLength(1);
|
||
expect(mentions[0]!.slug).toBe('people/nguyen-van-duc');
|
||
});
|
||
|
||
test('VN first-mention-only cap — repeated name → single link', () => {
|
||
const g = gazetteerFromEntries([
|
||
{ slug: 'people/nguyen-van-duc', source_id: 'default', title: 'Nguyễn Văn Đức' },
|
||
]);
|
||
const body = 'Nguyễn Văn Đức nói. Sau đó Nguyễn Văn Đức nói tiếp.';
|
||
// The cap must be capping a WHOLE-name match, not a fragment run.
|
||
expect(tokenizeForScan(body).map(t => t.text).slice(0, 3)).toEqual(['nguyễn', 'văn', 'đức']);
|
||
const mentions = findMentionedEntities(body, g, {
|
||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||
});
|
||
expect(mentions).toHaveLength(1);
|
||
});
|
||
|
||
test('VN determinism — identical output across 10 calls', () => {
|
||
const g = gazetteerFromEntries([
|
||
{ slug: 'people/nguyen-van-duc', source_id: 'default', title: 'Nguyễn Văn Đức' },
|
||
{ slug: 'places/da-nang', source_id: 'default', title: 'Đà Nẵng' },
|
||
]);
|
||
const body = 'Thầy Nguyễn Văn Đức ở Đà Nẵng. Nguyễn Văn Đức lần nữa.';
|
||
expect(tokenizeForScan(body).map(t => t.text))
|
||
.toEqual(['thầy', 'nguyễn', 'văn', 'đức', 'ở', 'đà', 'nẵng', 'nguyễn', 'văn', 'đức', 'lần', 'nữa']);
|
||
const refs = new Set<string>();
|
||
for (let i = 0; i < 10; i++) {
|
||
refs.add(JSON.stringify(findMentionedEntities(body, g, {
|
||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||
})));
|
||
}
|
||
expect(refs.size).toBe(1);
|
||
});
|
||
});
|
||
|
||
// ============================================================
|
||
// Tokenizer boundaries — non-word glyphs
|
||
// ============================================================
|
||
|
||
// Guards for the two ways a Unicode tokenizer regresses against the ASCII one
|
||
// it replaces. Both were found in review of the first version of this change,
|
||
// which used /[[\p{L}\p{M}\p{N}]--[CJK]]+/gv: \p{M} let a token consist of
|
||
// combining marks alone, and \p{N} minted tokens the ASCII regex never emitted.
|
||
describe('tokenizer boundaries — marks and non-ASCII numerics', () => {
|
||
const opts = { fromSlug: 'writing/post-1', fromSourceId: 'default' };
|
||
|
||
test('a token can never be combining marks alone (U+FE0F does not become a key)', () => {
|
||
// VARIATION SELECTOR-16 is \p{Mn} and rides on most emoji. Allowing a
|
||
// mark-only token made every emoji-prefixed entity title key on a bare
|
||
// U+FE0F, collapsing them into one mutually-confusable bucket.
|
||
expect(tokenizeTitle('❤️ Health Notes')).toEqual(['health', 'notes']);
|
||
expect(tokenizeTitle('⭐️ Budget Notes')).toEqual(['budget', 'notes']);
|
||
expect(tokenizeForScan('❤️').map(t => t.text)).toEqual([]);
|
||
|
||
const g = gazetteerFromEntries([
|
||
{ slug: 'companies/health-notes', source_id: 'default', title: '❤️ Health Notes' },
|
||
{ slug: 'companies/budget-notes', source_id: 'default', title: '⭐️ Budget Notes' },
|
||
]);
|
||
expect([...g.keys()].sort()).toEqual(['budget', 'health']);
|
||
|
||
// The plain-text link survives...
|
||
expect(findMentionedEntities('Plain health notes, no emoji.', g, opts).map(m => m.slug))
|
||
.toEqual(['companies/health-notes']);
|
||
// ...and an unrelated emoji in the body does not drag in the other entity.
|
||
expect(findMentionedEntities('Sprint ⚠️ health notes were fine.', g, opts).map(m => m.slug))
|
||
.toEqual(['companies/health-notes']);
|
||
});
|
||
|
||
test('non-ASCII numerics do not break strict token adjacency of an ASCII name', () => {
|
||
// findMentionedEntities requires an entry's tokens to be STRICTLY
|
||
// ADJACENT in the body, so any glyph that newly tokenizes between the
|
||
// words of "Acme Corp" silently kills a match that used to work.
|
||
const g = gazetteerFromEntries([
|
||
{ slug: 'companies/acme-corp', source_id: 'default', title: 'Acme Corp' },
|
||
]);
|
||
for (const body of [
|
||
'We met Acme Corp today.', // control
|
||
'We met Acme¹ Corp today.', // U+00B9 superscript one (No)
|
||
'Acme ½ Corp', // U+00BD vulgar fraction (No)
|
||
'Acme １ Corp', // U+FF11 fullwidth digit one (Nd)
|
||
'Acme ❤️ Corp', // emoji + VS16 (So + Mn)
|
||
]) {
|
||
expect(findMentionedEntities(body, g, opts)).toHaveLength(1);
|
||
}
|
||
// ASCII digits still tokenize exactly as /[a-zA-Z0-9]+/ did.
|
||
expect(tokenizeForScan('web3 and h2o').map(t => t.text)).toEqual(['web3', 'and', 'h2o']);
|
||
});
|
||
|
||
test('Han Extension A is no longer CJK here — aligned with cjk.ts scope', () => {
|
||
// Deliberate behaviour change. by-mention's walkers used to carry their
|
||
// own range copy covering Ext-A (U+3400–4DBF); cjk.ts scopes Ext-A out
|
||
// repo-wide, and this module now uses CJK_SLUG_CHARS verbatim. Ext-A
|
||
// therefore tokenizes as a word run instead of per character, and an
|
||
// Ext-A-only title is one sub-MIN_NAME_LENGTH token rather than N
|
||
// char-level ones. Search, chunking and slug grammar already ignore
|
||
// Ext-A, so this removes by-mention as the lone subsystem that disagreed.
|
||
expect(tokenizeForScan('㐀㐁').map(t => t.text)).toEqual(['㐀㐁']);
|
||
expect(tokenizeTitle('㐀㐁')).toEqual(['㐀㐁']);
|
||
// In-scope CJK is untouched: still char-level.
|
||
expect(tokenizeTitle('纳瓦尔')).toEqual(['纳', '瓦', '尔']);
|
||
expect(tokenizeForScan('纳瓦尔说').map(t => t.text)).toEqual(['纳', '瓦', '尔', '说']);
|
||
});
|
||
});
|
||
|
||
// ============================================================
|
||
// buildGazetteer — engine-backed tests
|
||
// ============================================================
|
||
|
||
describe('buildGazetteer — engine integration', () => {
|
||
test('7. min-length filter — title "AI" (length 2) not in gazetteer', async () => {
|
||
await engine.putPage('companies/ai', {
|
||
type: 'company', title: 'AI', compiled_truth: 'b', timeline: '', frontmatter: {},
|
||
});
|
||
await engine.putPage('companies/acme', {
|
||
type: 'company', title: 'Acme', compiled_truth: 'b', timeline: '', frontmatter: {},
|
||
});
|
||
const g = await buildGazetteer(engine);
|
||
expect(g.has('acme')).toBe(true);
|
||
expect(g.has('ai')).toBe(false);
|
||
});
|
||
|
||
test('8. ignore-list at build — "Apple" suppressed when no companies/apple page', async () => {
|
||
// Seed a different entity page named "Apple" — but importantly NO
|
||
// companies/apple slug exists. Wait — actually the ignore-list keys
|
||
// on TITLE not slug. So even a non-companies slug with title="Apple"
|
||
// would be in the gazetteer because `existingTitles.has("Apple")` is true.
|
||
// The ignore list only fires when NO row has title="Apple". To exercise
|
||
// suppression: seed no entity with title="Apple" — duh, then there's
|
||
// nothing in the gazetteer for Apple anyway. The ignore-list rule is
|
||
// only meaningful if a HYPOTHETICAL entity named "Apple" would otherwise
|
||
// appear; in practice, the ignore-list short-circuits ANY row whose
|
||
// title is in the ignore set AND whose title isn't in existingTitles.
|
||
// For a deterministic test: seed one entity with title="Apple" and
|
||
// verify it IS in the gazetteer (per CK12 inverse rule); seed another
|
||
// run with no Apple entity and verify the ignore-list doesn't add one.
|
||
// Both behaviors covered by the existingTitles vs ignore_set logic.
|
||
await engine.putPage('people/alice', {
|
||
type: 'person', title: 'Alice Example', compiled_truth: 'b', timeline: '', frontmatter: {},
|
||
});
|
||
const g = await buildGazetteer(engine);
|
||
// No Apple entity seeded → 'apple' not a gazetteer key (trivially).
|
||
expect(g.has('apple')).toBe(false);
|
||
// Alice IS in gazetteer.
|
||
expect(g.has('alice')).toBe(true);
|
||
});
|
||
|
||
test('9. ignore-list inverse — title "Apple" matches when companies/apple exists (CK12)', async () => {
|
||
await engine.putPage('companies/apple', {
|
||
type: 'company', title: 'Apple', compiled_truth: 'b', timeline: '', frontmatter: {},
|
||
});
|
||
const g = await buildGazetteer(engine);
|
||
// existingTitles has "Apple" so the ignore-list does NOT suppress;
|
||
// gazetteer presence wins per CK12 rule.
|
||
expect(g.has('apple')).toBe(true);
|
||
expect(g.get('apple')![0]!.slug).toBe('companies/apple');
|
||
});
|
||
|
||
test('13. all entity pages soft-deleted → empty gazetteer', async () => {
|
||
await engine.putPage('people/alice', {
|
||
type: 'person', title: 'Alice Example', compiled_truth: 'b', timeline: '', frontmatter: {},
|
||
});
|
||
await engine.softDeletePage('people/alice');
|
||
const g = await buildGazetteer(engine);
|
||
expect(g.size).toBe(0);
|
||
});
|
||
|
||
test('18. hardcoded type filter — page with type=meeting NOT in gazetteer', async () => {
|
||
await engine.putPage('meetings/2026-01-15', {
|
||
type: 'meeting' as any, title: 'Weekly Sync',
|
||
compiled_truth: 'b', timeline: '', frontmatter: {},
|
||
});
|
||
await engine.putPage('people/bob', {
|
||
type: 'person', title: 'Robert Builder', compiled_truth: 'b', timeline: '', frontmatter: {},
|
||
});
|
||
const g = await buildGazetteer(engine);
|
||
expect(g.has('weekly')).toBe(false); // meeting type filtered out
|
||
expect(g.has('robert')).toBe(true); // person type included
|
||
});
|
||
|
||
test('19. min-length + ignore-list interaction — "YC" (2 chars) filtered by min-length BEFORE ignore-list', async () => {
|
||
// YC isn't in DEFAULT_IGNORE_LIST. But "Box" (3 chars) is. "Box" length
|
||
// = 3 < MIN_NAME_LENGTH (4), so it's filtered by min-length first. The
|
||
// ignore-list never fires. We test the regression that the min-length
|
||
// gate runs BEFORE the ignore-list (so adding Box to ignore-list
|
||
// doesn't accidentally change the filter ordering).
|
||
await engine.putPage('companies/box', {
|
||
type: 'company', title: 'Box', compiled_truth: 'b', timeline: '', frontmatter: {},
|
||
});
|
||
const g = await buildGazetteer(engine);
|
||
// "Box" is 3 chars → min-length filter drops it (whether or not in ignore-list).
|
||
expect(g.has('box')).toBe(false);
|
||
});
|
||
|
||
test('extraIgnore — user-supplied additional ignore tokens', async () => {
|
||
await engine.putPage('people/john', {
|
||
type: 'person', title: 'John', compiled_truth: 'b', timeline: '', frontmatter: {},
|
||
});
|
||
// No companies/john exists, so adding John to extraIgnore should suppress.
|
||
const g1 = await buildGazetteer(engine);
|
||
expect(g1.has('john')).toBe(true); // baseline: in gazetteer
|
||
const g2 = await buildGazetteer(engine, { extraIgnore: ['John'] });
|
||
// But title "John" IS the entity title — existingTitles.has('John') is true.
|
||
// Per CK12 rule, gazetteer presence wins → John IS still in.
|
||
expect(g2.has('john')).toBe(true);
|
||
});
|
||
|
||
test('LINKABLE_ENTITY_TYPES exposes the hardcoded contract', () => {
|
||
// Regression: if anyone changes the hardcoded type list, this test
|
||
// forces a deliberate change (and a corresponding test update).
|
||
expect(LINKABLE_ENTITY_TYPES).toEqual(['person', 'company', 'organization', 'entity']);
|
||
});
|
||
|
||
// CJK — engine-backed tests
|
||
test('CJK entity with 2-char title enters gazetteer with char-level tokens', async () => {
|
||
await engine.putPage('people/naval', {
|
||
type: 'person', title: '纳瓦尔', compiled_truth: 'b', timeline: '', frontmatter: {},
|
||
});
|
||
const g = await buildGazetteer(engine);
|
||
// "纳瓦尔" tokenized as ["纳","瓦","尔"] → key is "纳"
|
||
expect(g.has('纳')).toBe(true);
|
||
const bucket = g.get('纳')!;
|
||
expect(bucket.length).toBe(1);
|
||
expect(bucket[0]!.tokens).toEqual(['纳', '瓦', '尔']);
|
||
expect(bucket[0]!.slug).toBe('people/naval');
|
||
});
|
||
|
||
test('CJK single-char title (cjkCharCount < 2) excluded from gazetteer', async () => {
|
||
await engine.putPage('people/x', {
|
||
type: 'person', title: '谢', compiled_truth: 'b', timeline: '', frontmatter: {},
|
||
});
|
||
const g = await buildGazetteer(engine);
|
||
expect(g.size).toBe(0);
|
||
});
|
||
|
||
test('VN person title enters gazetteer keyed on first diacritic-preserving token', async () => {
|
||
await engine.putPage('people/nguyen-van-duc', {
|
||
type: 'person', title: 'Nguyễn Văn Đức', compiled_truth: 'b', timeline: '', frontmatter: {},
|
||
});
|
||
const g = await buildGazetteer(engine);
|
||
// "Nguyễn Văn Đức" → ["nguyễn","văn","đức"], keyed on "nguyễn" (NOT fragmented to "nguy").
|
||
expect(g.has('nguyễn')).toBe(true);
|
||
const bucket = g.get('nguyễn')!;
|
||
expect(bucket[0]!.tokens).toEqual(['nguyễn', 'văn', 'đức']);
|
||
expect(bucket[0]!.slug).toBe('people/nguyen-van-duc');
|
||
// Regression guard: the old ASCII tokenizer would have keyed on "nguy".
|
||
expect(g.has('nguy')).toBe(false);
|
||
});
|
||
});
|