mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-14 08:53:22 +00:00
fix(enrich): keep entity-name candidates on a single line (#3991)
* fix(enrich): keep entity-name candidates on a single line extractEntities()'s namePattern used \s+ between capitalized words, which matches across line breaks. A capitalized word ending one paragraph and a capitalized word starting the next were spliced into one bogus multi-word candidate whose name embedded the line break(s). Scope the pattern to same-line whitespace: [^\S\r\n ]+ (spaces, tabs, NBSP, other Unicode space separators, but not \n, \r, or the Unicode line/paragraph separators U+2028/U+2029). Same-line multi-word names and NBSP-separated names still match; any line break between two capitalized words now stops the match, same as it already does for punctuation. Surfaced during single-user dogfooding on a ~100-page install (v0.44.0.0): extractEntities produced ~1,620 raw candidates, and these paragraph-spanning splices were the largest single source of noise. * fix(enrich): exclude vertical tab and form feed from name-splice guard The line-scoping fix in the prior commit excluded CR, LF, and the Unicode line/paragraph separators (U+2028/U+2029) from the same-line whitespace class, but still treated VT (U+000B) and FF (U+000C) as same-line whitespace since \s matches them and they weren't in the excluded set. Winters<VT>Reyes and Winters<FF>Reyes were still spliced into one bogus candidate, contradicting the "any line break stops the match" intent. Add \v and \f to the excluded set, so the pattern now excludes every vertical whitespace code point (CR, LF, VT, FF, U+2028, U+2029) while still retaining horizontal whitespace (spaces, tabs, NBSP, other Unicode space separators) between same-line words.
This commit is contained in:
@@ -284,8 +284,14 @@ export function extractEntities(text: string): Array<{ name: string; type: 'pers
|
||||
const seen = new Set<string>();
|
||||
|
||||
// Match capitalized multi-word names (likely people or companies)
|
||||
// Pattern: 2-4 capitalized words in sequence
|
||||
const namePattern = /\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+){1,3})\b/g;
|
||||
// Pattern: 2-4 capitalized words in sequence, scoped to a single line so a
|
||||
// paragraph break (or any line break) can't splice unrelated capitalized
|
||||
// words from adjacent sentences into one bogus candidate. The pattern
|
||||
// excludes all vertical whitespace -- CR, LF, VT (U+000B), FF (U+000C),
|
||||
// and the Unicode line/paragraph separators (U+2028/U+2029) -- while
|
||||
// retaining horizontal whitespace (spaces, tabs, NBSP, other Unicode
|
||||
// space separators), unlike a plain `[ \t]+`.
|
||||
const namePattern = /\b([A-Z][a-z]+(?:[^\S\r\n\v\f\u2028\u2029]+[A-Z][a-z]+){1,3})\b/g;
|
||||
let match;
|
||||
while ((match = namePattern.exec(text)) !== null) {
|
||||
const name = match[1];
|
||||
|
||||
@@ -78,6 +78,47 @@ describe('enrichment-service', () => {
|
||||
const entities = extractEntities('Mary Jane Watson Parker joined the team.');
|
||||
expect(entities.some(e => e.name.split(' ').length >= 3)).toBe(true);
|
||||
});
|
||||
|
||||
test('does not merge capitalized words across a paragraph break', () => {
|
||||
const entities = extractEntities('We spoke with Winters\n\nReyes continued the analysis.');
|
||||
expect(entities.some(e => e.name.includes('\n'))).toBe(false);
|
||||
expect(entities.some(e => e.name.replace(/\s+/g, ' ') === 'Winters Reyes')).toBe(false);
|
||||
});
|
||||
|
||||
test('does not merge capitalized words across a single line break', () => {
|
||||
const entities = extractEntities('Winters\nReyes discussed the proof.');
|
||||
expect(entities.some(e => e.name.includes('\n'))).toBe(false);
|
||||
});
|
||||
|
||||
test('still matches same-line multi-word capitalized names', () => {
|
||||
const entities = extractEntities('Casey Morgan visited the lab.');
|
||||
expect(entities.some(e => e.name === 'Casey Morgan')).toBe(true);
|
||||
});
|
||||
|
||||
test('still matches names separated by a non-breaking space on the same line', () => {
|
||||
const entities = extractEntities('Jordan Blake requested access.');
|
||||
expect(entities.some(e => e.name === 'Jordan Blake')).toBe(true);
|
||||
});
|
||||
|
||||
test('does not merge capitalized words across a Unicode line separator (U+2028)', () => {
|
||||
const entities = extractEntities('Winters\u2028Reyes discussed the proof.');
|
||||
expect(entities.some(e => e.name.includes('\u2028'))).toBe(false);
|
||||
});
|
||||
|
||||
test('does not merge capitalized words across a Unicode paragraph separator (U+2029)', () => {
|
||||
const entities = extractEntities('Winters\u2029Reyes discussed the proof.');
|
||||
expect(entities.some(e => e.name.includes('\u2029'))).toBe(false);
|
||||
});
|
||||
|
||||
test('does not merge capitalized words across a vertical tab (U+000B)', () => {
|
||||
const entities = extractEntities('Winters\vReyes discussed the proof.');
|
||||
expect(entities.some(e => e.name.includes('\v'))).toBe(false);
|
||||
});
|
||||
|
||||
test('does not merge capitalized words across a form feed (U+000C)', () => {
|
||||
const entities = extractEntities('Winters\fReyes discussed the proof.');
|
||||
expect(entities.some(e => e.name.includes('\f'))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('enrichEntity (mock)', () => {
|
||||
|
||||
Reference in New Issue
Block a user