mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-14 17:02:19 +00:00
Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a579c3686e | ||
|
|
4b76e6f13a | ||
|
|
9a9195fffe | ||
|
|
17fa84cda5 | ||
|
|
5df66c3855 |
+13
-8
@@ -1,7 +1,7 @@
|
||||
---
|
||||
id: x-to-brain
|
||||
name: X-to-Brain
|
||||
version: 0.8.1
|
||||
version: 0.8.2
|
||||
description: Twitter timeline, mentions, and keyword monitoring flow into brain pages. Tracks deletions, engagement velocity, OCR on images, and real-time alerts.
|
||||
category: sense
|
||||
requires: []
|
||||
@@ -9,9 +9,12 @@ secrets:
|
||||
- name: X_BEARER_TOKEN
|
||||
description: X API v2 Bearer token (Basic tier minimum, $200/mo for full archive search)
|
||||
where: https://developer.x.com/en/portal/dashboard — create a project + app, copy the Bearer Token from "Keys and tokens"
|
||||
- name: X_HANDLE
|
||||
description: Your X username without the @ (used for the app-only health check — /users/me requires user-context OAuth, which app-only bearer tokens don't have)
|
||||
where: Your X profile — the handle in your profile URL, e.g. x.com/yourhandle → yourhandle
|
||||
health_checks:
|
||||
- type: http
|
||||
url: "https://api.x.com/2/users/me"
|
||||
url: "https://api.x.com/2/users/by/username/$X_HANDLE"
|
||||
auth: bearer
|
||||
auth_token: "$X_BEARER_TOKEN"
|
||||
label: "X API"
|
||||
@@ -110,15 +113,17 @@ Tell the user:
|
||||
4. Inside the project, create a new App
|
||||
5. Go to the app's 'Keys and tokens' tab
|
||||
6. Under 'Bearer Token', click 'Generate' (or 'Regenerate')
|
||||
7. Copy the Bearer Token and paste it to me
|
||||
7. Copy the Bearer Token and paste it to me, along with your X handle (without the @)
|
||||
|
||||
Note: Free tier gives read-only access with low limits. Basic tier ($200/mo)
|
||||
gives search/recent endpoint and higher limits. Pro tier gets full archive search."
|
||||
|
||||
Validate immediately:
|
||||
Set both `X_BEARER_TOKEN` and `X_HANDLE` in the environment. Validate immediately
|
||||
(app-only bearer tokens cannot call `/users/me` — that endpoint requires
|
||||
user-context OAuth — so validation uses the by-username lookup):
|
||||
```bash
|
||||
curl -sf -H "Authorization: Bearer $X_BEARER_TOKEN" \
|
||||
"https://api.x.com/2/users/me" \
|
||||
"https://api.x.com/2/users/by/username/$X_HANDLE" \
|
||||
&& echo "PASS: X API connected" \
|
||||
|| echo "FAIL: X API token invalid"
|
||||
```
|
||||
@@ -134,10 +139,10 @@ starting with 'AAA...', (3) if you just created the app, the token is valid imme
|
||||
```bash
|
||||
# Look up the user's X user ID from their handle
|
||||
curl -sf -H "Authorization: Bearer $X_BEARER_TOKEN" \
|
||||
"https://api.x.com/2/users/by/username/USERNAME" | grep -o '"id":"[^"]*"'
|
||||
"https://api.x.com/2/users/by/username/$X_HANDLE" | grep -o '"id":"[^"]*"'
|
||||
```
|
||||
|
||||
Ask the user for their X handle (e.g., @yourhandle). Look up their user ID.
|
||||
Look up the user ID from the handle collected in Step 1.
|
||||
Save it — the collector needs the numeric ID, not the handle.
|
||||
|
||||
### Step 3: Configure the Collector
|
||||
@@ -205,7 +210,7 @@ The agent should review collected data 2-3x daily and run enrichment.
|
||||
|
||||
```bash
|
||||
mkdir -p ~/.gbrain/integrations/x-to-brain
|
||||
echo '{"ts":"'$(date -u +%Y-%m-%dT%H:%M:%SZ)'","event":"setup_complete","source_version":"0.8.1","status":"ok","details":{"user_id":"X_USER_ID"}}' >> ~/.gbrain/integrations/x-to-brain/heartbeat.jsonl
|
||||
echo '{"ts":"'$(date -u +%Y-%m-%dT%H:%M:%SZ)'","event":"setup_complete","source_version":"0.8.2","status":"ok","details":{"user_id":"X_USER_ID"}}' >> ~/.gbrain/integrations/x-to-brain/heartbeat.jsonl
|
||||
```
|
||||
|
||||
## Production Patterns (v0.8.1)
|
||||
|
||||
@@ -62,7 +62,7 @@ gbrain capture "..." --json # structured output for agents
|
||||
- **Slug:** `inbox/YYYY-MM-DD-<hash8>` (stable for same content; the daemon's 24h dedup catches re-captures).
|
||||
- **Type:** `note` (override with `--type idea` etc.).
|
||||
- **Frontmatter stamps:** `captured_via: capture-cli`, `captured_at: <ISO>`.
|
||||
- **Title:** first non-empty line of the body, capped at 80 chars.
|
||||
- **Title:** first non-empty line of the body, capped at 80 chars (truncation appends `…`).
|
||||
|
||||
## Output Format
|
||||
|
||||
|
||||
@@ -233,14 +233,18 @@ export function maybeRewriteSourceFkError(err: unknown, sourceId: string | undef
|
||||
|
||||
/**
|
||||
* Derive a title from the first non-empty, non-`---` line of the body,
|
||||
* stripping leading markdown heading marks, capped at 80 chars.
|
||||
* stripping leading markdown heading marks, capped at 80 chars. Truncation
|
||||
* is codepoint-aware (never splits an astral surrogate pair) and appends an
|
||||
* ellipsis so a cut title is visibly cut.
|
||||
* Falls back to 'Capture' when no usable line exists.
|
||||
*/
|
||||
function deriveTitle(rawBody: string): string {
|
||||
const firstLine = rawBody
|
||||
.split('\n')
|
||||
.find((l) => l.trim().length > 0 && l.trim() !== '---') ?? '';
|
||||
return firstLine.replace(/^#+\s*/, '').slice(0, 80) || 'Capture';
|
||||
const stripped = firstLine.replace(/^#+\s*/, '');
|
||||
const cps = [...stripped];
|
||||
return (cps.length > 80 ? cps.slice(0, 79).join('') + '…' : stripped) || 'Capture';
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -167,6 +167,33 @@ export function deriveDirectUrl(url: string): string | null {
|
||||
}
|
||||
}
|
||||
|
||||
/** True when the URL targets Supavisor transaction mode (port 6543). */
|
||||
function isTransactionPoolerUrl(url: string): boolean {
|
||||
try {
|
||||
return new URL(url.replace(/^postgres(ql)?:\/\//, 'http://')).port === '6543';
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the direct-pool URL from an explicit/env override + the primary URL.
|
||||
*
|
||||
* A direct override still pointing at the TRANSACTION-mode pooler (port 6543,
|
||||
* usually a copy-paste of the primary URL) is a misconfiguration: the manager
|
||||
* would believe DDL/bulk work runs on a long-timeout direct connection while
|
||||
* still routing through Supavisor transaction mode (short timeouts, no
|
||||
* prepared statements). Normalize it via deriveDirectUrl. Session-mode pooler
|
||||
* URLs (pooler host, port 5432) pass through — they are a legitimate
|
||||
* direct-ish target when the db.<ref> host is unreachable.
|
||||
*/
|
||||
export function normalizeDirectUrl(primaryUrl: string, override?: string | null): string | null {
|
||||
const candidate = override ?? deriveDirectUrl(primaryUrl);
|
||||
if (!candidate) return null;
|
||||
if (!isTransactionPoolerUrl(candidate)) return candidate;
|
||||
return deriveDirectUrl(candidate) ?? deriveDirectUrl(primaryUrl);
|
||||
}
|
||||
|
||||
/**
|
||||
* Read kill-switch state from env. Subordinate to parent manager's state
|
||||
* when present (A2 inheritance).
|
||||
@@ -213,9 +240,10 @@ export class ConnectionManager {
|
||||
} else {
|
||||
this._killSwitch = readKillSwitchEnv();
|
||||
this._isSupabase = isSupabasePoolerUrl(opts.url);
|
||||
// Direct URL: explicit override > env > derive > null
|
||||
// Direct URL: explicit override > env > derive > null. Pooler-shaped
|
||||
// overrides are normalized to a real direct host (or dropped).
|
||||
const envOverride = process.env.GBRAIN_DIRECT_DATABASE_URL;
|
||||
this._directUrl = opts.directUrl ?? envOverride ?? deriveDirectUrl(opts.url);
|
||||
this._directUrl = normalizeDirectUrl(opts.url, opts.directUrl ?? envOverride);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -83,6 +83,14 @@ const SYNTHESIS_OUTPUT_TYPES = new Set<string>(['atom', 'concept']);
|
||||
|
||||
const PAGE_DISCOVERY_BUDGET = 50;
|
||||
const MIN_PAGE_CHARS_FOR_EXTRACTION = 500;
|
||||
// Source pages whose frontmatter declares a `raw` payload pointer hold raw
|
||||
// import data, not extractable prose. Extraction on them yields zero atoms,
|
||||
// so no atom row is ever written and they re-enter discovery + the doctor
|
||||
// backlog count on every cycle — a permanent no-progress loop. Shared by
|
||||
// discoverExtractablePages and countExtractAtomsBacklog so the phase and the
|
||||
// doctor check can't drift.
|
||||
const RAW_SOURCE_HOLDER_EXCLUSION_SQL =
|
||||
`AND NOT (p.type = 'source' AND COALESCE(p.frontmatter ? 'raw', false))`;
|
||||
|
||||
/**
|
||||
* Pure allowlist policy: the legacy floor UNION the pack's `extractable: true`
|
||||
@@ -207,6 +215,10 @@ interface DiscoveredPage {
|
||||
* participate in the NOT EXISTS check anyway.
|
||||
* #4 dream_generated exclusion — prevents the phase from chewing
|
||||
* its own output (e.g. dream-generated originals).
|
||||
* #5 raw source-holder exclusion — source pages that only point at a raw
|
||||
* import payload are not extractable prose; counting them creates a
|
||||
* permanent backlog/no-progress loop (see
|
||||
* RAW_SOURCE_HOLDER_EXCLUSION_SQL).
|
||||
*/
|
||||
export async function discoverExtractablePages(
|
||||
engine: BrainEngine,
|
||||
@@ -225,6 +237,7 @@ export async function discoverExtractablePages(
|
||||
AND p.content_hash IS NOT NULL
|
||||
AND COALESCE(p.frontmatter->>'imported_from', '') <> 'markdown-greenfield'
|
||||
AND COALESCE(p.frontmatter->>'dream_generated', '') <> 'true'
|
||||
${RAW_SOURCE_HOLDER_EXCLUSION_SQL}
|
||||
AND length(COALESCE(p.compiled_truth, '')) >= $3
|
||||
${hasFilter ? "AND p.slug = ANY($5::text[])" : ''}
|
||||
AND NOT EXISTS (
|
||||
@@ -297,6 +310,7 @@ export async function countExtractAtomsBacklog(
|
||||
AND p.content_hash IS NOT NULL
|
||||
AND COALESCE(p.frontmatter->>'imported_from', '') <> 'markdown-greenfield'
|
||||
AND COALESCE(p.frontmatter->>'dream_generated', '') <> 'true'
|
||||
${RAW_SOURCE_HOLDER_EXCLUSION_SQL}
|
||||
AND length(COALESCE(p.compiled_truth, '')) >= $3
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM pages atom
|
||||
@@ -310,6 +324,7 @@ export async function countExtractAtomsBacklog(
|
||||
AND p.content_hash IS NOT NULL
|
||||
AND COALESCE(p.frontmatter->>'imported_from', '') <> 'markdown-greenfield'
|
||||
AND COALESCE(p.frontmatter->>'dream_generated', '') <> 'true'
|
||||
${RAW_SOURCE_HOLDER_EXCLUSION_SQL}
|
||||
AND length(COALESCE(p.compiled_truth, '')) >= $2
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM pages atom
|
||||
|
||||
@@ -145,6 +145,8 @@ export interface ProposeTakesOpts extends BasePhaseOpts {
|
||||
model?: string;
|
||||
/** Skip pages that already have a complete takes fence. Default: true. */
|
||||
skipPagesWithFence?: boolean;
|
||||
/** Override the phase wall-clock deadline (tests). Default: 30 min. */
|
||||
deadlineMs?: number;
|
||||
}
|
||||
|
||||
export interface ProposeTakesResult {
|
||||
@@ -153,6 +155,8 @@ export interface ProposeTakesResult {
|
||||
cache_misses: number;
|
||||
proposals_inserted: number;
|
||||
budget_exhausted: boolean;
|
||||
/** True when the phase deadline fired before the page loop completed (partial result). */
|
||||
deadline_hit?: boolean;
|
||||
warnings: string[];
|
||||
}
|
||||
|
||||
@@ -210,6 +214,9 @@ export function extractExistingTakesForDedup(pageBody: string): Array<{
|
||||
return rows;
|
||||
}
|
||||
|
||||
/** Per-call wall-clock timeout for the extractor LLM call. */
|
||||
const EXTRACTOR_CALL_TIMEOUT_MS = 90_000;
|
||||
|
||||
/**
|
||||
* Production extractor — calls gateway.chat with the EXTRACT_TAKES_PROMPT
|
||||
* and parses the JSON array output. Returns [] on parse failure (logged as
|
||||
@@ -227,10 +234,14 @@ export async function defaultExtractor(
|
||||
.replace('{EXISTING_TAKES_JSON}', JSON.stringify(input.existingTakes, null, 2))
|
||||
.replace('{PAGE_BODY}', input.pageBody);
|
||||
|
||||
// Bound each call so one stalled provider socket can't pin the phase for the
|
||||
// full gateway default (GBRAIN_AI_CHAT_TIMEOUT_MS, 300s) x pageLimit. The
|
||||
// caller already catches per-page errors, logs a warning, and continues.
|
||||
const result = await gatewayChat({
|
||||
messages: [{ role: 'user', content: prompt }],
|
||||
...(input.modelHint ? { model: input.modelHint } : {}),
|
||||
maxTokens: 2048,
|
||||
abortSignal: AbortSignal.timeout(EXTRACTOR_CALL_TIMEOUT_MS),
|
||||
});
|
||||
|
||||
// ChatResult.text is already the concatenated text content.
|
||||
@@ -287,6 +298,14 @@ class ProposeTakesPhase extends BaseCyclePhase {
|
||||
readonly name = 'propose_takes' as CyclePhase;
|
||||
protected readonly budgetUsdKey = 'cycle.propose_takes.budget_usd';
|
||||
protected readonly budgetUsdDefault = 5.0;
|
||||
/**
|
||||
* Hard wall-clock deadline for the phase. Even with the per-call timeout in
|
||||
* defaultExtractor, a long tail of slow-but-completing calls can accumulate.
|
||||
* The phase breaks cleanly and returns a partial result with
|
||||
* `deadline_hit: true` instead of being killed mid-write by an outer
|
||||
* `timeout` wrapper (the recurring SIGTERM in nightly dream runs).
|
||||
*/
|
||||
private static readonly PHASE_DEADLINE_MS = 30 * 60 * 1000;
|
||||
|
||||
protected override mapErrorCode(err: unknown): string {
|
||||
if (err instanceof GBrainError) return err.problem;
|
||||
@@ -307,6 +326,8 @@ class ProposeTakesPhase extends BaseCyclePhase {
|
||||
const promptVersion = opts.promptVersion ?? PROPOSE_TAKES_PROMPT_VERSION;
|
||||
const pageLimit = opts.pageLimit ?? 100;
|
||||
const skipPagesWithFence = opts.skipPagesWithFence ?? false;
|
||||
const deadlineMs = opts.deadlineMs ?? ProposeTakesPhase.PHASE_DEADLINE_MS;
|
||||
const phaseStartMs = Date.now();
|
||||
const proposalRunId = `propose-${new Date().toISOString().slice(0, 19).replace(/[-:T]/g, '')}-${randomUUID().slice(0, 8)}`;
|
||||
|
||||
const result: ProposeTakesResult = {
|
||||
@@ -333,6 +354,18 @@ class ProposeTakesPhase extends BaseCyclePhase {
|
||||
const modelId = opts.model ?? getChatModel();
|
||||
|
||||
for (const page of pages) {
|
||||
// Phase deadline check. Break (not throw) so the phase returns a
|
||||
// partial result with deadline_hit:true; work already banked stays.
|
||||
const elapsedMs = Date.now() - phaseStartMs;
|
||||
if (elapsedMs > deadlineMs) {
|
||||
result.warnings.push(
|
||||
`phase deadline hit at page ${result.pages_scanned}/${pages.length} ` +
|
||||
`after ${(elapsedMs / 1000).toFixed(0)}s (cap ${(deadlineMs / 1000).toFixed(0)}s); partial completion`,
|
||||
);
|
||||
result.deadline_hit = true;
|
||||
break;
|
||||
}
|
||||
|
||||
result.pages_scanned += 1;
|
||||
this.tick(opts);
|
||||
|
||||
@@ -440,17 +473,20 @@ class ProposeTakesPhase extends BaseCyclePhase {
|
||||
console.error(`[propose_takes] receipt write failed: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
// A deadline-hit run halted mid-list the same way a budget-exhausted one
|
||||
// does — record it as a halt, not a completed round.
|
||||
const halted = result.budget_exhausted || result.deadline_hit === true;
|
||||
await upsertExtractRollup(engine, {
|
||||
kind: 'takes.proposed',
|
||||
source_id: sourceIdForReceipt,
|
||||
round_completed_delta: result.budget_exhausted ? 0 : 1,
|
||||
halt_delta: result.budget_exhausted ? 1 : 0,
|
||||
round_completed_delta: halted ? 0 : 1,
|
||||
halt_delta: halted ? 1 : 0,
|
||||
});
|
||||
|
||||
return {
|
||||
summary: `propose_takes: scanned ${result.pages_scanned} pages, ${result.cache_hits} cached, ${result.proposals_inserted} new proposals (run ${proposalRunId})`,
|
||||
details: { ...result, proposal_run_id: proposalRunId, prompt_version: promptVersion },
|
||||
status: result.budget_exhausted ? 'warn' : 'ok',
|
||||
status: result.budget_exhausted || result.deadline_hit ? 'warn' : 'ok',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -190,9 +190,23 @@ describe('deriveTitle (no-frontmatter path)', () => {
|
||||
expect(deriveTitle('### Triple hash\nrest')).toBe('Triple hash');
|
||||
});
|
||||
|
||||
test('caps at 80 chars', () => {
|
||||
test('caps at 80 chars with explicit ellipsis', () => {
|
||||
const long = 'a'.repeat(120);
|
||||
expect(deriveTitle(long)).toBe('a'.repeat(80));
|
||||
expect(deriveTitle(long)).toBe('a'.repeat(79) + '…');
|
||||
expect([...deriveTitle(long)].length).toBe(80);
|
||||
});
|
||||
|
||||
test('exactly 80 chars is not truncated', () => {
|
||||
const exact = 'a'.repeat(80);
|
||||
expect(deriveTitle(exact)).toBe(exact);
|
||||
});
|
||||
|
||||
test('does not split astral Unicode while truncating', () => {
|
||||
const title = deriveTitle('😀'.repeat(120));
|
||||
expect(title.endsWith('…')).toBe(true);
|
||||
expect([...title].length).toBe(80);
|
||||
// No lone surrogate halves left behind by the cut.
|
||||
expect([...title].some((ch) => ch.length === 1 && /[\uD800-\uDFFF]/.test(ch))).toBe(false);
|
||||
});
|
||||
|
||||
test('falls back to Capture for empty input', () => {
|
||||
|
||||
@@ -136,12 +136,13 @@ describe('capture — buildContent', () => {
|
||||
expect(parsed.data.title).toBe('Real first line');
|
||||
});
|
||||
|
||||
test('caps title at 80 chars', () => {
|
||||
test('caps title at 80 chars with explicit ellipsis', () => {
|
||||
const longLine = 'x'.repeat(200);
|
||||
const result = __testing.buildContent(longLine, {});
|
||||
const parsed = matter(result);
|
||||
expect(typeof parsed.data.title).toBe('string');
|
||||
expect((parsed.data.title as string).length).toBeLessThanOrEqual(80);
|
||||
expect((parsed.data.title as string).endsWith('…')).toBe(true);
|
||||
});
|
||||
|
||||
test('honors --source via captured_via', () => {
|
||||
|
||||
@@ -2,6 +2,7 @@ import { describe, expect, test, beforeEach, afterEach } from 'bun:test';
|
||||
import {
|
||||
isSupabasePoolerUrl,
|
||||
deriveDirectUrl,
|
||||
normalizeDirectUrl,
|
||||
readKillSwitchEnv,
|
||||
resolveDirectPoolSize,
|
||||
ConnectionManager,
|
||||
@@ -78,6 +79,44 @@ describe('deriveDirectUrl', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('normalizeDirectUrl', () => {
|
||||
test('normalizes a transaction-pooler (6543) override to the real direct host', () => {
|
||||
const direct = normalizeDirectUrl(
|
||||
'postgresql://postgres.abcxyz:p@aws-0-us-west-2.pooler.supabase.com:6543/postgres',
|
||||
'postgresql://postgres.abcxyz:p@aws-0-us-west-2.pooler.supabase.com:6543/postgres',
|
||||
);
|
||||
expect(direct).toBe('postgresql://postgres:p@db.abcxyz.supabase.co:5432/postgres');
|
||||
});
|
||||
|
||||
test('keeps a non-pooler direct override', () => {
|
||||
const direct = normalizeDirectUrl(
|
||||
'postgresql://postgres.abc:p@aws.pooler.supabase.com:6543/db',
|
||||
'postgresql://u:p@custom-direct.example.com:5432/db',
|
||||
);
|
||||
expect(direct).toBe('postgresql://u:p@custom-direct.example.com:5432/db');
|
||||
});
|
||||
|
||||
test('keeps a session-mode pooler override (pooler host, port 5432)', () => {
|
||||
const sessionUrl = 'postgresql://postgres.abc:p@aws.pooler.supabase.com:5432/db';
|
||||
const direct = normalizeDirectUrl(
|
||||
'postgresql://postgres.abc:p@aws.pooler.supabase.com:6543/db',
|
||||
sessionUrl,
|
||||
);
|
||||
expect(direct).toBe(sessionUrl);
|
||||
});
|
||||
|
||||
test('no override: derives from the primary as before', () => {
|
||||
const direct = normalizeDirectUrl(
|
||||
'postgresql://postgres.abc:p@aws.pooler.supabase.com:6543/db',
|
||||
);
|
||||
expect(direct).toContain('db.abc.supabase.co:5432');
|
||||
});
|
||||
|
||||
test('no override, non-Supabase primary: null', () => {
|
||||
expect(normalizeDirectUrl('postgresql://u:p@localhost:5432/db')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('readKillSwitchEnv', () => {
|
||||
let original: string | undefined;
|
||||
beforeEach(() => { original = process.env.GBRAIN_DISABLE_DIRECT_POOL; });
|
||||
@@ -145,13 +184,18 @@ describe('resolveDirectPoolSize', () => {
|
||||
|
||||
describe('ConnectionManager — describeMode + dual-pool routing', () => {
|
||||
let originalKillSwitch: string | undefined;
|
||||
let originalDirectUrl: string | undefined;
|
||||
beforeEach(() => {
|
||||
originalKillSwitch = process.env.GBRAIN_DISABLE_DIRECT_POOL;
|
||||
originalDirectUrl = process.env.GBRAIN_DIRECT_DATABASE_URL;
|
||||
delete process.env.GBRAIN_DISABLE_DIRECT_POOL;
|
||||
delete process.env.GBRAIN_DIRECT_DATABASE_URL;
|
||||
});
|
||||
afterEach(() => {
|
||||
if (originalKillSwitch === undefined) delete process.env.GBRAIN_DISABLE_DIRECT_POOL;
|
||||
else process.env.GBRAIN_DISABLE_DIRECT_POOL = originalKillSwitch;
|
||||
if (originalDirectUrl === undefined) delete process.env.GBRAIN_DIRECT_DATABASE_URL;
|
||||
else process.env.GBRAIN_DIRECT_DATABASE_URL = originalDirectUrl;
|
||||
});
|
||||
|
||||
test('non-Supabase URL → single mode', () => {
|
||||
@@ -190,6 +234,25 @@ describe('ConnectionManager — describeMode + dual-pool routing', () => {
|
||||
expect(cm.resolveDirectUrl()).toContain('custom-direct.example.com');
|
||||
});
|
||||
|
||||
test('explicit transaction-pooler directUrl override is normalized to direct host', () => {
|
||||
const cm = new ConnectionManager({
|
||||
url: 'postgresql://postgres.abc:p@aws.pooler.supabase.com:6543/db',
|
||||
directUrl: 'postgresql://postgres.abc:p@aws.pooler.supabase.com:6543/db',
|
||||
});
|
||||
expect(cm.resolveDirectUrl()).toContain('db.abc.supabase.co:5432');
|
||||
expect(cm.resolveDirectUrl()).not.toContain(':6543');
|
||||
});
|
||||
|
||||
test('env transaction-pooler directUrl override is normalized to direct host', () => {
|
||||
process.env.GBRAIN_DIRECT_DATABASE_URL =
|
||||
'postgresql://postgres.abc:p@aws.pooler.supabase.com:6543/db';
|
||||
const cm = new ConnectionManager({
|
||||
url: 'postgresql://postgres.abc:p@aws.pooler.supabase.com:6543/db',
|
||||
});
|
||||
expect(cm.resolveDirectUrl()).toContain('db.abc.supabase.co:5432');
|
||||
expect(cm.resolveDirectUrl()).not.toContain(':6543');
|
||||
});
|
||||
|
||||
test('host string contains creds neither in describeMode nor resolveDirectUrl logging', () => {
|
||||
const cm = new ConnectionManager({
|
||||
url: 'postgresql://postgres.abc:secret@aws.pooler.supabase.com:6543/db',
|
||||
|
||||
@@ -76,6 +76,17 @@ describe('countExtractAtomsBacklog (issue #1678)', () => {
|
||||
});
|
||||
expect(await countExtractAtomsBacklog(engine)).toBe(0);
|
||||
});
|
||||
|
||||
it('ignores raw source-holder pages (permanent no-progress backlog otherwise)', async () => {
|
||||
await engine.putPage('wiki/raw-email-source', {
|
||||
type: 'source',
|
||||
title: 'Raw email source',
|
||||
compiled_truth: BODY,
|
||||
frontmatter: { raw: 'raw/email/example.md' },
|
||||
});
|
||||
expect(await countExtractAtomsBacklog(engine)).toBe(0);
|
||||
expect(await countExtractAtomsBacklog(engine, 'default')).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('computeExtractAtomsBacklogCheck (issue #1678)', () => {
|
||||
|
||||
@@ -178,6 +178,18 @@ describe('v0.41.2.1: discoverExtractablePages SQL contract', () => {
|
||||
expect(discovered.map((d) => d.slug)).toEqual(['original/normal']);
|
||||
});
|
||||
|
||||
test('raw source-holder pages excluded (#5 — no permanent no-progress backlog)', async () => {
|
||||
await seedPage({ slug: 'source/normal', type: 'source' });
|
||||
await seedPage({
|
||||
slug: 'wiki/raw-email-source',
|
||||
type: 'source',
|
||||
frontmatter: { raw: 'raw/email/example.md' },
|
||||
});
|
||||
|
||||
const discovered = await discoverExtractablePages(engine, 'default');
|
||||
expect(discovered.map((d) => d.slug)).toEqual(['source/normal']);
|
||||
});
|
||||
|
||||
test('pages with NULL content_hash excluded (D9 #3 — no .slice crash)', async () => {
|
||||
await seedPage({ slug: 'meeting/with-hash', type: 'meeting' });
|
||||
await seedPage({ slug: 'meeting/no-hash', type: 'meeting', content_hash: null });
|
||||
|
||||
@@ -263,6 +263,35 @@ describe('twilio-voice-brain recipe', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('x-to-brain recipe', () => {
|
||||
test('health check works with an app-only bearer token (#2343)', () => {
|
||||
const { readFileSync } = require('fs');
|
||||
const content = readFileSync(
|
||||
new URL('../recipes/x-to-brain.md', import.meta.url),
|
||||
'utf-8'
|
||||
);
|
||||
const recipe = parseRecipe(content, 'x-to-brain.md');
|
||||
expect(recipe).not.toBeNull();
|
||||
const httpChecks = recipe!.frontmatter.health_checks
|
||||
.filter((c: any) => typeof c === 'object' && c.type === 'http');
|
||||
expect(httpChecks.length).toBeGreaterThan(0);
|
||||
const secretNames = new Set(recipe!.frontmatter.secrets.map((s: any) => s.name));
|
||||
for (const check of httpChecks as any[]) {
|
||||
// /users/me requires user-context OAuth; the recipe only collects an
|
||||
// app-only bearer token, so probing it always fails.
|
||||
expect(check.url).not.toContain('/users/me');
|
||||
// Every $VAR the check expands must be declared in secrets, or the
|
||||
// installer never prompts for it and the check fails for everyone.
|
||||
const vars = [check.url, check.auth_token, check.auth_user, check.auth_pass]
|
||||
.filter((v: unknown): v is string => typeof v === 'string')
|
||||
.flatMap((v: string) => v.match(/\$[A-Z_][A-Z0-9_]*/g) ?? [])
|
||||
.map((v: string) => v.slice(1));
|
||||
expect(vars.length).toBeGreaterThan(0);
|
||||
for (const name of vars) expect(secretNames.has(name)).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// --- All recipes parse without error ---
|
||||
|
||||
describe('all recipes', () => {
|
||||
|
||||
@@ -368,6 +368,47 @@ New prose appended here.`;
|
||||
expect(extractorCalls).toBe(1);
|
||||
});
|
||||
|
||||
test('phase deadline breaks the page loop with a partial result (deadline_hit)', async () => {
|
||||
const pages = [
|
||||
buildPage({ slug: 'wiki/slow-a', body: 'page a' }),
|
||||
buildPage({ slug: 'wiki/slow-b', body: 'page b' }),
|
||||
];
|
||||
const { engine, captured } = buildMockEngine({ pages });
|
||||
let extractorCalls = 0;
|
||||
const extractor: ProposeTakesExtractor = async () => {
|
||||
extractorCalls++;
|
||||
await new Promise((r) => setTimeout(r, 10));
|
||||
return [{ claim_text: 'x', kind: 'take', holder: 'brain', weight: 0.5 }];
|
||||
};
|
||||
// 5ms deadline: page 1 processes (elapsed 0 at check), the 10ms extractor
|
||||
// call pushes elapsed past the cap, page 2 is never scanned.
|
||||
const result = await runPhaseProposeTakes(buildCtx(engine), { extractor, deadlineMs: 5 });
|
||||
|
||||
expect(result.status).toBe('warn');
|
||||
const details = result.details as Record<string, unknown>;
|
||||
expect(details.deadline_hit).toBe(true);
|
||||
expect(details.pages_scanned).toBe(1);
|
||||
expect(extractorCalls).toBe(1);
|
||||
expect((details.warnings as string[]).some(w => w.includes('phase deadline hit'))).toBe(true);
|
||||
|
||||
// Rollup records the deadline break as a halt, not a completed round
|
||||
// (same posture as budget exhaustion). Params: $5 = halt, $8 = completed.
|
||||
const rollup = captured.find((c) => c.sql.includes('extract_rollup_7d'));
|
||||
expect(rollup).toBeDefined();
|
||||
expect(rollup!.params[4]).toBe(1); // halt_count delta
|
||||
expect(rollup!.params[7]).toBe(0); // round_completed delta
|
||||
});
|
||||
|
||||
test('default deadline does not fire on a fast run', async () => {
|
||||
const pages = [buildPage({ slug: 'wiki/fast', body: 'quick page' })];
|
||||
const { engine } = buildMockEngine({ pages });
|
||||
const extractor: ProposeTakesExtractor = async () => [];
|
||||
const result = await runPhaseProposeTakes(buildCtx(engine), { extractor });
|
||||
const details = result.details as Record<string, unknown>;
|
||||
expect(details.deadline_hit).toBeUndefined();
|
||||
expect(details.pages_scanned).toBe(1);
|
||||
});
|
||||
|
||||
test('proposal_run_id is stable across all proposals from one phase invocation', async () => {
|
||||
const pages = [
|
||||
buildPage({ slug: 'wiki/a', body: 'page a' }),
|
||||
|
||||
Reference in New Issue
Block a user