mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-14 17:02:19 +00:00
Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
92656a221b | ||
|
|
6ec762cbcd | ||
|
|
4a81c017a0 |
@@ -206,11 +206,7 @@ jobs:
|
||||
needs: cache-check
|
||||
if: needs.cache-check.outputs.hit != 'true'
|
||||
runs-on: ubuntu-latest
|
||||
# 22, not 15: under parallel PR load the PGLite WASM cold-starts stretch a
|
||||
# shard past 15 min while every test is still passing — the timeout then
|
||||
# cancels the job and the test-status gate reads it as a failure. 13 runs
|
||||
# died this way on 2026-07-21/22 alone.
|
||||
timeout-minutes: 22
|
||||
timeout-minutes: 15
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
|
||||
@@ -71,8 +71,8 @@ GBrain is designed to be installed and operated by an AI agent. The fastest path
|
||||
|
||||
If you don't already have an AI agent platform running, start with one of these. Both are designed to read GBrain's install protocol and execute it:
|
||||
|
||||
- **[OpenClaw](https://github.com/openclaw/openclaw)** — deploy [AlphaClaw on Render](https://render.com/deploy?repo=https://github.com/chrysb/alphaclaw) (one click, 8GB+ RAM)
|
||||
- **[Hermes](https://github.com/NousResearch/hermes-agent)** — deploy on [Railway](https://github.com/praveen-ks-2001/hermes-agent-template) (one click)
|
||||
- **[OpenClaw](https://github.com/openclawagents/openclaw)** — deploy [AlphaClaw on Render](https://render.com/deploy?repo=https://github.com/chrysb/alphaclaw) (one click, 8GB+ RAM)
|
||||
- **[Hermes](https://github.com/openclawagents/hermes)** — deploy on [Railway](https://github.com/praveen-ks-2001/hermes-agent-template) (one click)
|
||||
|
||||
Then paste this into your agent:
|
||||
|
||||
|
||||
Vendored
+56
File diff suppressed because one or more lines are too long
Vendored
-56
File diff suppressed because one or more lines are too long
Vendored
+1
-1
@@ -7,7 +7,7 @@
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com" />
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
||||
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet" />
|
||||
<script type="module" crossorigin src="/admin/assets/index-CoGEje3-.js"></script>
|
||||
<script type="module" crossorigin src="/admin/assets/index-BpDk4NI4.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/admin/assets/index-GxkWX7v3.css">
|
||||
</head>
|
||||
<body>
|
||||
|
||||
+6
-2
@@ -5,13 +5,14 @@ import { AgentsPage } from './pages/Agents';
|
||||
import { RequestLogPage } from './pages/RequestLog';
|
||||
import { CalibrationPage } from './pages/Calibration';
|
||||
import { JobsWatchPage } from './pages/JobsWatch';
|
||||
import { SourcesPage } from './pages/Sources';
|
||||
import { api } from './api';
|
||||
|
||||
type Page = 'login' | 'dashboard' | 'agents' | 'log' | 'calibration' | 'jobs';
|
||||
type Page = 'login' | 'dashboard' | 'agents' | 'sources' | 'log' | 'calibration' | 'jobs';
|
||||
|
||||
function getPage(): Page {
|
||||
const hash = window.location.hash.replace('#', '') || 'dashboard';
|
||||
if (['login', 'dashboard', 'agents', 'log', 'calibration', 'jobs'].includes(hash)) return hash as Page;
|
||||
if (['login', 'dashboard', 'agents', 'sources', 'log', 'calibration', 'jobs'].includes(hash)) return hash as Page;
|
||||
return 'dashboard';
|
||||
}
|
||||
|
||||
@@ -54,6 +55,8 @@ export function App() {
|
||||
onClick={() => navigate('dashboard')}>Dashboard</a>
|
||||
<a className={`nav-item ${page === 'agents' ? 'active' : ''}`}
|
||||
onClick={() => navigate('agents')}>Agents</a>
|
||||
<a className={`nav-item ${page === 'sources' ? 'active' : ''}`}
|
||||
onClick={() => navigate('sources')}>Sources</a>
|
||||
<a className={`nav-item ${page === 'log' ? 'active' : ''}`}
|
||||
onClick={() => navigate('log')}>Request Log</a>
|
||||
<a className={`nav-item ${page === 'calibration' ? 'active' : ''}`}
|
||||
@@ -83,6 +86,7 @@ export function App() {
|
||||
<main className="main">
|
||||
{page === 'dashboard' && <DashboardPage />}
|
||||
{page === 'agents' && <AgentsPage />}
|
||||
{page === 'sources' && <SourcesPage />}
|
||||
{page === 'log' && <RequestLogPage />}
|
||||
{page === 'calibration' && <CalibrationPage />}
|
||||
{page === 'jobs' && <JobsWatchPage />}
|
||||
|
||||
@@ -52,4 +52,22 @@ export const api = {
|
||||
apiFetchText(`/admin/api/calibration/charts/${encodeURIComponent(type)}${holder ? `?holder=${encodeURIComponent(holder)}` : ''}`),
|
||||
// v0.41 D2 — live minion-jobs dashboard snapshot.
|
||||
jobsWatch: () => apiFetch('/admin/api/jobs/watch'),
|
||||
// v0.41.29 Sources tab + federated-read management
|
||||
sources: () => apiFetch('/admin/api/sources'),
|
||||
agentsFederatedRead: () => apiFetch('/admin/api/agents/federated-read'),
|
||||
grantRead: (clientId: string, sourceId: string) =>
|
||||
apiFetch(`/admin/api/agents/${encodeURIComponent(clientId)}/grant-read`, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ source_id: sourceId }),
|
||||
}),
|
||||
revokeRead: (clientId: string, sourceId: string) =>
|
||||
apiFetch(`/admin/api/agents/${encodeURIComponent(clientId)}/revoke-read`, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ source_id: sourceId }),
|
||||
}),
|
||||
setFederatedRead: (clientId: string, sourceIds: string[]) =>
|
||||
apiFetch(`/admin/api/agents/${encodeURIComponent(clientId)}/set-federated-read`, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ source_ids: sourceIds }),
|
||||
}),
|
||||
};
|
||||
|
||||
@@ -381,8 +381,16 @@ function CredentialsModal({ credentials, onClose }: {
|
||||
);
|
||||
}
|
||||
|
||||
interface FederationState {
|
||||
source_id: string | null;
|
||||
federated_read: string[];
|
||||
}
|
||||
|
||||
function AgentDrawer({ agent, onClose, onRevoked }: { agent: Agent; onClose: () => void; onRevoked: () => void }) {
|
||||
const [tab, setTab] = useState<'claude-code' | 'chatgpt' | 'claude-cowork' | 'perplexity' | 'cursor' | 'json'>('claude-code');
|
||||
const [federation, setFederation] = useState<FederationState | null>(null);
|
||||
const [allSources, setAllSources] = useState<string[]>([]);
|
||||
const [showFederation, setShowFederation] = useState(false);
|
||||
const copy = (text: string) => navigator.clipboard.writeText(text);
|
||||
const serverUrl = window.location.origin;
|
||||
|
||||
@@ -390,6 +398,30 @@ function AgentDrawer({ agent, onClose, onRevoked }: { agent: Agent; onClose: ()
|
||||
const isOAuth = agent.auth_type === 'oauth';
|
||||
const agentName = agent.name || agent.client_name || 'unknown';
|
||||
|
||||
// Lazy-load federation state when the drawer opens for an OAuth client.
|
||||
// The /admin/api/agents endpoint doesn't carry source_id / federated_read,
|
||||
// so we fetch /admin/api/agents/federated-read separately and pair by id.
|
||||
useEffect(() => {
|
||||
if (!isOAuth || !cid) return;
|
||||
let cancelled = false;
|
||||
Promise.all([
|
||||
api.agentsFederatedRead().catch(() => ({ clients: [] })),
|
||||
api.sources().catch(() => ({ sources: [] })),
|
||||
]).then(([feds, srcs]: any) => {
|
||||
if (cancelled) return;
|
||||
const me = (feds.clients || []).find((c: any) => c.client_id === cid);
|
||||
setFederation(me ? { source_id: me.source_id, federated_read: me.federated_read || [] } : null);
|
||||
setAllSources((srcs.sources || []).map((s: any) => s.source_id));
|
||||
});
|
||||
return () => { cancelled = true; };
|
||||
}, [cid, isOAuth]);
|
||||
|
||||
const reloadFederation = async () => {
|
||||
const feds: any = await api.agentsFederatedRead().catch(() => ({ clients: [] }));
|
||||
const me = (feds.clients || []).find((c: any) => c.client_id === cid);
|
||||
setFederation(me ? { source_id: me.source_id, federated_read: me.federated_read || [] } : null);
|
||||
};
|
||||
|
||||
// For API keys, we can't show the actual token (it was shown once at creation).
|
||||
// For OAuth, we show the client_id and tell them to use their secret.
|
||||
|
||||
@@ -553,6 +585,34 @@ function AgentDrawer({ agent, onClose, onRevoked }: { agent: Agent; onClose: ()
|
||||
<span>{agent.token_ttl ? (agent.token_ttl >= 31536000 ? 'No expiry' : agent.token_ttl >= 86400 ? `${Math.floor(agent.token_ttl / 86400)}d` : agent.token_ttl >= 3600 ? `${Math.floor(agent.token_ttl / 3600)}h` : `${agent.token_ttl}s`) : '1h (default)'}</span>
|
||||
</div>
|
||||
|
||||
{isOAuth && federation && (
|
||||
<>
|
||||
<div className="section-title" style={{ display: 'flex', alignItems: 'center', justifyContent: 'space-between' }}>
|
||||
<span>Federation</span>
|
||||
<button
|
||||
className="btn btn-secondary"
|
||||
style={{ padding: '4px 10px', fontSize: 12 }}
|
||||
onClick={() => setShowFederation(true)}
|
||||
>
|
||||
Manage reads
|
||||
</button>
|
||||
</div>
|
||||
<div style={{ display: 'grid', gridTemplateColumns: '120px 1fr', gap: '6px 12px', fontSize: 13 }}>
|
||||
<span style={{ color: 'var(--text-secondary)' }}>Write source</span>
|
||||
<span className="mono">{federation.source_id || '(none)'}</span>
|
||||
<span style={{ color: 'var(--text-secondary)' }}>Federated reads</span>
|
||||
<span style={{ fontSize: 12 }}>
|
||||
{federation.federated_read.length === 0
|
||||
? <span style={{ color: 'var(--text-muted)' }}>(empty — no federated reads)</span>
|
||||
: federation.federated_read.map((s) => (
|
||||
<span key={s} className="badge badge-read" style={{ marginRight: 4, marginBottom: 2 }}>{s}</span>
|
||||
))
|
||||
}
|
||||
</span>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
|
||||
{/*
|
||||
Config Export visible for both auth_type=oauth AND auth_type=api_key.
|
||||
Claude Code + Cursor + JSON tabs render real snippets regardless
|
||||
@@ -628,6 +688,142 @@ function AgentDrawer({ agent, onClose, onRevoked }: { agent: Agent; onClose: ()
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{showFederation && federation && (
|
||||
<FederationModal
|
||||
clientId={cid}
|
||||
clientName={agentName}
|
||||
allSources={allSources}
|
||||
currentReads={federation.federated_read}
|
||||
writeSource={federation.source_id}
|
||||
onClose={() => setShowFederation(false)}
|
||||
onSaved={async () => {
|
||||
await reloadFederation();
|
||||
setShowFederation(false);
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* FederationModal — admin counterpart of `gbrain auth set-federated-read`.
|
||||
* Source checkbox list; "Save" submits the full new list via the
|
||||
* race-safe atomic SQL path in setFederatedReadCore. Per the CLI's
|
||||
* documented contract, this is wholesale-replace semantics — concurrent
|
||||
* grant/revoke from a CLI operator would be last-writer-wins against
|
||||
* a Save here.
|
||||
*/
|
||||
function FederationModal({
|
||||
clientId, clientName, allSources, currentReads, writeSource, onClose, onSaved,
|
||||
}: {
|
||||
clientId: string;
|
||||
clientName: string;
|
||||
allSources: string[];
|
||||
currentReads: string[];
|
||||
writeSource: string | null;
|
||||
onClose: () => void;
|
||||
onSaved: () => Promise<void> | void;
|
||||
}) {
|
||||
const [selected, setSelected] = useState<Set<string>>(new Set(currentReads));
|
||||
const [saving, setSaving] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const toggle = (id: string) => {
|
||||
const next = new Set(selected);
|
||||
if (next.has(id)) next.delete(id); else next.add(id);
|
||||
setSelected(next);
|
||||
};
|
||||
|
||||
const handleSave = async () => {
|
||||
setSaving(true);
|
||||
setError(null);
|
||||
try {
|
||||
await api.setFederatedRead(clientId, Array.from(selected));
|
||||
await onSaved();
|
||||
} catch (e: any) {
|
||||
setError(e.message || 'save failed');
|
||||
setSaving(false);
|
||||
}
|
||||
};
|
||||
|
||||
// Union: all known sources + any current reads not in the source list
|
||||
// (e.g. orphan entries from before the source was deleted). The latter
|
||||
// surface as "(missing source)" so operators can revoke them.
|
||||
const allKnown = new Set([...allSources, ...currentReads]);
|
||||
const ordered = Array.from(allKnown).sort();
|
||||
|
||||
return (
|
||||
<div className="modal-overlay" onClick={onClose}>
|
||||
<div className="modal" onClick={(e) => e.stopPropagation()} style={{ maxWidth: 520 }}>
|
||||
<div className="modal-header">
|
||||
<div style={{ fontSize: 16, fontWeight: 600 }}>Manage federated reads</div>
|
||||
<div style={{ fontSize: 13, color: 'var(--text-secondary)', marginTop: 4 }}>
|
||||
<strong>{clientName}</strong> — pick which sources this client can read in addition to its
|
||||
{writeSource ? <> write source <code className="mono">{writeSource}</code></> : <> write source</>}.
|
||||
</div>
|
||||
</div>
|
||||
<div className="modal-body" style={{ maxHeight: '50vh', overflowY: 'auto' }}>
|
||||
{ordered.length === 0 && (
|
||||
<div style={{ color: 'var(--text-muted)', fontSize: 13 }}>
|
||||
No sources registered. Use <code>gbrain sources add <id> --path <dir></code> from the CLI first.
|
||||
</div>
|
||||
)}
|
||||
{ordered.map((id) => {
|
||||
const isOrphan = !allSources.includes(id);
|
||||
const isWriteSource = id === writeSource;
|
||||
return (
|
||||
<label
|
||||
key={id}
|
||||
style={{
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: 10,
|
||||
padding: '8px 10px',
|
||||
borderBottom: '1px solid var(--border)',
|
||||
cursor: 'pointer',
|
||||
fontSize: 13,
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={selected.has(id)}
|
||||
onChange={() => toggle(id)}
|
||||
style={{ width: 16, height: 16, margin: 0, flexShrink: 0, cursor: 'pointer' }}
|
||||
/>
|
||||
<span className="mono" style={{ flex: 1, minWidth: 0, overflow: 'hidden', textOverflow: 'ellipsis', whiteSpace: 'nowrap' }}>{id}</span>
|
||||
{isWriteSource && <span className="badge badge-write" style={{ fontSize: 10, flexShrink: 0 }}>write source</span>}
|
||||
{isOrphan && <span className="badge badge-danger" style={{ fontSize: 10, flexShrink: 0 }}>missing source</span>}
|
||||
</label>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
{error && (
|
||||
<div style={{
|
||||
background: 'rgba(239,68,68,0.08)',
|
||||
border: '1px solid rgba(239,68,68,0.3)',
|
||||
color: '#ef4444',
|
||||
padding: '10px 12px',
|
||||
borderRadius: 6,
|
||||
margin: '12px 0',
|
||||
fontSize: 12,
|
||||
}}>
|
||||
{error}
|
||||
</div>
|
||||
)}
|
||||
<div className="modal-footer">
|
||||
<button type="button" className="btn btn-secondary" onClick={onClose} disabled={saving}>Cancel</button>
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-primary"
|
||||
onClick={handleSave}
|
||||
disabled={saving}
|
||||
>
|
||||
{saving ? 'Saving…' : `Save (${selected.size} source${selected.size === 1 ? '' : 's'})`}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
import React, { useState, useEffect } from 'react';
|
||||
import { api } from '../api';
|
||||
|
||||
interface SourceRow {
|
||||
source_id: string;
|
||||
name: string;
|
||||
local_path: string | null;
|
||||
sync_enabled: boolean;
|
||||
last_sync_at: string | null;
|
||||
staleness_hours: number | null;
|
||||
staleness_class: 'fresh' | 'stale' | 'severe' | 'unknown';
|
||||
last_commit: string | null;
|
||||
pages: number;
|
||||
chunks_total: number;
|
||||
chunks_unembedded: number;
|
||||
embedding_coverage_pct: number;
|
||||
}
|
||||
|
||||
interface FederatedClient {
|
||||
client_id: string;
|
||||
client_name: string;
|
||||
source_id: string | null;
|
||||
federated_read: string[];
|
||||
}
|
||||
|
||||
function timeAgo(iso: string | null): string {
|
||||
if (!iso) return 'never';
|
||||
const s = Math.floor((Date.now() - new Date(iso).getTime()) / 1000);
|
||||
if (s < 0) return 'in the future?';
|
||||
if (s < 60) return 'just now';
|
||||
if (s < 3600) return `${Math.floor(s / 60)}m ago`;
|
||||
if (s < 86400) return `${Math.floor(s / 3600)}h ago`;
|
||||
return `${Math.floor(s / 86400)}d ago`;
|
||||
}
|
||||
|
||||
function stalenessColor(cls: string): string {
|
||||
switch (cls) {
|
||||
case 'fresh': return '#4ade80';
|
||||
case 'stale': return '#fbbf24';
|
||||
case 'severe': return '#ef4444';
|
||||
default: return 'var(--text-muted)';
|
||||
}
|
||||
}
|
||||
|
||||
function coverageColor(pct: number): string {
|
||||
if (pct >= 99) return '#4ade80';
|
||||
if (pct >= 90) return '#fbbf24';
|
||||
return '#ef4444';
|
||||
}
|
||||
|
||||
export function SourcesPage() {
|
||||
const [sources, setSources] = useState<SourceRow[]>([]);
|
||||
const [clients, setClients] = useState<FederatedClient[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const load = async () => {
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
try {
|
||||
const [srcReport, clientsResp] = await Promise.all([
|
||||
api.sources(),
|
||||
api.agentsFederatedRead(),
|
||||
]);
|
||||
setSources(srcReport.sources || []);
|
||||
setClients(clientsResp.clients || []);
|
||||
} catch (e: any) {
|
||||
setError(e.message || 'load failed');
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => { load(); }, []);
|
||||
|
||||
// Reverse-lookup: for each source, which clients can read it?
|
||||
const readersBySource = (sourceId: string): string[] =>
|
||||
clients.filter((c) => c.federated_read.includes(sourceId)).map((c) => c.client_name);
|
||||
|
||||
// Reverse-lookup: which clients WRITE to this source (source_id == sourceId)?
|
||||
const writersBySource = (sourceId: string): string[] =>
|
||||
clients.filter((c) => c.source_id === sourceId).map((c) => c.client_name);
|
||||
|
||||
return (
|
||||
<div style={{ padding: 24, maxWidth: 1200 }}>
|
||||
<div style={{ display: 'flex', alignItems: 'center', justifyContent: 'space-between', marginBottom: 24 }}>
|
||||
<h1 style={{ fontSize: 24, margin: 0 }}>Sources</h1>
|
||||
<button
|
||||
onClick={load}
|
||||
style={{
|
||||
background: 'transparent',
|
||||
border: '1px solid var(--border)',
|
||||
color: 'var(--text-secondary)',
|
||||
padding: '6px 12px',
|
||||
borderRadius: 6,
|
||||
fontSize: 12,
|
||||
cursor: 'pointer',
|
||||
}}
|
||||
>
|
||||
Refresh
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{loading && <div style={{ color: 'var(--text-muted)' }}>Loading…</div>}
|
||||
{error && (
|
||||
<div style={{
|
||||
background: 'rgba(239,68,68,0.08)',
|
||||
border: '1px solid rgba(239,68,68,0.3)',
|
||||
color: '#ef4444',
|
||||
padding: 12,
|
||||
borderRadius: 6,
|
||||
marginBottom: 16,
|
||||
fontSize: 13,
|
||||
}}>
|
||||
Failed to load sources: {error}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{!loading && !error && sources.length === 0 && (
|
||||
<div style={{ color: 'var(--text-muted)', padding: 16 }}>
|
||||
No active sources with a local_path. Use{' '}
|
||||
<code style={{ background: 'var(--bg-elevated)', padding: '2px 6px', borderRadius: 4 }}>
|
||||
gbrain sources add <id> --path <dir>
|
||||
</code>{' '}
|
||||
to register one.
|
||||
</div>
|
||||
)}
|
||||
|
||||
{!loading && !error && sources.length > 0 && (
|
||||
<div style={{ overflowX: 'auto' }}>
|
||||
<table style={{ width: '100%', borderCollapse: 'collapse', fontSize: 13 }}>
|
||||
<thead>
|
||||
<tr style={{ borderBottom: '1px solid var(--border)', textAlign: 'left', color: 'var(--text-muted)' }}>
|
||||
<th style={{ padding: '10px 12px' }}>ID</th>
|
||||
<th style={{ padding: '10px 12px', textAlign: 'right' }}>Pages</th>
|
||||
<th style={{ padding: '10px 12px', textAlign: 'right' }}>Chunks</th>
|
||||
<th style={{ padding: '10px 12px', textAlign: 'right' }}>Embed%</th>
|
||||
<th style={{ padding: '10px 12px' }}>Last Sync</th>
|
||||
<th style={{ padding: '10px 12px' }}>Writers</th>
|
||||
<th style={{ padding: '10px 12px' }}>Readers (federated)</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{sources.map((s) => {
|
||||
const readers = readersBySource(s.source_id);
|
||||
const writers = writersBySource(s.source_id);
|
||||
return (
|
||||
<tr key={s.source_id} style={{ borderBottom: '1px solid var(--border)' }}>
|
||||
<td style={{ padding: '10px 12px', fontFamily: 'JetBrains Mono, monospace' }}>
|
||||
<div>{s.source_id}</div>
|
||||
{s.name !== s.source_id && (
|
||||
<div style={{ fontSize: 11, color: 'var(--text-muted)', fontFamily: 'inherit' }}>{s.name}</div>
|
||||
)}
|
||||
</td>
|
||||
<td style={{ padding: '10px 12px', textAlign: 'right', fontFamily: 'JetBrains Mono, monospace' }}>{s.pages.toLocaleString()}</td>
|
||||
<td style={{ padding: '10px 12px', textAlign: 'right', fontFamily: 'JetBrains Mono, monospace' }}>{s.chunks_total.toLocaleString()}</td>
|
||||
<td style={{ padding: '10px 12px', textAlign: 'right', color: coverageColor(s.embedding_coverage_pct), fontFamily: 'JetBrains Mono, monospace' }}>
|
||||
{s.embedding_coverage_pct.toFixed(0)}%
|
||||
</td>
|
||||
<td style={{ padding: '10px 12px', color: s.local_path == null ? 'var(--text-muted)' : stalenessColor(s.staleness_class) }}>
|
||||
{s.local_path == null ? 'push-only' : timeAgo(s.last_sync_at)}
|
||||
</td>
|
||||
<td style={{ padding: '10px 12px', fontSize: 12, color: 'var(--text-secondary)' }}>
|
||||
{writers.length === 0 ? <span style={{ color: 'var(--text-muted)' }}>none</span> : writers.join(', ')}
|
||||
</td>
|
||||
<td style={{ padding: '10px 12px', fontSize: 12, color: 'var(--text-secondary)' }}>
|
||||
{readers.length === 0 ? <span style={{ color: 'var(--text-muted)' }}>none</span> : readers.join(', ')}
|
||||
</td>
|
||||
</tr>
|
||||
);
|
||||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div style={{
|
||||
marginTop: 24,
|
||||
padding: 12,
|
||||
background: 'var(--bg-elevated)',
|
||||
border: '1px solid var(--border)',
|
||||
borderRadius: 6,
|
||||
fontSize: 12,
|
||||
color: 'var(--text-muted)',
|
||||
lineHeight: 1.6,
|
||||
}}>
|
||||
<strong style={{ color: 'var(--text-secondary)' }}>Two scopes per OAuth client:</strong>{' '}
|
||||
<em>Writers</em> = clients with this source as their <code>source_id</code> (write authority).{' '}
|
||||
<em>Readers</em> = clients with this source in their <code>federated_read</code> list (read access via federation).
|
||||
Manage federation per-client from the <a href="#agents" style={{ color: '#60a5fa' }}>Agents</a> tab using the
|
||||
"Manage reads" action.
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -131,9 +131,7 @@ into gbrain so other clients can scaffold it. Default behavior:
|
||||
`~/.gbrain/harvest-private-patterns.txt` plus built-in defaults
|
||||
(canonical private fork name, common email regex, Slack channel pattern). Any
|
||||
match → rollback (delete the harvested files) and exit non-zero.
|
||||
- `openclaw.plugin.json` updated with the new slug, sorted. Harvest must preserve
|
||||
the top-level OpenClaw-native plugin fields (`id`, `configSchema`, `contracts`)
|
||||
because OpenClaw validates those before it can install the package.
|
||||
- `openclaw.plugin.json` updated with the new slug, sorted.
|
||||
- `--no-lint` bypasses the linter (after a manual editorial scrub).
|
||||
|
||||
Use the `skillpack-harvest` skill (its companion editorial workflow)
|
||||
|
||||
@@ -233,14 +233,13 @@ keep it or `git checkout` to throw it away. Nothing is committed for you.
|
||||
|
||||
**For a skill that ships with gbrain** (anything under the gbrain repo's own
|
||||
`skills/`): SkillOpt refuses to overwrite it by default and writes the winner to
|
||||
`skills/<name>/skillopt/proposed.md` instead (while keeping `best.md` as the
|
||||
optimizer's current-best pointer), so an optimization pass can never silently
|
||||
mutate a skill other people depend on. Two ways to handle that:
|
||||
`skills/<name>/skillopt/best.md` instead, so an optimization pass can never
|
||||
silently mutate a skill other people depend on. Two ways to handle that:
|
||||
|
||||
```bash
|
||||
# See the proposed improvement without touching SKILL.md (works for ANY skill):
|
||||
gbrain skillopt meeting-prep --split 1:1:1 --no-mutate
|
||||
# → writes skills/meeting-prep/skillopt/proposed.md, updates best.md, and prints the proposal path.
|
||||
# → writes skills/meeting-prep/skillopt/best.md (the proposed rewrite), prints its path. Copy what you want.
|
||||
|
||||
# Actually rewrite a bundled skill (explicit opt-in + an independent held-out set):
|
||||
gbrain skillopt brain-ops --split 1:1:1 --allow-mutate-bundled \
|
||||
|
||||
+2
-2
@@ -1565,8 +1565,8 @@ GBrain is designed to be installed and operated by an AI agent. The fastest path
|
||||
|
||||
If you don't already have an AI agent platform running, start with one of these. Both are designed to read GBrain's install protocol and execute it:
|
||||
|
||||
- **[OpenClaw](https://github.com/openclaw/openclaw)** — deploy [AlphaClaw on Render](https://render.com/deploy?repo=https://github.com/chrysb/alphaclaw) (one click, 8GB+ RAM)
|
||||
- **[Hermes](https://github.com/NousResearch/hermes-agent)** — deploy on [Railway](https://github.com/praveen-ks-2001/hermes-agent-template) (one click)
|
||||
- **[OpenClaw](https://github.com/openclawagents/openclaw)** — deploy [AlphaClaw on Render](https://render.com/deploy?repo=https://github.com/chrysb/alphaclaw) (one click, 8GB+ RAM)
|
||||
- **[Hermes](https://github.com/openclawagents/hermes)** — deploy on [Railway](https://github.com/praveen-ks-2001/hermes-agent-template) (one click)
|
||||
|
||||
Then paste this into your agent:
|
||||
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
{
|
||||
"id": "gbrain-context-engine",
|
||||
"name": "gbrain",
|
||||
"version": "0.32.3.0",
|
||||
"description": "Personal knowledge brain with Postgres + pgvector hybrid search",
|
||||
|
||||
@@ -266,5 +266,4 @@ editorial pass.
|
||||
(e.g. `src/commands/<slug>.ts` if the host SKILL.md declares it
|
||||
in frontmatter)
|
||||
- gbrain's `openclaw.plugin.json` — adds the slug to `skills:`
|
||||
array, sorted alphabetically, without removing OpenClaw-native plugin fields
|
||||
like `id`, `configSchema`, or `contracts`
|
||||
array, sorted alphabetically
|
||||
|
||||
@@ -57,8 +57,6 @@ This mode guarantees:
|
||||
- `skills/manifest.json` lists every skill directory
|
||||
- `skills/RESOLVER.md` references every skill in the manifest
|
||||
- `openclaw.plugin.json` `skills[]` round-trips with both
|
||||
- `openclaw.plugin.json` keeps OpenClaw install-required native plugin fields
|
||||
(`id`, object `configSchema`, and `contracts.contextEngines` when applicable)
|
||||
- No MECE violations (duplicate triggers across skills)
|
||||
|
||||
### Phases
|
||||
@@ -74,7 +72,7 @@ This mode guarantees:
|
||||
### Automation
|
||||
|
||||
```bash
|
||||
bun test test/skills-conformance.test.ts test/resolver.test.ts test/openclaw-plugin-manifest.test.ts
|
||||
bun test test/skills-conformance.test.ts test/resolver.test.ts
|
||||
```
|
||||
|
||||
The CI-gated check is the package.json `test` script.
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
// AUTO-GENERATED — do not edit by hand.
|
||||
// Run `bun run scripts/build-admin-embedded.ts` to regenerate.
|
||||
// Source: admin/dist/ at 2026-05-27.
|
||||
// Source: admin/dist/ at 2026-07-22.
|
||||
//
|
||||
// Bun resolves the file: imports to a path that works at runtime even
|
||||
// inside a compiled binary (`bun build --compile`). The manifest maps
|
||||
// the request path the express handler sees to (resolved-path, mime).
|
||||
|
||||
// @ts-ignore — type: 'file' is Bun ESM, not in lib.d.ts
|
||||
import A_0_assets_index_CoGEje3__js from '../admin/dist/assets/index-CoGEje3-.js' with { type: 'file' };
|
||||
import A_0_assets_index_BpDk4NI4_js from '../admin/dist/assets/index-BpDk4NI4.js' with { type: 'file' };
|
||||
// @ts-ignore — type: 'file' is Bun ESM, not in lib.d.ts
|
||||
import A_1_assets_index_GxkWX7v3_css from '../admin/dist/assets/index-GxkWX7v3.css' with { type: 'file' };
|
||||
// @ts-ignore — type: 'file' is Bun ESM, not in lib.d.ts
|
||||
@@ -19,7 +19,7 @@ export interface AdminAsset {
|
||||
}
|
||||
|
||||
export const ADMIN_ASSETS: Record<string, AdminAsset> = {
|
||||
"/admin/assets/index-CoGEje3-.js": { path: A_0_assets_index_CoGEje3__js as unknown as string, mime: "application/javascript; charset=utf-8" },
|
||||
"/admin/assets/index-BpDk4NI4.js": { path: A_0_assets_index_BpDk4NI4_js as unknown as string, mime: "application/javascript; charset=utf-8" },
|
||||
"/admin/assets/index-GxkWX7v3.css": { path: A_1_assets_index_GxkWX7v3_css as unknown as string, mime: "text/css; charset=utf-8" },
|
||||
"/admin/index.html": { path: A_2_index_html as unknown as string, mime: "text/html; charset=utf-8" },
|
||||
};
|
||||
|
||||
+8
-51
@@ -54,7 +54,7 @@ export function bigintToStringReplacer(_key: string, value: unknown): unknown {
|
||||
}
|
||||
|
||||
// CLI-only commands that bypass the operation layer
|
||||
export const CLI_ONLY = new Set(['init', 'reinit-pglite', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'extract-conversation-facts', 'enrich', 'features', 'autopilot', 'graph-query', 'jobs', 'agent', 'apply-migrations', 'skillpack-check', 'skillpack', 'resolvers', 'integrity', 'repair-jsonb', 'orphans', 'maintain', 'sources', 'mounts', 'dream', 'check-resolvable', 'routing-eval', 'skillify', 'smoke-test', 'providers', 'storage', 'repos', 'code-def', 'code-refs', 'reindex', 'reindex-code', 'reindex-frontmatter', 'code-callers', 'code-callees', 'reconcile-links', 'frontmatter', 'auth', 'friction', 'claw-test', 'book-mirror', 'takes', 'think', 'salience', 'anomalies', 'calibration', 'transcripts', 'models', 'remote', 'recall', 'forget', 'edges-backfill', 'backfill', 'cache', 'ze-switch', 'founder', 'brainstorm', 'lsd', 'schema', 'capture', 'onboard', 'conversation-parser', 'status', 'connect', 'skillopt', 'quarantine', 'self-upgrade', 'advisor', 'watch', 'reindex-search-vector']);
|
||||
export const CLI_ONLY = new Set(['init', 'reinit-pglite', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'extract-conversation-facts', 'enrich', 'features', 'autopilot', 'graph-query', 'jobs', 'agent', 'apply-migrations', 'skillpack-check', 'skillpack', 'resolvers', 'integrity', 'repair-jsonb', 'orphans', 'sources', 'mounts', 'dream', 'check-resolvable', 'routing-eval', 'skillify', 'smoke-test', 'providers', 'storage', 'repos', 'code-def', 'code-refs', 'reindex', 'reindex-code', 'reindex-frontmatter', 'code-callers', 'code-callees', 'reconcile-links', 'frontmatter', 'auth', 'friction', 'claw-test', 'book-mirror', 'takes', 'think', 'salience', 'anomalies', 'calibration', 'transcripts', 'models', 'remote', 'recall', 'forget', 'edges-backfill', 'cache', 'ze-switch', 'founder', 'brainstorm', 'lsd', 'schema', 'capture', 'onboard', 'conversation-parser', 'status', 'connect', 'skillopt', 'quarantine', 'self-upgrade', 'advisor', 'watch', 'reindex-search-vector']);
|
||||
// CLI-only commands whose handlers print their own --help text. These are
|
||||
// excluded from the generic short-circuit so detailed per-command and
|
||||
// per-subcommand usage stays reachable.
|
||||
@@ -78,8 +78,6 @@ const CLI_ONLY_SELF_HELP = new Set([
|
||||
'capture',
|
||||
// v0.42 self-upgrade ships its own usage (flags + the agent-skill story).
|
||||
'self-upgrade',
|
||||
// maintain (#3015) prints its own usage block (modes + not-auto-applied list).
|
||||
'maintain',
|
||||
// v0.43 (#2095): watch ships WATCH_HELP (flags + the stdin-turn protocol).
|
||||
'watch',
|
||||
// v0.37 fix wave (Lane D.4 + CDX2-12): sync's --no-embed flag was
|
||||
@@ -106,11 +104,6 @@ const CLI_ONLY_SELF_HELP = new Set([
|
||||
// `gbrain connect --help` prints its own usage (flags + examples) from
|
||||
// runConnect; route around the generic one-line short-circuit.
|
||||
'connect',
|
||||
// #3224 — `backfill` was missing from CLI_ONLY entirely (dispatch never
|
||||
// reached runBackfillCommand). Once added there, the generic short-circuit
|
||||
// still shadowed its own printHelp() (kind list + flags, same WARN-5 class
|
||||
// as capture); route around it so `gbrain backfill --help` reaches it.
|
||||
'backfill',
|
||||
]);
|
||||
|
||||
// v114 (#1941): alias -> operation lookup, kept separate from `cliOps` so
|
||||
@@ -1012,17 +1005,6 @@ const THIN_CLIENT_REFUSED_COMMANDS = new Set([
|
||||
// it gets a partial dispatch (list/get route over MCP engine-free, the
|
||||
// rest refuse) in the main dispatch before connectEngine().
|
||||
'config',
|
||||
// #3224: `backfill` (like migrate/apply-migrations/repair-jsonb) opens
|
||||
// and mutates the local engine directly with no MCP equivalent op. On a
|
||||
// thin-client install it would otherwise fabricate the same kind of
|
||||
// ephemeral scratch-local PGLite `config` did before this audit —
|
||||
// refuse cleanly instead. This check runs before backfill's own
|
||||
// pre-connectEngine dispatch branch, so `--help` is ALSO refused on a
|
||||
// thin client rather than showing help — the same pre-existing tradeoff
|
||||
// `sync`/`enrich` already make (both are also THIN_CLIENT_REFUSED_COMMANDS
|
||||
// with their own pre-connectEngine `--help` branch further down); not a
|
||||
// new inconsistency introduced here.
|
||||
'backfill',
|
||||
]);
|
||||
|
||||
/**
|
||||
@@ -1063,8 +1045,6 @@ const THIN_CLIENT_REFUSE_HINTS: Record<string, string> = {
|
||||
// scratch-DB audit additions
|
||||
config: "config reads/writes the host brain's config plane. Edit the host's .gbrain/config.json (file-plane keys) or run on the host with GBRAIN_HOME set.",
|
||||
jobs: '`jobs list` and `jobs get <id>` are thin-client routable; this subcommand runs against the host queue. Use the submit_job / list_jobs / get_job MCP tools from your agent, or run on the host with GBRAIN_HOME set.',
|
||||
// #3224
|
||||
backfill: 'backfill mutates the host brain directly with no MCP equivalent. Run `gbrain backfill` on the host machine.',
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -1543,24 +1523,6 @@ async function handleCliOnly(command: string, args: string[]) {
|
||||
return;
|
||||
}
|
||||
|
||||
// #3224: `backfill` dispatches unconditionally here (not just --help).
|
||||
// runBackfillCommand manages its own engine end-to-end (createEngine +
|
||||
// connect at commands/backfill.ts, disconnect when done) — it takes no
|
||||
// engine parameter at all. The old `case 'backfill':` further down sits
|
||||
// behind this function's shared `const engine = await connectEngine()`;
|
||||
// reaching it there would open a SECOND PGLite connection to the same
|
||||
// database while the first is still held, and PGLite's single-writer
|
||||
// lock would make every real (non---help) backfill run hang for 30s and
|
||||
// fail. Dispatching before that shared connect — same as schema/init/
|
||||
// auth/remote above — avoids the double-connect entirely and, as a
|
||||
// bonus, makes `--help` reachable from a fresh tmpdir with no configured
|
||||
// brain (same motivation as the sync/capture/enrich branches below).
|
||||
if (command === 'backfill') {
|
||||
const { runBackfillCommand } = await import('./commands/backfill.ts');
|
||||
await runBackfillCommand(args);
|
||||
return;
|
||||
}
|
||||
|
||||
// v0.41.6.0 D3 (per outside-voice F1): connect-time + dispatch-time wallclock
|
||||
// timeouts for read-only commands whose hang would otherwise spin at 100% CPU
|
||||
// (the production "10-day zombie gbrain search ping" bug class). The wrap
|
||||
@@ -1795,11 +1757,6 @@ async function handleCliOnly(command: string, args: string[]) {
|
||||
await runOrphans(engine, args);
|
||||
break;
|
||||
}
|
||||
case 'maintain': {
|
||||
const { runMaintain } = await import('./commands/maintain.ts');
|
||||
await runMaintain(engine, args);
|
||||
break;
|
||||
}
|
||||
// v0.32.7 CJK wave — post-upgrade markdown re-chunk sweep.
|
||||
// v0.36 Phase 3 wave — `gbrain reindex --multimodal` re-embeds content_chunks
|
||||
// into the unified Voyage multimodal-3 column.
|
||||
@@ -2097,13 +2054,13 @@ async function handleCliOnly(command: string, args: string[]) {
|
||||
await reindexFrontmatterCli(args);
|
||||
return; // reindexFrontmatterCli handles its own engine lifecycle
|
||||
}
|
||||
// #3224: `backfill` (v0.30.1's first-class generic backfill command)
|
||||
// used to have its `case` right here, but runBackfillCommand manages
|
||||
// its own engine end-to-end and dispatching it AFTER this switch's
|
||||
// shared `connectEngine()` double-connects PGLite's single-writer
|
||||
// lock to itself (hangs 30s, then fails on every real run). Moved to
|
||||
// an unconditional pre-connectEngine branch above, next to schema/
|
||||
// init/auth/remote/sync/capture/enrich — see that branch's comment.
|
||||
case 'backfill': {
|
||||
// v0.30.1: first-class generic backfill command. Subcommand dispatch
|
||||
// is inside runBackfillCommand (kind | list | --help).
|
||||
const { runBackfillCommand } = await import('./commands/backfill.ts');
|
||||
await runBackfillCommand(args);
|
||||
return;
|
||||
}
|
||||
case 'code-callers': {
|
||||
// v0.20.0 Cathedral II Layer 10 (C4): "who calls <symbol>?"
|
||||
const { runCodeCallers } = await import('./commands/code-callers.ts');
|
||||
|
||||
@@ -133,15 +133,14 @@ function indexCompleted(entries: CompletedMigrationEntry[]): CompletedIndex {
|
||||
* Returns the resolved status for a migration based on its entries.
|
||||
*
|
||||
* Semantics (Bug 3 — keep "complete wins" safety):
|
||||
* - If the latest entry is `retry`, the version is pending. This is the
|
||||
* explicit escape hatch written by `--force-retry`, and it overrides an
|
||||
* earlier `complete` entry without hand-editing the ledger.
|
||||
* - Otherwise, if any entry is `complete`, the version is complete.
|
||||
* - If any entry is `complete`, the version is complete. Terminal state.
|
||||
* - Otherwise, if the latest entry is `retry`, the version is pending
|
||||
* (user requested a fresh attempt).
|
||||
* - Otherwise, if any entry is `partial`, the version is partial.
|
||||
* - Otherwise, pending.
|
||||
*
|
||||
* `complete` never regresses accidentally. A later `partial` append cannot
|
||||
* undo a completed migration; only a trailing, explicit `retry` marker can.
|
||||
* `complete` never regresses. A later accidental `partial` append cannot
|
||||
* undo a completed migration.
|
||||
*/
|
||||
function statusForVersion(
|
||||
version: string,
|
||||
@@ -149,9 +148,9 @@ function statusForVersion(
|
||||
): 'complete' | 'partial' | 'pending' | 'wedged' {
|
||||
const entries = idx.byVersion.get(version) ?? [];
|
||||
if (entries.length === 0) return 'pending';
|
||||
if (entries.some(e => e.status === 'complete')) return 'complete';
|
||||
const latest = entries[entries.length - 1];
|
||||
if (latest.status === 'retry') return 'pending';
|
||||
if (entries.some(e => e.status === 'complete')) return 'complete';
|
||||
// Bug 3 attempt cap — count consecutive partials from the end (stopping
|
||||
// at any 'retry' or 'complete'). If we hit MAX_CONSECUTIVE_PARTIALS,
|
||||
// the migration is wedged and needs explicit --force-retry to try again.
|
||||
|
||||
+586
-1
@@ -24,6 +24,8 @@ import { loadConfig, toEngineConfig } from '../core/config.ts';
|
||||
import { createEngine } from '../core/engine-factory.ts';
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import { sqlQueryForEngine, executeRawJsonb, type SqlQuery } from '../core/sql-query.ts';
|
||||
import { pgArray } from '../core/oauth-provider.ts';
|
||||
import { assertValidSourceId } from '../core/source-id.ts';
|
||||
|
||||
function hashToken(token: string): string {
|
||||
return createHash('sha256').update(token).digest('hex');
|
||||
@@ -165,6 +167,100 @@ async function list() {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* `gbrain auth list-clients [--json]` — read surface for OAuth 2.1 clients.
|
||||
*
|
||||
* The existing `gbrain auth list` shows LEGACY bearer tokens from
|
||||
* `access_tokens`; this is the parallel for v0.26+ OAuth clients. Separate
|
||||
* commands rather than merged output because the two models have different
|
||||
* field sets (legacy: lifecycle dates; OAuth: scopes + source_id +
|
||||
* federated_read).
|
||||
*
|
||||
* Human output is card-style (multi-line per client) instead of a fixed-
|
||||
* width table — federated_read can hold many ids per client and a wide
|
||||
* single-line layout truncates / wraps badly on terminals < 200 cols.
|
||||
* JSON output uses a `schema_version: 1` envelope; additive only.
|
||||
*/
|
||||
async function listClients(args: string[]) {
|
||||
const json = args.includes('--json');
|
||||
const includeDeleted = args.includes('--include-deleted');
|
||||
await withConfiguredSql(async (sql) => {
|
||||
// Codex finding #2 (medium): default-hide soft-deleted clients so admin
|
||||
// soft-deletes are honored by the CLI surface. Opt-in via flag.
|
||||
const rows = includeDeleted
|
||||
? await sql`
|
||||
SELECT client_id, client_name, scope, source_id, federated_read,
|
||||
grant_types, created_at, deleted_at
|
||||
FROM oauth_clients
|
||||
ORDER BY client_name
|
||||
`
|
||||
: await sql`
|
||||
SELECT client_id, client_name, scope, source_id, federated_read,
|
||||
grant_types, created_at, deleted_at
|
||||
FROM oauth_clients
|
||||
WHERE deleted_at IS NULL
|
||||
ORDER BY client_name
|
||||
`;
|
||||
if (json) {
|
||||
const clients = rows.map((r) => ({
|
||||
client_id: String(r.client_id),
|
||||
client_name: String(r.client_name),
|
||||
scope: r.scope == null ? null : String(r.scope),
|
||||
source_id: r.source_id == null ? null : String(r.source_id),
|
||||
federated_read: Array.isArray(r.federated_read)
|
||||
? (r.federated_read as string[]).map(String)
|
||||
: [],
|
||||
grant_types: Array.isArray(r.grant_types)
|
||||
? (r.grant_types as string[]).map(String)
|
||||
: [],
|
||||
created_at:
|
||||
r.created_at instanceof Date
|
||||
? r.created_at.toISOString()
|
||||
: r.created_at == null
|
||||
? null
|
||||
: String(r.created_at),
|
||||
deleted_at:
|
||||
r.deleted_at instanceof Date
|
||||
? r.deleted_at.toISOString()
|
||||
: r.deleted_at == null
|
||||
? null
|
||||
: String(r.deleted_at),
|
||||
}));
|
||||
process.stdout.write(JSON.stringify({ schema_version: 1, clients }, null, 2) + '\n');
|
||||
return;
|
||||
}
|
||||
if (rows.length === 0) {
|
||||
console.log(
|
||||
includeDeleted
|
||||
? 'No OAuth clients found (including deleted). Register one: gbrain auth register-client <name>'
|
||||
: 'No active OAuth clients found. Register one: gbrain auth register-client <name>'
|
||||
+ '\n(Use --include-deleted to also show soft-deleted clients.)',
|
||||
);
|
||||
return;
|
||||
}
|
||||
for (let i = 0; i < rows.length; i++) {
|
||||
const r = rows[i];
|
||||
const fed = Array.isArray(r.federated_read)
|
||||
? (r.federated_read as string[]).map(String)
|
||||
: [];
|
||||
const grants = Array.isArray(r.grant_types)
|
||||
? (r.grant_types as string[]).map(String)
|
||||
: [];
|
||||
const deletedAt = r.deleted_at;
|
||||
const status = deletedAt == null
|
||||
? ''
|
||||
: ` [SOFT-DELETED ${deletedAt instanceof Date ? deletedAt.toISOString() : String(deletedAt)}]`;
|
||||
console.log(`${sanitizeForTerminal(String(r.client_name))}${status}`);
|
||||
console.log(` client_id: ${sanitizeForTerminal(String(r.client_id))}`);
|
||||
console.log(` scope: ${r.scope == null ? '(none)' : sanitizeForTerminal(String(r.scope))}`);
|
||||
console.log(` grant types: ${grants.length ? sanitizeForTerminal(grants.join(', ')) : '(none)'}`);
|
||||
console.log(` write source: ${r.source_id == null ? '(none)' : sanitizeForTerminal(String(r.source_id))}`);
|
||||
console.log(` federated: ${fed.length ? sanitizeForTerminal(fed.join(', ')) : '(empty)'}`);
|
||||
if (i < rows.length - 1) console.log('');
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function revoke(name: string) {
|
||||
if (!name) { console.error('Usage: auth revoke <name>'); process.exit(1); }
|
||||
await withConfiguredSql(async (sql) => {
|
||||
@@ -301,6 +397,475 @@ async function test(url: string, token: string) {
|
||||
console.log(`\n🧠 Your brain is live! (${elapsed}s)`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip ANSI escapes + C0/C1 control characters from a string before
|
||||
* printing it to the operator's terminal. Defense for the
|
||||
* codex-flagged terminal-control-injection class: a client_name or
|
||||
* source_id registered via DCR with `\x1b[2J` (clear-screen) or
|
||||
* `\x1b]0;TITLE\x07` (OSC title-change) would poison
|
||||
* `gbrain auth list-clients` output otherwise.
|
||||
*
|
||||
* Replaces unsafe bytes with their `\xNN` hex escape so the operator
|
||||
* sees that something weird is in the field, instead of silent
|
||||
* mutilation. Tab and newline are preserved as-is so legitimate
|
||||
* multi-line values render.
|
||||
*/
|
||||
export function sanitizeForTerminal(s: string): string {
|
||||
// ALL C0/C1 controls + DEL get escaped. Codex re-review caught that
|
||||
// preserving `\n` lets a DCR-registered client_name spoof additional
|
||||
// human-output lines in list-clients (a real attack — newline in the
|
||||
// name visually adds a fake row to the operator's terminal). Tab is
|
||||
// also escaped for the same reason — field-separator spoofing.
|
||||
// C0: 0x00-0x1F. DEL: 0x7F. C1: 0x80-0x9F.
|
||||
return s.replace(/[\x00-\x1f\x7f-\x9f]/g, (ch) =>
|
||||
`\\x${ch.charCodeAt(0).toString(16).padStart(2, '0')}`,
|
||||
);
|
||||
}
|
||||
|
||||
export interface ResolvedClient {
|
||||
client_id: string;
|
||||
client_name: string;
|
||||
source_id: string | null;
|
||||
federated_read: string[];
|
||||
deleted_at: Date | string | null;
|
||||
}
|
||||
|
||||
export type FederatedReadOutcome =
|
||||
| { kind: 'noop'; reason: 'already-granted' | 'not-present' | 'same-list'; client: ResolvedClient; current: string[] }
|
||||
| { kind: 'updated'; client: ResolvedClient; before: string[]; after: string[] };
|
||||
|
||||
/**
|
||||
* Resolve an OAuth client by client_id (exact) or client_name (unique).
|
||||
* Errors on no-match and on ambiguous client_name (>1 row). client_id
|
||||
* takes precedence — if a long hash is passed and matches, returns
|
||||
* immediately without ever querying by name.
|
||||
*
|
||||
* Legacy bearer tokens in `access_tokens` are NOT searched. Federated read
|
||||
* scope is an OAuth-client concept (oauth_clients.federated_read column);
|
||||
* legacy bearers have no source scope.
|
||||
*/
|
||||
/**
|
||||
* Resolve an OAuth client. Codex finding #2 (medium): default-hide
|
||||
* soft-deleted clients so admin-soft-deleted rows aren't mutated by the
|
||||
* CLI. The `includeDeleted` opt is reserved for future read-side surfaces;
|
||||
* grant/revoke/set ALWAYS filter active rows only.
|
||||
*/
|
||||
export async function resolveClient(
|
||||
sql: SqlQuery,
|
||||
nameOrId: string,
|
||||
opts: { includeDeleted?: boolean } = {},
|
||||
): Promise<ResolvedClient> {
|
||||
const allowDeleted = opts.includeDeleted === true;
|
||||
const byId = allowDeleted
|
||||
? await sql`
|
||||
SELECT client_id, client_name, source_id, federated_read, deleted_at
|
||||
FROM oauth_clients WHERE client_id = ${nameOrId} LIMIT 1
|
||||
`
|
||||
: await sql`
|
||||
SELECT client_id, client_name, source_id, federated_read, deleted_at
|
||||
FROM oauth_clients WHERE client_id = ${nameOrId} AND deleted_at IS NULL LIMIT 1
|
||||
`;
|
||||
if (byId.length === 1) return normalizeClientRow(byId[0]);
|
||||
const byName = allowDeleted
|
||||
? await sql`
|
||||
SELECT client_id, client_name, source_id, federated_read, deleted_at
|
||||
FROM oauth_clients WHERE client_name = ${nameOrId}
|
||||
`
|
||||
: await sql`
|
||||
SELECT client_id, client_name, source_id, federated_read, deleted_at
|
||||
FROM oauth_clients WHERE client_name = ${nameOrId} AND deleted_at IS NULL
|
||||
`;
|
||||
if (byName.length === 0) {
|
||||
throw new Error(
|
||||
`No active OAuth client found with name or id "${nameOrId}". ` +
|
||||
`Run \`gbrain auth register-client <name>\` to create one, ` +
|
||||
`or \`gbrain auth list-clients\` to see what exists. ` +
|
||||
`(Soft-deleted clients are hidden by default.)`,
|
||||
);
|
||||
}
|
||||
if (byName.length > 1) {
|
||||
const ids = byName.map((r) => ` ${String(r.client_id)}`).join('\n');
|
||||
throw new Error(
|
||||
`Multiple active OAuth clients named "${nameOrId}". Pass the full client_id instead:\n${ids}`,
|
||||
);
|
||||
}
|
||||
return normalizeClientRow(byName[0]);
|
||||
}
|
||||
|
||||
function normalizeClientRow(row: Record<string, unknown>): ResolvedClient {
|
||||
const fed = row.federated_read;
|
||||
return {
|
||||
client_id: String(row.client_id),
|
||||
client_name: String(row.client_name),
|
||||
source_id: row.source_id == null ? null : String(row.source_id),
|
||||
federated_read: Array.isArray(fed) ? (fed as string[]).map(String) : [],
|
||||
deleted_at: row.deleted_at == null
|
||||
? null
|
||||
: (row.deleted_at as Date | string),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate the source_id shape AND DB existence. Codex finding #3 (medium):
|
||||
* a manually-INSERTed source row with weird chars (e.g. comma, quote)
|
||||
* would otherwise land in oauth_clients.federated_read as a never-deletable
|
||||
* malformed entry. Fail at the boundary before the existence query so
|
||||
* malformed input gets the validator's hint, not a "does not exist" hint
|
||||
* pointing at a non-creatable id.
|
||||
*/
|
||||
export async function assertSourceExists(sql: SqlQuery, sourceId: string): Promise<void> {
|
||||
assertValidSourceId(sourceId);
|
||||
const rows = await sql`SELECT id FROM sources WHERE id = ${sourceId} LIMIT 1`;
|
||||
if (rows.length === 0) {
|
||||
throw new Error(
|
||||
`Source "${sourceId}" does not exist. Run \`gbrain sources list\` to see registered sources, ` +
|
||||
`or \`gbrain sources add ${sourceId}\` to create it.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Atomic append: array_append + NOT-ANY guard so the row-lock fully
|
||||
* serializes concurrent grant/revoke against the same client. Codex
|
||||
* finding #1 (HIGH): the previous read-modify-write shape allowed a
|
||||
* concurrent revoke to be silently UNDONE by a racing grant.
|
||||
*
|
||||
* Returns the post-write federated_read array, or null when no rows
|
||||
* matched (already-granted, soft-deleted, or missing client). Callers
|
||||
* disambiguate via prior resolveClient + includes() check.
|
||||
*
|
||||
* `WHERE deleted_at IS NULL` is part of the atomic guard so a client
|
||||
* soft-deleted between resolveClient and the UPDATE can't be mutated.
|
||||
*/
|
||||
async function appendFederatedReadAtomic(
|
||||
sql: SqlQuery,
|
||||
clientId: string,
|
||||
sourceId: string,
|
||||
): Promise<string[] | null> {
|
||||
const rows = await sql`
|
||||
UPDATE oauth_clients
|
||||
SET federated_read = array_append(federated_read, ${sourceId})
|
||||
WHERE client_id = ${clientId}
|
||||
AND deleted_at IS NULL
|
||||
AND NOT (${sourceId} = ANY(federated_read))
|
||||
RETURNING federated_read
|
||||
`;
|
||||
if (rows.length === 0) return null;
|
||||
const fed = rows[0].federated_read;
|
||||
return Array.isArray(fed) ? (fed as string[]).map(String) : [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Atomic remove: array_remove + ANY guard. Same race-correctness story
|
||||
* as appendFederatedReadAtomic. Returns post-write array or null.
|
||||
*/
|
||||
async function removeFederatedReadAtomic(
|
||||
sql: SqlQuery,
|
||||
clientId: string,
|
||||
sourceId: string,
|
||||
): Promise<string[] | null> {
|
||||
const rows = await sql`
|
||||
UPDATE oauth_clients
|
||||
SET federated_read = array_remove(federated_read, ${sourceId})
|
||||
WHERE client_id = ${clientId}
|
||||
AND deleted_at IS NULL
|
||||
AND ${sourceId} = ANY(federated_read)
|
||||
RETURNING federated_read
|
||||
`;
|
||||
if (rows.length === 0) return null;
|
||||
const fed = rows[0].federated_read;
|
||||
return Array.isArray(fed) ? (fed as string[]).map(String) : [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Wholesale array overwrite for `set-federated-read`. Honors the
|
||||
* deleted_at filter. Last-writer-wins semantics under concurrent
|
||||
* `set` calls is acceptable — the user is asserting "this exact list"
|
||||
* intent; concurrent set+set just means whichever ran second wins.
|
||||
* Concurrent set+grant or set+revoke is also last-writer-wins, which
|
||||
* is the documented contract for `set`.
|
||||
*/
|
||||
async function replaceFederatedReadAtomic(
|
||||
sql: SqlQuery,
|
||||
clientId: string,
|
||||
next: string[],
|
||||
): Promise<string[] | null> {
|
||||
// TEXT[] binding via pgArray() string-literal escaping (see helper
|
||||
// for the security note). Our narrow SqlQuery surface
|
||||
// (src/core/sql-query.ts) doesn't bind JS arrays directly.
|
||||
const literal = pgArray(next);
|
||||
const rows = await sql`
|
||||
UPDATE oauth_clients
|
||||
SET federated_read = ${literal}
|
||||
WHERE client_id = ${clientId}
|
||||
AND deleted_at IS NULL
|
||||
RETURNING federated_read
|
||||
`;
|
||||
if (rows.length === 0) return null;
|
||||
const fed = rows[0].federated_read;
|
||||
return Array.isArray(fed) ? (fed as string[]).map(String) : [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure helper: dedupe a comma-separated source-id list while preserving
|
||||
* insertion order. Empty input → empty array. Exported so the CLI parser
|
||||
* and tests share one normalizer.
|
||||
*/
|
||||
export function parseSourceCsv(csv: string): string[] {
|
||||
const requested = csv.split(',').map((s) => s.trim()).filter(Boolean);
|
||||
const seen = new Set<string>();
|
||||
const out: string[] = [];
|
||||
for (const s of requested) {
|
||||
if (!seen.has(s)) {
|
||||
seen.add(s);
|
||||
out.push(s);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export interface FederatedReadOpts {
|
||||
/** When true, compute the outcome but skip the persisting UPDATE. */
|
||||
dryRun?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Core: append a source to the client's federated_read.
|
||||
*
|
||||
* Atomicity contract (Codex finding #1, HIGH):
|
||||
* The actual write goes through `appendFederatedReadAtomic` which
|
||||
* serializes at the row-lock so concurrent grant/revoke against the
|
||||
* same client cannot lose updates. The race vector that previously
|
||||
* silently restored revoked access is closed: under two operators
|
||||
* racing `revoke-read sensitive` + `grant-read harmless`, postgres
|
||||
* serializes the two UPDATEs and BOTH ops apply (sensitive removed,
|
||||
* harmless added), instead of one clobbering the other.
|
||||
*
|
||||
* The reported `before` is the snapshot at resolveClient time, which
|
||||
* may be stale relative to a concurrent racer. The `after` reflects
|
||||
* the post-UPDATE state from RETURNING (always fresh).
|
||||
*/
|
||||
export async function grantReadCore(
|
||||
sql: SqlQuery,
|
||||
nameOrId: string,
|
||||
sourceId: string,
|
||||
opts: FederatedReadOpts = {},
|
||||
): Promise<FederatedReadOutcome> {
|
||||
const client = await resolveClient(sql, nameOrId);
|
||||
await assertSourceExists(sql, sourceId);
|
||||
if (client.federated_read.includes(sourceId)) {
|
||||
return { kind: 'noop', reason: 'already-granted', client, current: client.federated_read };
|
||||
}
|
||||
if (opts.dryRun) {
|
||||
// Compute the would-be result without touching the row. Last-known
|
||||
// snapshot is best-effort under concurrent writes.
|
||||
const projected = [...client.federated_read, sourceId];
|
||||
return { kind: 'updated', client, before: client.federated_read, after: projected };
|
||||
}
|
||||
const after = await appendFederatedReadAtomic(sql, client.client_id, sourceId);
|
||||
if (after === null) {
|
||||
// Two equivalent failure modes: (a) racing grant-read already added
|
||||
// the source and the NOT-ANY guard suppressed our UPDATE, or
|
||||
// (b) the client was soft-deleted between resolveClient and UPDATE.
|
||||
// (a) is the more common path. Re-resolve to confirm + report.
|
||||
const reresolved = await resolveClient(sql, client.client_id, { includeDeleted: true });
|
||||
if (reresolved.deleted_at != null) {
|
||||
throw new Error(`Client "${client.client_name}" was soft-deleted before write could land.`);
|
||||
}
|
||||
return { kind: 'noop', reason: 'already-granted', client: reresolved, current: reresolved.federated_read };
|
||||
}
|
||||
return { kind: 'updated', client, before: client.federated_read, after };
|
||||
}
|
||||
|
||||
/**
|
||||
* Core: remove a source from the client's federated_read. Atomic via
|
||||
* array_remove + ANY-guard. Same race-correctness rationale as
|
||||
* grantReadCore — concurrent ops serialize at the row lock.
|
||||
*/
|
||||
export async function revokeReadCore(
|
||||
sql: SqlQuery,
|
||||
nameOrId: string,
|
||||
sourceId: string,
|
||||
opts: FederatedReadOpts = {},
|
||||
): Promise<FederatedReadOutcome> {
|
||||
const client = await resolveClient(sql, nameOrId);
|
||||
if (!client.federated_read.includes(sourceId)) {
|
||||
return { kind: 'noop', reason: 'not-present', client, current: client.federated_read };
|
||||
}
|
||||
if (opts.dryRun) {
|
||||
const projected = client.federated_read.filter((s) => s !== sourceId);
|
||||
return { kind: 'updated', client, before: client.federated_read, after: projected };
|
||||
}
|
||||
const after = await removeFederatedReadAtomic(sql, client.client_id, sourceId);
|
||||
if (after === null) {
|
||||
// Same disambiguation as grant: either a concurrent revoke already
|
||||
// removed the source (most common) or the client was soft-deleted.
|
||||
const reresolved = await resolveClient(sql, client.client_id, { includeDeleted: true });
|
||||
if (reresolved.deleted_at != null) {
|
||||
throw new Error(`Client "${client.client_name}" was soft-deleted before write could land.`);
|
||||
}
|
||||
return { kind: 'noop', reason: 'not-present', client: reresolved, current: reresolved.federated_read };
|
||||
}
|
||||
return { kind: 'updated', client, before: client.federated_read, after };
|
||||
}
|
||||
|
||||
/**
|
||||
* Core: replace the whole federated_read list. Idempotent on same list.
|
||||
*
|
||||
* Race semantics: wholesale-overwrite + deleted_at guard. Concurrent
|
||||
* set+set is last-writer-wins (documented contract for `set` — the
|
||||
* operator is asserting the exact list). Concurrent set+grant or
|
||||
* set+revoke is also last-writer-wins. If a strict-merge semantics is
|
||||
* needed, use grant-read / revoke-read individually.
|
||||
*/
|
||||
export async function setFederatedReadCore(
|
||||
sql: SqlQuery,
|
||||
nameOrId: string,
|
||||
sourceCsv: string,
|
||||
opts: FederatedReadOpts = {},
|
||||
): Promise<FederatedReadOutcome> {
|
||||
const next = parseSourceCsv(sourceCsv);
|
||||
const client = await resolveClient(sql, nameOrId);
|
||||
for (const s of next) {
|
||||
await assertSourceExists(sql, s);
|
||||
}
|
||||
const prev = client.federated_read;
|
||||
const same = prev.length === next.length && prev.every((v, i) => v === next[i]);
|
||||
if (same) {
|
||||
return { kind: 'noop', reason: 'same-list', client, current: prev };
|
||||
}
|
||||
if (opts.dryRun) {
|
||||
return { kind: 'updated', client, before: prev, after: next };
|
||||
}
|
||||
const after = await replaceFederatedReadAtomic(sql, client.client_id, next);
|
||||
if (after === null) {
|
||||
throw new Error(`Client "${client.client_name}" was soft-deleted before write could land.`);
|
||||
}
|
||||
return { kind: 'updated', client, before: prev, after };
|
||||
}
|
||||
|
||||
function printOutcome(
|
||||
verb: 'grant' | 'revoke' | 'set',
|
||||
sourceArg: string,
|
||||
outcome: FederatedReadOutcome,
|
||||
dryRun: boolean,
|
||||
): void {
|
||||
// Terminal-injection defense (Codex finding #5, low): a client_name
|
||||
// registered via DCR with ANSI escapes or control chars would
|
||||
// otherwise poison this output. Sanitize ALL strings that round-trip
|
||||
// from the DB before printing.
|
||||
const s = sanitizeForTerminal;
|
||||
const prefix = dryRun ? '[dry-run] ' : '';
|
||||
if (outcome.kind === 'noop') {
|
||||
const name = s(outcome.client.client_name);
|
||||
if (outcome.reason === 'already-granted') {
|
||||
console.log(`${prefix}No change: "${name}" already reads "${s(sourceArg)}".`);
|
||||
} else if (outcome.reason === 'not-present') {
|
||||
console.log(`${prefix}No change: "${name}" did not read "${s(sourceArg)}".`);
|
||||
} else {
|
||||
console.log(`${prefix}No change: "${name}" federated_read already matches.`);
|
||||
}
|
||||
console.log(` federated_read: ${outcome.current.map(s).join(', ') || '(empty)'}`);
|
||||
return;
|
||||
}
|
||||
const { client, before, after } = outcome;
|
||||
const name = s(client.client_name);
|
||||
const wouldOrDid = dryRun ? 'Would' : 'Did';
|
||||
if (verb === 'grant') {
|
||||
console.log(`${prefix}${wouldOrDid} grant: "${name}" can now read "${s(sourceArg)}".`);
|
||||
console.log(` federated_read: ${after.map(s).join(', ')}`);
|
||||
} else if (verb === 'revoke') {
|
||||
console.log(`${prefix}${wouldOrDid} revoke: "${name}" no longer reads "${s(sourceArg)}".`);
|
||||
console.log(` federated_read: ${after.map(s).join(', ') || '(empty — client has no federated reads)'}`);
|
||||
} else {
|
||||
console.log(`${prefix}${wouldOrDid} update "${name}" federated_read:`);
|
||||
console.log(` before: ${before.map(s).join(', ') || '(empty)'}`);
|
||||
console.log(` after: ${after.map(s).join(', ') || '(empty)'}`);
|
||||
}
|
||||
if (after.length === 0) {
|
||||
console.log(
|
||||
'Warning: client now reads no sources via federation. Queries through this ' +
|
||||
'client will only see content scoped explicitly via its write source.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip `--dry-run` from a positional-arg list. Returns the filtered list
|
||||
* plus the flag value. Kept positional-tolerant — the existing
|
||||
* `auth grant-read alice source` shape MUST keep working, AND
|
||||
* `auth grant-read alice source --dry-run` AND `auth grant-read --dry-run alice source`.
|
||||
*/
|
||||
export function extractDryRun(args: string[]): { dryRun: boolean; rest: string[] } {
|
||||
let dryRun = false;
|
||||
const rest: string[] = [];
|
||||
for (const a of args) {
|
||||
if (a === '--dry-run') {
|
||||
dryRun = true;
|
||||
continue;
|
||||
}
|
||||
rest.push(a);
|
||||
}
|
||||
return { dryRun, rest };
|
||||
}
|
||||
|
||||
async function grantRead(args: string[]): Promise<void> {
|
||||
const { dryRun, rest } = extractDryRun(args);
|
||||
const [nameOrId, sourceId] = rest;
|
||||
if (!nameOrId || !sourceId) {
|
||||
console.error('Usage: gbrain auth grant-read <client-name-or-id> <source-id> [--dry-run]');
|
||||
process.exit(1);
|
||||
}
|
||||
try {
|
||||
await withConfiguredSql(async (sql) => {
|
||||
const outcome = await grantReadCore(sql, nameOrId, sourceId, { dryRun });
|
||||
printOutcome('grant', sourceId, outcome, dryRun);
|
||||
});
|
||||
} catch (e: any) {
|
||||
console.error('Error:', e.message);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
async function revokeRead(args: string[]): Promise<void> {
|
||||
const { dryRun, rest } = extractDryRun(args);
|
||||
const [nameOrId, sourceId] = rest;
|
||||
if (!nameOrId || !sourceId) {
|
||||
console.error('Usage: gbrain auth revoke-read <client-name-or-id> <source-id> [--dry-run]');
|
||||
process.exit(1);
|
||||
}
|
||||
try {
|
||||
await withConfiguredSql(async (sql) => {
|
||||
const outcome = await revokeReadCore(sql, nameOrId, sourceId, { dryRun });
|
||||
printOutcome('revoke', sourceId, outcome, dryRun);
|
||||
});
|
||||
} catch (e: any) {
|
||||
console.error('Error:', e.message);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
async function setFederatedRead(args: string[]): Promise<void> {
|
||||
const { dryRun, rest } = extractDryRun(args);
|
||||
const [nameOrId, sourceCsv] = rest;
|
||||
if (!nameOrId || sourceCsv === undefined) {
|
||||
console.error(
|
||||
'Usage: gbrain auth set-federated-read <client-name-or-id> <source-id1,source-id2,...> [--dry-run]',
|
||||
);
|
||||
console.error('Pass an empty string ("") to clear all federated reads.');
|
||||
process.exit(1);
|
||||
}
|
||||
try {
|
||||
await withConfiguredSql(async (sql) => {
|
||||
const outcome = await setFederatedReadCore(sql, nameOrId, sourceCsv, { dryRun });
|
||||
printOutcome('set', sourceCsv, outcome, dryRun);
|
||||
});
|
||||
} catch (e: any) {
|
||||
console.error('Error:', e.message);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
async function revokeClient(clientId: string) {
|
||||
if (!clientId) {
|
||||
console.error('Usage: auth revoke-client <client_id>');
|
||||
@@ -319,7 +884,7 @@ async function revokeClient(clientId: string) {
|
||||
console.error(`No client found with id "${clientId}"`);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(`OAuth client revoked: "${rows[0].client_name}" (${clientId})`);
|
||||
console.log(`OAuth client revoked: "${sanitizeForTerminal(String(rows[0].client_name))}" (${clientId})`);
|
||||
console.log('Tokens and authorization codes purged via cascade.');
|
||||
});
|
||||
} catch (e: any) {
|
||||
@@ -440,6 +1005,15 @@ export function parseRegisterClientArgs(args: string[]): RegisterClientArgs {
|
||||
if (!grantTypesSet && out.redirectUris.length > 0) {
|
||||
out.grantTypes = ['authorization_code', 'refresh_token'];
|
||||
}
|
||||
// Codex re-review (medium): validate source_id shape at the CLI boundary
|
||||
// so register-client can't seed malformed entries into source_id /
|
||||
// federated_read that subsequent grant/revoke/set commands can't manage.
|
||||
assertValidSourceId(out.sourceId);
|
||||
if (out.federatedRead) {
|
||||
for (const s of out.federatedRead) {
|
||||
assertValidSourceId(s);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
@@ -557,6 +1131,10 @@ export async function runAuth(args: string[]): Promise<void> {
|
||||
}
|
||||
case 'register-client': await registerClient(rest[0], rest.slice(1)); return;
|
||||
case 'revoke-client': await revokeClient(rest[0]); return;
|
||||
case 'list-clients': await listClients(rest); return;
|
||||
case 'grant-read': await grantRead(rest); return;
|
||||
case 'revoke-read': await revokeRead(rest); return;
|
||||
case 'set-federated-read': await setFederatedRead(rest); return;
|
||||
case 'test': {
|
||||
const tokenIdx = rest.indexOf('--token');
|
||||
const url = rest.find(a => !a.startsWith('--') && a !== rest[tokenIdx + 1]);
|
||||
@@ -594,6 +1172,13 @@ Usage:
|
||||
--bound-max-concurrent <n> Bound submit_agent concurrency (default: 1)
|
||||
--budget-usd-per-day <usd> Bound submit_agent daily spend cap
|
||||
gbrain auth revoke-client <client_id> Hard-delete an OAuth 2.1 client (cascades to tokens + codes)
|
||||
gbrain auth list-clients [--json] List OAuth 2.1 clients with scope + write source + federated_read.
|
||||
gbrain auth grant-read <name|client_id> <source-id> [--dry-run]
|
||||
Add a source to the client's federated_read list (idempotent).
|
||||
gbrain auth revoke-read <name|client_id> <source-id> [--dry-run]
|
||||
Remove a source from the client's federated_read list (idempotent).
|
||||
gbrain auth set-federated-read <name|client_id> "<id1,id2,...>" [--dry-run]
|
||||
Replace the client's whole federated_read list. Pass "" to clear.
|
||||
gbrain auth test <url> --token <token> Smoke-test a remote MCP server
|
||||
`);
|
||||
}
|
||||
|
||||
@@ -37,19 +37,9 @@ export async function findCodeDef(
|
||||
// trigger) are first-class definitions in the SQL sense. The chunker's
|
||||
// normalizeSymbolType maps create_table → 'table' etc, so adding the SQL
|
||||
// kinds here is what makes `gbrain code-def users` work against SQL.
|
||||
// Method-level + member definitions. normalizeSymbolType only canonicalizes
|
||||
// some node types; the rest fall through `type.replace(/_/g, ' ')`, so
|
||||
// tree-sitter's method_declaration → 'method declaration', struct_specifier →
|
||||
// 'struct specifier', protocol_declaration → 'protocol declaration', etc.
|
||||
// Without these, code-def is blind to every method, constructor, field, C
|
||||
// struct, and Swift protocol — which is most of an OO codebase. The plain
|
||||
// 'struct' entry above never matched for the same reason (C emits the
|
||||
// 'struct specifier' fallback form).
|
||||
const DEF_TYPES = [
|
||||
'function', 'class', 'interface', 'type', 'enum', 'struct', 'trait', 'module', 'contract',
|
||||
'table', 'view', 'index', 'procedure', 'schema', 'database', 'trigger',
|
||||
'method declaration', 'method definition', 'constructor declaration',
|
||||
'field declaration', 'field definition', 'struct specifier', 'protocol declaration',
|
||||
];
|
||||
const params: unknown[] = [symbol, limit];
|
||||
let whereLang = '';
|
||||
|
||||
@@ -4963,7 +4963,8 @@ export async function buildChecks(
|
||||
message:
|
||||
`${unmatched}/${sample.length} conversation pages (${unmatchedPct.toFixed(1)}%) match NO built-in pattern. ` +
|
||||
`Breakdown: ${breakdown}. ` +
|
||||
`Investigate: gbrain conversation-parser scan <slug>`,
|
||||
`Investigate: gbrain conversation-parser scan <slug> | ` +
|
||||
`Enable LLM fallback (opt-in): gbrain config set conversation_parser.llm_fallback_enabled true`,
|
||||
});
|
||||
} else {
|
||||
checks.push({
|
||||
|
||||
+4
-34
@@ -581,7 +581,7 @@ async function embedPage(
|
||||
for (let j = 0; j < toEmbed.length; j++) {
|
||||
embeddingMap.set(toEmbed[j].chunk_index, embeddings[j]);
|
||||
}
|
||||
const updated: ChunkInput[] = chunks.map(c => preserveCodeMetadata(c, {
|
||||
const updated: ChunkInput[] = chunks.map(c => ({
|
||||
chunk_index: c.chunk_index,
|
||||
chunk_text: c.chunk_text,
|
||||
chunk_source: c.chunk_source,
|
||||
@@ -605,31 +605,6 @@ async function embedPage(
|
||||
slog(`${slug}: embedded ${toEmbed.length} chunks`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Carry code-chunk metadata (language, symbol_name, symbol_type, line range,
|
||||
* parent scope, doc comment, qualified name) from a loaded Chunk back into a
|
||||
* ChunkInput destined for upsertChunks.
|
||||
*
|
||||
* Issue #769: every re-embed used to strip these fields, and upsertChunks
|
||||
* overwrites (does not COALESCE) the metadata columns from EXCLUDED, so
|
||||
* each pass clobbered code-def's primary index to NULL. Pulling the
|
||||
* preservation into one helper keeps the three re-embed call sites
|
||||
* (embedPage, embedAll non-stale, embedAllStale) in lock-step.
|
||||
*/
|
||||
function preserveCodeMetadata(loaded: any, base: ChunkInput): ChunkInput {
|
||||
return {
|
||||
...base,
|
||||
language: loaded.language ?? undefined,
|
||||
symbol_name: loaded.symbol_name ?? undefined,
|
||||
symbol_type: loaded.symbol_type ?? undefined,
|
||||
start_line: loaded.start_line ?? undefined,
|
||||
end_line: loaded.end_line ?? undefined,
|
||||
parent_symbol_path: loaded.parent_symbol_path ?? undefined,
|
||||
doc_comment: loaded.doc_comment ?? undefined,
|
||||
symbol_name_qualified: loaded.symbol_name_qualified ?? undefined,
|
||||
};
|
||||
}
|
||||
|
||||
async function embedAll(
|
||||
engine: BrainEngine,
|
||||
staleOnly: boolean,
|
||||
@@ -742,10 +717,8 @@ async function embedAll(
|
||||
for (let j = 0; j < toEmbed.length; j++) {
|
||||
embeddingMap.set(toEmbed[j].chunk_index, embeddings[j]);
|
||||
}
|
||||
// Preserve ALL chunks, only update embeddings for stale ones.
|
||||
// preserveCodeMetadata threads code-chunk metadata (#769) so re-embed
|
||||
// doesn't clobber language/symbol_name/symbol_type to NULL.
|
||||
const updated: ChunkInput[] = chunks.map(c => preserveCodeMetadata(c, {
|
||||
// Preserve ALL chunks, only update embeddings for stale ones
|
||||
const updated: ChunkInput[] = chunks.map(c => ({
|
||||
chunk_index: c.chunk_index,
|
||||
chunk_text: c.chunk_text,
|
||||
chunk_source: c.chunk_source,
|
||||
@@ -1039,10 +1012,7 @@ async function embedAllStale(
|
||||
for (let j = 0; j < stale.length; j++) {
|
||||
staleIdxToEmbedding.set(stale[j].chunk_index, embeddings[j]);
|
||||
}
|
||||
// preserveCodeMetadata threads code-chunk metadata (#769) so the
|
||||
// autopilot --stale path doesn't clobber language/symbol_name/etc
|
||||
// to NULL on every cycle.
|
||||
const merged: ChunkInput[] = existing.map(c => preserveCodeMetadata(c, {
|
||||
const merged: ChunkInput[] = existing.map(c => ({
|
||||
chunk_index: c.chunk_index,
|
||||
chunk_text: c.chunk_text,
|
||||
chunk_source: c.chunk_source,
|
||||
|
||||
+4
-53
@@ -469,53 +469,15 @@ export async function extractLinksFromFile(
|
||||
|
||||
// --- Timeline extraction ---
|
||||
|
||||
/**
|
||||
* Index of the first dash (—, –, -) that can serve as the Source — Summary
|
||||
* delimiter: it must have whitespace on both sides and sit outside every
|
||||
* markdown-link span. Hyphens inside link targets
|
||||
* (`../people/alice-example.md`) and dashes inside link labels
|
||||
* (`[Deals — Q1 Review](...)`) are content, not delimiters — splitting on
|
||||
* them shatters one entry into two fragments whose halves re-insert on
|
||||
* every sync (the (page_id, date, summary, source) uniqueness sees each
|
||||
* fragment shape as a new row). Returns -1 when the line has no delimiter.
|
||||
*/
|
||||
function findDelimiterOutsideLinks(text: string): number {
|
||||
let depth = 0;
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
const c = text[i];
|
||||
if (c === '[' || c === '(') depth++;
|
||||
else if (c === ']' || c === ')') { if (depth > 0) depth--; }
|
||||
else if (
|
||||
depth === 0 &&
|
||||
(c === '—' || c === '–' || c === '-') &&
|
||||
i > 0 && /\s/.test(text[i - 1]) &&
|
||||
i + 1 < text.length && /\s/.test(text[i + 1])
|
||||
) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
/** Extract timeline entries from markdown content */
|
||||
export function extractTimelineFromContent(content: string, slug: string): ExtractedTimelineEntry[] {
|
||||
const entries: ExtractedTimelineEntry[] = [];
|
||||
|
||||
// Format 1: Bullet — - **YYYY-MM-DD** | Source — Summary
|
||||
// The delimiter search is link-aware (see findDelimiterOutsideLinks); a
|
||||
// bullet with no delimiter (e.g. an auto-generated backlink line
|
||||
// `- **date** | Referenced in [X](y.md)`) is kept whole as the summary
|
||||
// rather than dropped or fragmented.
|
||||
const bulletPattern = /^-\s+\*\*(\d{4}-\d{2}-\d{2})\*\*\s*\|\s*(.+)$/gm;
|
||||
const bulletPattern = /^-\s+\*\*(\d{4}-\d{2}-\d{2})\*\*\s*\|\s*(.+?)\s*[—–-]\s*(.+)$/gm;
|
||||
let match;
|
||||
while ((match = bulletPattern.exec(content)) !== null) {
|
||||
const rest = match[2].trim();
|
||||
const at = findDelimiterOutsideLinks(rest);
|
||||
if (at >= 0) {
|
||||
entries.push({ slug, date: match[1], source: rest.slice(0, at).trim(), summary: rest.slice(at + 1).trim() });
|
||||
} else {
|
||||
entries.push({ slug, date: match[1], source: 'markdown', summary: rest });
|
||||
}
|
||||
entries.push({ slug, date: match[1], source: match[2].trim(), summary: match[3].trim() });
|
||||
}
|
||||
|
||||
// Format 2: Header — ### YYYY-MM-DD — Title
|
||||
@@ -1689,7 +1651,7 @@ async function extractTimelineFromDB(
|
||||
* make re-extraction idempotent). EVERY processed page is stamped, including
|
||||
* zero-link pages — they WERE processed.
|
||||
*/
|
||||
export async function extractStaleFromDB(
|
||||
async function extractStaleFromDB(
|
||||
engine: BrainEngine,
|
||||
opts: {
|
||||
dryRun: boolean;
|
||||
@@ -1781,18 +1743,7 @@ export async function extractStaleFromDB(
|
||||
// `page.updated_at.toISOString()` — the JS Date is ms-truncated, so the
|
||||
// µs-precision DB updated_at stayed strictly greater and the page never
|
||||
// cleared on Postgres. Stamping the exact value makes them equal.
|
||||
//
|
||||
// BUT the stamp must also clear the version-staleness clause
|
||||
// (`links_extracted_at < versionTs`). A page whose updated_at predates
|
||||
// versionTs would otherwise be stamped below the threshold and read as
|
||||
// stale forever — a permanent re-extract loop that never clears the lag.
|
||||
// GREATEST(updated_at, versionTs) preserves the race semantics (a real
|
||||
// future edit advances updated_at > versionTs >= stamp → re-extracts)
|
||||
// while lifting old pages to the threshold so they clear.
|
||||
const stampIso = page.updated_at.getTime() >= Date.parse(versionTs)
|
||||
? page.updated_at_iso
|
||||
: versionTs;
|
||||
processedRefs.push({ slug: page.slug, source_id: page.source_id, extractedAt: stampIso });
|
||||
processedRefs.push({ slug: page.slug, source_id: page.source_id, extractedAt: page.updated_at_iso });
|
||||
}
|
||||
|
||||
// Flush NON-swallowing (CDX-4): a throw here propagates out of the sweep so
|
||||
|
||||
@@ -98,17 +98,8 @@ export function findBareTweetHits(compiledTruth: string, slug: string): BareTwee
|
||||
}
|
||||
// If the line already contains a tweet URL, it's cited — skip
|
||||
if (URL_NEARBY_RE.test(line)) continue;
|
||||
// If the line carries an explicit source citation (e.g.
|
||||
// "[Source: X, @handle, 2026-05-28]"), it's already attributed — skip.
|
||||
// Catches instructional/example lines in recipe docs that demonstrate
|
||||
// the CORRECT citation format. (v0.42.x)
|
||||
if (/\[\s*source:/i.test(line)) continue;
|
||||
// Strip inline-code spans (`...`) before matching: phrases shown as
|
||||
// inline-code templates in docs are examples, not bare claims. The
|
||||
// fenced-code skip above only covers ``` blocks, not inline backticks.
|
||||
const lineForMatch = line.replace(/`[^`]*`/g, '');
|
||||
for (const re of BARE_TWEET_PHRASES) {
|
||||
const m = lineForMatch.match(re);
|
||||
const m = line.match(re);
|
||||
if (m) {
|
||||
hits.push({ slug, line: i + 1, rawLine: line.trim(), phrase: m[0] });
|
||||
break; // one finding per line is enough
|
||||
|
||||
@@ -1664,13 +1664,7 @@ export async function registerBuiltinHandlers(
|
||||
|
||||
worker.register('backlinks', async (job) => {
|
||||
const { runBacklinksCore } = await import('./backlinks.ts');
|
||||
// Default to 'check', not 'fix': backlinks jobs submitted with an empty
|
||||
// payload (e.g. the sync→embed→backlinks chains enqueued after ingestion)
|
||||
// must never rewrite tracked brain pages with generated "Referenced in"
|
||||
// timeline bullets. Mirrors the documented intent in src/core/cycle.ts
|
||||
// (runPhaseBacklinks). The filesystem fixer stays available explicitly
|
||||
// via '{"action":"fix"}' or `gbrain check-backlinks fix`.
|
||||
const action: 'check' | 'fix' = job.data.action === 'fix' ? 'fix' : 'check';
|
||||
const action: 'check' | 'fix' = job.data.action === 'check' ? 'check' : 'fix';
|
||||
const dir = typeof job.data.dir === 'string'
|
||||
? job.data.dir
|
||||
: (await engine.getConfig('sync.repo_path')) ?? '.';
|
||||
|
||||
@@ -127,12 +127,7 @@ export function lintContent(content: string, filePath: string, opts: LintContent
|
||||
}
|
||||
|
||||
// Rule: Wrapping code fences (```markdown ... ```)
|
||||
// Detector intentionally has NO /m flag so ^/$ match start/end of the whole
|
||||
// file, not inner lines. Keeps detector in sync with fixContent() below,
|
||||
// which also has no /m flag. Without this, lint reports "fixable" false
|
||||
// positives on any page that simply contains a ```markdown code block, but
|
||||
// fixContent can never strip them (its regex only matches whole-file wrappers).
|
||||
if (content.match(/^```(?:markdown|md)\s*\n/) && content.match(/\n```\s*$/)) {
|
||||
if (content.match(/^```(?:markdown|md)\s*\n/m) && content.match(/\n```\s*$/m)) {
|
||||
issues.push({
|
||||
file: filePath, line: 1, rule: 'code-fence-wrap',
|
||||
message: 'Page wrapped in ```markdown code fences (LLM artifact)',
|
||||
|
||||
@@ -1,224 +0,0 @@
|
||||
/**
|
||||
* gbrain maintain — conservative self-healing maintenance.
|
||||
*
|
||||
* This command automates the safe parts of the operator runbook:
|
||||
* - stale link/timeline extraction
|
||||
* - stale per-source dream cycles when doctor reports cycle_freshness
|
||||
*
|
||||
* It deliberately does NOT mutate source files, apply schema-pack upgrades, or
|
||||
* invent semantic hub links. Those need review or a separate command with an
|
||||
* auditable proposal surface.
|
||||
*/
|
||||
|
||||
import { existsSync } from 'fs';
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import type { BrainHealth } from '../core/types.ts';
|
||||
import { buildChecks, computeDoctorReport, type DoctorReport, type Check } from './doctor.ts';
|
||||
import { extractStaleFromDB } from './extract.ts';
|
||||
import { runCycle, type CycleReport } from '../core/cycle.ts';
|
||||
|
||||
type ActionStatus = 'ok' | 'would_apply' | 'applied' | 'blocked' | 'skipped';
|
||||
|
||||
export interface MaintenanceAction {
|
||||
name: string;
|
||||
status: ActionStatus;
|
||||
message: string;
|
||||
details?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface MaintainOptions {
|
||||
json: boolean;
|
||||
safe: boolean;
|
||||
dryRun: boolean;
|
||||
help: boolean;
|
||||
}
|
||||
|
||||
export interface MaintainReport {
|
||||
mode: 'dry-run' | 'safe';
|
||||
before: {
|
||||
health: BrainHealth;
|
||||
doctor: DoctorReport;
|
||||
};
|
||||
actions: MaintenanceAction[];
|
||||
after: {
|
||||
health: BrainHealth;
|
||||
doctor: DoctorReport;
|
||||
};
|
||||
}
|
||||
|
||||
export function parseMaintainArgs(args: string[]): MaintainOptions {
|
||||
const safe = args.includes('--safe');
|
||||
return {
|
||||
json: args.includes('--json'),
|
||||
safe,
|
||||
dryRun: args.includes('--dry-run') || !safe,
|
||||
help: args.includes('--help') || args.includes('-h'),
|
||||
};
|
||||
}
|
||||
|
||||
export function extractCycleFreshnessSourceIds(checks: Check[]): string[] {
|
||||
const ids = new Set<string>();
|
||||
for (const check of checks) {
|
||||
if (check.name !== 'cycle_freshness' || check.status === 'ok') continue;
|
||||
const re = /Source '([^']+)' last cycled/g;
|
||||
for (const match of check.message.matchAll(re)) {
|
||||
const id = match[1]?.trim();
|
||||
if (id) ids.add(id);
|
||||
}
|
||||
}
|
||||
return [...ids].sort();
|
||||
}
|
||||
|
||||
async function buildDoctorReport(engine: BrainEngine): Promise<DoctorReport> {
|
||||
const checks = await buildChecks(engine, ['--json', '--scope=brain']);
|
||||
return computeDoctorReport(checks);
|
||||
}
|
||||
|
||||
async function runStaleExtraction(
|
||||
engine: BrainEngine,
|
||||
beforeHealth: BrainHealth,
|
||||
dryRun: boolean,
|
||||
): Promise<MaintenanceAction> {
|
||||
if (beforeHealth.stale_pages <= 0) {
|
||||
return { name: 'extract_stale', status: 'ok', message: 'No stale pages.' };
|
||||
}
|
||||
|
||||
if (dryRun) {
|
||||
return {
|
||||
name: 'extract_stale',
|
||||
status: 'would_apply',
|
||||
message: `Would run DB-backed stale extraction for ${beforeHealth.stale_pages} page(s).`,
|
||||
details: { stale_pages: beforeHealth.stale_pages },
|
||||
};
|
||||
}
|
||||
|
||||
const result = await extractStaleFromDB(engine, {
|
||||
dryRun: false,
|
||||
jsonMode: false,
|
||||
includeFrontmatter: false,
|
||||
catchUp: false,
|
||||
});
|
||||
|
||||
return {
|
||||
name: 'extract_stale',
|
||||
status: 'applied',
|
||||
message: `Processed ${result.pagesProcessed} stale page(s); ${result.staleRemaining} remain.`,
|
||||
details: {
|
||||
links_created: result.linksCreated,
|
||||
timeline_created: result.timelineCreated,
|
||||
pages_processed: result.pagesProcessed,
|
||||
stale_remaining: result.staleRemaining,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function runCycleFreshnessMaintenance(
|
||||
engine: BrainEngine,
|
||||
beforeDoctor: DoctorReport,
|
||||
dryRun: boolean,
|
||||
): Promise<MaintenanceAction[]> {
|
||||
const sourceIds = extractCycleFreshnessSourceIds(beforeDoctor.checks);
|
||||
if (sourceIds.length === 0) {
|
||||
return [{ name: 'cycle_freshness', status: 'ok', message: 'All sources cycled recently.' }];
|
||||
}
|
||||
|
||||
if (dryRun) {
|
||||
return sourceIds.map((sourceId) => ({
|
||||
name: 'cycle_freshness',
|
||||
status: 'would_apply',
|
||||
message: `Would run source-scoped dream cycle for ${sourceId}.`,
|
||||
details: { source_id: sourceId },
|
||||
}));
|
||||
}
|
||||
|
||||
const sources = await engine.listAllSources();
|
||||
const actions: MaintenanceAction[] = [];
|
||||
|
||||
for (const sourceId of sourceIds) {
|
||||
const source = sources.find((s) => s.id === sourceId);
|
||||
const localPath = source?.local_path ?? null;
|
||||
const brainDir = localPath && existsSync(localPath) ? localPath : null;
|
||||
const report: CycleReport = await runCycle(engine, {
|
||||
brainDir,
|
||||
dryRun: false,
|
||||
pull: false,
|
||||
sourceId,
|
||||
});
|
||||
actions.push({
|
||||
name: 'cycle_freshness',
|
||||
status: report.status === 'failed' ? 'blocked' : 'applied',
|
||||
message: `Ran source-scoped dream cycle for ${sourceId}: ${report.status}.`,
|
||||
details: {
|
||||
source_id: sourceId,
|
||||
brain_dir: brainDir,
|
||||
cycle_status: report.status,
|
||||
phases: report.phases.map((p) => ({ phase: p.phase, status: p.status })),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return actions;
|
||||
}
|
||||
|
||||
export async function runMaintain(engine: BrainEngine, args: string[]): Promise<MaintainReport | void> {
|
||||
const opts = parseMaintainArgs(args);
|
||||
if (opts.help) {
|
||||
console.log(`Usage: gbrain maintain [--safe] [--dry-run] [--json]
|
||||
|
||||
Conservative self-healing maintenance.
|
||||
|
||||
Modes:
|
||||
--dry-run Preview safe actions without writes. Default when --safe is absent.
|
||||
--safe Apply safe actions: stale extraction and source cycle freshness.
|
||||
--json Emit a structured before/action/after report.
|
||||
|
||||
Not auto-applied:
|
||||
source-file frontmatter fixes, schema-pack upgrades, atom-pack changes,
|
||||
semantic hub-link guesses, and destructive cleanup.
|
||||
`);
|
||||
return;
|
||||
}
|
||||
|
||||
const beforeHealth = await engine.getHealth();
|
||||
const beforeDoctor = await buildDoctorReport(engine);
|
||||
const actions: MaintenanceAction[] = [];
|
||||
|
||||
actions.push(await runStaleExtraction(engine, beforeHealth, opts.dryRun));
|
||||
actions.push(...await runCycleFreshnessMaintenance(engine, beforeDoctor, opts.dryRun));
|
||||
|
||||
const afterHealth = await engine.getHealth();
|
||||
const afterDoctor = await buildDoctorReport(engine);
|
||||
const report: MaintainReport = {
|
||||
mode: opts.dryRun ? 'dry-run' : 'safe',
|
||||
before: { health: beforeHealth, doctor: beforeDoctor },
|
||||
actions,
|
||||
after: { health: afterHealth, doctor: afterDoctor },
|
||||
};
|
||||
|
||||
if (opts.json) {
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
} else {
|
||||
printMaintainReport(report);
|
||||
}
|
||||
return report;
|
||||
}
|
||||
|
||||
function printMaintainReport(report: MaintainReport): void {
|
||||
console.log(`GBrain maintain (${report.mode})`);
|
||||
console.log(
|
||||
`Before: brain_score=${Math.round(report.before.health.brain_score)}/100 ` +
|
||||
`stale=${report.before.health.stale_pages} islands=${report.before.health.orphan_pages} ` +
|
||||
`doctor=${report.before.doctor.status}`,
|
||||
);
|
||||
for (const action of report.actions) {
|
||||
console.log(` ${action.status}: ${action.name} — ${action.message}`);
|
||||
}
|
||||
console.log(
|
||||
`After: brain_score=${Math.round(report.after.health.brain_score)}/100 ` +
|
||||
`stale=${report.after.health.stale_pages} islands=${report.after.health.orphan_pages} ` +
|
||||
`doctor=${report.after.doctor.status}`,
|
||||
);
|
||||
if (report.mode === 'dry-run') {
|
||||
console.log('Run `gbrain maintain --safe` to apply safe actions.');
|
||||
}
|
||||
}
|
||||
+1
-14
@@ -536,20 +536,7 @@ function shouldSkipProvider(modelStr: string, skip: string[]): boolean {
|
||||
|
||||
export async function runModels(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
const json = args.includes('--json');
|
||||
// args is `subArgs` from cli.ts `handleCliOnly` — the leading 'models'
|
||||
// token has already been stripped. The subcommand is at args[0], NOT
|
||||
// args[1]. Pre-fix this check was `args[1]`, so `gbrain models doctor`
|
||||
// silently fell through to the read view. The doctor probe path was
|
||||
// unreachable from the CLI.
|
||||
//
|
||||
// --help honored FIRST so `gbrain models doctor --help` shows usage
|
||||
// instead of running network probes (which would spend tokens or
|
||||
// exit nonzero when the user only asked for help). Pre-fix the
|
||||
// args[1] ternary happened to dodge this by always falling through
|
||||
// to the args.includes('--help') branch; the args[0] rewrite needs
|
||||
// explicit ordering to preserve that behavior.
|
||||
const hasHelp = args.includes('--help') || args.includes('-h') || args[0] === 'help';
|
||||
const sub = hasHelp ? 'help' : args[0] === 'doctor' ? 'doctor' : 'read';
|
||||
const sub = args[1] === 'doctor' ? 'doctor' : args[1] === 'help' || args.includes('--help') || args.includes('-h') ? 'help' : 'read';
|
||||
|
||||
if (sub === 'help') {
|
||||
process.stdout.write(
|
||||
|
||||
+55
-10
@@ -15,11 +15,6 @@
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import { createProgress, startHeartbeat } from '../core/progress.ts';
|
||||
import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts';
|
||||
import {
|
||||
shouldExcludeFromOrphanReporting,
|
||||
loadOrphanPolicyOverrides,
|
||||
type OrphanPolicyOverrides,
|
||||
} from '../core/orphan-policy.ts';
|
||||
|
||||
// --- Types ---
|
||||
|
||||
@@ -37,14 +32,65 @@ export interface OrphanResult {
|
||||
excluded: number;
|
||||
}
|
||||
|
||||
// --- Filter constants ---
|
||||
|
||||
/** Slug suffixes that are always auto-generated root files */
|
||||
const AUTO_SUFFIX_PATTERNS = ['/_index', '/log'];
|
||||
|
||||
/** Page slugs that are pseudo-pages by convention */
|
||||
const PSEUDO_SLUGS = new Set(['_atlas', '_index', '_stats', '_orphans', '_scratch', 'claude']);
|
||||
|
||||
/** Slug segment that marks raw sources */
|
||||
const RAW_SEGMENT = '/raw/';
|
||||
|
||||
/** Slug prefixes where no inbound links is expected */
|
||||
const DENY_PREFIXES = [
|
||||
'output/',
|
||||
'dashboards/',
|
||||
'scripts/',
|
||||
'templates/',
|
||||
'openclaw/config/',
|
||||
];
|
||||
|
||||
/** First slug segments where no inbound links is expected */
|
||||
const FIRST_SEGMENT_EXCLUSIONS = new Set([
|
||||
'scratch',
|
||||
'thoughts',
|
||||
'catalog',
|
||||
'entities',
|
||||
'raw',
|
||||
'atoms',
|
||||
'skills',
|
||||
]);
|
||||
|
||||
// --- Filter logic ---
|
||||
|
||||
/**
|
||||
* Returns true if a slug should be excluded from orphan reporting by default.
|
||||
* These are pages where having no inbound links is expected / not a content problem.
|
||||
*/
|
||||
export function shouldExclude(slug: string, overrides?: OrphanPolicyOverrides): boolean {
|
||||
return shouldExcludeFromOrphanReporting(slug, overrides);
|
||||
export function shouldExclude(slug: string): boolean {
|
||||
// Pseudo-pages (exact match)
|
||||
if (PSEUDO_SLUGS.has(slug)) return true;
|
||||
|
||||
// Auto-generated suffix patterns
|
||||
for (const suffix of AUTO_SUFFIX_PATTERNS) {
|
||||
if (slug.endsWith(suffix)) return true;
|
||||
}
|
||||
|
||||
// Raw source slugs
|
||||
if (slug.includes(RAW_SEGMENT)) return true;
|
||||
|
||||
// Deny-prefix slugs
|
||||
for (const prefix of DENY_PREFIXES) {
|
||||
if (slug.startsWith(prefix)) return true;
|
||||
}
|
||||
|
||||
// First-segment exclusions
|
||||
const firstSegment = slug.split('/')[0];
|
||||
if (FIRST_SEGMENT_EXCLUSIONS.has(firstSegment)) return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -110,7 +156,6 @@ export async function findOrphans(
|
||||
let allOrphans: { slug: string; title: string; domain: string | null }[];
|
||||
let total: number;
|
||||
let excludedAll: number;
|
||||
const overrides = includePseudo ? undefined : await loadOrphanPolicyOverrides(engine);
|
||||
try {
|
||||
allOrphans = await engine.findOrphanPages(
|
||||
sourceIds ? { sourceIds } : sourceId ? { sourceId } : undefined,
|
||||
@@ -139,7 +184,7 @@ export async function findOrphans(
|
||||
total = liveRows.length;
|
||||
excludedAll = includePseudo
|
||||
? 0
|
||||
: liveRows.reduce((n, r) => n + (shouldExclude(r.slug, overrides) ? 1 : 0), 0);
|
||||
: liveRows.reduce((n, r) => n + (shouldExclude(r.slug) ? 1 : 0), 0);
|
||||
} finally {
|
||||
stopHb();
|
||||
progress.finish();
|
||||
@@ -147,7 +192,7 @@ export async function findOrphans(
|
||||
|
||||
const filtered = includePseudo
|
||||
? allOrphans
|
||||
: allOrphans.filter(row => !shouldExclude(row.slug, overrides));
|
||||
: allOrphans.filter(row => !shouldExclude(row.slug));
|
||||
|
||||
const orphans: OrphanPage[] = filtered.map(row => ({
|
||||
slug: row.slug,
|
||||
|
||||
+143
-17
@@ -365,6 +365,42 @@ export interface AgentClientSpend {
|
||||
inflight_count: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* `/admin/api/sources` source list — the input rows for buildSyncStatusReport.
|
||||
*
|
||||
* Queries the JSONB config column directly (listSources doesn't carry it,
|
||||
* but buildSyncStatusReport needs syncEnabled / strategy fields).
|
||||
*
|
||||
* Deliberately does NOT filter on local_path: in a push-only deployment
|
||||
* (content arrives via MCP put_page / capture / ingest, not `gbrain sync`
|
||||
* of a server checkout) every source has a null local_path — filtering on
|
||||
* it would empty both the Sources tab AND the federation source-picker.
|
||||
* buildSyncStatusReport does no disk I/O, so null-local_path sources
|
||||
* report fine (pages/chunks from SQL, staleness 'unknown' / never-synced).
|
||||
*/
|
||||
export async function queryAdminSources(engine: BrainEngine): Promise<
|
||||
Array<{ id: string; name: string; local_path: string | null; config: Record<string, unknown> }>
|
||||
> {
|
||||
const rows = await engine.executeRaw<{
|
||||
id: string;
|
||||
name: string;
|
||||
local_path: string | null;
|
||||
config: Record<string, unknown> | string | null;
|
||||
}>(
|
||||
`SELECT id, name, local_path, config FROM sources
|
||||
WHERE archived IS NOT TRUE
|
||||
ORDER BY id`,
|
||||
);
|
||||
return rows.map((r) => ({
|
||||
id: r.id,
|
||||
name: r.name,
|
||||
local_path: r.local_path,
|
||||
config: typeof r.config === 'string'
|
||||
? (JSON.parse(r.config) as Record<string, unknown>)
|
||||
: (r.config ?? {}),
|
||||
}));
|
||||
}
|
||||
|
||||
export async function queryAgentClientSpend(engine: BrainEngine): Promise<AgentClientSpend[]> {
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
const rows = await sql`
|
||||
@@ -843,21 +879,6 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
// reverse proxies / tunnels; default to localhost for dev.
|
||||
const issuerUrl = new URL(publicUrl || `http://localhost:${port}`);
|
||||
|
||||
// MCP authorization spec (2025-06-18 draft §5.1) and RFC 9728 require the
|
||||
// protected resource server to return its discovery metadata URL in the
|
||||
// WWW-Authenticate header on 401 responses:
|
||||
//
|
||||
// WWW-Authenticate: Bearer resource_metadata="<URL>"
|
||||
//
|
||||
// Clients (claude.ai, Cursor, every other MCP-aware OAuth client) use that
|
||||
// URL to find the authorization-server discovery doc + token endpoint
|
||||
// without the user having to paste those URLs manually. Pre-fix the header
|
||||
// shipped `Bearer error="invalid_token", ...` with no resource_metadata
|
||||
// parameter, so MCP clients couldn't begin the OAuth flow from a fresh
|
||||
// 401 — they would silently fail to connect with a generic "couldn't
|
||||
// reach the MCP server" error.
|
||||
const resourceMetadataUrl = `${issuerUrl.toString().replace(/\/$/, '')}/.well-known/oauth-protected-resource`;
|
||||
|
||||
// F9: cookie `secure` flag honors both the request's TLS state (req.secure
|
||||
// is set when express trust-proxy lands an X-Forwarded-Proto: https) AND
|
||||
// the operator's declared issuer protocol (so a Cloudflare-tunnel deploy
|
||||
@@ -1526,6 +1547,111 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Sources tab — read-only view of registered sources with sync + embed
|
||||
// coverage stats. Drives the admin SPA's `Sources` page.
|
||||
//
|
||||
// Returns the same shape `gbrain sources status --json` prints, so the
|
||||
// SPA stays in lockstep with the CLI surface.
|
||||
// ---------------------------------------------------------------------------
|
||||
app.get('/admin/api/sources', requireAdmin, async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const { buildSyncStatusReport } = await import('./sync.ts');
|
||||
const report = await buildSyncStatusReport(engine, await queryAdminSources(engine));
|
||||
res.json(report);
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
res.status(503).json({ error: 'service_unavailable', detail: msg });
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Federated-read management (admin-side counterparts of the CLI commands
|
||||
// `gbrain auth grant-read / revoke-read / set-federated-read`). All three
|
||||
// route through the same *Core helpers as the CLI so race-safety,
|
||||
// soft-delete filter, and source-id shape validation apply uniformly.
|
||||
//
|
||||
// The admin SPA's `Agents` page renders "Manage reads" actions per
|
||||
// client backed by these endpoints.
|
||||
// ---------------------------------------------------------------------------
|
||||
app.get('/admin/api/agents/federated-read', requireAdmin, async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const rows = await sql`
|
||||
SELECT client_id, client_name, source_id, federated_read
|
||||
FROM oauth_clients
|
||||
WHERE deleted_at IS NULL
|
||||
ORDER BY client_name
|
||||
`;
|
||||
const clients = rows.map((r) => ({
|
||||
client_id: String(r.client_id),
|
||||
client_name: String(r.client_name),
|
||||
source_id: r.source_id == null ? null : String(r.source_id),
|
||||
federated_read: Array.isArray(r.federated_read)
|
||||
? (r.federated_read as string[]).map(String)
|
||||
: [],
|
||||
}));
|
||||
res.json({ clients });
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
res.status(503).json({ error: 'service_unavailable', detail: msg });
|
||||
}
|
||||
});
|
||||
|
||||
app.post('/admin/api/agents/:clientId/grant-read', requireAdmin, express.json(), async (req: Request, res: Response) => {
|
||||
const clientId = String(req.params.clientId ?? '');
|
||||
const sourceId = String(req.body?.source_id ?? '').trim();
|
||||
if (!clientId || !sourceId) {
|
||||
res.status(400).json({ error: 'invalid_request', detail: 'clientId path param + source_id body required' });
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const { grantReadCore } = await import('./auth.ts');
|
||||
const outcome = await grantReadCore(sql, clientId, sourceId);
|
||||
res.json({ outcome });
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
res.status(400).json({ error: 'mutation_failed', detail: msg });
|
||||
}
|
||||
});
|
||||
|
||||
app.post('/admin/api/agents/:clientId/revoke-read', requireAdmin, express.json(), async (req: Request, res: Response) => {
|
||||
const clientId = String(req.params.clientId ?? '');
|
||||
const sourceId = String(req.body?.source_id ?? '').trim();
|
||||
if (!clientId || !sourceId) {
|
||||
res.status(400).json({ error: 'invalid_request', detail: 'clientId path param + source_id body required' });
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const { revokeReadCore } = await import('./auth.ts');
|
||||
const outcome = await revokeReadCore(sql, clientId, sourceId);
|
||||
res.json({ outcome });
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
res.status(400).json({ error: 'mutation_failed', detail: msg });
|
||||
}
|
||||
});
|
||||
|
||||
app.post('/admin/api/agents/:clientId/set-federated-read', requireAdmin, express.json(), async (req: Request, res: Response) => {
|
||||
const clientId = String(req.params.clientId ?? '');
|
||||
const rawIds = req.body?.source_ids;
|
||||
if (!clientId || !Array.isArray(rawIds)) {
|
||||
res.status(400).json({ error: 'invalid_request', detail: 'clientId path param + source_ids[] body required' });
|
||||
return;
|
||||
}
|
||||
// Encode the array as CSV so the same setFederatedReadCore signature
|
||||
// (string CSV input) the CLI uses applies here. Empty array → empty
|
||||
// string → clears the list.
|
||||
const csv = rawIds.map((s) => String(s).trim()).filter(Boolean).join(',');
|
||||
try {
|
||||
const { setFederatedReadCore } = await import('./auth.ts');
|
||||
const outcome = await setFederatedReadCore(sql, clientId, csv);
|
||||
res.json({ outcome });
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
res.status(400).json({ error: 'mutation_failed', detail: msg });
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// SSE live activity feed
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -1616,7 +1742,7 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
res.status(405).json({ jsonrpc: '2.0', error: { code: -32000, message: 'Method not allowed' }, id: null });
|
||||
});
|
||||
|
||||
app.post('/mcp', requireBearerAuth({ verifier: oauthProvider, resourceMetadataUrl }), async (req: Request, res: Response) => {
|
||||
app.post('/mcp', requireBearerAuth({ verifier: oauthProvider }), async (req: Request, res: Response) => {
|
||||
const startTime = Date.now();
|
||||
const authInfo = (req as any).auth as AuthInfo;
|
||||
|
||||
@@ -1959,7 +2085,7 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
app.post(
|
||||
'/ingest',
|
||||
ingestRateLimiter,
|
||||
requireBearerAuth({ verifier: oauthProvider, requiredScopes: ['write'], resourceMetadataUrl }),
|
||||
requireBearerAuth({ verifier: oauthProvider, requiredScopes: ['write'] }),
|
||||
express.raw({ type: '*/*', limit: ingestMaxBytes }),
|
||||
async (req: Request, res: Response) => {
|
||||
const startTime = Date.now();
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
* degrades to gather-only output with a warning if missing.
|
||||
*/
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import { runThink, persistSynthesis, stripGapsSection } from '../core/think/index.ts';
|
||||
import { runThink, persistSynthesis } from '../core/think/index.ts';
|
||||
import { loadConfig, isThinClient } from '../core/config.ts';
|
||||
import { callRemoteTool, unpackToolResult } from '../core/mcp-client.ts';
|
||||
|
||||
@@ -157,7 +157,7 @@ prints what would have been the input (exit 0).
|
||||
|
||||
// Human-readable output
|
||||
console.log(`# ${question}\n`);
|
||||
console.log(stripGapsSection(result.answer));
|
||||
console.log(result.answer);
|
||||
console.log('');
|
||||
if (result.gaps.length > 0) {
|
||||
console.log('## Gaps');
|
||||
|
||||
@@ -263,15 +263,6 @@ export function dimsProviderOptions(
|
||||
if (modelId === 'text-embedding-v3' || modelId === 'embedding-3') {
|
||||
return { openaiCompatible: { dimensions: dims } };
|
||||
}
|
||||
// Qwen3-Embedding family on Ollama (and any other openai-compatible
|
||||
// provider serving it) supports Matryoshka truncation via `dimensions`.
|
||||
// Native sizes: 0.6B=1024, 4B=2560, 8B=4096. Without `dimensions`,
|
||||
// Ollama returns the native size and brains configured for narrower
|
||||
// widths hard-fail with a dim-mismatch error. Pattern match the bare
|
||||
// model name + any `:tag` (e.g. `qwen3-embedding:4b`, `qwen3-embedding:0.6b`).
|
||||
if (modelId === 'qwen3-embedding' || modelId.startsWith('qwen3-embedding:')) {
|
||||
return { openaiCompatible: { dimensions: dims } };
|
||||
}
|
||||
// MiniMax embo-01 takes a `type: 'db' | 'query'` field for asymmetric
|
||||
// retrieval. Today still hardcoded to 'db' for back-compat — opting
|
||||
// into the new inputType seam is a follow-up (see plan's deferred
|
||||
|
||||
+1
-27
@@ -599,8 +599,6 @@ function warnRecipesMissingBatchTokens(): void {
|
||||
// LiteLLM proxy, llama-server) — they ship without a static cap because
|
||||
// the cap depends on a user-launched server. Warning is noise for them.
|
||||
if (embedding.no_batch_cap === true) continue;
|
||||
// A declared item-count cap is a real batch cap — no warning needed.
|
||||
if (embedding.max_batch_items !== undefined) continue;
|
||||
if (_warnedRecipes.has(recipe.id)) continue;
|
||||
_warnedRecipes.add(recipe.id);
|
||||
// eslint-disable-next-line no-console
|
||||
@@ -1519,17 +1517,10 @@ export async function embed(texts: string[], opts?: EmbedOpts): Promise<Float32A
|
||||
|
||||
// Pre-split is gated on max_batch_tokens. Recipes without it (e.g. OpenAI)
|
||||
// ride the fast path: one embedMany call, no recursion safety net.
|
||||
const tokenBatches = maxBatchTokens
|
||||
const batches = maxBatchTokens
|
||||
? splitByTokenBudget(truncated, Math.floor(maxBatchTokens * effectiveSafetyFactor(recipe)), charsPerToken)
|
||||
: [truncated];
|
||||
|
||||
// Hard COUNT cap (e.g. llama-server's "maximum allowed batch size 32").
|
||||
// Token budget can't bound item count, so re-split any oversized batch.
|
||||
const maxBatchItems = embedding?.max_batch_items;
|
||||
const batches = maxBatchItems
|
||||
? tokenBatches.flatMap(b => capBatchItems(b, maxBatchItems))
|
||||
: tokenBatches;
|
||||
|
||||
const allEmbeddings: Float32Array[] = [];
|
||||
let _embedThrew = false;
|
||||
try {
|
||||
@@ -1605,23 +1596,6 @@ export function splitByTokenBudget(
|
||||
return batches;
|
||||
}
|
||||
|
||||
/**
|
||||
* Split a batch into sub-batches of at most `maxItems` inputs. Enforces a
|
||||
* hard COUNT cap that the token-budget split can't (many tiny inputs fit
|
||||
* under any token budget). Used for endpoints like llama.cpp's llama-server
|
||||
* that reject requests exceeding their launch batch size.
|
||||
*
|
||||
* @internal exported for tests; not part of the public gateway API.
|
||||
*/
|
||||
export function capBatchItems(texts: string[], maxItems: number): string[][] {
|
||||
if (maxItems <= 0 || texts.length <= maxItems) return [texts];
|
||||
const batches: string[][] = [];
|
||||
for (let i = 0; i < texts.length; i += maxItems) {
|
||||
batches.push(texts.slice(i, i + maxItems));
|
||||
}
|
||||
return batches;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if the error looks like a provider batch-token-limit error.
|
||||
*
|
||||
|
||||
@@ -35,12 +35,9 @@ export const llamaServer: Recipe = {
|
||||
trust_custom_dims: true, // #2271: user knows the launched model's native dim
|
||||
cost_per_1m_tokens_usd: 0,
|
||||
price_last_verified: '2026-05-10',
|
||||
// llama-server enforces a hard request-COUNT cap equal to its launch
|
||||
// batch size (`--batch-size`, default 32): it rejects requests with
|
||||
// more inputs with `batch size N > maximum allowed batch size 32`.
|
||||
// The token-budget split can't bound item count, so cap it here. A
|
||||
// server launched with a larger `-b` can raise this. v0.32 (#779).
|
||||
max_batch_items: 32,
|
||||
// llama-server's batch capacity is set by `--ctx-size` at launch
|
||||
// time; no static cap to declare. v0.32 (#779).
|
||||
no_batch_cap: true,
|
||||
},
|
||||
},
|
||||
/**
|
||||
|
||||
@@ -54,16 +54,6 @@ export interface EmbeddingTouchpoint {
|
||||
* `max_batch_tokens` is also set.
|
||||
*/
|
||||
safety_factor?: number;
|
||||
/**
|
||||
* Maximum number of inputs per embedding request. Some endpoints enforce a
|
||||
* hard COUNT cap independent of token budget — notably llama.cpp's
|
||||
* `llama-server`, which rejects requests with more inputs than its launch
|
||||
* batch size (e.g. `batch size 100 > maximum allowed batch size 32`). The
|
||||
* token-budget pre-split cannot bound item count (many tiny chunks fit under
|
||||
* any token budget), so this is enforced as a separate hard re-split after
|
||||
* the token split. When unset, no count cap is applied.
|
||||
*/
|
||||
max_batch_items?: number;
|
||||
/**
|
||||
* v0.27.1: when true, at least one model in this recipe accepts image
|
||||
* inputs via a multimodal embedding endpoint (e.g. Voyage's
|
||||
|
||||
@@ -166,13 +166,9 @@ const FREE_LOCAL_EMBED_PROVIDERS: ReadonlySet<string> = new Set([
|
||||
* local-inference providers (FREE_LOCAL_EMBED_PROVIDERS) price at $0 so
|
||||
* `--max-cost` callers don't hard-fail.
|
||||
* - Rerank: try ANTHROPIC_PRICING (legacy path for any Claude-priced
|
||||
* rerank); else try lookupEmbeddingPrice — paid rerank providers (e.g.
|
||||
* ZeroEntropy's zerank-2) share the same provider:model-keyed,
|
||||
* $/1M-token table as their embedding siblings, so it's reused here
|
||||
* rather than duplicated into a third table; else if the provider half
|
||||
* is in FREE_LOCAL_RERANK_PROVIDERS, return zero pricing so `--max-cost`
|
||||
* callers don't TX2 hard-fail on local inference recipes (electricity,
|
||||
* not tokens); else unknown.
|
||||
* rerank); else if the provider half is in FREE_LOCAL_RERANK_PROVIDERS,
|
||||
* return zero pricing so `--max-cost` callers don't TX2 hard-fail on
|
||||
* local inference recipes (electricity, not tokens); else unknown.
|
||||
*/
|
||||
function lookupPricing(modelId: string, kind: BudgetKind): ModelPricing | null {
|
||||
if (kind === 'embed') {
|
||||
@@ -198,14 +194,6 @@ function lookupPricing(modelId: string, kind: BudgetKind): ModelPricing | null {
|
||||
const tailHit = ANTHROPIC_PRICING[modelTail];
|
||||
if (tailHit) return tailHit;
|
||||
}
|
||||
// Paid rerank providers (e.g. ZeroEntropy's zerank-2) aren't Claude-priced,
|
||||
// so they miss the ANTHROPIC_PRICING checks above. Reuse the embedding
|
||||
// pricing table (issue #3223) — same provider:model key shape, same
|
||||
// $/1M-token unit — instead of hand-copying a third pricing surface.
|
||||
if (kind === 'rerank') {
|
||||
const hit = lookupEmbeddingPrice(modelId);
|
||||
if (hit.kind === 'known') return { input: hit.pricePerMTok, output: 0 };
|
||||
}
|
||||
// v0.40.6.1: zero-price local-inference rerank providers so the budget
|
||||
// tracker's TX2 hard-fail doesn't trip on `llama-server-reranker:<model>`
|
||||
// under `--max-cost`. Only the rerank kind — chat/embed already have
|
||||
|
||||
@@ -217,19 +217,9 @@ export function parseResolverEntries(resolverContent: string): ResolverEntry[] {
|
||||
// `skillsDir/*/SKILL.md` when manifest.json is missing — the scenario
|
||||
// needed for AGENTS.md-only OpenClaw deployments. See D-CX-12 / F-ENG-1.
|
||||
|
||||
/**
|
||||
* Simple YAML frontmatter parser — extracts triggers array if present.
|
||||
*
|
||||
* Normalizes CRLF → LF before parsing so Windows checkouts (where
|
||||
* `core.autocrlf=true` is the default) parse correctly. Without this,
|
||||
* the `^---\n` and `^triggers:\s*\n` regexes never match because the
|
||||
* file content is `---\r\n` / `triggers:\r\n`, and every skill on
|
||||
* Windows is reported as `mece_gap` regardless of its actual content.
|
||||
* CI runs on Ubuntu-only so the bug only surfaces in user environments.
|
||||
*/
|
||||
export function extractTriggers(skillContent: string): string[] {
|
||||
const content = skillContent.replace(/\r\n/g, '\n');
|
||||
const fmMatch = content.match(/^---\n([\s\S]*?)\n---/);
|
||||
/** Simple YAML frontmatter parser — extracts triggers array if present. */
|
||||
function extractTriggers(skillContent: string): string[] {
|
||||
const fmMatch = skillContent.match(/^---\n([\s\S]*?)\n---/);
|
||||
if (!fmMatch) return [];
|
||||
const fm = fmMatch[1];
|
||||
const triggersMatch = fm.match(/^triggers:\s*\n((?:\s+-\s+.+\n?)*)/m);
|
||||
|
||||
@@ -168,15 +168,6 @@ export interface CodeChunkOptions {
|
||||
largeChunkThresholdTokens?: number;
|
||||
fallbackChunkSizeWords?: number;
|
||||
fallbackOverlapWords?: number;
|
||||
/**
|
||||
* Hard upper bound (estimated tokens) on any single emitted chunk. A node
|
||||
* the AST splitter can't break up (a giant object/array literal, a single
|
||||
* huge assignment, a massive template literal) would otherwise be emitted
|
||||
* whole and rejected by the embedder ("input exceeds context length").
|
||||
* Chunks over this budget are recursively re-split. Default 2000 fits the
|
||||
* smallest common embedder context (e.g. nomic-embed-text, 2048).
|
||||
*/
|
||||
maxChunkTokens?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -558,7 +549,6 @@ export function parseWithTimeout(
|
||||
}
|
||||
|
||||
const DEFAULT_CHUNKER_TIMEOUT_MS = 30_000;
|
||||
const DEFAULT_MAX_CHUNK_TOKENS = 2000;
|
||||
|
||||
function resolveChunkerTimeoutMs(): number {
|
||||
const raw = process.env.GBRAIN_CHUNKER_TIMEOUT_MS;
|
||||
@@ -716,9 +706,9 @@ export async function chunkCodeTextFull(
|
||||
}
|
||||
|
||||
if (chunks.length === 0) {
|
||||
return { chunks: capOversizedChunks(fallbackChunks(source, filePath, language, opts), filePath, language, opts), edges: rawEdges };
|
||||
return { chunks: fallbackChunks(source, filePath, language, opts), edges: rawEdges };
|
||||
}
|
||||
return { chunks: capOversizedChunks(mergeSmallSiblings(chunks, chunkTarget), filePath, language, opts), edges: rawEdges };
|
||||
return { chunks: mergeSmallSiblings(chunks, chunkTarget), edges: rawEdges };
|
||||
} catch {
|
||||
return { chunks: fallbackChunks(source, filePath, language, opts), edges: [] };
|
||||
} finally {
|
||||
@@ -824,73 +814,6 @@ function buildMergedChunk(group: CodeChunk[], index: number): CodeChunk {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Final safety net: guarantee no emitted chunk exceeds the embedder's context
|
||||
* budget. tree-sitter splitting (splitLargeNode) can only break up a node that
|
||||
* exposes a `body` with >= 2 named children. A node without one — a giant
|
||||
* object/array literal, a single huge assignment, a massive template literal —
|
||||
* is emitted whole, producing a chunk far larger than the embedder accepts.
|
||||
* The embedder then rejects it ("input exceeds context length") and the chunk
|
||||
* is never embedded. Recursively re-split any over-budget chunk; fall back to a
|
||||
* hard character split for pathological no-whitespace content (e.g. a minified
|
||||
* one-liner) where word/line splitting can't get under budget.
|
||||
*/
|
||||
function capOversizedChunks(
|
||||
chunks: CodeChunk[],
|
||||
filePath: string,
|
||||
language: SupportedCodeLanguage,
|
||||
opts: CodeChunkOptions,
|
||||
): CodeChunk[] {
|
||||
const cap = opts.maxChunkTokens ?? DEFAULT_MAX_CHUNK_TOKENS;
|
||||
if (!chunks.some((c) => estimateTokens(c.text) > cap)) return chunks;
|
||||
const out: CodeChunk[] = [];
|
||||
for (const c of chunks) {
|
||||
if (estimateTokens(c.text) <= cap) {
|
||||
out.push({ ...c, index: out.length });
|
||||
continue;
|
||||
}
|
||||
// Strip the structured header ("[Lang] path:N-M symbol\n\n") so the splitter
|
||||
// works on the raw body; buildChunk re-adds a header to each piece.
|
||||
const body = c.text.replace(/^\[[^\]]+\] [^\n]+\n\n/, '');
|
||||
for (const piece of splitToTokenBudget(body, cap, opts)) {
|
||||
if (!piece.trim()) continue;
|
||||
out.push(buildChunk({
|
||||
body: piece,
|
||||
filePath,
|
||||
language,
|
||||
symbolName: c.metadata.symbolName,
|
||||
symbolType: c.metadata.symbolType,
|
||||
startLine: c.metadata.startLine,
|
||||
endLine: c.metadata.endLine,
|
||||
index: out.length,
|
||||
parentSymbolPath: c.metadata.parentSymbolPath,
|
||||
}));
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Split `text` into pieces each estimated <= cap tokens. Word/line-aware
|
||||
* (recursiveChunk) first; a hard character split is the last resort for
|
||||
* content with no whitespace to break on. */
|
||||
function splitToTokenBudget(text: string, cap: number, opts: CodeChunkOptions): string[] {
|
||||
const out: string[] = [];
|
||||
const pieces = recursiveChunk(text, {
|
||||
chunkSize: opts.fallbackChunkSizeWords ?? 300,
|
||||
chunkOverlap: opts.fallbackOverlapWords ?? 50,
|
||||
}).map((p) => p.text);
|
||||
for (const piece of pieces) {
|
||||
if (estimateTokens(piece) <= cap) {
|
||||
out.push(piece);
|
||||
continue;
|
||||
}
|
||||
// ~3.5 chars/token is a conservative cl100k estimate for source text.
|
||||
const charBudget = Math.max(1, Math.floor(cap * 3.5));
|
||||
for (let i = 0; i < piece.length; i += charBudget) out.push(piece.slice(i, i + charBudget));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---------- Internals ----------
|
||||
|
||||
function fallbackChunks(
|
||||
|
||||
+1
-33
@@ -620,10 +620,7 @@ export function loadConfig(): GBrainConfig | null {
|
||||
* size the schema and must be stable across engine connect.
|
||||
*/
|
||||
export async function loadConfigWithEngine(
|
||||
engine: {
|
||||
getConfig(key: string): Promise<string | null | undefined>;
|
||||
listConfigKeys?(prefix: string): Promise<string[]>;
|
||||
},
|
||||
engine: { getConfig(key: string): Promise<string | null | undefined> },
|
||||
base?: GBrainConfig | null,
|
||||
): Promise<GBrainConfig | null> {
|
||||
// Codex /ship finding #3: when there's no file config AND no env DB URL,
|
||||
@@ -660,31 +657,11 @@ export async function loadConfigWithEngine(
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
async function dbPrefixMap(prefix: string): Promise<Record<string, string> | undefined> {
|
||||
if (typeof engine.listConfigKeys !== 'function') return undefined;
|
||||
let keys: string[];
|
||||
try {
|
||||
keys = await engine.listConfigKeys(prefix);
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const out: Record<string, string> = {};
|
||||
for (const key of keys.sort()) {
|
||||
if (!key.startsWith(prefix)) continue;
|
||||
const leaf = key.slice(prefix.length);
|
||||
if (!leaf) continue;
|
||||
const value = await dbStr(key);
|
||||
if (value !== undefined) out[leaf] = value;
|
||||
}
|
||||
return Object.keys(out).length > 0 ? out : undefined;
|
||||
}
|
||||
|
||||
const dbMultimodal = await dbBool('embedding_multimodal');
|
||||
const dbMultimodalModel = await dbStr('embedding_multimodal_model');
|
||||
const dbOcr = await dbBool('embedding_image_ocr');
|
||||
const dbOcrModel = await dbStr('embedding_image_ocr_model');
|
||||
const dbProviderBaseUrls = await dbPrefixMap('provider_base_urls.');
|
||||
// v0.36 (D7) — embedding-column registry merge. Stored as JSON string in
|
||||
// the config table. Parse + shape-check here; full registry validation
|
||||
// (regex on keys, type/dim/provider field shapes) runs in the resolver at
|
||||
@@ -708,15 +685,6 @@ export async function loadConfigWithEngine(
|
||||
if (merged.embedding_image_ocr_model === undefined && dbOcrModel !== undefined) {
|
||||
merged.embedding_image_ocr_model = dbOcrModel;
|
||||
}
|
||||
if (dbProviderBaseUrls !== undefined) {
|
||||
const next = { ...(merged.provider_base_urls ?? {}) };
|
||||
for (const [providerId, baseUrl] of Object.entries(dbProviderBaseUrls)) {
|
||||
if (next[providerId] === undefined) next[providerId] = baseUrl;
|
||||
}
|
||||
if (Object.keys(next).length > 0) {
|
||||
merged.provider_base_urls = next;
|
||||
}
|
||||
}
|
||||
if (merged.embedding_columns === undefined && dbEmbeddingColumns !== undefined) {
|
||||
try {
|
||||
const parsed = JSON.parse(dbEmbeddingColumns);
|
||||
|
||||
@@ -54,7 +54,6 @@ import {
|
||||
import {
|
||||
generatePerChunkSynopsis,
|
||||
SYNOPSIS_PROMPT_VERSION,
|
||||
SYNOPSIS_DOC_MAX_CHARS,
|
||||
type GeneratePerChunkSynopsisResult,
|
||||
} from './page-summary.ts';
|
||||
import {
|
||||
@@ -104,17 +103,8 @@ function getEmbeddingModelTag(): string {
|
||||
export function computeCorpusGeneration(args: {
|
||||
crMode: CRMode;
|
||||
haikuModel: string;
|
||||
/**
|
||||
* Resolved `SYNOPSIS_DOC_MAX_CHARS` for per_chunk_synopsis runs. When
|
||||
* present, folded into the hash so changes to
|
||||
* `GBRAIN_SYNOPSIS_DOC_MAX_CHARS` invalidate the prior cache cleanly.
|
||||
* Omit for `crMode !== 'per_chunk_synopsis'` — title / none modes
|
||||
* don't consult the cap and the field stays out of the hash for
|
||||
* back-compat with pre-cap embeddings.
|
||||
*/
|
||||
synopsisDocMaxChars?: number;
|
||||
}): string {
|
||||
const h = createHash('sha256')
|
||||
return createHash('sha256')
|
||||
.update(args.crMode)
|
||||
.update('|')
|
||||
.update(String(SYNOPSIS_PROMPT_VERSION))
|
||||
@@ -123,11 +113,9 @@ export function computeCorpusGeneration(args: {
|
||||
.update('|')
|
||||
.update(String(TITLE_WRAPPER_VERSION))
|
||||
.update('|')
|
||||
.update(getEmbeddingModelTag());
|
||||
if (args.synopsisDocMaxChars !== undefined) {
|
||||
h.update('|doc_cap=').update(String(args.synopsisDocMaxChars));
|
||||
}
|
||||
return h.digest('hex').slice(0, 16);
|
||||
.update(getEmbeddingModelTag())
|
||||
.digest('hex')
|
||||
.slice(0, 16);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -265,11 +253,7 @@ export async function reembedPageWithContextualRetrieval(
|
||||
args.pageSlug,
|
||||
args.sourceId,
|
||||
resolution.mode,
|
||||
computeCorpusGeneration({
|
||||
crMode: resolution.mode,
|
||||
haikuModel: args.haikuModel ?? DEFAULT_HAIKU_MODEL,
|
||||
synopsisDocMaxChars: resolution.mode === 'per_chunk_synopsis' ? SYNOPSIS_DOC_MAX_CHARS : undefined,
|
||||
}),
|
||||
computeCorpusGeneration({ crMode: resolution.mode, haikuModel: args.haikuModel ?? DEFAULT_HAIKU_MODEL }),
|
||||
);
|
||||
return { kind: 'skipped', reason: 'no_chunks' };
|
||||
}
|
||||
@@ -298,7 +282,6 @@ export async function reembedPageWithContextualRetrieval(
|
||||
const corpus_generation = computeCorpusGeneration({
|
||||
crMode: attemptMode,
|
||||
haikuModel,
|
||||
synopsisDocMaxChars: attemptMode === 'per_chunk_synopsis' ? SYNOPSIS_DOC_MAX_CHARS : undefined,
|
||||
});
|
||||
|
||||
// ── PHASE 2: single DB transaction ───────────────────────────
|
||||
|
||||
@@ -23,24 +23,14 @@
|
||||
* page coordinate only; legacy NULL-source_markdown_slug rows survive
|
||||
* because deleteFactsForPage targets source_markdown_slug = slug only.
|
||||
*
|
||||
* Empty-fence guard (Codex R2-#7; #2484): the phase refuses to do its
|
||||
* destructive reconciliation pass when genuinely-backfillable legacy
|
||||
* rows still exist — `row_num IS NULL` (never fenced) AND `entity_slug`
|
||||
* resolves to a live page in this source (so the v0_32_2 migration's
|
||||
* Phase B could fence them). Status returns `warn` with a hint to run
|
||||
* `gbrain apply-migrations --yes`. Without the guard, an interrupted
|
||||
* upgrade where v0_32_2 hasn't run could leave the cycle silently
|
||||
* misreporting "0 facts on people/alice" while legacy rows linger.
|
||||
*
|
||||
* The live-page requirement (#2484) is load-bearing: the inline facts
|
||||
* writer keeps producing `row_num IS NULL, entity_slug IS NOT NULL`
|
||||
* rows AFTER the migration completes, whenever a resolved slug has no
|
||||
* fenceable page (slugify-floor / stub-guard-blocked unprefixed slugs).
|
||||
* Those are structurally unfenceable — no page to fence onto, and the
|
||||
* ledger-complete migration won't re-run — so they must NOT gate, or
|
||||
* the phase jams forever (~16/day observed). Requiring a backing page
|
||||
* keeps genuine pre-v0.32.2 rows (whose entity page exists) gating
|
||||
* while excluding the inline-writer's permanent-unfenceable rows.
|
||||
* Empty-fence guard (Codex R2-#7): the phase refuses to do its
|
||||
* destructive reconciliation pass when legacy rows (row_num IS NULL,
|
||||
* entity_slug IS NOT NULL) still exist in the brain — they're the
|
||||
* v0.31 hot-memory facts pending the v0_32_2 backfill. Status returns
|
||||
* `warn` with a hint to run `gbrain apply-migrations --yes`. Without
|
||||
* the guard, an interrupted upgrade where v0_32_2 hasn't run could
|
||||
* leave the cycle silently misreporting "0 facts on people/alice"
|
||||
* while legacy rows linger in the DB.
|
||||
*/
|
||||
|
||||
import type { BrainEngine } from '../engine.ts';
|
||||
@@ -173,48 +163,22 @@ export async function runExtractFacts(
|
||||
phantomsMorePending: false,
|
||||
};
|
||||
|
||||
// ── Empty-fence guard (Codex R2-#7; #2484) ─────────────────────
|
||||
// Pre-check: if any genuinely-backfillable legacy fact rows exist,
|
||||
// refuse to run the destructive reconciliation pass — the v0_32_2
|
||||
// orchestrator must fence them first.
|
||||
//
|
||||
// A row is a real backfill candidate only when `row_num IS NULL`
|
||||
// (never fenced) AND its `entity_slug` resolves to a LIVE page in
|
||||
// this source (the migration's Phase B only fences rows whose
|
||||
// entity_slug maps to a writable page). #2484: the original
|
||||
// predicate was just `row_num IS NULL AND entity_slug IS NOT NULL`,
|
||||
// which ALSO matched structurally-unfenceable hot-memory rows the
|
||||
// inline writer keeps producing post-migration: the legacy DB-only
|
||||
// fallback (backstop.ts) writes `entity_slug` (a resolved slug, e.g.
|
||||
// a slugify-floor or stub-guard-blocked unprefixed slug like
|
||||
// `people-jane-doe`) with `row_num` NULL whenever the slug has no
|
||||
// fenceable page. Those rows can never satisfy the migration's exit
|
||||
// condition (no page to fence onto, and `apply-migrations` is a
|
||||
// ledger-complete no-op for them), so they jammed the phase forever
|
||||
// — ~16/day, mislabeled "v0.31 pending backfill." We now require a
|
||||
// live backing page, which both genuine pre-v0.32.2 rows (their
|
||||
// entity page exists) satisfy and inline-writer unfenceable rows do
|
||||
// not.
|
||||
// ── Empty-fence guard (Codex R2-#7) ────────────────────────────
|
||||
// Pre-check: if any legacy fact rows exist (row_num NULL but
|
||||
// entity_slug NOT NULL), refuse to run the destructive
|
||||
// reconciliation pass. The v0_32_2 orchestrator must complete
|
||||
// first.
|
||||
const legacy = await engine.executeRaw<{ n: string }>(
|
||||
`SELECT COUNT(*) AS n
|
||||
FROM facts f
|
||||
WHERE f.row_num IS NULL
|
||||
AND f.entity_slug IS NOT NULL
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM pages p
|
||||
WHERE p.source_id = f.source_id
|
||||
AND p.slug = f.entity_slug
|
||||
AND p.deleted_at IS NULL
|
||||
)`,
|
||||
`SELECT COUNT(*) AS n FROM facts WHERE row_num IS NULL AND entity_slug IS NOT NULL`,
|
||||
);
|
||||
const legacyCount = parseInt(legacy[0]?.n ?? '0', 10);
|
||||
result.legacyRowsPending = legacyCount;
|
||||
if (legacyCount > 0) {
|
||||
result.guardTriggered = true;
|
||||
result.warnings.push(
|
||||
`extract_facts: ${legacyCount} legacy v0.31 fact rows (entity page present, not yet ` +
|
||||
`fenced) pending fence backfill. Run \`gbrain apply-migrations --yes\` to complete ` +
|
||||
`v0_32_2 before this phase can safely reconcile fence → DB.`,
|
||||
`extract_facts: ${legacyCount} legacy v0.31 fact rows pending fence backfill. ` +
|
||||
`Run \`gbrain apply-migrations --yes\` to complete v0_32_2 before this phase ` +
|
||||
`can safely reconcile fence → DB.`,
|
||||
);
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -55,17 +55,6 @@ import type { PhaseStatus, CyclePhase } from '../cycle.ts';
|
||||
*/
|
||||
export const PROPOSE_TAKES_PROMPT_VERSION = 'v0.36.1.0-tuned-cat15';
|
||||
|
||||
/**
|
||||
* Sentinel claim_text for the tombstone row written when a page extracts
|
||||
* ZERO gradeable claims. Without a tombstone the idempotency tuple is never
|
||||
* recorded, so every cycle re-spends an LLM call on unchanged zero-claim
|
||||
* prose — the "unchanged page never re-spends tokens" contract only held
|
||||
* for pages that produced >=1 claim. The tombstone is inserted with
|
||||
* status='rejected' so no pending-review query surfaces it as a live
|
||||
* proposal; its only job is to make the next cycle a cache hit.
|
||||
*/
|
||||
export const EMPTY_EXTRACTION_TOMBSTONE_TEXT = '(no gradeable claims)';
|
||||
|
||||
/**
|
||||
* Tuned extractor prompt, validated against the hand-labeled synthetic
|
||||
* corpus at test/fixtures/calibration/. Measured F1 on first live run
|
||||
@@ -163,8 +152,6 @@ export interface ProposeTakesResult {
|
||||
cache_hits: number;
|
||||
cache_misses: number;
|
||||
proposals_inserted: number;
|
||||
/** Idempotency rows written for pages that extracted zero claims. */
|
||||
tombstones_written: number;
|
||||
budget_exhausted: boolean;
|
||||
warnings: string[];
|
||||
}
|
||||
@@ -247,43 +234,7 @@ export async function defaultExtractor(
|
||||
});
|
||||
|
||||
// ChatResult.text is already the concatenated text content.
|
||||
const takes = parseExtractorOutput(result.text);
|
||||
// A parse-level `[]` is AMBIGUOUS: it means either "the model genuinely
|
||||
// found no gradeable claims" OR "the model returned malformed/prose/
|
||||
// truncated output we couldn't parse." The caller memoizes empty
|
||||
// extractions with a tombstone, so a transient parse failure would
|
||||
// PERMANENTLY suppress a page that actually has claims. Only a cleanly
|
||||
// parsed empty array is a real "no claims" result worth memoizing; treat
|
||||
// anything else as a transient error and throw, so the phase's catch
|
||||
// retries the page next cycle (writing no tombstone).
|
||||
if (takes.length === 0 && !isWellFormedEmptyExtraction(result.text)) {
|
||||
throw new Error('propose_takes extractor: no parseable takes JSON (transient — retry)');
|
||||
}
|
||||
return takes;
|
||||
}
|
||||
|
||||
/**
|
||||
* True only when `raw` is a cleanly-parseable EMPTY JSON array — the
|
||||
* well-behaved "no gradeable claims" response (the prompt instructs the model
|
||||
* to return `[]`). Distinguishes a genuine empty extraction (safe to memoize
|
||||
* via a tombstone) from malformed / prose / truncated output (transient —
|
||||
* must be retried, never tombstoned). Mirrors parseExtractorOutput's
|
||||
* fence-strip + first-array handling so both agree on what "the model
|
||||
* returned []" means.
|
||||
*/
|
||||
export function isWellFormedEmptyExtraction(raw: string): boolean {
|
||||
if (!raw || raw.trim().length === 0) return false;
|
||||
let text = raw.trim();
|
||||
const fenced = text.match(/^```(?:json)?\s*\n?([\s\S]*?)\n?```$/);
|
||||
if (fenced) text = (fenced[1] ?? '').trim();
|
||||
const arrStart = text.indexOf('[');
|
||||
if (arrStart === -1) return false;
|
||||
try {
|
||||
const parsed = JSON.parse(text.slice(arrStart));
|
||||
return Array.isArray(parsed) && parsed.length === 0;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
return parseExtractorOutput(result.text);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -295,8 +246,6 @@ export function isWellFormedEmptyExtraction(raw: string): boolean {
|
||||
export function parseExtractorOutput(raw: string): ProposedTake[] {
|
||||
if (!raw || raw.trim().length === 0) return [];
|
||||
let text = raw.trim();
|
||||
// Strip <think>...</think> reasoning tags (MiniMax-M3, DeepSeek-R1, etc.).
|
||||
text = text.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
|
||||
// Strip markdown code fence wrapper.
|
||||
const fenced = text.match(/^```(?:json)?\s*\n?([\s\S]*?)\n?```$/);
|
||||
if (fenced) text = (fenced[1] ?? '').trim();
|
||||
@@ -309,21 +258,7 @@ export function parseExtractorOutput(raw: string): ProposedTake[] {
|
||||
try {
|
||||
parsed = JSON.parse(text.slice(start));
|
||||
} catch {
|
||||
// Fallback: truncate at last ] or } to handle trailing noise (e.g. leftover
|
||||
// markdown fences after <think> stripping). Try array-closing first.
|
||||
const sliced = text.slice(start);
|
||||
const lastArr = sliced.lastIndexOf(']');
|
||||
const lastObj = sliced.lastIndexOf('}');
|
||||
const end = Math.max(lastArr, lastObj);
|
||||
if (end > 0) {
|
||||
try {
|
||||
parsed = JSON.parse(sliced.slice(0, end + 1));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
} else {
|
||||
return [];
|
||||
}
|
||||
return [];
|
||||
}
|
||||
const arr = Array.isArray(parsed) ? parsed : [parsed];
|
||||
const out: ProposedTake[] = [];
|
||||
@@ -379,7 +314,6 @@ class ProposeTakesPhase extends BaseCyclePhase {
|
||||
cache_hits: 0,
|
||||
cache_misses: 0,
|
||||
proposals_inserted: 0,
|
||||
tombstones_written: 0,
|
||||
budget_exhausted: false,
|
||||
warnings: [],
|
||||
};
|
||||
@@ -481,40 +415,6 @@ class ProposeTakesPhase extends BaseCyclePhase {
|
||||
);
|
||||
result.proposals_inserted += 1;
|
||||
}
|
||||
|
||||
// Memoize the empty case too. A page that extracted zero claims gets
|
||||
// NO row from the loop above, so without this its idempotency tuple is
|
||||
// never recorded and the next cycle re-spends an LLM call on unchanged
|
||||
// prose (the idle-cost bug). Write one tombstone row keyed by the same
|
||||
// (source, slug, content_hash, prompt_version) tuple. status='rejected'
|
||||
// keeps it out of any pending-review query; its sole purpose is to make
|
||||
// the next cycle a cache hit. Only reached on a SUCCESSFUL empty extract
|
||||
// — the extractor-throw path `continue`s above, so failed pages are
|
||||
// retried rather than tombstoned.
|
||||
if (proposals.length === 0) {
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO take_proposals
|
||||
(source_id, page_slug, content_hash, prompt_version, proposal_run_id,
|
||||
claim_text, kind, holder, weight, domain, dedup_against_fence_rows, model_id, status)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, 'rejected')
|
||||
ON CONFLICT (source_id, page_slug, content_hash, prompt_version) DO NOTHING`,
|
||||
[
|
||||
sourceId,
|
||||
page.slug,
|
||||
ch,
|
||||
promptVersion,
|
||||
proposalRunId,
|
||||
EMPTY_EXTRACTION_TOMBSTONE_TEXT,
|
||||
'fact',
|
||||
'brain',
|
||||
0,
|
||||
null,
|
||||
JSON.stringify(existingTakes),
|
||||
opts.model ?? 'claude-sonnet-4-6',
|
||||
],
|
||||
);
|
||||
result.tombstones_written += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (opts.reporter) opts.reporter.finish();
|
||||
@@ -548,7 +448,7 @@ class ProposeTakesPhase extends BaseCyclePhase {
|
||||
});
|
||||
|
||||
return {
|
||||
summary: `propose_takes: scanned ${result.pages_scanned} pages, ${result.cache_hits} cached, ${result.proposals_inserted} new proposals, ${result.tombstones_written} empty (run ${proposalRunId})`,
|
||||
summary: `propose_takes: scanned ${result.pages_scanned} pages, ${result.cache_hits} cached, ${result.proposals_inserted} new proposals (run ${proposalRunId})`,
|
||||
details: { ...result, proposal_run_id: proposalRunId, prompt_version: promptVersion },
|
||||
status: result.budget_exhausted ? 'warn' : 'ok',
|
||||
};
|
||||
|
||||
@@ -37,10 +37,6 @@ export const EMBEDDING_PRICING: Record<string, EmbeddingPricing> = {
|
||||
'voyage:voyage-4-large': { pricePerMTok: 0.18 },
|
||||
// ZeroEntropy (https://zeroentropy.dev/pricing — zembed-1)
|
||||
'zeroentropyai:zembed-1': { pricePerMTok: 0.05 },
|
||||
// ZeroEntropy reranker (docs/ai-providers/zeroentropy.md — $0.025/1M tokens).
|
||||
// Reused here (not a separate rerank table) because budget-tracker.ts's
|
||||
// rerank-kind lookup falls back to this same table for paid providers.
|
||||
'zeroentropyai:zerank-2': { pricePerMTok: 0.025 },
|
||||
// Mistral (https://mistral.ai/pricing/api/, verified 2026-07-19)
|
||||
'mistral:mistral-embed': { pricePerMTok: 0.10 },
|
||||
'mistral:mistral-embed-2312': { pricePerMTok: 0.10 },
|
||||
|
||||
@@ -1218,21 +1218,11 @@ export interface BrainEngine {
|
||||
*
|
||||
* Uses the `%` trigram operator (GIN-indexed) + the standard `similarity()`
|
||||
* function. Both engines support pg_trgm (PGLite 0.3+, Postgres always).
|
||||
*
|
||||
* `sourceId` constrains the search to a single source and filters out
|
||||
* soft-deleted pages. Mirrors the same filters `tryFuzzyMatch` in
|
||||
* `src/core/entities/resolve.ts` got via #1436 (v0.41.13.0). Omit for the
|
||||
* historical unscoped behavior — live-mode callers that already know
|
||||
* the source should pass it to avoid cross-source slug suggestions that
|
||||
* get silently dropped at the FK filter downstream. Batch-mode callers
|
||||
* (e.g. `gbrain extract`) intentionally omit it to build a cross-source
|
||||
* resolution map.
|
||||
*/
|
||||
findByTitleFuzzy(
|
||||
name: string,
|
||||
dirPrefix?: string,
|
||||
minSimilarity?: number,
|
||||
sourceId?: string,
|
||||
): Promise<{ slug: string; similarity: number } | null>;
|
||||
/**
|
||||
* v0.34.1 (#861 — P0 leak seal): `opts.sourceId` / `opts.sourceIds`
|
||||
|
||||
@@ -733,11 +733,6 @@ export async function importFromContent(
|
||||
: computeCorpusGeneration({
|
||||
crMode: effectiveCRMode,
|
||||
haikuModel: 'anthropic:claude-haiku-4-5-20251001',
|
||||
// Inline import-file path never uses per_chunk_synopsis (refuses
|
||||
// upstream); pass undefined so the doc-cap field stays out of
|
||||
// the hash here. Per_chunk_synopsis runs through the Minion
|
||||
// backfill handler which threads SYNOPSIS_DOC_MAX_CHARS through
|
||||
// the service layer.
|
||||
});
|
||||
|
||||
// Transaction wraps all DB writes. Every per-page tx call carries the
|
||||
|
||||
@@ -489,22 +489,7 @@ export async function extractPageLinks(
|
||||
// text inside `[[...]]` before any `|`), NOT the display alias
|
||||
// (ref.name = match[2]). `[[struktura|the project]]` must resolve
|
||||
// `struktura`, not "the project". The display text is for context only.
|
||||
//
|
||||
// The literal may be path-qualified (`[[notes/struktura]]`). The FS
|
||||
// path (resolveSlugAll) strips the dirname before its basename lookup,
|
||||
// but this path passed the raw literal to an index keyed by final
|
||||
// segments only — so every slash-containing wikilink outside
|
||||
// DIR_PATTERN silently resolved to nothing. Query by the final
|
||||
// segment, then use the written path as a disambiguation filter
|
||||
// (the analogue of the FS ancestor walk honoring the written path):
|
||||
// a match must end with the literal, so `[[notes/struktura]]` can
|
||||
// resolve to `vault/notes/struktura` but never to `wiki/struktura`.
|
||||
const slashIdx = ref.slug.lastIndexOf('/');
|
||||
const basename = slashIdx === -1 ? ref.slug : ref.slug.slice(slashIdx + 1);
|
||||
let matches = await resolver.resolveBasenameMatches(basename);
|
||||
if (slashIdx !== -1) {
|
||||
matches = matches.filter(m => m === ref.slug || m.endsWith(`/${ref.slug}`));
|
||||
}
|
||||
const matches = await resolver.resolveBasenameMatches(ref.slug);
|
||||
if (matches.length === 0) continue;
|
||||
const idx = content.indexOf(ref.slug);
|
||||
const context = idx >= 0 ? excerpt(content, idx, 240) : ref.name;
|
||||
@@ -980,14 +965,10 @@ export function makeResolver(
|
||||
|
||||
// Step 3: pg_trgm fuzzy title match — both modes. Tries each hint in
|
||||
// order; first hint with a ≥0.55 similarity match wins. If no hints,
|
||||
// try the whole pages table. When opts.sourceId is set, the fuzzy
|
||||
// search is constrained to that source (and skips soft-deleted pages)
|
||||
// so cross-source slug suggestions don't get silently dropped at the
|
||||
// FK filter downstream. Mirrors the same scope fix `tryFuzzyMatch` got
|
||||
// via #1436.
|
||||
// try the whole pages table.
|
||||
const searchHints = hints.length > 0 ? hints : [undefined];
|
||||
for (const hint of searchHints) {
|
||||
const match = await engine.findByTitleFuzzy(trimmed, hint, 0.55, opts.sourceId);
|
||||
const match = await engine.findByTitleFuzzy(trimmed, hint, 0.55);
|
||||
if (match) {
|
||||
cache.set(cacheKey, match.slug);
|
||||
return match.slug;
|
||||
|
||||
+1
-35
@@ -135,16 +135,7 @@ export function parseMarkdown(
|
||||
const type = coerceFrontmatterString(frontmatter.type) || (
|
||||
opts?.activePack ? inferTypeFromPack(filePath, opts.activePack) : inferType(filePath)
|
||||
);
|
||||
// #2446: title precedence is frontmatter `title:` > the body's first H1 >
|
||||
// the slug/filename-humanized fallback. Slug-based imports (contacts,
|
||||
// calendar) write a correct `# Heading` but no frontmatter title; without
|
||||
// the H1 fallback they get junk titles humanized from the slug
|
||||
// (`Contact 20170928 5 John Defalco`), which also breaks anything keyed on
|
||||
// the title (e.g. the by-mention gazetteer's first-token bucketing).
|
||||
const title =
|
||||
coerceFrontmatterString(frontmatter.title).trim() ||
|
||||
inferTitleFromBody(body) ||
|
||||
inferTitle(filePath);
|
||||
const title = coerceFrontmatterString(frontmatter.title).trim() || inferTitle(filePath);
|
||||
const tags = extractTags(frontmatter);
|
||||
const slug = coerceFrontmatterString(frontmatter.slug) || inferSlug(filePath);
|
||||
|
||||
@@ -611,31 +602,6 @@ function inferTypeWithPrefixes(
|
||||
return 'concept';
|
||||
}
|
||||
|
||||
/**
|
||||
* #2446: derive a title from the body's first ATX H1 (`# Heading`).
|
||||
*
|
||||
* Returns the trimmed heading text with the leading `# ` and any decorative
|
||||
* trailing `#` run stripped, or '' if the body has no H1. Only a SINGLE leading
|
||||
* `#` matches — `##`+ (h2 and deeper) are skipped — and lines inside a fenced
|
||||
* code block (```/~~~) are ignored so a `# comment` in a shell snippet can't be
|
||||
* mistaken for the page title.
|
||||
*/
|
||||
function inferTitleFromBody(body: string): string {
|
||||
let inFence = false;
|
||||
for (const raw of body.split('\n')) {
|
||||
const fence = /^\s*(`{3,}|~{3,})/.exec(raw);
|
||||
if (fence) {
|
||||
inFence = !inFence;
|
||||
continue;
|
||||
}
|
||||
if (inFence) continue;
|
||||
// Exactly one leading `#`, then whitespace, then the heading text.
|
||||
const m = /^#(?!#)\s+(.+?)\s*$/.exec(raw);
|
||||
if (m) return m[1].replace(/\s+#+\s*$/, '').trim();
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
function inferTitle(filePath?: string): string {
|
||||
if (!filePath) return 'Untitled';
|
||||
|
||||
|
||||
@@ -24,7 +24,6 @@
|
||||
*/
|
||||
|
||||
const THIRTY_MIN_MS = 30 * 60 * 1000;
|
||||
const SIXTY_MIN_MS = 60 * 60 * 1000;
|
||||
const TEN_MIN_MS = 10 * 60 * 1000;
|
||||
|
||||
/**
|
||||
@@ -43,10 +42,6 @@ export const HANDLER_DEFAULT_TIMEOUT_MS: Readonly<Record<string, number>> = {
|
||||
// few writes. Generous 10-min budget (vs the tight null-default) covers a
|
||||
// slow gateway without the 30-min loop budget.
|
||||
chronicle_extract: TEN_MIN_MS,
|
||||
// Per-page contextual reindex jobs process chunks sequentially with one
|
||||
// rate-leased LLM synopsis call per chunk; large transcript pages need more
|
||||
// than the standard 30-min long-job budget.
|
||||
contextual_reindex_per_chunk: SIXTY_MIN_MS,
|
||||
};
|
||||
|
||||
/**
|
||||
|
||||
@@ -87,11 +87,6 @@ function walkMarkdownAndMdxFiles(
|
||||
for (const entry of entries) {
|
||||
if (truncated) return;
|
||||
if (entry.startsWith('.')) continue;
|
||||
// Skip heavy non-content dirs so the walk doesn't exhaust the time
|
||||
// budget on dependency/build trees (node_modules can be 50k+ files
|
||||
// with zero .md). These are never gbrain page sources.
|
||||
if (entry === 'node_modules' || entry === 'dist' || entry === 'build' ||
|
||||
entry === '.next' || entry === 'vendor' || entry === 'target') continue;
|
||||
const full = join(d, entry);
|
||||
let isDir = false;
|
||||
try {
|
||||
@@ -100,9 +95,6 @@ function walkMarkdownAndMdxFiles(
|
||||
continue;
|
||||
}
|
||||
if (isDir) {
|
||||
// Time check on directory descent too, so a deep dependency-free
|
||||
// tree still respects the deadline even before any .md is found.
|
||||
if (Date.now() >= deadlineMs) { truncated = true; return; }
|
||||
walk(full);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -55,7 +55,7 @@ export interface AgentClientBindings {
|
||||
* `redirect_uri` containing `,`) would be parsed by Postgres as MULTIPLE
|
||||
* array elements, smuggling values past validation. See CSO finding #5.
|
||||
*/
|
||||
function pgArray(arr: string[]): string {
|
||||
export function pgArray(arr: string[]): string {
|
||||
if (!arr || arr.length === 0) return '{}';
|
||||
const escaped = arr.map(s => `"${s.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`);
|
||||
return `{${escaped.join(',')}}`;
|
||||
|
||||
+3
-37
@@ -1153,11 +1153,7 @@ async function runAutoLink(
|
||||
|
||||
// Live-mode resolver: per-put throwaway cache, pg_trgm + optional search.
|
||||
// Issue #972 (codex [P1]): pass sourceId so basename resolution stays
|
||||
// within this page's source — no cross-source basename edges. Also scopes
|
||||
// the fuzzy fallback (findByTitleFuzzy) to the same source the put_page is
|
||||
// targeting — without it, cross-source slug suggestions get silently dropped
|
||||
// at the FK filter and the link looks like it failed to resolve. Twin of
|
||||
// #1436's `tryFuzzyMatch` fix.
|
||||
// within this page's source — no cross-source basename edges.
|
||||
const resolver = makeResolver(engine, { mode: 'live', sourceId: opts?.sourceId });
|
||||
// Issue #972: opt-in bare-wikilink basename resolution. Off by default.
|
||||
const globalBasename = await isGlobalBasenameEnabled(engine);
|
||||
@@ -1388,11 +1384,7 @@ const list_pages: Operation = {
|
||||
params: {
|
||||
type: { type: 'string', description: 'Filter by page type' },
|
||||
tag: { type: 'string', description: 'Filter by tag' },
|
||||
limit: { type: 'number', description: 'Max results (default 50; remote callers are capped at 100)' },
|
||||
offset: {
|
||||
type: 'number',
|
||||
description: 'Skip first N rows (pagination). Engine-supported since PageFilters gained offset; previously accepted at the CLI and silently dropped.',
|
||||
},
|
||||
limit: { type: 'number', description: 'Max results (default 50)' },
|
||||
// v0.29 — surface filter that already exists on PageFilters.
|
||||
updated_after: {
|
||||
type: 'string',
|
||||
@@ -1419,36 +1411,10 @@ const list_pages: Operation = {
|
||||
// were ignored at this op handler and the engine returned every source's
|
||||
// pages indiscriminately.
|
||||
const scope = sourceScopeOpts(ctx);
|
||||
// The 100-row cap exists to protect remote MCP/OAuth transports from
|
||||
// unbounded result dumps. Local CLI callers (ctx.remote === false — the
|
||||
// same trust boundary that already bypasses scope enforcement, see the
|
||||
// Operation.scope doc above) own the machine, and a full enumeration is a
|
||||
// legitimate local operation, so an explicit limit above 100 is honored.
|
||||
// Anything that is not strictly `false` stays remote/untrusted (defense
|
||||
// in depth, matching the ctx.remote contract).
|
||||
const requestedLimit = p.limit as number | undefined;
|
||||
const isLocal = ctx.remote === false;
|
||||
const limit = isLocal
|
||||
? clampSearchLimit(requestedLimit, 50, Number.MAX_SAFE_INTEGER)
|
||||
: clampSearchLimit(requestedLimit, 50, 100);
|
||||
if (!isLocal && requestedLimit !== undefined && Number.isFinite(requestedLimit) && requestedLimit > limit) {
|
||||
// Loud clamp, parity with the three search paths ("search limit clamped
|
||||
// from N to 100"). logger.warn goes to stderr — `list` stdout is
|
||||
// tab-separated and consumed by scripts, so it must stay clean.
|
||||
ctx.logger.warn(`[gbrain] Warning: list limit clamped from ${requestedLimit} to ${limit}; use offset to paginate`);
|
||||
}
|
||||
// Thread offset through — PageFilters has supported it all along; the op
|
||||
// layer just never passed it, so `--offset` was accepted and ignored.
|
||||
const requestedOffset = p.offset as number | undefined;
|
||||
const offset =
|
||||
requestedOffset !== undefined && Number.isFinite(requestedOffset) && requestedOffset > 0
|
||||
? Math.floor(requestedOffset)
|
||||
: undefined;
|
||||
const pages = await ctx.engine.listPages({
|
||||
type: p.type as any,
|
||||
tag: p.tag as string,
|
||||
limit,
|
||||
offset,
|
||||
limit: clampSearchLimit(p.limit as number | undefined, 50, 100),
|
||||
includeDeleted: (p.include_deleted as boolean) === true,
|
||||
updated_after: typeof p.updated_after === 'string' ? p.updated_after : undefined,
|
||||
sort,
|
||||
|
||||
@@ -1,116 +0,0 @@
|
||||
/**
|
||||
* Shared orphan-reporting exclusion policy.
|
||||
*
|
||||
* These are pages where "no inbound links" is expected and should not count
|
||||
* against health. Keep this in core so the CLI orphan report and engine health
|
||||
* dashboard cannot drift.
|
||||
*
|
||||
* Defaults are GBrain-wide conventions only. Brain-specific exclusions
|
||||
* (private folder names, one-off fixture slugs) belong in the brain's own
|
||||
* config, not here:
|
||||
*
|
||||
* gbrain config set orphans.exclude_prefixes "my-private-folder/,archive/"
|
||||
* gbrain config set orphans.exclude_slugs "some-one-off-page"
|
||||
*/
|
||||
|
||||
const AUTO_SUFFIX_PATTERNS = ['/_index', '/log'];
|
||||
|
||||
const PSEUDO_SLUGS = new Set(['_atlas', '_index', '_stats', '_orphans', '_scratch', 'claude']);
|
||||
|
||||
const RAW_SEGMENT = '/raw/';
|
||||
|
||||
const DENY_PREFIXES = [
|
||||
'output/',
|
||||
'dashboards/',
|
||||
'scripts/',
|
||||
'templates/',
|
||||
'_templates/',
|
||||
'openclaw/config/',
|
||||
'extracts/',
|
||||
];
|
||||
|
||||
const FIRST_SEGMENT_EXCLUSIONS = new Set([
|
||||
'scratch',
|
||||
'thoughts',
|
||||
'catalog',
|
||||
'entities',
|
||||
'raw',
|
||||
'atoms',
|
||||
'skills',
|
||||
'dreaming',
|
||||
'daily',
|
||||
]);
|
||||
|
||||
const ROOT_DATE_SLUG = /^\d{4}-\d{2}-\d{2}(?:-.+)?$/;
|
||||
|
||||
function isAgentWorkspaceConvention(slug: string): boolean {
|
||||
if (!slug.startsWith('agents/')) return false;
|
||||
if (slug.includes('/memory/dreaming/')) return true;
|
||||
return /^agents\/[^/]+\/(?:agents|identity|soul|tools|user|heartbeat|dreams|dormant)$/.test(slug);
|
||||
}
|
||||
|
||||
/** Per-brain additions to the convention defaults (from config). */
|
||||
export interface OrphanPolicyOverrides {
|
||||
excludePrefixes?: string[];
|
||||
excludeSlugs?: string[];
|
||||
}
|
||||
|
||||
/** Config keys for per-brain orphan exclusions (comma-separated values). */
|
||||
export const ORPHAN_EXCLUDE_PREFIXES_KEY = 'orphans.exclude_prefixes';
|
||||
export const ORPHAN_EXCLUDE_SLUGS_KEY = 'orphans.exclude_slugs';
|
||||
|
||||
function parseList(value: string | null): string[] {
|
||||
if (!value) return [];
|
||||
return value.split(',').map(s => s.trim()).filter(Boolean);
|
||||
}
|
||||
|
||||
/**
|
||||
* Load per-brain orphan exclusions from the brain config table. Callers with
|
||||
* an engine in hand (getHealth, `gbrain orphans`) pass the result as the
|
||||
* second argument to shouldExcludeFromOrphanReporting.
|
||||
*/
|
||||
export async function loadOrphanPolicyOverrides(
|
||||
engine: { getConfig(key: string): Promise<string | null> },
|
||||
): Promise<OrphanPolicyOverrides> {
|
||||
const [prefixes, slugs] = await Promise.all([
|
||||
engine.getConfig(ORPHAN_EXCLUDE_PREFIXES_KEY),
|
||||
engine.getConfig(ORPHAN_EXCLUDE_SLUGS_KEY),
|
||||
]);
|
||||
return { excludePrefixes: parseList(prefixes), excludeSlugs: parseList(slugs) };
|
||||
}
|
||||
|
||||
export function shouldExcludeFromOrphanReporting(
|
||||
slug: string,
|
||||
overrides?: OrphanPolicyOverrides,
|
||||
): boolean {
|
||||
if (PSEUDO_SLUGS.has(slug)) return true;
|
||||
|
||||
for (const suffix of AUTO_SUFFIX_PATTERNS) {
|
||||
if (slug.endsWith(suffix)) return true;
|
||||
}
|
||||
|
||||
if (slug.includes(RAW_SEGMENT)) return true;
|
||||
if (slug.includes('/daily/')) return true;
|
||||
|
||||
for (const prefix of DENY_PREFIXES) {
|
||||
if (slug.startsWith(prefix)) return true;
|
||||
}
|
||||
|
||||
const firstSegment = slug.split('/')[0];
|
||||
if (FIRST_SEGMENT_EXCLUSIONS.has(firstSegment)) return true;
|
||||
|
||||
if (ROOT_DATE_SLUG.test(slug)) return true;
|
||||
|
||||
if (slug.startsWith('_brain-')) return true;
|
||||
|
||||
if (isAgentWorkspaceConvention(slug)) return true;
|
||||
|
||||
if (overrides) {
|
||||
if (overrides.excludeSlugs?.includes(slug)) return true;
|
||||
for (const prefix of overrides.excludePrefixes ?? []) {
|
||||
if (slug.startsWith(prefix)) return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
@@ -44,33 +44,6 @@ const HAIKU_MAX_TOKENS = 200;
|
||||
/** Default model when caller doesn't override. Resolves through the gateway. */
|
||||
const DEFAULT_SYNOPSIS_MODEL = 'anthropic:claude-haiku-4-5-20251001';
|
||||
|
||||
/**
|
||||
* Hard cap on `documentText` length (chars) before send.
|
||||
*
|
||||
* 2026-05-25 fix wave: small local chat models (Gemma 4 E2B, Qwen3 4B) get
|
||||
* dramatically slower on long contexts even with 131K-token windows declared.
|
||||
* A 73K-char page synopsis on Gemma 4 E2B takes 60-120s, exceeding the
|
||||
* worker's default 30s `lockDuration` and tripping `lock-lost` errors.
|
||||
*
|
||||
* Truncate to a budget that fits a small model's effective throughput while
|
||||
* preserving enough document context for the synopsis to be useful. Truncates
|
||||
* the TAIL because the head (title, frontmatter, intro) carries the
|
||||
* document-level anchor the synopsis needs.
|
||||
*
|
||||
* Override per workload via `GBRAIN_SYNOPSIS_DOC_MAX_CHARS`. Default 32768
|
||||
* (~8K tokens at 4 chars/tok) keeps small-model synopsis under ~30s.
|
||||
* Anthropic Haiku is unaffected at this cap; bump higher when running
|
||||
* frontier models if you want richer document anchoring.
|
||||
*/
|
||||
export const SYNOPSIS_DOC_MAX_CHARS = (() => {
|
||||
const env = process.env.GBRAIN_SYNOPSIS_DOC_MAX_CHARS;
|
||||
if (env && /^\d+$/.test(env)) {
|
||||
const n = parseInt(env, 10);
|
||||
if (n >= 512 && n <= 1_048_576) return n;
|
||||
}
|
||||
return 32768;
|
||||
})();
|
||||
|
||||
/**
|
||||
* Synopsis prompt version. Folded into corpus_generation so prompt edits
|
||||
* invalidate prior embeddings via the v0.40.3.0 query_cache.page_generations
|
||||
@@ -215,19 +188,11 @@ function buildUserPrompt(
|
||||
documentText: string,
|
||||
chunkText: string,
|
||||
): string {
|
||||
// Tail-truncate `documentText` to `SYNOPSIS_DOC_MAX_CHARS` so small local
|
||||
// chat models don't stall on >100KB pages. Head preserved (title block,
|
||||
// frontmatter, intro paragraphs carry the document-level anchor).
|
||||
let trimmedDoc = documentText;
|
||||
if (documentText.length > SYNOPSIS_DOC_MAX_CHARS) {
|
||||
trimmedDoc = documentText.slice(0, SYNOPSIS_DOC_MAX_CHARS) +
|
||||
`\n\n[... ${documentText.length - SYNOPSIS_DOC_MAX_CHARS} chars truncated for synopsis budget ...]`;
|
||||
}
|
||||
return [
|
||||
`<page_title>${pageTitle}</page_title>`,
|
||||
'',
|
||||
'<full_document>',
|
||||
trimmedDoc,
|
||||
documentText,
|
||||
'</full_document>',
|
||||
'',
|
||||
'<chunk>',
|
||||
|
||||
+28
-74
@@ -57,8 +57,6 @@ import { finalizeLastSeen } from './chronicle/last-seen.ts';
|
||||
import { computeAnomaliesFromBuckets } from './cycle/anomaly.ts';
|
||||
import { resolveBoostMap, resolveHardExcludes } from './search/source-boost.ts';
|
||||
import { buildSourceFactorCase, buildHardExcludeClause, buildVisibilityClause, buildRecencyComponentSql, buildBestPerPagePoolCte, buildOrFallbackWebsearchQuery } from './search/sql-ranking.ts';
|
||||
import { shouldExcludeFromOrphanReporting, loadOrphanPolicyOverrides } from './orphan-policy.ts';
|
||||
import { LINK_EXTRACTOR_VERSION_TS } from './link-extraction.ts';
|
||||
import {
|
||||
normalizeEngineColumn,
|
||||
buildVectorCastFragment,
|
||||
@@ -1060,16 +1058,6 @@ export class PGLiteEngine implements BrainEngine {
|
||||
RETURNING id, source_id, slug, type, title, compiled_truth, timeline, frontmatter, content_hash, created_at, updated_at, effective_date, effective_date_source, import_filename, source_kind, source_uri, ingested_via, ingested_at`,
|
||||
[sourceId, slug, page.type, pageKind, page.title, page.compiled_truth, page.timeline || '', JSON.stringify(frontmatter), hash, effectiveDate, effectiveDateSource, importFilename, chunkerVersion, sourcePath, sourceKind, sourceUri, ingestedVia, ingestedAt]
|
||||
);
|
||||
// PGLite can return zero rows from INSERT ... ON CONFLICT DO UPDATE ...
|
||||
// RETURNING in no-op/trigger edge cases, which made rowToPage(undefined)
|
||||
// throw "undefined is not an object (evaluating 'row.deleted_at')" and
|
||||
// skip the file during sync. The row WAS written, so re-read instead of
|
||||
// crashing.
|
||||
if (rows.length === 0) {
|
||||
const reread = await this.getPage(slug, { sourceId });
|
||||
if (reread) return reread;
|
||||
throw new Error(`putPage: RETURNING produced no row for ${sourceId}/${slug}`);
|
||||
}
|
||||
return rowToPage(rows[0] as Record<string, unknown>);
|
||||
}
|
||||
|
||||
@@ -2334,10 +2322,6 @@ export class PGLiteEngine implements BrainEngine {
|
||||
// v0.40.3.0 D24 NULL→non-NULL race fix mirrors postgres-engine.ts. Two writers
|
||||
// racing on the same chunk previously raced last-write-wins; the fix lets the
|
||||
// fresher `embedded_at` win in the text-unchanged branch.
|
||||
//
|
||||
// Code-chunk metadata columns follow the same chunk_text-gated CASE pattern as `embedding`
|
||||
// (#769). Re-chunk trusts EXCLUDED outright; pure re-embed COALESCEs so a caller carrying
|
||||
// only embedding-shaped fields doesn't clobber metadata to NULL.
|
||||
await this.db.query(
|
||||
`INSERT INTO content_chunks ${cols} VALUES ${rowParts.join(', ')}
|
||||
ON CONFLICT (page_id, chunk_index) DO UPDATE SET
|
||||
@@ -2361,14 +2345,14 @@ export class PGLiteEngine implements BrainEngine {
|
||||
THEN EXCLUDED.embedded_at
|
||||
ELSE content_chunks.embedded_at
|
||||
END,
|
||||
language = CASE WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.language ELSE COALESCE(EXCLUDED.language, content_chunks.language) END,
|
||||
symbol_name = CASE WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.symbol_name ELSE COALESCE(EXCLUDED.symbol_name, content_chunks.symbol_name) END,
|
||||
symbol_type = CASE WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.symbol_type ELSE COALESCE(EXCLUDED.symbol_type, content_chunks.symbol_type) END,
|
||||
start_line = CASE WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.start_line ELSE COALESCE(EXCLUDED.start_line, content_chunks.start_line) END,
|
||||
end_line = CASE WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.end_line ELSE COALESCE(EXCLUDED.end_line, content_chunks.end_line) END,
|
||||
parent_symbol_path = CASE WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.parent_symbol_path ELSE COALESCE(EXCLUDED.parent_symbol_path, content_chunks.parent_symbol_path) END,
|
||||
doc_comment = CASE WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.doc_comment ELSE COALESCE(EXCLUDED.doc_comment, content_chunks.doc_comment) END,
|
||||
symbol_name_qualified = CASE WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.symbol_name_qualified ELSE COALESCE(EXCLUDED.symbol_name_qualified, content_chunks.symbol_name_qualified) END,
|
||||
language = EXCLUDED.language,
|
||||
symbol_name = EXCLUDED.symbol_name,
|
||||
symbol_type = EXCLUDED.symbol_type,
|
||||
start_line = EXCLUDED.start_line,
|
||||
end_line = EXCLUDED.end_line,
|
||||
parent_symbol_path = EXCLUDED.parent_symbol_path,
|
||||
doc_comment = EXCLUDED.doc_comment,
|
||||
symbol_name_qualified = EXCLUDED.symbol_name_qualified,
|
||||
modality = EXCLUDED.modality,
|
||||
embedding_image = COALESCE(EXCLUDED.embedding_image, content_chunks.embedding_image)`,
|
||||
params
|
||||
@@ -2922,41 +2906,22 @@ export class PGLiteEngine implements BrainEngine {
|
||||
name: string,
|
||||
dirPrefix?: string,
|
||||
minSimilarity: number = 0.55,
|
||||
sourceId?: string,
|
||||
): Promise<{ slug: string; similarity: number } | null> {
|
||||
// Inline threshold comparison instead of `SET LOCAL pg_trgm.similarity_threshold`.
|
||||
// The GUC only scopes to the current transaction and pglite auto-commits each
|
||||
// .query() call, so the SET LOCAL would be a no-op. Using similarity() >= $N
|
||||
// directly gives predictable behavior. Tie-breaker: sort by slug so re-runs
|
||||
// pick the same winner.
|
||||
//
|
||||
// `sourceId` + `deleted_at IS NULL` mirror the filters `tryFuzzyMatch` in
|
||||
// `src/core/entities/resolve.ts` got via #1436 (v0.41.13.0). Without them,
|
||||
// fuzzy resolution could suggest cross-source slugs that the caller then
|
||||
// silently drops at the FK filter — making it look like the match failed
|
||||
// when in fact it picked the wrong page.
|
||||
const prefixPattern = dirPrefix ? `${dirPrefix}/%` : '%';
|
||||
const { rows } = sourceId
|
||||
? await this.db.query(
|
||||
`SELECT slug, similarity(title, $1) AS sim
|
||||
FROM pages
|
||||
WHERE similarity(title, $1) >= $3
|
||||
AND slug LIKE $2
|
||||
AND source_id = $4
|
||||
AND deleted_at IS NULL
|
||||
ORDER BY sim DESC, slug ASC
|
||||
LIMIT 1`,
|
||||
[name, prefixPattern, minSimilarity, sourceId]
|
||||
)
|
||||
: await this.db.query(
|
||||
`SELECT slug, similarity(title, $1) AS sim
|
||||
FROM pages
|
||||
WHERE similarity(title, $1) >= $3
|
||||
AND slug LIKE $2
|
||||
ORDER BY sim DESC, slug ASC
|
||||
LIMIT 1`,
|
||||
[name, prefixPattern, minSimilarity]
|
||||
);
|
||||
const { rows } = await this.db.query(
|
||||
`SELECT slug, similarity(title, $1) AS sim
|
||||
FROM pages
|
||||
WHERE similarity(title, $1) >= $3
|
||||
AND slug LIKE $2
|
||||
ORDER BY sim DESC, slug ASC
|
||||
LIMIT 1`,
|
||||
[name, prefixPattern, minSimilarity]
|
||||
);
|
||||
if (rows.length === 0) return null;
|
||||
const row = rows[0] as { slug: string; sim: number };
|
||||
return { slug: row.slug, similarity: row.sim };
|
||||
@@ -5242,10 +5207,15 @@ export class PGLiteEngine implements BrainEngine {
|
||||
(SELECT count(*) FROM pages) as page_count,
|
||||
(SELECT count(*) FROM content_chunks WHERE embedded_at IS NOT NULL)::float /
|
||||
GREATEST((SELECT count(*) FROM content_chunks), 1)::float as embed_coverage,
|
||||
0 as stale_pages,
|
||||
-- Bug 11 — orphan = islanded (no inbound AND no outbound). The raw
|
||||
-- list is filtered in TS using the shared orphan-reporting policy.
|
||||
0 as orphan_pages,
|
||||
(SELECT count(*) FROM pages p
|
||||
WHERE p.updated_at < (SELECT MAX(te.created_at) FROM timeline_entries te WHERE te.page_id = p.id)
|
||||
) as stale_pages,
|
||||
-- Bug 11 — orphan = islanded (no inbound AND no outbound).
|
||||
-- See BrainHealth.orphan_pages docstring; docs updated to match this.
|
||||
(SELECT count(*) FROM pages p
|
||||
WHERE NOT EXISTS (SELECT 1 FROM links l WHERE l.to_page_id = p.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM links l WHERE l.from_page_id = p.id)
|
||||
) as orphan_pages,
|
||||
(SELECT count(*) FROM links l
|
||||
WHERE NOT EXISTS (SELECT 1 FROM pages p WHERE p.id = l.to_page_id)
|
||||
) as dead_links,
|
||||
@@ -5270,20 +5240,10 @@ export class PGLiteEngine implements BrainEngine {
|
||||
LIMIT 5
|
||||
`);
|
||||
|
||||
const { rows: islandedRows } = await this.db.query(`
|
||||
SELECT p.slug
|
||||
FROM pages p
|
||||
WHERE NOT EXISTS (SELECT 1 FROM links l WHERE l.to_page_id = p.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM links l WHERE l.from_page_id = p.id)
|
||||
`);
|
||||
|
||||
const r = h as Record<string, unknown>;
|
||||
const pageCount = Number(r.page_count);
|
||||
const embedCoverage = Number(r.embed_coverage);
|
||||
const stalePages = await this.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS });
|
||||
const orphanOverrides = await loadOrphanPolicyOverrides(this);
|
||||
const orphanPages = (islandedRows as { slug: string }[])
|
||||
.filter(row => !shouldExcludeFromOrphanReporting(row.slug, orphanOverrides)).length;
|
||||
const orphanPages = Number(r.orphan_pages);
|
||||
const deadLinks = Number(r.dead_links);
|
||||
const linkCount = Number(r.link_count);
|
||||
const pagesWithTimeline = Number(r.pages_with_timeline);
|
||||
@@ -5311,7 +5271,7 @@ export class PGLiteEngine implements BrainEngine {
|
||||
return {
|
||||
page_count: pageCount,
|
||||
embed_coverage: embedCoverage,
|
||||
stale_pages: stalePages,
|
||||
stale_pages: Number(r.stale_pages),
|
||||
orphan_pages: orphanPages,
|
||||
missing_embeddings: Number(r.missing_embeddings),
|
||||
brain_score: brainScore,
|
||||
@@ -5866,11 +5826,6 @@ export class PGLiteEngine implements BrainEngine {
|
||||
params.push(escaped);
|
||||
prefixCondition = `AND p.slug LIKE $${params.length} ESCAPE '\\'`;
|
||||
}
|
||||
// TIM-37: exclude briefing pages from their own Brain Pulse. See the
|
||||
// matching block in postgres-engine.ts getRecentSalience() for context.
|
||||
const excludeBriefings = !(slugPrefix && slugPrefix.startsWith('briefings'))
|
||||
? `AND p.slug NOT LIKE 'briefings/%'`
|
||||
: '';
|
||||
params.push(limit);
|
||||
const limitParam = `$${params.length}`;
|
||||
|
||||
@@ -5906,7 +5861,6 @@ export class PGLiteEngine implements BrainEngine {
|
||||
LEFT JOIN takes t ON t.page_id = p.id AND t.active = TRUE
|
||||
WHERE GREATEST(p.updated_at, COALESCE(p.salience_touched_at, p.updated_at)) >= $1::timestamptz
|
||||
${prefixCondition}
|
||||
${excludeBriefings}
|
||||
GROUP BY p.id
|
||||
ORDER BY score DESC
|
||||
LIMIT ${limitParam}`,
|
||||
|
||||
+30
-68
@@ -67,8 +67,6 @@ import { resolveBoostMap, resolveHardExcludes } from './search/source-boost.ts';
|
||||
import { buildSourceFactorCase, buildHardExcludeClause, buildVisibilityClause, buildRecencyComponentSql, buildBestPerPagePoolCte, buildOrFallbackWebsearchQuery } from './search/sql-ranking.ts';
|
||||
import { DEFAULT_EMBEDDING_MODEL, DEFAULT_EMBEDDING_DIMENSIONS } from './ai/defaults.ts';
|
||||
import { DELETE_BATCH_SIZE } from './engine-constants.ts';
|
||||
import { shouldExcludeFromOrphanReporting, loadOrphanPolicyOverrides } from './orphan-policy.ts';
|
||||
import { LINK_EXTRACTOR_VERSION_TS } from './link-extraction.ts';
|
||||
|
||||
function escapeSqlStringLiteral(value: string): string {
|
||||
return value.replace(/'/g, "''");
|
||||
@@ -2475,13 +2473,6 @@ export class PostgresEngine implements BrainEngine {
|
||||
// - new is fresher (embedded_at > existing.embedded_at) → take new
|
||||
// - otherwise → keep existing (slower writer with stale embedding loses)
|
||||
// Mirrored in pglite-engine.ts; pinned by test/e2e/concurrent-embed-race.test.ts.
|
||||
//
|
||||
// Code-chunk metadata columns (language / symbol_name / symbol_type / line range /
|
||||
// parent_symbol_path / doc_comment / symbol_name_qualified) follow the SAME chunk_text-gated
|
||||
// CASE pattern as `embedding` (#769). Re-chunk (chunk_text changed) trusts EXCLUDED outright;
|
||||
// pure re-embed (chunk_text unchanged) COALESCEs so a caller that only carries embedding
|
||||
// doesn't clobber metadata to NULL. Without this, every embed --stale pass nuked code-def's
|
||||
// primary index for thousands of chunks at once.
|
||||
await sql.unsafe(
|
||||
`INSERT INTO content_chunks ${cols} VALUES ${rows.join(', ')}
|
||||
ON CONFLICT (page_id, chunk_index) DO UPDATE SET
|
||||
@@ -2505,14 +2496,14 @@ export class PostgresEngine implements BrainEngine {
|
||||
THEN EXCLUDED.embedded_at
|
||||
ELSE content_chunks.embedded_at
|
||||
END,
|
||||
language = CASE WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.language ELSE COALESCE(EXCLUDED.language, content_chunks.language) END,
|
||||
symbol_name = CASE WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.symbol_name ELSE COALESCE(EXCLUDED.symbol_name, content_chunks.symbol_name) END,
|
||||
symbol_type = CASE WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.symbol_type ELSE COALESCE(EXCLUDED.symbol_type, content_chunks.symbol_type) END,
|
||||
start_line = CASE WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.start_line ELSE COALESCE(EXCLUDED.start_line, content_chunks.start_line) END,
|
||||
end_line = CASE WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.end_line ELSE COALESCE(EXCLUDED.end_line, content_chunks.end_line) END,
|
||||
parent_symbol_path = CASE WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.parent_symbol_path ELSE COALESCE(EXCLUDED.parent_symbol_path, content_chunks.parent_symbol_path) END,
|
||||
doc_comment = CASE WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.doc_comment ELSE COALESCE(EXCLUDED.doc_comment, content_chunks.doc_comment) END,
|
||||
symbol_name_qualified = CASE WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.symbol_name_qualified ELSE COALESCE(EXCLUDED.symbol_name_qualified, content_chunks.symbol_name_qualified) END,
|
||||
language = EXCLUDED.language,
|
||||
symbol_name = EXCLUDED.symbol_name,
|
||||
symbol_type = EXCLUDED.symbol_type,
|
||||
start_line = EXCLUDED.start_line,
|
||||
end_line = EXCLUDED.end_line,
|
||||
parent_symbol_path = EXCLUDED.parent_symbol_path,
|
||||
doc_comment = EXCLUDED.doc_comment,
|
||||
symbol_name_qualified = EXCLUDED.symbol_name_qualified,
|
||||
modality = EXCLUDED.modality,
|
||||
embedding_image = COALESCE(EXCLUDED.embedding_image, content_chunks.embedding_image)`,
|
||||
params as Parameters<typeof sql.unsafe>[1],
|
||||
@@ -3082,7 +3073,6 @@ export class PostgresEngine implements BrainEngine {
|
||||
name: string,
|
||||
dirPrefix?: string,
|
||||
minSimilarity: number = 0.55,
|
||||
sourceId?: string,
|
||||
): Promise<{ slug: string; similarity: number } | null> {
|
||||
const sql = this.sql;
|
||||
// Use the `similarity()` function directly with an explicit threshold
|
||||
@@ -3095,33 +3085,15 @@ export class PostgresEngine implements BrainEngine {
|
||||
// Tie-breaker: sort by slug after similarity so re-runs return the
|
||||
// same winner when multiple pages score equally (prevents churn
|
||||
// in put_page auto-link reconciliation).
|
||||
//
|
||||
// `sourceId` + `deleted_at IS NULL` mirror the filters `tryFuzzyMatch`
|
||||
// in `src/core/entities/resolve.ts` got via #1436 (v0.41.13.0). Without
|
||||
// them, fuzzy resolution could suggest cross-source slugs that the
|
||||
// caller then silently drops at the FK filter in
|
||||
// `operations.ts:reconcileLinks` (the `allSlugs` filter) — making it
|
||||
// look like the match failed when in fact it picked the wrong page.
|
||||
const prefixPattern = dirPrefix ? `${dirPrefix}/%` : '%';
|
||||
const rows = sourceId
|
||||
? await sql`
|
||||
SELECT slug, similarity(title, ${name}) AS sim
|
||||
FROM pages
|
||||
WHERE similarity(title, ${name}) >= ${minSimilarity}
|
||||
AND slug LIKE ${prefixPattern}
|
||||
AND source_id = ${sourceId}
|
||||
AND deleted_at IS NULL
|
||||
ORDER BY sim DESC, slug ASC
|
||||
LIMIT 1
|
||||
`
|
||||
: await sql`
|
||||
SELECT slug, similarity(title, ${name}) AS sim
|
||||
FROM pages
|
||||
WHERE similarity(title, ${name}) >= ${minSimilarity}
|
||||
AND slug LIKE ${prefixPattern}
|
||||
ORDER BY sim DESC, slug ASC
|
||||
LIMIT 1
|
||||
`;
|
||||
const rows = await sql`
|
||||
SELECT slug, similarity(title, ${name}) AS sim
|
||||
FROM pages
|
||||
WHERE similarity(title, ${name}) >= ${minSimilarity}
|
||||
AND slug LIKE ${prefixPattern}
|
||||
ORDER BY sim DESC, slug ASC
|
||||
LIMIT 1
|
||||
`;
|
||||
if (rows.length === 0) return null;
|
||||
const row = rows[0] as { slug: string; sim: number };
|
||||
return { slug: row.slug, similarity: row.sim };
|
||||
@@ -5341,9 +5313,11 @@ export class PostgresEngine implements BrainEngine {
|
||||
async getHealth(): Promise<BrainHealth> {
|
||||
const sql = this.sql;
|
||||
// Bug 11 doc-drift fix — orphan_pages means "islanded" (no inbound AND
|
||||
// no outbound links). The raw islanded list is filtered through the same
|
||||
// policy as `gbrain orphans` so convention pages do not count against
|
||||
// dashboard health.
|
||||
// no outbound links), aligning both engines with the user-facing
|
||||
// definition. The type comment previously said "no inbound" but the
|
||||
// SQL required both — docs now match code so users can trust the
|
||||
// number. A hub page that links out to many but has no back-references
|
||||
// is working as intended, not an orphan.
|
||||
const [h] = await sql`
|
||||
WITH entity_pages AS (
|
||||
SELECT id, slug FROM pages WHERE type IN ('person', 'company')
|
||||
@@ -5352,8 +5326,13 @@ export class PostgresEngine implements BrainEngine {
|
||||
(SELECT count(*) FROM pages) as page_count,
|
||||
(SELECT count(*) FROM content_chunks WHERE embedded_at IS NOT NULL)::float /
|
||||
GREATEST((SELECT count(*) FROM content_chunks), 1)::float as embed_coverage,
|
||||
0 as stale_pages,
|
||||
0 as orphan_pages,
|
||||
(SELECT count(*) FROM pages p
|
||||
WHERE p.updated_at < (SELECT MAX(te.created_at) FROM timeline_entries te WHERE te.page_id = p.id)
|
||||
) as stale_pages,
|
||||
(SELECT count(*) FROM pages p
|
||||
WHERE NOT EXISTS (SELECT 1 FROM links l WHERE l.to_page_id = p.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM links l WHERE l.from_page_id = p.id)
|
||||
) as orphan_pages,
|
||||
(SELECT count(*) FROM links l
|
||||
WHERE NOT EXISTS (SELECT 1 FROM pages p WHERE p.id = l.to_page_id)
|
||||
) as dead_links,
|
||||
@@ -5377,18 +5356,9 @@ export class PostgresEngine implements BrainEngine {
|
||||
LIMIT 5
|
||||
`;
|
||||
|
||||
const islandedRows = await sql<{ slug: string }[]>`
|
||||
SELECT p.slug
|
||||
FROM pages p
|
||||
WHERE NOT EXISTS (SELECT 1 FROM links l WHERE l.to_page_id = p.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM links l WHERE l.from_page_id = p.id)
|
||||
`;
|
||||
|
||||
const pageCount = Number(h.page_count);
|
||||
const embedCoverage = Number(h.embed_coverage);
|
||||
const stalePages = await this.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS });
|
||||
const orphanOverrides = await loadOrphanPolicyOverrides(this);
|
||||
const orphanPages = islandedRows.filter(row => !shouldExcludeFromOrphanReporting(row.slug, orphanOverrides)).length;
|
||||
const orphanPages = Number(h.orphan_pages);
|
||||
const deadLinks = Number(h.dead_links);
|
||||
const linkCount = Number(h.link_count);
|
||||
const pagesWithTimeline = Number(h.pages_with_timeline);
|
||||
@@ -5416,7 +5386,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
return {
|
||||
page_count: pageCount,
|
||||
embed_coverage: embedCoverage,
|
||||
stale_pages: stalePages,
|
||||
stale_pages: Number(h.stale_pages),
|
||||
orphan_pages: orphanPages,
|
||||
missing_embeddings: Number(h.missing_embeddings),
|
||||
brain_score: brainScore,
|
||||
@@ -6172,13 +6142,6 @@ export class PostgresEngine implements BrainEngine {
|
||||
const prefixCondition = slugPrefix
|
||||
? sql`AND p.slug LIKE ${slugPrefix.replace(/[\\%_]/g, (c) => '\\' + c) + '%'} ESCAPE '\\'`
|
||||
: sql``;
|
||||
// TIM-37: exclude briefing pages from their own Brain Pulse. The cron
|
||||
// briefing writes to 90_Briefings/, gets re-ingested, and would otherwise
|
||||
// top tomorrow's salience as pure self-reference. Suppress unless the
|
||||
// caller explicitly asked for the briefings/ prefix.
|
||||
const excludeBriefings = !(slugPrefix && slugPrefix.startsWith('briefings'))
|
||||
? sql`AND p.slug NOT LIKE 'briefings/%'`
|
||||
: sql``;
|
||||
// v0.29.1: third score term via buildRecencyComponentSql. Default
|
||||
// 'flat' = v0.29.0 behavior (1 / (1 + days_old)). 'on' opts into the
|
||||
// per-prefix decay map (concepts/ evergreen, daily/ aggressive, etc.).
|
||||
@@ -6212,7 +6175,6 @@ export class PostgresEngine implements BrainEngine {
|
||||
LEFT JOIN takes t ON t.page_id = p.id AND t.active = TRUE
|
||||
WHERE GREATEST(p.updated_at, COALESCE(p.salience_touched_at, p.updated_at)) >= ${boundaryIso}::timestamptz
|
||||
${prefixCondition}
|
||||
${excludeBriefings}
|
||||
GROUP BY p.id
|
||||
ORDER BY score DESC
|
||||
LIMIT ${limit}
|
||||
|
||||
@@ -93,13 +93,7 @@ import { resolveLrSchedule } from './lr-schedule.ts';
|
||||
import { preflight, formatPreflightReport } from './preflight.ts';
|
||||
import { isRejected, loadRejectedBuffer, makeRejectedEntry, saveRejectedBuffer } from './rejected-buffer.ts';
|
||||
import { runReflect, runOneShotRewrite, describeJudges } from './reflect.ts';
|
||||
import {
|
||||
acceptCandidate,
|
||||
proposedPath as proposedFilePath,
|
||||
revertAllPending,
|
||||
skillPath,
|
||||
writeProposed,
|
||||
} from './version-store.ts';
|
||||
import { acceptCandidate, bestPath, revertAllPending, skillPath, writeProposed } from './version-store.ts';
|
||||
import { runValidationGate, scoreSkillOnTasks } from './validate-gate.ts';
|
||||
import { ROLLOUT_SUCCESS_THRESHOLD } from './types.ts';
|
||||
import type { SkillOptOpts, EditOp, RunReceipt, BenchmarkTask } from './types.ts';
|
||||
@@ -708,9 +702,9 @@ async function runOptimizationLoop(
|
||||
// to the catch's assignment values only (it can't prove the async callback ran).
|
||||
const finalOutcome = outcome as 'accepted' | 'no_improvement' | 'aborted' | 'errored';
|
||||
if (!mutateDecision.mutate && finalOutcome === 'accepted') {
|
||||
// writeProposed() emitted both the best pointer and the stable review
|
||||
// artifact in the accept branch. SKILL.md remains untouched.
|
||||
proposedPath = proposedFilePath(skillsDir, skillName);
|
||||
// best.md was written by writeProposed() in the accept branch (no-mutate
|
||||
// path); it doubles as proposed.md for human review. SKILL.md untouched.
|
||||
proposedPath = bestPath(skillsDir, skillName);
|
||||
} else if (mutateDecision.mutate) {
|
||||
mutatedSkillFile = finalOutcome === 'accepted';
|
||||
}
|
||||
|
||||
@@ -23,7 +23,6 @@
|
||||
*
|
||||
* history.json
|
||||
* best.md
|
||||
* proposed.md
|
||||
* versions/
|
||||
* v0001_e1_s1.md
|
||||
* v0002_e1_s2.md
|
||||
@@ -53,10 +52,6 @@ export function bestPath(skillsDir: string, skillName: string): string {
|
||||
return path.join(skilloptDir(skillsDir, skillName), 'best.md');
|
||||
}
|
||||
|
||||
export function proposedPath(skillsDir: string, skillName: string): string {
|
||||
return path.join(skilloptDir(skillsDir, skillName), 'proposed.md');
|
||||
}
|
||||
|
||||
export function skillPath(skillsDir: string, skillName: string): string {
|
||||
return path.join(skillsDir, skillName, 'SKILL.md');
|
||||
}
|
||||
@@ -176,18 +171,17 @@ export function acceptCandidate(input: AcceptInput): AcceptResult {
|
||||
}
|
||||
|
||||
/**
|
||||
* Write the candidate to both `best.md` and `proposed.md` WITHOUT touching
|
||||
* SKILL.md or the history ledger. `best.md` remains the optimizer's current
|
||||
* best pointer; `proposed.md` is the stable human-review artifact promised by
|
||||
* `--no-mutate`. Returns the proposal path. Each write is atomic (.tmp + rename).
|
||||
* Write the candidate to `best.md` (which doubles as `proposed.md`) WITHOUT
|
||||
* touching SKILL.md or the history ledger. Used by the `--no-mutate` /
|
||||
* bundled-without-allow paths: the optimizer found a better candidate but the
|
||||
* caller opted out of in-place mutation, so we surface it for human review.
|
||||
* Returns the path written. Atomic (.tmp + rename).
|
||||
*/
|
||||
export function writeProposed(skillsDir: string, skillName: string, candidateText: string): string {
|
||||
const best = bestPath(skillsDir, skillName);
|
||||
const proposed = proposedPath(skillsDir, skillName);
|
||||
fs.mkdirSync(path.dirname(best), { recursive: true });
|
||||
atomicWrite(best, candidateText);
|
||||
atomicWrite(proposed, candidateText);
|
||||
return proposed;
|
||||
const p = bestPath(skillsDir, skillName);
|
||||
fs.mkdirSync(path.dirname(p), { recursive: true });
|
||||
atomicWrite(p, candidateText);
|
||||
return p;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -195,14 +195,7 @@ export class SupabaseStorage implements StorageBackend {
|
||||
throw new Error(`Supabase signed URL failed: ${res.status} ${body}`);
|
||||
}
|
||||
const result = await res.json() as { signedURL: string };
|
||||
// Supabase returns `signedURL` relative to the Storage API root, e.g.
|
||||
// "/object/sign/<bucket>/<path>?token=...". Prepend projectUrl + "/storage/v1"
|
||||
// (not just projectUrl) or the link 404s. Tolerate an already-absolute URL or a
|
||||
// value that already carries the /storage/v1 prefix.
|
||||
const signed = result.signedURL;
|
||||
if (/^https?:\/\//.test(signed)) return signed;
|
||||
if (signed.startsWith('/storage/v1')) return `${this.projectUrl}${signed}`;
|
||||
return `${this.projectUrl}/storage/v1${signed.startsWith('/') ? '' : '/'}${signed}`;
|
||||
return `${this.projectUrl}${result.signedURL}`;
|
||||
}
|
||||
|
||||
async getUrl(path: string): Promise<string> {
|
||||
|
||||
+1
-35
@@ -553,40 +553,6 @@ export async function runThink(
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip a "## Gaps" section from an answer body.
|
||||
*
|
||||
* `think` returns gaps in the structured `gaps` array, which the CLI and the
|
||||
* persisted synthesis page render exactly once. The system prompt also used to
|
||||
* ask for a "Gaps" section inside the answer prose, so a model that still emits
|
||||
* one would make the output show "## Gaps" twice — once from the prose, once
|
||||
* from the structured array. This removes the prose section so the structured
|
||||
* array stays the single source of truth.
|
||||
*
|
||||
* Matches a heading line `## Gaps` (level 2-6, case-insensitive) and removes it
|
||||
* through the next heading of the same-or-higher level, or end of string.
|
||||
* Returns the input unchanged when there is no such section.
|
||||
*/
|
||||
export function stripGapsSection(answer: string): string {
|
||||
if (!answer) return answer;
|
||||
const lines = answer.split('\n');
|
||||
let start = -1;
|
||||
let level = 0;
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const m = /^(#{2,6})\s+gaps\s*$/i.exec(lines[i]);
|
||||
if (m) { start = i; level = m[1].length; break; }
|
||||
}
|
||||
if (start === -1) return answer;
|
||||
let end = lines.length;
|
||||
for (let i = start + 1; i < lines.length; i++) {
|
||||
const h = /^(#{1,6})\s+\S/.exec(lines[i]);
|
||||
if (h && h[1].length <= level) { end = i; break; }
|
||||
}
|
||||
const kept = [...lines.slice(0, start), ...lines.slice(end)].join('\n');
|
||||
// Drop trailing blank lines left by removing a trailing section.
|
||||
return kept.replace(/\s+$/, '');
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist a synthesis page + its evidence. Returns the saved slug.
|
||||
* Synthesis pages are written under `synthesis/<slugified-question>-<date>.md`.
|
||||
@@ -616,7 +582,7 @@ export async function persistSynthesis(
|
||||
const body = [
|
||||
`# ${result.question}`,
|
||||
'',
|
||||
stripGapsSection(result.answer),
|
||||
result.answer,
|
||||
'',
|
||||
result.gaps.length > 0 ? '## Gaps\n\n' + result.gaps.map(g => `- ${g}`).join('\n') : '',
|
||||
].filter(Boolean).join('\n');
|
||||
|
||||
@@ -52,19 +52,19 @@ Hard rules:
|
||||
rather than asserting it as established. Confidence is part of the data.
|
||||
- If two takes contradict (different holders, opposite claims), surface BOTH in a "Conflicts"
|
||||
section. Never silently pick one.
|
||||
- If the brain doesn't contain data needed to answer, do NOT make it up. Record each
|
||||
missing piece in the structured "gaps" array (below), not as a section in the answer prose.
|
||||
- If you cannot answer because the brain doesn't contain the relevant data, say so in the
|
||||
"Gaps" section. List the specific missing pieces. Do not make up answers.
|
||||
- Never instruct the user (no "you should" / "I recommend X"). The brain reports; the user decides.
|
||||
- Output MUST be valid JSON matching the schema below. No prose outside JSON.
|
||||
|
||||
Output schema:
|
||||
{
|
||||
"answer": "<markdown body. Inline citations like [slug#row] or [slug]. Sections: Answer, Conflicts (optional). Do NOT add a Gaps section here — gaps belong in the gaps array.>",
|
||||
"answer": "<markdown body. Inline citations like [slug#row] or [slug]. Sections: Answer, Conflicts (optional), Gaps>",
|
||||
"citations": [
|
||||
{"page_slug": "people/alice-example", "row_num": 3, "citation_index": 1},
|
||||
{"page_slug": "companies/acme-example", "row_num": null, "citation_index": 2}
|
||||
],
|
||||
"gaps": ["a specific, self-contained missing-or-stale data point, citing the [slug] where relevant", "another specific gap"]
|
||||
"gaps": ["specific missing data point 1", "specific missing data point 2"]
|
||||
}
|
||||
|
||||
The "row_num" field is required for take citations and MUST be null for page-only citations.`;
|
||||
@@ -83,7 +83,7 @@ export function buildThinkSystemPrompt(opts: ThinkSystemPromptOpts = {}): string
|
||||
lines.push(`\nThis is a temporal question. Order key claims chronologically when it helps the reader.`);
|
||||
}
|
||||
if (opts.willSave) {
|
||||
lines.push(`\nThis synthesis will be persisted as a brain page. Aim for completeness — cover the Answer and any Conflicts thoroughly, and list every missing piece in the structured "gaps" array.`);
|
||||
lines.push(`\nThis synthesis will be persisted as a brain page. Aim for completeness — cover Answer, Conflicts, and Gaps thoroughly.`);
|
||||
}
|
||||
if (opts.withCalibration) {
|
||||
lines.push(
|
||||
@@ -92,7 +92,7 @@ export function buildThinkSystemPrompt(opts: ThinkSystemPromptOpts = {}): string
|
||||
lines.push(`- Name both the user's PRIOR (default reasoning) AND the COUNTER-PRIOR from their hedged-domain self.`);
|
||||
lines.push(`- Reference active bias tags by name when relevant ("this fits the over-confident-geography pattern").`);
|
||||
lines.push(`- Do NOT silently substitute the debiased answer. ALWAYS surface both priors transparently.`);
|
||||
lines.push(`- Track-record sentences belong in a "Calibration" section in the answer body, after the Conflicts section (if present).`);
|
||||
lines.push(`- Track-record sentences belong in a "Calibration" section in the answer body, between Conflicts and Gaps.`);
|
||||
}
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
@@ -63,26 +63,25 @@ interface PluginCtx {
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export function register(api: PluginApi) {
|
||||
api.registerContextEngine(ENGINE_ID, (ctx: PluginCtx) => {
|
||||
const hostResolver =
|
||||
typeof ctx.resolveEntities === 'function'
|
||||
? ctx.resolveEntities
|
||||
: typeof ctx.brainQuery === 'function'
|
||||
? ctx.brainQuery
|
||||
: undefined;
|
||||
return createGBrainContextEngine({
|
||||
workspaceDir: ctx.workspaceDir,
|
||||
resolveEntities: hostResolver,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
const entry: PluginEntry = {
|
||||
id: 'gbrain-context-engine',
|
||||
name: 'GBrain Context Engine',
|
||||
description: 'Deterministic temporal/spatial context injection on every turn',
|
||||
register,
|
||||
|
||||
register(api: PluginApi) {
|
||||
api.registerContextEngine(ENGINE_ID, (ctx: PluginCtx) => {
|
||||
const hostResolver =
|
||||
typeof ctx.resolveEntities === 'function'
|
||||
? ctx.resolveEntities
|
||||
: typeof ctx.brainQuery === 'function'
|
||||
? ctx.brainQuery
|
||||
: undefined;
|
||||
return createGBrainContextEngine({
|
||||
workspaceDir: ctx.workspaceDir,
|
||||
resolveEntities: hostResolver,
|
||||
});
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
export default entry;
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { resetPgliteState } from './helpers/reset-pglite.ts';
|
||||
import { queryAdminSources } from '../src/commands/serve-http.ts';
|
||||
import { buildSyncStatusReport } from '../src/commands/sync.ts';
|
||||
|
||||
/**
|
||||
* v0.41.29 Sources tab — `/admin/api/sources` endpoint SQL.
|
||||
*
|
||||
* The endpoint is a thin Express handler over `queryAdminSources` +
|
||||
* `buildSyncStatusReport`; the source-selection SQL is the load-bearing
|
||||
* surface (same pattern as test/admin-agents-spend.test.ts).
|
||||
*
|
||||
* Pinned behaviors:
|
||||
* - Excludes archived sources
|
||||
* - INCLUDES sources with null local_path (push-only brains: filtering
|
||||
* on local_path emptied the Sources tab + federation source-picker)
|
||||
* - JSONB config surfaces as an object, defaulting to {}
|
||||
* - Deterministic ORDER BY id
|
||||
* - buildSyncStatusReport accepts the rows (no disk I/O on null paths)
|
||||
*/
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await resetPgliteState(engine);
|
||||
});
|
||||
|
||||
describe('queryAdminSources (/admin/api/sources SQL)', () => {
|
||||
it('includes push-only sources with null local_path', async () => {
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO sources (id, name, local_path, config)
|
||||
VALUES ('push-only', 'push-only', NULL, '{}'::jsonb)`,
|
||||
);
|
||||
const sources = await queryAdminSources(engine);
|
||||
const ids = sources.map((s) => s.id);
|
||||
expect(ids).toContain('push-only');
|
||||
expect(sources.find((s) => s.id === 'push-only')!.local_path).toBe(null);
|
||||
});
|
||||
|
||||
it('excludes archived sources', async () => {
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO sources (id, name, archived) VALUES ('gone', 'gone', true)`,
|
||||
);
|
||||
const sources = await queryAdminSources(engine);
|
||||
expect(sources.map((s) => s.id)).not.toContain('gone');
|
||||
});
|
||||
|
||||
it('surfaces JSONB config as an object and orders by id', async () => {
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO sources (id, name, config)
|
||||
VALUES ('bbb', 'bbb', '{"syncEnabled": true}'::jsonb),
|
||||
('aaa', 'aaa', '{}'::jsonb)`,
|
||||
);
|
||||
const sources = await queryAdminSources(engine);
|
||||
const ids = sources.map((s) => s.id);
|
||||
expect(ids.indexOf('aaa')).toBeLessThan(ids.indexOf('bbb'));
|
||||
expect(sources.find((s) => s.id === 'bbb')!.config).toEqual({ syncEnabled: true });
|
||||
expect(sources.find((s) => s.id === 'aaa')!.config).toEqual({});
|
||||
});
|
||||
|
||||
it('buildSyncStatusReport accepts the rows (null local_path does not throw)', async () => {
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO sources (id, name, local_path, config)
|
||||
VALUES ('push-only', 'push-only', NULL, '{}'::jsonb)`,
|
||||
);
|
||||
const report = await buildSyncStatusReport(engine, await queryAdminSources(engine));
|
||||
expect(report.schema_version).toBe(1);
|
||||
const row = report.sources.find((s) => s.source_id === 'push-only');
|
||||
expect(row).toBeDefined();
|
||||
});
|
||||
});
|
||||
@@ -34,7 +34,6 @@ import {
|
||||
resetGateway,
|
||||
embed,
|
||||
splitByTokenBudget,
|
||||
capBatchItems,
|
||||
isTokenLimitError,
|
||||
__setEmbedTransportForTests,
|
||||
__getShrinkStateForTests,
|
||||
@@ -152,41 +151,6 @@ describe('splitByTokenBudget (pure helper)', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('capBatchItems (hard COUNT cap helper)', () => {
|
||||
test('batch at or under the cap is returned as a single batch (no copy of contents)', () => {
|
||||
const texts = ['a', 'b', 'c'];
|
||||
expect(capBatchItems(texts, 3)).toEqual([texts]);
|
||||
expect(capBatchItems(texts, 10)).toEqual([texts]);
|
||||
});
|
||||
|
||||
test('oversized batch splits into chunks of at most maxItems', () => {
|
||||
const texts = Array.from({ length: 100 }, (_, i) => `t${i}`);
|
||||
const result = capBatchItems(texts, 32);
|
||||
expect(result.map(b => b.length)).toEqual([32, 32, 32, 4]);
|
||||
expect(result.every(b => b.length <= 32)).toBe(true);
|
||||
});
|
||||
|
||||
test('exact multiple splits evenly with no trailing empty batch', () => {
|
||||
const texts = Array.from({ length: 64 }, (_, i) => `t${i}`);
|
||||
expect(capBatchItems(texts, 32).map(b => b.length)).toEqual([32, 32]);
|
||||
});
|
||||
|
||||
test('order is preserved across the split (concatenation round-trips)', () => {
|
||||
const texts = Array.from({ length: 70 }, (_, i) => `t${i}`);
|
||||
expect(capBatchItems(texts, 32).flat()).toEqual(texts);
|
||||
});
|
||||
|
||||
test('maxItems <= 0 is a no-op (single batch) — never produces empty/infinite batches', () => {
|
||||
const texts = ['a', 'b', 'c'];
|
||||
expect(capBatchItems(texts, 0)).toEqual([texts]);
|
||||
expect(capBatchItems(texts, -5)).toEqual([texts]);
|
||||
});
|
||||
|
||||
test('empty input returns a single empty batch', () => {
|
||||
expect(capBatchItems([], 32)).toEqual([[]]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('isTokenLimitError (pure helper)', () => {
|
||||
test('matches Voyage error format', () => {
|
||||
expect(isTokenLimitError(VOYAGE_TOKEN_LIMIT_ERROR)).toBe(true);
|
||||
|
||||
@@ -28,8 +28,8 @@ describe('v0.32 #779: no_batch_cap suppresses the missing-max_batch_tokens warni
|
||||
resetGateway();
|
||||
});
|
||||
|
||||
test('Ollama, LiteLLM declare no_batch_cap: true', () => {
|
||||
for (const id of ['ollama', 'litellm']) {
|
||||
test('Ollama, LiteLLM, llama-server all declare no_batch_cap: true', () => {
|
||||
for (const id of ['ollama', 'litellm', 'llama-server']) {
|
||||
const r = getRecipe(id);
|
||||
expect(r, `${id} not registered`).toBeDefined();
|
||||
expect(
|
||||
@@ -39,16 +39,6 @@ describe('v0.32 #779: no_batch_cap suppresses the missing-max_batch_tokens warni
|
||||
}
|
||||
});
|
||||
|
||||
test('llama-server declares a hard item-count cap (max_batch_items: 32)', () => {
|
||||
// llama.cpp enforces a request-COUNT cap equal to its launch --batch-size
|
||||
// (default 32); declaring max_batch_items both bounds batches AND suppresses
|
||||
// the missing-max_batch_tokens warning. Replaces the prior no_batch_cap flag.
|
||||
const r = getRecipe('llama-server');
|
||||
expect(r, 'llama-server not registered').toBeDefined();
|
||||
expect(r!.touchpoints.embedding?.max_batch_items).toBe(32);
|
||||
expect(r!.touchpoints.embedding?.no_batch_cap).toBeUndefined();
|
||||
});
|
||||
|
||||
test('configureGateway does NOT warn for ollama/litellm/llama-server', () => {
|
||||
warnSpy.mockClear();
|
||||
resetGateway();
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
/**
|
||||
* Ollama Matryoshka dims passthrough.
|
||||
*
|
||||
* Several embedding models served via Ollama (Qwen3-Embedding family) support
|
||||
* Matryoshka truncation through the `dimensions` field on /v1/embeddings.
|
||||
* Without this passthrough, gbrain ignores user-selected reduced dims and the
|
||||
* provider returns its native size, causing dim-mismatch failures against
|
||||
* brains configured for smaller widths.
|
||||
*/
|
||||
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { dimsProviderOptions } from '../../src/core/ai/dims.ts';
|
||||
|
||||
describe('dims: ollama Matryoshka models', () => {
|
||||
test('qwen3-embedding:4b threads dimensions=1536', () => {
|
||||
expect(dimsProviderOptions('openai-compatible', 'qwen3-embedding:4b', 1536))
|
||||
.toEqual({ openaiCompatible: { dimensions: 1536 } });
|
||||
});
|
||||
|
||||
test('qwen3-embedding:0.6b threads dimensions=512', () => {
|
||||
expect(dimsProviderOptions('openai-compatible', 'qwen3-embedding:0.6b', 512))
|
||||
.toEqual({ openaiCompatible: { dimensions: 512 } });
|
||||
});
|
||||
|
||||
test('qwen3-embedding:8b threads dimensions=2048', () => {
|
||||
expect(dimsProviderOptions('openai-compatible', 'qwen3-embedding:8b', 2048))
|
||||
.toEqual({ openaiCompatible: { dimensions: 2048 } });
|
||||
});
|
||||
|
||||
test('bare qwen3-embedding (no quant tag) also recognized', () => {
|
||||
expect(dimsProviderOptions('openai-compatible', 'qwen3-embedding', 1024))
|
||||
.toEqual({ openaiCompatible: { dimensions: 1024 } });
|
||||
});
|
||||
|
||||
test('unrelated openai-compat model returns undefined (regression guard)', () => {
|
||||
expect(dimsProviderOptions('openai-compatible', 'nomic-embed-text', 768))
|
||||
.toBeUndefined();
|
||||
expect(dimsProviderOptions('openai-compatible', 'mxbai-embed-large', 1024))
|
||||
.toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -167,41 +167,6 @@ describe('buildPlan — diff against completed + installed VERSION', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('force-retry escape hatch', () => {
|
||||
test("complete then retry-latest → pending and buildPlan lists the version as pending", () => {
|
||||
const idx = indexCompleted([
|
||||
{ version: '0.11.0', status: 'complete' },
|
||||
{ version: '0.11.0', status: 'retry' },
|
||||
]);
|
||||
|
||||
expect(statusForVersion('0.11.0', idx)).toBe('pending');
|
||||
const plan = buildPlan(idx, '0.11.1', '0.11.0');
|
||||
expect(plan.pending.map(m => m.version)).toEqual(['0.11.0']);
|
||||
expect(plan.applied).toEqual([]);
|
||||
expect(plan.partial).toEqual([]);
|
||||
expect(plan.wedged).toEqual([]);
|
||||
});
|
||||
|
||||
test('complete then stray partial without retry → still complete', () => {
|
||||
const idx = indexCompleted([
|
||||
{ version: '0.11.0', status: 'complete' },
|
||||
{ version: '0.11.0', status: 'partial' },
|
||||
]);
|
||||
|
||||
expect(statusForVersion('0.11.0', idx)).toBe('complete');
|
||||
});
|
||||
|
||||
test('retry followed by a newer complete → complete', () => {
|
||||
const idx = indexCompleted([
|
||||
{ version: '0.11.0', status: 'complete' },
|
||||
{ version: '0.11.0', status: 'retry' },
|
||||
{ version: '0.11.0', status: 'complete' },
|
||||
]);
|
||||
|
||||
expect(statusForVersion('0.11.0', idx)).toBe('complete');
|
||||
});
|
||||
});
|
||||
|
||||
// v0.36.1.x (cherry-pick #1062): list, dry-run, and "all migrations up to
|
||||
// date" paths must exit 0 so shell scripts gating on the exit code work.
|
||||
// Pre-fix, these `return` statements left the CLI dispatcher's implicit
|
||||
|
||||
@@ -0,0 +1,568 @@
|
||||
/**
|
||||
* Tests for `gbrain auth grant-read|revoke-read|set-federated-read`.
|
||||
*
|
||||
* Pure helper: parseSourceCsv (no DB).
|
||||
* DB-coupled: resolveClient, assertSourceExists, *Core fns — exercised
|
||||
* against a real PGLite via the canonical block.
|
||||
*/
|
||||
import { describe, expect, test, beforeAll, afterAll, beforeEach } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { resetPgliteState } from './helpers/reset-pglite.ts';
|
||||
import { sqlQueryForEngine } from '../src/core/sql-query.ts';
|
||||
import { pgArray } from '../src/core/oauth-provider.ts';
|
||||
import {
|
||||
parseSourceCsv,
|
||||
resolveClient,
|
||||
assertSourceExists,
|
||||
grantReadCore,
|
||||
revokeReadCore,
|
||||
setFederatedReadCore,
|
||||
extractDryRun,
|
||||
sanitizeForTerminal,
|
||||
} from '../src/commands/auth.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await resetPgliteState(engine);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// pure helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('parseSourceCsv', () => {
|
||||
test('splits and trims', () => {
|
||||
expect(parseSourceCsv('a,b,c')).toEqual(['a', 'b', 'c']);
|
||||
expect(parseSourceCsv(' a , b ')).toEqual(['a', 'b']);
|
||||
});
|
||||
|
||||
test('drops empty segments', () => {
|
||||
expect(parseSourceCsv('a,,b,')).toEqual(['a', 'b']);
|
||||
expect(parseSourceCsv(',,')).toEqual([]);
|
||||
expect(parseSourceCsv('')).toEqual([]);
|
||||
});
|
||||
|
||||
test('dedupes while preserving first-seen order', () => {
|
||||
expect(parseSourceCsv('a,b,a,c,b')).toEqual(['a', 'b', 'c']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractDryRun', () => {
|
||||
test('absent flag → false', () => {
|
||||
expect(extractDryRun(['alice', 'proj-x'])).toEqual({
|
||||
dryRun: false,
|
||||
rest: ['alice', 'proj-x'],
|
||||
});
|
||||
});
|
||||
|
||||
test('flag at end', () => {
|
||||
expect(extractDryRun(['alice', 'proj-x', '--dry-run'])).toEqual({
|
||||
dryRun: true,
|
||||
rest: ['alice', 'proj-x'],
|
||||
});
|
||||
});
|
||||
|
||||
test('flag at start', () => {
|
||||
expect(extractDryRun(['--dry-run', 'alice', 'proj-x'])).toEqual({
|
||||
dryRun: true,
|
||||
rest: ['alice', 'proj-x'],
|
||||
});
|
||||
});
|
||||
|
||||
test('flag in middle', () => {
|
||||
expect(extractDryRun(['alice', '--dry-run', 'proj-x'])).toEqual({
|
||||
dryRun: true,
|
||||
rest: ['alice', 'proj-x'],
|
||||
});
|
||||
});
|
||||
|
||||
test('no args', () => {
|
||||
expect(extractDryRun([])).toEqual({ dryRun: false, rest: [] });
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// DB-coupled
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function seedSource(id: string): Promise<void> {
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
await sql`INSERT INTO sources (id, name) VALUES (${id}, ${id}) ON CONFLICT (id) DO NOTHING`;
|
||||
}
|
||||
|
||||
async function seedClient(name: string, federated: string[] = []): Promise<string> {
|
||||
// Ensure write source FK is satisfied — every seeded client points at 'default'.
|
||||
await seedSource('default');
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
const clientId = `gbrain_cl_test_${name}_${Date.now()}_${Math.random().toString(16).slice(2, 8)}`;
|
||||
const fedLit = pgArray(federated);
|
||||
await sql`
|
||||
INSERT INTO oauth_clients (client_id, client_name, client_secret_hash,
|
||||
redirect_uris, grant_types, scope,
|
||||
client_id_issued_at, source_id, federated_read)
|
||||
VALUES (${clientId}, ${name}, ${'dummy-hash'},
|
||||
${pgArray([])}, ${pgArray(['client_credentials'])}, ${'read'},
|
||||
${Date.now()}, ${'default'}, ${fedLit})
|
||||
`;
|
||||
return clientId;
|
||||
}
|
||||
|
||||
async function readFederated(clientId: string): Promise<string[]> {
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
const rows = await sql`SELECT federated_read FROM oauth_clients WHERE client_id = ${clientId}`;
|
||||
const fed = rows[0]?.federated_read;
|
||||
return Array.isArray(fed) ? (fed as string[]).map(String) : [];
|
||||
}
|
||||
|
||||
describe('resolveClient', () => {
|
||||
test('matches by client_id', async () => {
|
||||
await seedSource('default');
|
||||
const id = await seedClient('alice', ['default']);
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
const c = await resolveClient(sql, id);
|
||||
expect(c.client_name).toBe('alice');
|
||||
expect(c.federated_read).toEqual(['default']);
|
||||
});
|
||||
|
||||
test('matches by client_name', async () => {
|
||||
await seedSource('default');
|
||||
await seedClient('alice', ['default']);
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
const c = await resolveClient(sql, 'alice');
|
||||
expect(c.client_name).toBe('alice');
|
||||
});
|
||||
|
||||
test('errors loudly on no-match', async () => {
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
await expect(resolveClient(sql, 'nobody')).rejects.toThrow(/No active OAuth client found/);
|
||||
});
|
||||
|
||||
test('errors loudly on ambiguous client_name', async () => {
|
||||
await seedSource('default');
|
||||
await seedClient('bob', ['default']);
|
||||
await seedClient('bob', ['default']);
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
await expect(resolveClient(sql, 'bob')).rejects.toThrow(/Multiple active OAuth clients named/);
|
||||
});
|
||||
|
||||
test('null source_id is preserved as null (legacy row tolerance)', async () => {
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
const clientId = `gbrain_cl_test_null_${Date.now()}`;
|
||||
await sql`
|
||||
INSERT INTO oauth_clients (client_id, client_name, client_secret_hash,
|
||||
redirect_uris, grant_types, scope,
|
||||
client_id_issued_at, source_id, federated_read)
|
||||
VALUES (${clientId}, ${'legacy'}, ${'dummy'},
|
||||
${pgArray([])}, ${pgArray(['client_credentials'])}, ${'read'},
|
||||
${Date.now()}, ${null}, ${pgArray([])})
|
||||
`;
|
||||
const c = await resolveClient(sql, clientId);
|
||||
expect(c.source_id).toBeNull();
|
||||
expect(c.federated_read).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('assertSourceExists', () => {
|
||||
test('passes when present', async () => {
|
||||
await seedSource('proj-x');
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
await expect(assertSourceExists(sql, 'proj-x')).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
test('throws with paste-ready hint when missing', async () => {
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
await expect(assertSourceExists(sql, 'ghost')).rejects.toThrow(
|
||||
/Source "ghost" does not exist.*gbrain sources add ghost/s,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('grantReadCore', () => {
|
||||
test('appends when not present and persists', async () => {
|
||||
await seedSource('default');
|
||||
await seedSource('proj-x');
|
||||
const id = await seedClient('alice', ['default']);
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
const outcome = await grantReadCore(sql, 'alice', 'proj-x');
|
||||
expect(outcome.kind).toBe('updated');
|
||||
if (outcome.kind === 'updated') {
|
||||
expect(outcome.before).toEqual(['default']);
|
||||
expect(outcome.after).toEqual(['default', 'proj-x']);
|
||||
}
|
||||
expect(await readFederated(id)).toEqual(['default', 'proj-x']);
|
||||
});
|
||||
|
||||
test('is idempotent — second call is a noop, list unchanged', async () => {
|
||||
await seedSource('default');
|
||||
await seedSource('proj-x');
|
||||
const id = await seedClient('alice', ['default']);
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
await grantReadCore(sql, 'alice', 'proj-x');
|
||||
const outcome = await grantReadCore(sql, 'alice', 'proj-x');
|
||||
expect(outcome.kind).toBe('noop');
|
||||
if (outcome.kind === 'noop') {
|
||||
expect(outcome.reason).toBe('already-granted');
|
||||
}
|
||||
expect(await readFederated(id)).toEqual(['default', 'proj-x']);
|
||||
});
|
||||
|
||||
test('refuses unknown source (fails BEFORE mutating)', async () => {
|
||||
await seedSource('default');
|
||||
const id = await seedClient('alice', ['default']);
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
await expect(grantReadCore(sql, 'alice', 'ghost')).rejects.toThrow(/does not exist/);
|
||||
expect(await readFederated(id)).toEqual(['default']);
|
||||
});
|
||||
|
||||
test('refuses unknown client', async () => {
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
await expect(grantReadCore(sql, 'nobody', 'whatever')).rejects.toThrow(/No active OAuth client found/);
|
||||
});
|
||||
|
||||
test('accepts client_id resolution too', async () => {
|
||||
await seedSource('default');
|
||||
await seedSource('proj-x');
|
||||
const id = await seedClient('alice', ['default']);
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
await grantReadCore(sql, id, 'proj-x');
|
||||
expect(await readFederated(id)).toEqual(['default', 'proj-x']);
|
||||
});
|
||||
|
||||
test('rejects malformed source_id BEFORE existence check (Codex finding #3)', async () => {
|
||||
await seedSource('default');
|
||||
const id = await seedClient('alice', ['default']);
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
// Even with a row in `sources` having a weird id, the validator at the
|
||||
// boundary refuses. Closes the "manual SQL plants a row, CLI lets it
|
||||
// become unmanageable in federated_read" vector.
|
||||
await sql`INSERT INTO sources (id, name) VALUES (${'has,"weird"-bits'}, ${'weird'})`;
|
||||
await expect(grantReadCore(sql, 'alice', 'has,"weird"-bits')).rejects.toThrow(/Invalid source_id/);
|
||||
// DB unchanged.
|
||||
expect(await readFederated(id)).toEqual(['default']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('revokeReadCore', () => {
|
||||
test('removes when present', async () => {
|
||||
await seedSource('default');
|
||||
await seedSource('proj-x');
|
||||
const id = await seedClient('alice', ['default', 'proj-x']);
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
const outcome = await revokeReadCore(sql, 'alice', 'proj-x');
|
||||
expect(outcome.kind).toBe('updated');
|
||||
expect(await readFederated(id)).toEqual(['default']);
|
||||
});
|
||||
|
||||
test('is idempotent — second call is a noop, list unchanged', async () => {
|
||||
await seedSource('default');
|
||||
const id = await seedClient('alice', ['default']);
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
const outcome = await revokeReadCore(sql, 'alice', 'ghost-source');
|
||||
expect(outcome.kind).toBe('noop');
|
||||
if (outcome.kind === 'noop') {
|
||||
expect(outcome.reason).toBe('not-present');
|
||||
}
|
||||
expect(await readFederated(id)).toEqual(['default']);
|
||||
});
|
||||
|
||||
test('allows clearing the list down to empty (no implicit guard)', async () => {
|
||||
await seedSource('default');
|
||||
const id = await seedClient('alice', ['default']);
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
await revokeReadCore(sql, 'alice', 'default');
|
||||
expect(await readFederated(id)).toEqual([]);
|
||||
});
|
||||
|
||||
test('does NOT validate the source exists — operator may revoke stale references', async () => {
|
||||
await seedSource('default');
|
||||
// federated_read carries 'proj-x' but the source row was deleted.
|
||||
const id = await seedClient('alice', ['default', 'proj-x']);
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
const outcome = await revokeReadCore(sql, 'alice', 'proj-x');
|
||||
expect(outcome.kind).toBe('updated');
|
||||
expect(await readFederated(id)).toEqual(['default']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('setFederatedReadCore', () => {
|
||||
test('replaces list wholesale', async () => {
|
||||
await seedSource('a');
|
||||
await seedSource('b');
|
||||
await seedSource('c');
|
||||
const id = await seedClient('alice', ['a']);
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
const outcome = await setFederatedReadCore(sql, 'alice', 'b,c');
|
||||
expect(outcome.kind).toBe('updated');
|
||||
expect(await readFederated(id)).toEqual(['b', 'c']);
|
||||
});
|
||||
|
||||
test('dedupes CSV input', async () => {
|
||||
await seedSource('a');
|
||||
await seedSource('b');
|
||||
const id = await seedClient('alice', []);
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
await setFederatedReadCore(sql, 'alice', 'a,b,a,b,a');
|
||||
expect(await readFederated(id)).toEqual(['a', 'b']);
|
||||
});
|
||||
|
||||
test('empty string clears the list', async () => {
|
||||
await seedSource('a');
|
||||
const id = await seedClient('alice', ['a']);
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
await setFederatedReadCore(sql, 'alice', '');
|
||||
expect(await readFederated(id)).toEqual([]);
|
||||
});
|
||||
|
||||
test('noop when result equals current list', async () => {
|
||||
await seedSource('a');
|
||||
await seedSource('b');
|
||||
const id = await seedClient('alice', ['a', 'b']);
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
const outcome = await setFederatedReadCore(sql, 'alice', 'a,b');
|
||||
expect(outcome.kind).toBe('noop');
|
||||
if (outcome.kind === 'noop') {
|
||||
expect(outcome.reason).toBe('same-list');
|
||||
}
|
||||
expect(await readFederated(id)).toEqual(['a', 'b']);
|
||||
});
|
||||
|
||||
test('refuses unknown source (fails BEFORE mutating)', async () => {
|
||||
await seedSource('a');
|
||||
const id = await seedClient('alice', ['a']);
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
await expect(setFederatedReadCore(sql, 'alice', 'a,ghost')).rejects.toThrow(/does not exist/);
|
||||
// Original list preserved.
|
||||
expect(await readFederated(id)).toEqual(['a']);
|
||||
});
|
||||
|
||||
test('order in CSV is the order persisted', async () => {
|
||||
await seedSource('a');
|
||||
await seedSource('b');
|
||||
await seedSource('c');
|
||||
const id = await seedClient('alice', ['a']);
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
await setFederatedReadCore(sql, 'alice', 'c,a,b');
|
||||
expect(await readFederated(id)).toEqual(['c', 'a', 'b']);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// --dry-run semantics
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Codex fixes: soft-delete filter, atomic-SQL race-safety, sanitizer
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('sanitizeForTerminal', () => {
|
||||
test('preserves printable ASCII unchanged', () => {
|
||||
expect(sanitizeForTerminal('alice')).toBe('alice');
|
||||
expect(sanitizeForTerminal('a b-c_d.e/f@g')).toBe('a b-c_d.e/f@g');
|
||||
});
|
||||
|
||||
test('escapes ANSI escape sequences', () => {
|
||||
expect(sanitizeForTerminal('\x1b[2J')).toBe('\\x1b[2J');
|
||||
expect(sanitizeForTerminal('\x1b]0;TITLE\x07')).toBe('\\x1b]0;TITLE\\x07');
|
||||
});
|
||||
|
||||
test('escapes ALL C0 controls including tab and newline', () => {
|
||||
// Codex re-review: preserving \n lets a DCR-registered name spoof
|
||||
// additional rows in list-clients output. Tab spoofs field separators.
|
||||
// Both are now escaped.
|
||||
expect(sanitizeForTerminal('\x00\x07\x08')).toBe('\\x00\\x07\\x08');
|
||||
expect(sanitizeForTerminal('line1\nline2')).toBe('line1\\x0aline2');
|
||||
expect(sanitizeForTerminal('col1\tcol2')).toBe('col1\\x09col2');
|
||||
});
|
||||
|
||||
test('escapes DEL and C1 controls', () => {
|
||||
expect(sanitizeForTerminal('\x7f')).toBe('\\x7f');
|
||||
expect(sanitizeForTerminal('\x9b[31m')).toBe('\\x9b[31m');
|
||||
});
|
||||
|
||||
test('passes through unicode', () => {
|
||||
expect(sanitizeForTerminal('café')).toBe('café');
|
||||
expect(sanitizeForTerminal('日本語')).toBe('日本語');
|
||||
});
|
||||
});
|
||||
|
||||
describe('soft-delete filter (Codex finding #2)', () => {
|
||||
async function softDeleteClient(clientId: string): Promise<void> {
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
await sql`UPDATE oauth_clients SET deleted_at = now() WHERE client_id = ${clientId}`;
|
||||
}
|
||||
|
||||
test('resolveClient hides soft-deleted clients by default', async () => {
|
||||
await seedSource('default');
|
||||
const id = await seedClient('alice', ['default']);
|
||||
await softDeleteClient(id);
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
await expect(resolveClient(sql, 'alice')).rejects.toThrow(/No active OAuth client found/);
|
||||
await expect(resolveClient(sql, id)).rejects.toThrow(/No active OAuth client found/);
|
||||
});
|
||||
|
||||
test('resolveClient with includeDeleted finds soft-deleted clients', async () => {
|
||||
await seedSource('default');
|
||||
const id = await seedClient('alice', ['default']);
|
||||
await softDeleteClient(id);
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
const c = await resolveClient(sql, id, { includeDeleted: true });
|
||||
expect(c.client_name).toBe('alice');
|
||||
expect(c.deleted_at).not.toBeNull();
|
||||
});
|
||||
|
||||
test('grantReadCore refuses to mutate soft-deleted clients', async () => {
|
||||
await seedSource('default');
|
||||
await seedSource('proj-x');
|
||||
const id = await seedClient('alice', ['default']);
|
||||
await softDeleteClient(id);
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
await expect(grantReadCore(sql, 'alice', 'proj-x')).rejects.toThrow(/No active OAuth client found/);
|
||||
expect(await readFederated(id)).toEqual(['default']);
|
||||
});
|
||||
|
||||
test('revokeReadCore refuses to mutate soft-deleted clients', async () => {
|
||||
await seedSource('default');
|
||||
const id = await seedClient('alice', ['default']);
|
||||
await softDeleteClient(id);
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
await expect(revokeReadCore(sql, 'alice', 'default')).rejects.toThrow(/No active OAuth client found/);
|
||||
expect(await readFederated(id)).toEqual(['default']);
|
||||
});
|
||||
|
||||
test('two clients with same name but only one active resolves to the active one', async () => {
|
||||
await seedSource('default');
|
||||
// Seed two clients with the same name; soft-delete the older one.
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
const oldId = await seedClient('alice', ['default']);
|
||||
await softDeleteClient(oldId);
|
||||
const newId = await seedClient('alice', ['default']); // same name, new row
|
||||
const c = await resolveClient(sql, 'alice');
|
||||
expect(c.client_id).toBe(newId); // active row wins; ambiguity error suppressed
|
||||
});
|
||||
});
|
||||
|
||||
describe('atomic SQL race-safety (Codex finding #1, HIGH)', () => {
|
||||
test('grant+revoke serialize at row-lock — sensitive stays revoked', async () => {
|
||||
await seedSource('default');
|
||||
await seedSource('sensitive');
|
||||
await seedSource('harmless');
|
||||
const id = await seedClient('alice', ['default', 'sensitive']);
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
|
||||
// Simulate concurrent revoke(sensitive) + grant(harmless). Real concurrency
|
||||
// would race at the JS event loop boundary; here we await sequentially but
|
||||
// each call goes through the ATOMIC SQL path. The contract: regardless of
|
||||
// ordering, the final state has sensitive REMOVED and harmless ADDED.
|
||||
await revokeReadCore(sql, 'alice', 'sensitive');
|
||||
await grantReadCore(sql, 'alice', 'harmless');
|
||||
const final1 = await readFederated(id);
|
||||
expect(final1.sort()).toEqual(['default', 'harmless']);
|
||||
|
||||
// Reverse order, same final state. The pre-fix read-modify-write shape
|
||||
// would have produced ['default', 'sensitive', 'harmless'] here (the
|
||||
// resurrection bug Codex caught).
|
||||
const id2 = await seedClient('bob', ['default', 'sensitive']);
|
||||
await grantReadCore(sql, 'bob', 'harmless');
|
||||
await revokeReadCore(sql, 'bob', 'sensitive');
|
||||
const final2 = await readFederated(id2);
|
||||
expect(final2.sort()).toEqual(['default', 'harmless']);
|
||||
});
|
||||
|
||||
test('grant uses RETURNING to surface the post-write state', async () => {
|
||||
await seedSource('default');
|
||||
await seedSource('proj-x');
|
||||
await seedClient('alice', ['default']);
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
const outcome = await grantReadCore(sql, 'alice', 'proj-x');
|
||||
expect(outcome.kind).toBe('updated');
|
||||
if (outcome.kind === 'updated') {
|
||||
// The `after` came from RETURNING, not from computing prev+sourceId
|
||||
// in JS — proves the atomic path returned authoritative state.
|
||||
expect(outcome.after).toEqual(['default', 'proj-x']);
|
||||
}
|
||||
});
|
||||
|
||||
test('grant noop path still survives without writing', async () => {
|
||||
await seedSource('default');
|
||||
const id = await seedClient('alice', ['default']);
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
const outcome = await grantReadCore(sql, 'alice', 'default');
|
||||
expect(outcome.kind).toBe('noop');
|
||||
if (outcome.kind === 'noop') expect(outcome.reason).toBe('already-granted');
|
||||
expect(await readFederated(id)).toEqual(['default']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('dryRun mode', () => {
|
||||
test('grantReadCore returns "updated" outcome but skips the write', async () => {
|
||||
await seedSource('default');
|
||||
await seedSource('proj-x');
|
||||
const id = await seedClient('alice', ['default']);
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
const outcome = await grantReadCore(sql, 'alice', 'proj-x', { dryRun: true });
|
||||
expect(outcome.kind).toBe('updated');
|
||||
if (outcome.kind === 'updated') {
|
||||
expect(outcome.before).toEqual(['default']);
|
||||
expect(outcome.after).toEqual(['default', 'proj-x']);
|
||||
}
|
||||
// Crucially: the DB row is UNCHANGED.
|
||||
expect(await readFederated(id)).toEqual(['default']);
|
||||
});
|
||||
|
||||
test('revokeReadCore returns "updated" outcome but skips the write', async () => {
|
||||
await seedSource('default');
|
||||
await seedSource('proj-x');
|
||||
const id = await seedClient('alice', ['default', 'proj-x']);
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
const outcome = await revokeReadCore(sql, 'alice', 'proj-x', { dryRun: true });
|
||||
expect(outcome.kind).toBe('updated');
|
||||
expect(await readFederated(id)).toEqual(['default', 'proj-x']);
|
||||
});
|
||||
|
||||
test('setFederatedReadCore returns "updated" outcome but skips the write', async () => {
|
||||
await seedSource('a');
|
||||
await seedSource('b');
|
||||
await seedSource('c');
|
||||
const id = await seedClient('alice', ['a']);
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
const outcome = await setFederatedReadCore(sql, 'alice', 'b,c', { dryRun: true });
|
||||
expect(outcome.kind).toBe('updated');
|
||||
expect(await readFederated(id)).toEqual(['a']);
|
||||
});
|
||||
|
||||
test('noop outcomes are surfaced identically with or without dryRun', async () => {
|
||||
await seedSource('default');
|
||||
await seedSource('proj-x');
|
||||
await seedClient('alice', ['default', 'proj-x']);
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
const live = await grantReadCore(sql, 'alice', 'proj-x', { dryRun: false });
|
||||
const dry = await grantReadCore(sql, 'alice', 'proj-x', { dryRun: true });
|
||||
expect(live.kind).toBe('noop');
|
||||
expect(dry.kind).toBe('noop');
|
||||
});
|
||||
|
||||
test('errors still fire in dryRun (operator sees the problem before commit)', async () => {
|
||||
await seedSource('default');
|
||||
const id = await seedClient('alice', ['default']);
|
||||
const sql = sqlQueryForEngine(engine);
|
||||
await expect(
|
||||
grantReadCore(sql, 'alice', 'ghost', { dryRun: true }),
|
||||
).rejects.toThrow(/does not exist/);
|
||||
await expect(
|
||||
grantReadCore(sql, 'nobody', 'default', { dryRun: true }),
|
||||
).rejects.toThrow(/No active OAuth client found/);
|
||||
// DB unchanged.
|
||||
expect(await readFederated(id)).toEqual(['default']);
|
||||
});
|
||||
});
|
||||
@@ -174,6 +174,29 @@ describe('parseRegisterClientArgs', () => {
|
||||
});
|
||||
|
||||
describe('error cases', () => {
|
||||
test('--source with malformed id throws (validates source_id shape — codex re-review)', () => {
|
||||
// Defense for the "register-client seeds an unmanageable
|
||||
// federated_read entry" vector. assertValidSourceId fires before the
|
||||
// function returns so DB never sees a row with bad source scope.
|
||||
expect(() => parseRegisterClientArgs(['--source', 'has,weird,bits'])).toThrow(/Invalid source_id/);
|
||||
expect(() => parseRegisterClientArgs(['--source', 'UPPER'])).toThrow(/Invalid source_id/);
|
||||
expect(() => parseRegisterClientArgs(['--source', ''])).toThrow(/Invalid source_id|requires a value/);
|
||||
});
|
||||
|
||||
test('--federated-read with any malformed id throws', () => {
|
||||
// Single-item bad.
|
||||
expect(() => parseRegisterClientArgs(['--federated-read', 'bad,source!'])).toThrow(/Invalid source_id/);
|
||||
// Mixed valid + invalid — fails on the first bad one.
|
||||
expect(() => parseRegisterClientArgs(['--federated-read', 'good,bad source'])).toThrow(/Invalid source_id/);
|
||||
});
|
||||
|
||||
test('--source default + --federated-read default,team passes (regression — common case)', () => {
|
||||
// Sanity: the canonical real-world invocation still parses cleanly.
|
||||
const out = parseRegisterClientArgs(['--source', 'default', '--federated-read', 'default,team']);
|
||||
expect(out.sourceId).toBe('default');
|
||||
expect(out.federatedRead).toEqual(['default', 'team']);
|
||||
});
|
||||
|
||||
test('--redirect-uri without value → throws', () => {
|
||||
expect(() => parseRegisterClientArgs(['--redirect-uri'])).toThrow(/requires a value/);
|
||||
});
|
||||
|
||||
@@ -1,47 +0,0 @@
|
||||
/**
|
||||
* Structural regression for the backlinks Minion handler default.
|
||||
*
|
||||
* Backlinks jobs submitted with an EMPTY payload (the sync→embed→backlinks
|
||||
* chains enqueued after every ingestion) must run as 'check', never 'fix'.
|
||||
* The pre-fix handler inverted the default (`=== 'check' ? 'check' : 'fix'`),
|
||||
* so every routine post-ingestion job rewrote tracked brain pages with
|
||||
* generated "Referenced in" timeline bullets — contradicting the documented
|
||||
* intent in src/core/cycle.ts (runPhaseBacklinks): "Maintenance cycles must
|
||||
* not rewrite tracked brain pages with generated 'Referenced in' timeline
|
||||
* bullets."
|
||||
*
|
||||
* Source-grep is the right tool here (see fix-wave-structural.test.ts): the
|
||||
* handler dynamically imports runBacklinksCore and walks a real repo dir, so
|
||||
* a behavioral test would require heavy mocking that hides the regression
|
||||
* behind a test seam. The rule is "this specific default must stay 'check'".
|
||||
*/
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { readFileSync } from 'fs';
|
||||
|
||||
describe('backlinks Minion handler — empty payload defaults to check, not fix', () => {
|
||||
const src = readFileSync('src/commands/jobs.ts', 'utf8');
|
||||
|
||||
// Isolate the backlinks register block so assertions can't accidentally
|
||||
// match another handler's action parsing.
|
||||
const blockMatch = src.match(
|
||||
/worker\.register\('backlinks',[\s\S]*?runBacklinksCore\(\{[\s\S]*?\}\);/
|
||||
);
|
||||
|
||||
test('the backlinks handler block exists', () => {
|
||||
expect(blockMatch).not.toBeNull();
|
||||
});
|
||||
|
||||
test("default action is 'check' (explicit opt-in required for 'fix')", () => {
|
||||
const block = blockMatch![0];
|
||||
expect(block).toMatch(
|
||||
/job\.data\.action\s*===\s*'fix'\s*\?\s*'fix'\s*:\s*'check'/
|
||||
);
|
||||
});
|
||||
|
||||
test('the inverted (fix-by-default) shape stays absent', () => {
|
||||
const block = blockMatch![0];
|
||||
expect(block).not.toMatch(
|
||||
/job\.data\.action\s*===\s*'check'\s*\?\s*'check'\s*:\s*'fix'/
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -6,7 +6,6 @@ import {
|
||||
checkResolvable,
|
||||
parseResolverEntries,
|
||||
extractDelegationTargets,
|
||||
extractTriggers,
|
||||
} from "../src/core/check-resolvable.ts";
|
||||
|
||||
const SKILLS_DIR = join(import.meta.dir, "..", "skills");
|
||||
@@ -196,39 +195,6 @@ describe("parseResolverEntries", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("extractTriggers", () => {
|
||||
const LF_FRONTMATTER =
|
||||
"---\nname: query\ndescription: Test\ntriggers:\n - \"what do we know\"\n - \"tell me about\"\ntools:\n - search\n---\n\n# Body\n";
|
||||
|
||||
test("parses triggers from LF-terminated frontmatter", () => {
|
||||
const triggers = extractTriggers(LF_FRONTMATTER);
|
||||
expect(triggers).toEqual(["what do we know", "tell me about"]);
|
||||
});
|
||||
|
||||
test("parses triggers from CRLF-terminated frontmatter (Windows checkouts)", () => {
|
||||
// Regression: `core.autocrlf=true` is the Windows default. Without
|
||||
// CRLF→LF normalization, every Windows skill is reported as a false
|
||||
// mece_gap warning because the `^---\n` regex never matches `---\r\n`.
|
||||
const crlf = LF_FRONTMATTER.replace(/\n/g, "\r\n");
|
||||
const triggers = extractTriggers(crlf);
|
||||
expect(triggers).toEqual(["what do we know", "tell me about"]);
|
||||
});
|
||||
|
||||
test("returns [] when frontmatter is missing", () => {
|
||||
expect(extractTriggers("# Just a body, no frontmatter\n")).toEqual([]);
|
||||
});
|
||||
|
||||
test("returns [] when triggers field is absent from frontmatter", () => {
|
||||
const fm = "---\nname: query\ndescription: Test\ntools:\n - search\n---\n";
|
||||
expect(extractTriggers(fm)).toEqual([]);
|
||||
});
|
||||
|
||||
test("strips surrounding quotes from trigger values", () => {
|
||||
const fm = "---\nname: x\ntriggers:\n - \"double quoted\"\n - 'single quoted'\n - unquoted\n---\n";
|
||||
expect(extractTriggers(fm)).toEqual(["double quoted", "single quoted", "unquoted"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("checkResolvable — real skills directory", () => {
|
||||
const report = checkResolvable(SKILLS_DIR);
|
||||
|
||||
|
||||
@@ -1,199 +0,0 @@
|
||||
/**
|
||||
* #3224 — `backfill` was missing from the `CLI_ONLY` set at src/cli.ts,
|
||||
* so the dispatcher rejected the command with "Unknown command: backfill"
|
||||
* before ever reaching the fully-implemented `case 'backfill':` inside
|
||||
* `handleCliOnly`'s switch (`runBackfillCommand` in
|
||||
* src/commands/backfill.ts). `edges-backfill` was registered, which is
|
||||
* why the omission went unnoticed.
|
||||
*
|
||||
* A second, deeper bug surfaced once `backfill` became reachable:
|
||||
* `runBackfillCommand` manages its own PGLite engine end-to-end
|
||||
* (createEngine + connect + disconnect) and takes no engine argument at
|
||||
* all. The old `case 'backfill':` sat behind handleCliOnly's SHARED
|
||||
* `const engine = await connectEngine()` — dispatching there would open a
|
||||
* second PGLite connection to the same data dir while the first was still
|
||||
* held, and PGLite's single-writer file lock (src/core/pglite-lock.ts)
|
||||
* has no same-process reentrancy check, so every real (non---help) run
|
||||
* would hang for the full 30s lock timeout and then fail. The fix moves
|
||||
* `backfill` dispatch to an unconditional pre-connectEngine branch (same
|
||||
* spot as schema/init/auth/remote) instead of just gating `--help` there.
|
||||
*
|
||||
* Three things are pinned here:
|
||||
* 1. The runtime repro from the issue (`gbrain backfill --help` /
|
||||
* `gbrain backfill`) no longer reports "Unknown command".
|
||||
* 2. A real backfill run against a configured PGLite brain completes
|
||||
* quickly instead of hanging on its own lock.
|
||||
* 3. The issue's own suggested class-guard: every `case` in
|
||||
* handleCliOnly's command switch must have a matching `CLI_ONLY`
|
||||
* entry, so the next one-word omission fails CI instead of shipping
|
||||
* silently disabled.
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { mkdtempSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
const CLI_TS_PATH = fileURLToPath(new URL('../src/cli.ts', import.meta.url));
|
||||
const REPO_ROOT = fileURLToPath(new URL('..', import.meta.url));
|
||||
|
||||
function runCli(
|
||||
args: string[],
|
||||
gbrainHome?: string,
|
||||
opts?: { timeoutMs?: number },
|
||||
): { stdout: string; stderr: string; status: number; timedOut: boolean } {
|
||||
const result = spawnSync('bun', ['run', 'src/cli.ts', ...args], {
|
||||
cwd: REPO_ROOT,
|
||||
encoding: 'utf8',
|
||||
env: {
|
||||
...process.env,
|
||||
GBRAIN_HOME: gbrainHome ?? '/tmp/gbrain-test-cli-backfill-dispatch-nonexistent',
|
||||
},
|
||||
// Explicit timeout: spawnSync blocks the whole (single-threaded) test
|
||||
// runner, so bun:test's own per-test timeout (the 3rd `test()` arg)
|
||||
// can't interrupt a hung child process — it only starts counting again
|
||||
// once spawnSync itself returns. Without this, a regression that
|
||||
// reintroduces the ~30s PGLite lock hang would silently wait out the
|
||||
// full 30s instead of the test failing fast on a bounded timeout.
|
||||
timeout: opts?.timeoutMs,
|
||||
});
|
||||
return {
|
||||
stdout: result.stdout ?? '',
|
||||
stderr: result.stderr ?? '',
|
||||
status: result.status ?? -1,
|
||||
// Node sets `error.code === 'ETIMEDOUT'` (and `signal` on the result)
|
||||
// when `timeout` fires and the child is killed.
|
||||
timedOut: result.signal !== null || (result.error as NodeJS.ErrnoException | undefined)?.code === 'ETIMEDOUT',
|
||||
};
|
||||
}
|
||||
|
||||
describe('#3224 — `gbrain backfill` is dispatchable', () => {
|
||||
test('`backfill --help` reaches the real HELP text, not "Unknown command"', () => {
|
||||
const { stdout, stderr, status } = runCli(['backfill', '--help']);
|
||||
expect(stderr).not.toContain('Unknown command');
|
||||
expect(status).toBe(0);
|
||||
// runBackfillCommand's own printHelp() (src/commands/backfill.ts) —
|
||||
// proves dispatch reached the real handler, not the generic CLI_ONLY
|
||||
// one-line stub (the WARN-5 class: registering `backfill` in CLI_ONLY
|
||||
// alone routes it behind the generic short-circuit / a `connectEngine()`
|
||||
// that a fresh tmpdir can't satisfy; the fix also needed the
|
||||
// pre-engine-bind `--help` short-circuit + a CLI_ONLY_SELF_HELP entry).
|
||||
expect(stdout).toContain('gbrain backfill list');
|
||||
expect(stdout).toContain('--batch-size N');
|
||||
});
|
||||
|
||||
test('`-h` short flag also works', () => {
|
||||
const { stdout, status } = runCli(['backfill', '-h']);
|
||||
expect(status).toBe(0);
|
||||
expect(stdout).toContain('gbrain backfill list');
|
||||
});
|
||||
|
||||
test('`backfill` with no brain configured fails on the config, not "Unknown command"', () => {
|
||||
// Without --help, runBackfillCommand needs a real engine, so a
|
||||
// fresh/nonexistent GBRAIN_HOME must still fail — but on "no brain
|
||||
// configured", never on dispatch rejecting the command outright.
|
||||
const { stderr, status } = runCli(['backfill']);
|
||||
expect(status).not.toBe(0);
|
||||
expect(stderr).not.toContain('Unknown command');
|
||||
});
|
||||
});
|
||||
|
||||
describe('#3224 — a real backfill run against a configured PGLite brain does not deadlock', () => {
|
||||
let gbrainHome: string;
|
||||
|
||||
beforeAll(() => {
|
||||
gbrainHome = mkdtempSync(join(tmpdir(), 'gbrain-test-cli-backfill-pglite-'));
|
||||
const init = runCli(['init', '--pglite', '--no-embedding'], gbrainHome);
|
||||
expect(init.status).toBe(0);
|
||||
}, 60000);
|
||||
|
||||
afterAll(() => {
|
||||
rmSync(gbrainHome, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test('`backfill effective_date --dry-run` connects, runs, and exits — no 30s PGLite lock hang', () => {
|
||||
// Pre-fix (case inside the switch, behind the shared connectEngine()):
|
||||
// this would hang for ~30s and then fail with "Timed out waiting for
|
||||
// PGLite lock. Process <own PID> has held it since ...". The explicit
|
||||
// spawnSync `timeoutMs` below (well under the 30s lock timeout) is what
|
||||
// actually bounds this — bun:test's own per-test timeout can't
|
||||
// interrupt a blocking spawnSync call, it only resumes counting once
|
||||
// spawnSync returns.
|
||||
const { stdout, stderr, status, timedOut } = runCli(
|
||||
['backfill', 'effective_date', '--dry-run'],
|
||||
gbrainHome,
|
||||
{ timeoutMs: 20_000 },
|
||||
);
|
||||
expect(timedOut).toBe(false);
|
||||
expect(stderr).not.toContain('Timed out waiting for PGLite lock');
|
||||
expect(status).toBe(0);
|
||||
expect(stdout).toContain('Running backfill: effective_date');
|
||||
expect(stdout).toContain('complete');
|
||||
}, 25000);
|
||||
});
|
||||
|
||||
describe('#3224 class guard — every handleCliOnly switch case is CLI_ONLY-registered', () => {
|
||||
test('no `case` label in the command switch is missing from CLI_ONLY (except documented pre-existing dead arms)', () => {
|
||||
const src = readFileSync(CLI_TS_PATH, 'utf-8');
|
||||
|
||||
const setMatch = src.match(/export const CLI_ONLY = new Set\(\[([^\]]*)\]\);/);
|
||||
expect(setMatch).not.toBeNull();
|
||||
const cliOnly = new Set([...setMatch![1].matchAll(/'([^']+)'/g)].map((m) => m[1]));
|
||||
expect(cliOnly.size).toBeGreaterThan(0);
|
||||
|
||||
// Scope tightly to the actual `switch (command) { ... }` body inside
|
||||
// handleCliOnly — NOT the whole function. The function also has a long
|
||||
// pre-engine if-chain (schema/init/auth/...) whose prose comments can
|
||||
// themselves contain the literal text `case 'xyz':` when documenting
|
||||
// this exact bug class (e.g. the `backfill` pre-connectEngine branch's
|
||||
// own comment references the old case it replaced) — matching against
|
||||
// the whole function body would misread that comment as a real,
|
||||
// CLI_ONLY-registered dispatch site and silently mask a genuine gap.
|
||||
const fnStartIdx = src.indexOf("async function handleCliOnly(command: string, args: string[]) {");
|
||||
expect(fnStartIdx).toBeGreaterThan(-1);
|
||||
const fnEndIdx = src.indexOf('\nasync function dispatchReadOnlyCommand', fnStartIdx);
|
||||
expect(fnEndIdx).toBeGreaterThan(fnStartIdx);
|
||||
const fnBody = src.slice(fnStartIdx, fnEndIdx);
|
||||
|
||||
const switchStartIdx = fnBody.indexOf('switch (command) {');
|
||||
expect(switchStartIdx).toBeGreaterThan(-1);
|
||||
// The switch's own closing brace is immediately followed by
|
||||
// ` } finally {` (the engine-teardown wrapper) — a marker specific
|
||||
// enough to bound the switch precisely without a full brace-matcher.
|
||||
const switchEndIdx = fnBody.indexOf('\n } finally {', switchStartIdx);
|
||||
expect(switchEndIdx).toBeGreaterThan(switchStartIdx);
|
||||
const switchBody = fnBody.slice(switchStartIdx, switchEndIdx);
|
||||
|
||||
// Strip comments before matching so prose that merely MENTIONS a case
|
||||
// label (block or line comments) can never be mistaken for a real one.
|
||||
const switchBodyNoComments = switchBody
|
||||
.replace(/\/\*[\s\S]*?\*\//g, '')
|
||||
.replace(/\/\/[^\n]*/g, '');
|
||||
|
||||
const cases = [...switchBodyNoComments.matchAll(/case '([^']+)':/g)].map((m) => m[1]);
|
||||
expect(cases.length).toBeGreaterThan(30); // sanity: the switch is large
|
||||
|
||||
const missing = cases.filter((c) => !cliOnly.has(c));
|
||||
|
||||
// Pre-existing switch arms that are unreachable for reasons OTHER than
|
||||
// #3224's bug class: each is shadowed by an earlier, unconditional
|
||||
// branch before the CLI_ONLY.has(command) dispatch gate is ever
|
||||
// consulted, so adding them to CLI_ONLY would not change behavior (and
|
||||
// fixing/removing the dead code is a separate, out-of-scope cleanup).
|
||||
// Documented here instead of silently re-hidden, per the issue's own
|
||||
// "next one-word omission" framing — if this allowlist needs to grow,
|
||||
// that growth itself is the signal a case was orphaned.
|
||||
const KNOWN_UNREACHABLE_DEAD_CASES = new Set([
|
||||
'search', // shadowed by the T5 special-case (modes/stats/tune/diagnose) + the generic op fallback for free-text search
|
||||
'pages', // shadowed by the generic op fallback (list_pages)
|
||||
'notability-eval', // superseded command name; no CLI_ONLY entry ever existed for it
|
||||
'whoknows', // superseded command name (now `find_experts`); no CLI_ONLY entry ever existed for it
|
||||
]);
|
||||
|
||||
const unexpected = missing.filter((c) => !KNOWN_UNREACHABLE_DEAD_CASES.has(c));
|
||||
expect(unexpected).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -8,15 +8,13 @@
|
||||
//
|
||||
// PGLite-only: in-memory engine, no DATABASE_URL needed.
|
||||
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { loadConfigWithEngine, type GBrainConfig } from '../src/core/config.ts';
|
||||
import {
|
||||
__setRerankTransportForTests,
|
||||
configureGateway,
|
||||
getEmbeddingModel,
|
||||
getMultimodalModel,
|
||||
rerank,
|
||||
resetGateway,
|
||||
} from '../src/core/ai/gateway.ts';
|
||||
import type { AIGatewayConfig } from '../src/core/ai/types.ts';
|
||||
@@ -54,16 +52,10 @@ afterAll(async () => {
|
||||
|
||||
beforeEach(async () => {
|
||||
resetGateway();
|
||||
__setRerankTransportForTests(null);
|
||||
// Clear any prior config rows so tests are independent. setConfig with
|
||||
// empty string is treated as undefined by loadConfigWithEngine (per
|
||||
// dbStr semantics), so this is safe to call between tests.
|
||||
await engine.setConfig('embedding_multimodal_model', '');
|
||||
await engine.setConfig('provider_base_urls.llama-server-reranker', '');
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
__setRerankTransportForTests(null);
|
||||
});
|
||||
|
||||
describe('cli connectEngine — embedding_multimodal_model DB→gateway plumbing', () => {
|
||||
@@ -130,34 +122,4 @@ describe('cli connectEngine — embedding_multimodal_model DB→gateway plumbing
|
||||
expect(getEmbeddingModel()).toBe('openai:text-embedding-3-large');
|
||||
expect(getMultimodalModel()).toBeUndefined();
|
||||
});
|
||||
|
||||
test('DB-set provider_base_urls.llama-server-reranker flows to gateway.rerank URL', async () => {
|
||||
await engine.setConfig('provider_base_urls.llama-server-reranker', 'http://127.0.0.1:8091/v1');
|
||||
|
||||
const baseConfig: GBrainConfig = {
|
||||
engine: 'pglite',
|
||||
embedding_model: 'openai:text-embedding-3-large',
|
||||
embedding_dimensions: 1536,
|
||||
};
|
||||
|
||||
const merged = await loadConfigWithEngine(engine, baseConfig);
|
||||
configureGateway(buildGatewayConfig(merged!));
|
||||
|
||||
let capturedUrl = '';
|
||||
__setRerankTransportForTests(async (url) => {
|
||||
capturedUrl = url;
|
||||
return new Response(JSON.stringify({ results: [{ index: 0, relevance_score: 0.9 }] }), {
|
||||
status: 200,
|
||||
headers: { 'content-type': 'application/json' },
|
||||
});
|
||||
});
|
||||
|
||||
await rerank({
|
||||
query: 'q',
|
||||
documents: ['d'],
|
||||
model: 'llama-server-reranker:qwen3-reranker-4b',
|
||||
});
|
||||
|
||||
expect(capturedUrl).toBe('http://127.0.0.1:8091/v1/rerank');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -248,37 +248,6 @@ describe('BudgetTracker.reserve', () => {
|
||||
expect((caught as BudgetExhausted).reason).toBe('no_pricing');
|
||||
});
|
||||
|
||||
test('#3223: rerank kind for zeroentropyai:zerank-2 prices from the embedding table (no TX2 throw under --max-cost)', () => {
|
||||
// Pre-fix: `search_mode: tokenmax` defaults the zerank-2 reranker ON
|
||||
// (docs/ai-providers/zeroentropy.md), but lookupPricing's rerank branch
|
||||
// never consulted the embedding pricing table (where ZeroEntropy's
|
||||
// provider:model-keyed prices live) — so any --max-cost run that
|
||||
// reranked TX2 hard-failed with "no pricing entry" even after adding
|
||||
// the entry to EMBEDDING_PRICING alone. Fixed by wiring the rerank
|
||||
// branch to fall back to lookupEmbeddingPrice.
|
||||
const t = new BudgetTracker({ maxCostUsd: 0.0001, label: 'test', auditPath });
|
||||
expect(() =>
|
||||
t.reserve({
|
||||
modelId: 'zeroentropyai:zerank-2',
|
||||
estimatedInputTokens: 3000,
|
||||
maxOutputTokens: 0,
|
||||
kind: 'rerank',
|
||||
}),
|
||||
).not.toThrow();
|
||||
expect(t.totalSpent).toBe(0); // reserve() only projects; record() below banks it.
|
||||
expect(() =>
|
||||
t.record({
|
||||
modelId: 'zeroentropyai:zerank-2',
|
||||
inputTokens: 3000,
|
||||
outputTokens: 0,
|
||||
kind: 'rerank',
|
||||
}),
|
||||
).not.toThrow();
|
||||
// $0.025/1M * 3000 tokens = $0.000075, under the $0.0001 cap — proves the
|
||||
// real ZeroEntropy price was used, not a $0 fallback.
|
||||
expect(t.totalSpent).toBeCloseTo(0.000075, 9);
|
||||
});
|
||||
|
||||
test('v0.40.x: local embed providers price at $0 (no TX2 throw under --max-cost)', () => {
|
||||
// FREE_LOCAL_EMBED_PROVIDERS — ollama / llama-server run on local inference
|
||||
// (electricity, not tokens). Pre-fix a --max-cost embed/reindex job
|
||||
|
||||
@@ -20,10 +20,6 @@ import {
|
||||
const skip = !hasDatabase();
|
||||
const describeE2E = skip ? describe.skip : describe;
|
||||
|
||||
if (skip) {
|
||||
console.log('Skipping E2E doctor --progress-json tests (DATABASE_URL not set)');
|
||||
}
|
||||
|
||||
const CLI = join(import.meta.dir, '..', '..', 'src', 'cli.ts');
|
||||
|
||||
describeE2E('gbrain doctor --progress-json (E2E)', () => {
|
||||
|
||||
@@ -172,59 +172,6 @@ describe('issue #972 — DB-source (gbrain extract links --source db)', () => {
|
||||
expect(strk!.link_type).toBe('wikilink_basename');
|
||||
});
|
||||
|
||||
test('flag ON → path-qualified wikilink outside DIR_PATTERN resolves via DB path', async () => {
|
||||
// `[[notes/struktura]]` — `notes` is not in DIR_PATTERN, so the ref
|
||||
// reaches the generic pass with its dirname intact. Regression: the DB
|
||||
// path queried the basename index with the raw literal (which is keyed
|
||||
// by final segments only), so path-qualified wikilinks outside
|
||||
// DIR_PATTERN silently produced zero edges while the FS path resolved
|
||||
// the identical content.
|
||||
await engine.putPage('notes/struktura', {
|
||||
type: 'concept' as any, title: 'Struktura Notes',
|
||||
compiled_truth: '', timeline: '',
|
||||
});
|
||||
await engine.putPage('concepts/knowledge-graph', {
|
||||
type: 'concept', title: 'Knowledge Graph',
|
||||
compiled_truth: 'Background in [[notes/struktura]].', timeline: '',
|
||||
});
|
||||
await engine.setConfig('link_resolution.global_basename', 'true');
|
||||
|
||||
await runExtract(engine, ['links', '--source', 'db']);
|
||||
|
||||
const outLinks = await engine.getLinks('concepts/knowledge-graph');
|
||||
const strk = outLinks.find(l => l.to_slug === 'notes/struktura');
|
||||
expect(strk).toBeDefined();
|
||||
expect(strk!.link_type).toBe('wikilink_basename');
|
||||
expect(strk!.link_source).toBe('wikilink-resolved');
|
||||
});
|
||||
|
||||
test('path-qualified wikilink never attaches to a basename-only sibling', async () => {
|
||||
// Both notes/struktura and wiki/struktura exist. The author wrote
|
||||
// `[[notes/struktura]]` — the written path must exclude wiki/struktura
|
||||
// (a bare `[[struktura]]` would legitimately match both).
|
||||
await engine.putPage('notes/struktura', {
|
||||
type: 'concept' as any, title: 'Struktura Notes',
|
||||
compiled_truth: '', timeline: '',
|
||||
});
|
||||
await engine.putPage('wiki/struktura', {
|
||||
type: 'concept' as any, title: 'Struktura Wiki',
|
||||
compiled_truth: '', timeline: '',
|
||||
});
|
||||
await engine.putPage('concepts/x', {
|
||||
type: 'concept', title: 'X',
|
||||
compiled_truth: 'See [[notes/struktura]].', timeline: '',
|
||||
});
|
||||
await engine.setConfig('link_resolution.global_basename', 'true');
|
||||
|
||||
await runExtract(engine, ['links', '--source', 'db']);
|
||||
|
||||
const outLinks = await engine.getLinks('concepts/x');
|
||||
const basenameLinks = outLinks
|
||||
.filter(l => l.link_type === 'wikilink_basename')
|
||||
.map(l => l.to_slug);
|
||||
expect(basenameLinks).toEqual(['notes/struktura']);
|
||||
});
|
||||
|
||||
test('flag OFF → no basename edges via DB path (back-compat)', async () => {
|
||||
await engine.putPage('projects/struktura', {
|
||||
type: 'project', title: 'Struktura',
|
||||
|
||||
@@ -29,15 +29,9 @@ afterAll(async () => {
|
||||
});
|
||||
|
||||
async function truncateAll() {
|
||||
for (const t of ['content_chunks', 'links', 'tags', 'raw_data', 'timeline_entries', 'page_versions', 'ingest_log', 'config', 'pages']) {
|
||||
for (const t of ['content_chunks', 'links', 'tags', 'raw_data', 'timeline_entries', 'page_versions', 'ingest_log', 'pages']) {
|
||||
await (engine as any).db.exec(`DELETE FROM ${t}`);
|
||||
}
|
||||
// Re-seed the two config keys this file touches back to their documented
|
||||
// defaults (both default to ON). This makes every test deterministic even if
|
||||
// an earlier test threw before its finally restored auto_link/auto_timeline,
|
||||
// and even though absent-key already resolves truthy via isAuto*Enabled.
|
||||
await engine.setConfig('auto_link', 'true');
|
||||
await engine.setConfig('auto_timeline', 'true');
|
||||
}
|
||||
|
||||
function makeContext(): OperationContext {
|
||||
@@ -83,12 +77,10 @@ describe('E2E graph quality (v0.10.1 pipeline)', () => {
|
||||
await runExtract(engine, ['links', '--source', 'db']);
|
||||
await runExtract(engine, ['timeline', '--source', 'db']);
|
||||
|
||||
// Verify graph populated. Concrete floors derived from the seeded fixtures:
|
||||
// resolvable entity refs: alice->acme, bob->acme, standup->alice, standup->bob = 4
|
||||
// timeline lines: alice(2) + bob(1) + acme(1) + standup(1) = 5
|
||||
// Verify graph populated.
|
||||
const stats = await engine.getStats();
|
||||
expect(stats.link_count).toBeGreaterThanOrEqual(4);
|
||||
expect(stats.timeline_entry_count).toBeGreaterThanOrEqual(5);
|
||||
expect(stats.link_count).toBeGreaterThan(0);
|
||||
expect(stats.timeline_entry_count).toBeGreaterThan(0);
|
||||
|
||||
// Verify typed link inference.
|
||||
const aliceLinks = await engine.getLinks('people/alice');
|
||||
@@ -99,16 +91,7 @@ describe('E2E graph quality (v0.10.1 pipeline)', () => {
|
||||
const bobAcme = bobLinks.find(l => l.to_slug === 'companies/acme');
|
||||
expect(bobAcme?.link_type).toBe('invested_in');
|
||||
|
||||
// The standup meeting references both Alice and Bob as attendees. Assert the
|
||||
// exact attendee edges are present and typed 'attended' (a plain .every()
|
||||
// would silently pass if a meeting->company edge were misclassified or if the
|
||||
// attendee edges were missing entirely).
|
||||
const meetingLinks = await engine.getLinks('meetings/standup');
|
||||
const attended = new Set(
|
||||
meetingLinks.filter(l => l.link_type === 'attended').map(l => l.to_slug),
|
||||
);
|
||||
expect(attended.has('people/alice')).toBe(true);
|
||||
expect(attended.has('people/bob')).toBe(true);
|
||||
expect(meetingLinks.every(l => l.link_type === 'attended')).toBe(true);
|
||||
});
|
||||
|
||||
@@ -135,9 +118,7 @@ Attendees: [Alice](people/alice). Discussed [Acme](companies/acme).
|
||||
// The response should include auto_links results.
|
||||
expect((result as any).auto_links).toBeDefined();
|
||||
const autoLinks = (result as any).auto_links;
|
||||
// The page references exactly two seeded, resolvable targets (Alice + Acme),
|
||||
// so exactly two links are created.
|
||||
expect(autoLinks.created).toBe(2);
|
||||
expect(autoLinks.created).toBeGreaterThan(0);
|
||||
expect(autoLinks.errors).toBe(0);
|
||||
|
||||
// Verify links actually exist in DB.
|
||||
@@ -302,53 +283,6 @@ Mention of [Alice](people/alice).
|
||||
expect(paths[0].link_type).toBe('works_at');
|
||||
});
|
||||
|
||||
test('graph-query traversal: direction out and both, plus depth:2 multi-hop', async () => {
|
||||
// Seed a 2-hop chain: alice -works_at-> acme -partnered_with-> beta.
|
||||
await engine.putPage('people/alice', { type: 'person', title: 'Alice', compiled_truth: '', timeline: '' });
|
||||
await engine.putPage('companies/acme', { type: 'company', title: 'Acme', compiled_truth: '', timeline: '' });
|
||||
await engine.putPage('companies/beta', { type: 'company', title: 'Beta', compiled_truth: '', timeline: '' });
|
||||
await engine.addLink('people/alice', 'companies/acme', '', 'works_at');
|
||||
await engine.addLink('companies/acme', 'companies/beta', '', 'partnered_with');
|
||||
|
||||
// direction:'out' from alice, depth 1 -> only the first hop.
|
||||
const out1 = await engine.traversePaths('people/alice', { direction: 'out', depth: 1 });
|
||||
expect(out1.length).toBe(1);
|
||||
expect(out1[0].from_slug).toBe('people/alice');
|
||||
expect(out1[0].to_slug).toBe('companies/acme');
|
||||
expect(out1[0].depth).toBe(1);
|
||||
|
||||
// depth:2 -> both hops, depths 1 and 2.
|
||||
const out2 = await engine.traversePaths('people/alice', { direction: 'out', depth: 2 });
|
||||
const out2Edges = new Set(out2.map(p => `${p.from_slug}->${p.to_slug}@${p.depth}`));
|
||||
expect(out2Edges.has('people/alice->companies/acme@1')).toBe(true);
|
||||
expect(out2Edges.has('companies/acme->companies/beta@2')).toBe(true);
|
||||
expect(out2.length).toBe(2);
|
||||
|
||||
// direction:'both' from acme depth 1 -> sees the inbound edge from alice AND
|
||||
// the outbound edge to beta. Edges keep their natural from->to orientation.
|
||||
const both = await engine.traversePaths('companies/acme', { direction: 'both', depth: 1 });
|
||||
const bothEdges = new Set(both.map(p => `${p.from_slug}->${p.to_slug}`));
|
||||
expect(bothEdges.has('people/alice->companies/acme')).toBe(true);
|
||||
expect(bothEdges.has('companies/acme->companies/beta')).toBe(true);
|
||||
});
|
||||
|
||||
test('graph-query cycle safety: A->B->A terminates and returns bounded results', async () => {
|
||||
await engine.putPage('people/alice', { type: 'person', title: 'Alice', compiled_truth: '', timeline: '' });
|
||||
await engine.putPage('people/bob', { type: 'person', title: 'Bob', compiled_truth: '', timeline: '' });
|
||||
// Create a 2-cycle: alice -> bob -> alice.
|
||||
await engine.addLink('people/alice', 'people/bob', '', 'knows');
|
||||
await engine.addLink('people/bob', 'people/alice', '', 'knows');
|
||||
|
||||
// High depth must NOT loop forever; the visited-set guard bounds the walk.
|
||||
const paths = await engine.traversePaths('people/alice', { direction: 'out', depth: 100 });
|
||||
const edges = new Set(paths.map(p => `${p.from_slug}->${p.to_slug}`));
|
||||
// Both edges of the cycle are reachable exactly once.
|
||||
expect(edges.has('people/alice->people/bob')).toBe(true);
|
||||
expect(edges.has('people/bob->people/alice')).toBe(true);
|
||||
// Bounded: there are only two edges in the graph, so no path explosion.
|
||||
expect(paths.length).toBe(2);
|
||||
});
|
||||
|
||||
test('search backlink boost: well-connected pages rank higher', async () => {
|
||||
// Create 3 pages all matching a search term, but with different inbound link counts.
|
||||
await engine.putPage('topic/popular', {
|
||||
|
||||
@@ -21,10 +21,6 @@ import { hasDatabase, setupDB, teardownDB, getEngine, getConn } from './helpers.
|
||||
const skip = !hasDatabase();
|
||||
const describeE2E = skip ? describe.skip : describe;
|
||||
|
||||
if (skip) {
|
||||
console.log('Skipping E2E JSONB roundtrip tests (DATABASE_URL not set)');
|
||||
}
|
||||
|
||||
describeE2E('E2E: JSONB roundtrip — v0.12.1 reliability wave', () => {
|
||||
beforeAll(async () => { await setupDB(); });
|
||||
afterAll(async () => { await teardownDB(); });
|
||||
|
||||
@@ -56,7 +56,6 @@ describe('E2E: MCP Tool Generation', () => {
|
||||
expect(names).toContain('get_health');
|
||||
expect(names).toContain('sync_brain');
|
||||
expect(names).toContain('file_upload');
|
||||
expect(names).toContain('find_orphans');
|
||||
});
|
||||
|
||||
test('MCP server module can be imported', async () => {
|
||||
|
||||
@@ -175,15 +175,6 @@ describeE2E('E2E: Search', () => {
|
||||
for (const [query, score] of Object.entries(scores)) {
|
||||
console.log(` "${query}": ${(score * 100).toFixed(0)}%`);
|
||||
}
|
||||
|
||||
// Guard value: every known-item query must surface at least one ground-truth
|
||||
// doc in the top 5. This is a deliberately loose floor (not a tuned P@5
|
||||
// threshold) — it catches a total keyword-retrieval regression without
|
||||
// breaking on every scoring/fixture tweak. Without it this test asserted
|
||||
// nothing and a 0%-precision result passed silently.
|
||||
for (const [query, score] of Object.entries(scores)) {
|
||||
expect(score).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -214,22 +205,10 @@ describeE2E('E2E: Links', () => {
|
||||
}, 30_000);
|
||||
|
||||
test('traverse_graph finds connected pages', async () => {
|
||||
// Self-contained: do not depend on a prior test's add_link. add_link is
|
||||
// idempotent (ON CONFLICT DO NOTHING), so re-adding here is safe whether or
|
||||
// not the round-trip test ran first, and the test no longer false-passes or
|
||||
// false-fails based on describe-block ordering.
|
||||
await callOp('add_link', {
|
||||
from: 'people/sarah-chen',
|
||||
to: 'companies/novamind',
|
||||
link_type: 'founded',
|
||||
});
|
||||
|
||||
const graph = await callOp('traverse_graph', { slug: 'people/sarah-chen', depth: 2 }) as any[];
|
||||
// Links should already be added from prior test in this describe block
|
||||
const graph = await callOp('traverse_graph', { slug: 'people/sarah-chen', depth: 2 }) as any;
|
||||
expect(Array.isArray(graph)).toBe(true);
|
||||
expect(graph.length).toBeGreaterThanOrEqual(1);
|
||||
// Content assertion, not just shape: the linked company must be reachable.
|
||||
const reachable = graph.map((n: any) => n.slug ?? n.to_slug ?? n.to_page_slug);
|
||||
expect(reachable).toContain('companies/novamind');
|
||||
});
|
||||
|
||||
test('remove_link removes the link', async () => {
|
||||
@@ -490,14 +469,8 @@ describeE2E('E2E: Admin', () => {
|
||||
test('get_health returns valid structure', async () => {
|
||||
const health = await callOp('get_health') as any;
|
||||
expect(health).toBeDefined();
|
||||
// Value bounds, not just types: page_count must match the fixture inventory
|
||||
// and embed_coverage is a 0..1 fraction (src/commands/doctor.ts multiplies
|
||||
// by 100 and compares to 0.9). Type-only checks let embed_coverage: -9999
|
||||
// through; these catch a genuinely broken health payload.
|
||||
expect(health.page_count).toBe(16);
|
||||
expect(Number.isFinite(health.embed_coverage)).toBe(true);
|
||||
expect(health.embed_coverage).toBeGreaterThanOrEqual(0);
|
||||
expect(health.embed_coverage).toBeLessThanOrEqual(1);
|
||||
expect(typeof health.page_count).toBe('number');
|
||||
expect(typeof health.embed_coverage).toBe('number');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -515,17 +488,7 @@ describeE2E('E2E: Chunks & Resolution', () => {
|
||||
test('get_chunks returns chunks for imported page', async () => {
|
||||
const chunks = await callOp('get_chunks', { slug: 'people/sarah-chen' }) as any[];
|
||||
expect(chunks.length).toBeGreaterThan(0);
|
||||
// Content + ordering, not just truthiness (a whitespace-only chunk is truthy):
|
||||
// every chunk has real text and a numeric index, the indexes are
|
||||
// non-decreasing in return order, and the page's own name appears somewhere.
|
||||
for (const c of chunks) {
|
||||
expect(typeof c.chunk_text).toBe('string');
|
||||
expect(c.chunk_text.trim().length).toBeGreaterThan(0);
|
||||
expect(typeof c.chunk_index).toBe('number');
|
||||
}
|
||||
const indexes = chunks.map((c: any) => c.chunk_index);
|
||||
expect(indexes).toEqual([...indexes].sort((x, y) => x - y));
|
||||
expect(chunks.some((c: any) => c.chunk_text.includes('Sarah'))).toBe(true);
|
||||
expect(chunks[0].chunk_text).toBeTruthy();
|
||||
}, 30_000);
|
||||
|
||||
test('resolve_slugs finds partial match', async () => {
|
||||
@@ -699,29 +662,9 @@ describeE2E('E2E: file_list LIMIT enforcement', () => {
|
||||
}, 30_000);
|
||||
|
||||
test('file_list without slug also respects LIMIT 100', async () => {
|
||||
// Self-sufficient: seed our own >100 rows rather than relying on the
|
||||
// previous test's 150 rows surviving in the DB. A bun reorder, a focused
|
||||
// `-t` run, or a failure mid-insert in the prior test would otherwise leave
|
||||
// this asserting against an indeterminate row count.
|
||||
const sql = getConn();
|
||||
const seedSlug = 'test-limit-noslug';
|
||||
await sql`
|
||||
INSERT INTO pages (slug, title, type, compiled_truth, frontmatter)
|
||||
VALUES (${seedSlug}, ${'Test Limit NoSlug'}, ${'note'}, ${'body'}, ${'{}'}::jsonb)
|
||||
ON CONFLICT (source_id, slug) DO NOTHING
|
||||
`;
|
||||
for (let i = 0; i < 120; i++) {
|
||||
await sql`
|
||||
INSERT INTO files (page_slug, filename, storage_path, mime_type, size_bytes, content_hash, metadata)
|
||||
VALUES (${seedSlug}, ${'nf-' + String(i).padStart(3, '0') + '.txt'}, ${seedSlug + '/nf-' + i + '.txt'}, ${'text/plain'}, ${100}, ${'nhash-' + i}, ${'{}'}::jsonb)
|
||||
ON CONFLICT (storage_path) DO NOTHING
|
||||
`;
|
||||
}
|
||||
const total = await sql`SELECT count(*)::int AS n FROM files`;
|
||||
expect(Number(total[0].n)).toBeGreaterThan(100); // cap is actually exercised
|
||||
|
||||
// The 150 rows from the previous test are still in the DB
|
||||
const files = await callOp('file_list', {}) as any[];
|
||||
expect(files.length).toBe(100);
|
||||
expect(files.length).toBeLessThanOrEqual(100);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
+111
-149
@@ -78,19 +78,6 @@ function freshTempHome(label: string) {
|
||||
return dir;
|
||||
}
|
||||
|
||||
// Restore HOME/PATH to the captured originals. Called from each test's
|
||||
// `finally` so a throw mid-test can never leave HOME/PATH pointed at a temp
|
||||
// dir for the rest of the bun process (which would silently break unrelated
|
||||
// suites that read HOME). PATH keeps the shim prepended because the
|
||||
// module-level shim install is what subsequent tests in this suite rely on;
|
||||
// afterAll does the final teardown to the pristine origPath.
|
||||
function restoreHomePath() {
|
||||
if (origHome === undefined) delete process.env.HOME;
|
||||
else process.env.HOME = origHome;
|
||||
if (origPath === undefined) delete process.env.PATH;
|
||||
else process.env.PATH = `${fakeBinDir}:${origPath ?? ''}`;
|
||||
}
|
||||
|
||||
beforeAll(() => {
|
||||
if (SKIP) {
|
||||
console.log('[migration-flow.e2e] DATABASE_URL not set — skipping.');
|
||||
@@ -113,15 +100,6 @@ afterAll(() => {
|
||||
|
||||
beforeEach(() => {
|
||||
if (SKIP) return;
|
||||
// Robust restore: if a prior test threw before its own finally ran (or
|
||||
// before afterAll), HOME/PATH could still point at a dead temp dir. Reset
|
||||
// them to the captured originals at the start of every test so a throw in
|
||||
// one test can never leak a temp HOME/PATH into sibling suites that read
|
||||
// them. freshTempHome() re-points HOME per test immediately after this.
|
||||
if (origHome === undefined) delete process.env.HOME;
|
||||
else process.env.HOME = origHome;
|
||||
if (origPath === undefined) delete process.env.PATH;
|
||||
else process.env.PATH = `${fakeBinDir}:${origPath ?? ''}`;
|
||||
try { if (tmp) rmSync(tmp, { recursive: true, force: true }); } catch { /* best-effort */ }
|
||||
});
|
||||
|
||||
@@ -136,160 +114,144 @@ const COMMON_OPTS = {
|
||||
describeE2E('E2E: v0.11.0 orchestrator against live Postgres', () => {
|
||||
test('fresh install flow: schema → smoke → prefs → host-rewrite → completed', async () => {
|
||||
tmp = freshTempHome('fresh');
|
||||
try {
|
||||
const result = await v0_11_0.orchestrator(COMMON_OPTS);
|
||||
const result = await v0_11_0.orchestrator(COMMON_OPTS);
|
||||
|
||||
// Orchestrator returns a structured result (status is `complete` when
|
||||
// no pending-host-work TODOs fired, `partial` otherwise).
|
||||
expect(result.version).toBe('0.11.0');
|
||||
expect(['complete', 'partial']).toContain(result.status);
|
||||
// Orchestrator returns a structured result (status is `complete` when
|
||||
// no pending-host-work TODOs fired, `partial` otherwise).
|
||||
expect(result.version).toBe('0.11.0');
|
||||
expect(['complete', 'partial']).toContain(result.status);
|
||||
|
||||
// Phase D: preferences.json exists with 0o600 + mode=pain_triggered.
|
||||
const prefsPath = join(tmp, '.gbrain', 'preferences.json');
|
||||
expect(existsSync(prefsPath)).toBe(true);
|
||||
expect(statSync(prefsPath).mode & 0o777).toBe(0o600);
|
||||
const prefs = loadPreferences();
|
||||
expect(prefs.minion_mode).toBe('pain_triggered');
|
||||
expect(prefs.set_at).toBeTruthy();
|
||||
expect(prefs.set_in_version).toBeTruthy();
|
||||
// Phase D: preferences.json exists with 0o600 + mode=pain_triggered.
|
||||
const prefsPath = join(tmp, '.gbrain', 'preferences.json');
|
||||
expect(existsSync(prefsPath)).toBe(true);
|
||||
expect(statSync(prefsPath).mode & 0o777).toBe(0o600);
|
||||
const prefs = loadPreferences();
|
||||
expect(prefs.minion_mode).toBe('pain_triggered');
|
||||
expect(prefs.set_at).toBeTruthy();
|
||||
expect(prefs.set_in_version).toBeTruthy();
|
||||
|
||||
// Bug 3 (v0.14.2) — orchestrator no longer writes completed.jsonl.
|
||||
// The runner (apply-migrations.ts) persists the result after the
|
||||
// orchestrator returns. A direct orchestrator call in E2E leaves the
|
||||
// ledger empty; the runner path is tested separately in
|
||||
// test/apply-migrations.test.ts + test/migration-resume.test.ts.
|
||||
const completed = loadCompletedMigrations();
|
||||
const v0110Entries = completed.filter(e => e.version === '0.11.0');
|
||||
expect(v0110Entries.length).toBe(0);
|
||||
// Bug 3 (v0.14.2) — orchestrator no longer writes completed.jsonl.
|
||||
// The runner (apply-migrations.ts) persists the result after the
|
||||
// orchestrator returns. A direct orchestrator call in E2E leaves the
|
||||
// ledger empty; the runner path is tested separately in
|
||||
// test/apply-migrations.test.ts + test/migration-resume.test.ts.
|
||||
const completed = loadCompletedMigrations();
|
||||
const v0110Entries = completed.filter(e => e.version === '0.11.0');
|
||||
expect(v0110Entries.length).toBe(0);
|
||||
|
||||
// Phase F is skipped per COMMON_OPTS — autopilot should NOT have been
|
||||
// installed on this host.
|
||||
expect(result.autopilot_installed).toBe(false);
|
||||
} finally {
|
||||
restoreHomePath();
|
||||
}
|
||||
// Phase F is skipped per COMMON_OPTS — autopilot should NOT have been
|
||||
// installed on this host.
|
||||
expect(result.autopilot_installed).toBe(false);
|
||||
}, 60_000);
|
||||
|
||||
test('idempotent rerun: second invocation is a safe no-op', async () => {
|
||||
tmp = freshTempHome('rerun');
|
||||
try {
|
||||
const first = await v0_11_0.orchestrator(COMMON_OPTS);
|
||||
expect(['complete', 'partial']).toContain(first.status);
|
||||
const first = await v0_11_0.orchestrator(COMMON_OPTS);
|
||||
expect(['complete', 'partial']).toContain(first.status);
|
||||
|
||||
const second = await v0_11_0.orchestrator(COMMON_OPTS);
|
||||
expect(['complete', 'partial']).toContain(second.status);
|
||||
const second = await v0_11_0.orchestrator(COMMON_OPTS);
|
||||
expect(['complete', 'partial']).toContain(second.status);
|
||||
|
||||
// Bug 3 (v0.14.2) — orchestrator does not write completed.jsonl, so
|
||||
// repeated direct invocations don't accumulate ledger entries. Assert
|
||||
// the preferences state stays stable (the real idempotency signal for
|
||||
// this orchestrator is "running again doesn't corrupt preferences").
|
||||
expect(loadPreferences().minion_mode).toBe('pain_triggered');
|
||||
const completed = loadCompletedMigrations();
|
||||
expect(completed.filter(e => e.version === '0.11.0').length).toBe(0);
|
||||
} finally {
|
||||
restoreHomePath();
|
||||
}
|
||||
// Bug 3 (v0.14.2) — orchestrator does not write completed.jsonl, so
|
||||
// repeated direct invocations don't accumulate ledger entries. Assert
|
||||
// the preferences state stays stable (the real idempotency signal for
|
||||
// this orchestrator is "running again doesn't corrupt preferences").
|
||||
expect(loadPreferences().minion_mode).toBe('pain_triggered');
|
||||
const completed = loadCompletedMigrations();
|
||||
expect(completed.filter(e => e.version === '0.11.0').length).toBe(0);
|
||||
}, 90_000);
|
||||
|
||||
test('host rewrite: builtin handlers auto-rewritten, non-builtins queued as JSONL TODOs', async () => {
|
||||
tmp = freshTempHome('host-rewrite');
|
||||
try {
|
||||
// Fixture: AGENTS.md + cron/jobs.json with a mix of gbrain-builtin and
|
||||
// non-builtin handlers.
|
||||
const claudeDir = join(tmp, '.claude');
|
||||
mkdirSync(claudeDir, { recursive: true });
|
||||
writeFileSync(
|
||||
join(claudeDir, 'AGENTS.md'),
|
||||
'# Test AGENTS.md\n\nSome existing content referencing sessions_spawn routing.\n',
|
||||
);
|
||||
mkdirSync(join(claudeDir, 'cron'), { recursive: true });
|
||||
writeFileSync(
|
||||
join(claudeDir, 'cron', 'jobs.json'),
|
||||
JSON.stringify({
|
||||
jobs: [
|
||||
{ schedule: '*/5 * * * *', kind: 'agentTurn', skill: 'sync' }, // builtin
|
||||
{ schedule: '0 */30 * * *', kind: 'agentTurn', skill: 'ea-inbox-sweep' }, // non-builtin
|
||||
{ schedule: '*/10 * * * *', kind: 'agentTurn', skill: 'embed' }, // builtin
|
||||
{ schedule: '0 8 * * *', kind: 'agentTurn', skill: 'morning-briefing' }, // non-builtin
|
||||
],
|
||||
}, null, 2) + '\n',
|
||||
);
|
||||
// Fixture: AGENTS.md + cron/jobs.json with a mix of gbrain-builtin and
|
||||
// non-builtin handlers.
|
||||
const claudeDir = join(tmp, '.claude');
|
||||
mkdirSync(claudeDir, { recursive: true });
|
||||
writeFileSync(
|
||||
join(claudeDir, 'AGENTS.md'),
|
||||
'# Test AGENTS.md\n\nSome existing content referencing sessions_spawn routing.\n',
|
||||
);
|
||||
mkdirSync(join(claudeDir, 'cron'), { recursive: true });
|
||||
writeFileSync(
|
||||
join(claudeDir, 'cron', 'jobs.json'),
|
||||
JSON.stringify({
|
||||
jobs: [
|
||||
{ schedule: '*/5 * * * *', kind: 'agentTurn', skill: 'sync' }, // builtin
|
||||
{ schedule: '0 */30 * * *', kind: 'agentTurn', skill: 'ea-inbox-sweep' }, // non-builtin
|
||||
{ schedule: '*/10 * * * *', kind: 'agentTurn', skill: 'embed' }, // builtin
|
||||
{ schedule: '0 8 * * *', kind: 'agentTurn', skill: 'morning-briefing' }, // non-builtin
|
||||
],
|
||||
}, null, 2) + '\n',
|
||||
);
|
||||
|
||||
const result = await v0_11_0.orchestrator(COMMON_OPTS);
|
||||
const result = await v0_11_0.orchestrator(COMMON_OPTS);
|
||||
|
||||
// Builtins rewritten in place; non-builtins left alone.
|
||||
const cronAfter = JSON.parse(readFileSync(join(claudeDir, 'cron', 'jobs.json'), 'utf-8'));
|
||||
expect(cronAfter.jobs[0].kind).toBe('shell'); // sync (builtin)
|
||||
expect(cronAfter.jobs[0].cmd).toContain('gbrain jobs submit sync');
|
||||
expect(cronAfter.jobs[1].kind).toBe('agentTurn'); // ea-inbox-sweep (non-builtin)
|
||||
expect(cronAfter.jobs[2].kind).toBe('shell'); // embed (builtin)
|
||||
expect(cronAfter.jobs[3].kind).toBe('agentTurn'); // morning-briefing (non-builtin)
|
||||
// Builtins rewritten in place; non-builtins left alone.
|
||||
const cronAfter = JSON.parse(readFileSync(join(claudeDir, 'cron', 'jobs.json'), 'utf-8'));
|
||||
expect(cronAfter.jobs[0].kind).toBe('shell'); // sync (builtin)
|
||||
expect(cronAfter.jobs[0].cmd).toContain('gbrain jobs submit sync');
|
||||
expect(cronAfter.jobs[1].kind).toBe('agentTurn'); // ea-inbox-sweep (non-builtin)
|
||||
expect(cronAfter.jobs[2].kind).toBe('shell'); // embed (builtin)
|
||||
expect(cronAfter.jobs[3].kind).toBe('agentTurn'); // morning-briefing (non-builtin)
|
||||
|
||||
// files_rewritten counts the 2 builtin rewrites.
|
||||
expect(result.files_rewritten).toBeGreaterThanOrEqual(2);
|
||||
// files_rewritten counts the 2 builtin rewrites.
|
||||
expect(result.files_rewritten).toBeGreaterThanOrEqual(2);
|
||||
|
||||
// pending_host_work counts the 2 non-builtin TODOs.
|
||||
expect(result.pending_host_work).toBe(2);
|
||||
// pending_host_work counts the 2 non-builtin TODOs.
|
||||
expect(result.pending_host_work).toBe(2);
|
||||
|
||||
// Status is "partial" because non-builtin TODOs remain.
|
||||
expect(result.status).toBe('partial');
|
||||
// Status is "partial" because non-builtin TODOs remain.
|
||||
expect(result.status).toBe('partial');
|
||||
|
||||
// AGENTS.md got the marker injected.
|
||||
const agentsMdAfter = readFileSync(join(claudeDir, 'AGENTS.md'), 'utf-8');
|
||||
expect(agentsMdAfter).toContain('gbrain:subagent-routing v0.11.0');
|
||||
expect(agentsMdAfter).toContain('skills/conventions/subagent-routing.md');
|
||||
// AGENTS.md got the marker injected.
|
||||
const agentsMdAfter = readFileSync(join(claudeDir, 'AGENTS.md'), 'utf-8');
|
||||
expect(agentsMdAfter).toContain('gbrain:subagent-routing v0.11.0');
|
||||
expect(agentsMdAfter).toContain('skills/conventions/subagent-routing.md');
|
||||
|
||||
// JSONL TODO file written under ~/.gbrain/migrations/.
|
||||
const jsonlPath = join(tmp, '.gbrain', 'migrations', 'pending-host-work.jsonl');
|
||||
expect(existsSync(jsonlPath)).toBe(true);
|
||||
const lines = readFileSync(jsonlPath, 'utf-8').split('\n').filter(l => l.trim());
|
||||
expect(lines.length).toBe(2);
|
||||
const todos = lines.map(l => JSON.parse(l));
|
||||
const handlers = todos.map(t => t.handler).sort();
|
||||
expect(handlers).toEqual(['ea-inbox-sweep', 'morning-briefing']);
|
||||
for (const todo of todos) {
|
||||
expect(todo.type).toBe('cron-handler-needs-host-registration');
|
||||
expect(todo.status).toBe('pending');
|
||||
expect(todo.manifest_path).toContain('cron/jobs.json');
|
||||
}
|
||||
} finally {
|
||||
restoreHomePath();
|
||||
// JSONL TODO file written under ~/.gbrain/migrations/.
|
||||
const jsonlPath = join(tmp, '.gbrain', 'migrations', 'pending-host-work.jsonl');
|
||||
expect(existsSync(jsonlPath)).toBe(true);
|
||||
const lines = readFileSync(jsonlPath, 'utf-8').split('\n').filter(l => l.trim());
|
||||
expect(lines.length).toBe(2);
|
||||
const todos = lines.map(l => JSON.parse(l));
|
||||
const handlers = todos.map(t => t.handler).sort();
|
||||
expect(handlers).toEqual(['ea-inbox-sweep', 'morning-briefing']);
|
||||
for (const todo of todos) {
|
||||
expect(todo.type).toBe('cron-handler-needs-host-registration');
|
||||
expect(todo.status).toBe('pending');
|
||||
expect(todo.manifest_path).toContain('cron/jobs.json');
|
||||
}
|
||||
}, 90_000);
|
||||
|
||||
test('resumable: partial run → orchestrator re-run → complete', async () => {
|
||||
tmp = freshTempHome('resumable');
|
||||
try {
|
||||
// Simulate a stopgap-written partial entry BEFORE running the orchestrator.
|
||||
mkdirSync(join(tmp, '.gbrain', 'migrations'), { recursive: true });
|
||||
writeFileSync(
|
||||
join(tmp, '.gbrain', 'migrations', 'completed.jsonl'),
|
||||
JSON.stringify({
|
||||
version: '0.11.0',
|
||||
status: 'partial',
|
||||
apply_migrations_pending: true,
|
||||
mode: 'pain_triggered',
|
||||
source: 'fix-v0.11.0.sh',
|
||||
ts: new Date().toISOString(),
|
||||
}) + '\n',
|
||||
);
|
||||
// Simulate a stopgap-written partial entry BEFORE running the orchestrator.
|
||||
mkdirSync(join(tmp, '.gbrain', 'migrations'), { recursive: true });
|
||||
writeFileSync(
|
||||
join(tmp, '.gbrain', 'migrations', 'completed.jsonl'),
|
||||
JSON.stringify({
|
||||
version: '0.11.0',
|
||||
status: 'partial',
|
||||
apply_migrations_pending: true,
|
||||
mode: 'pain_triggered',
|
||||
source: 'fix-v0.11.0.sh',
|
||||
ts: new Date().toISOString(),
|
||||
}) + '\n',
|
||||
);
|
||||
|
||||
// Orchestrator re-running on a partial → should succeed (schema apply
|
||||
// and smoke are idempotent; prefs are preserved from the partial
|
||||
// record; host-rewrite runs its safe-skip pass). Per Bug 3 (v0.14.2),
|
||||
// the orchestrator itself doesn't append to completed.jsonl — the
|
||||
// runner does. The stopgap's partial entry stays unchanged here.
|
||||
const result = await v0_11_0.orchestrator(COMMON_OPTS);
|
||||
expect(['complete', 'partial']).toContain(result.status);
|
||||
// Orchestrator re-running on a partial → should succeed (schema apply
|
||||
// and smoke are idempotent; prefs are preserved from the partial
|
||||
// record; host-rewrite runs its safe-skip pass). Per Bug 3 (v0.14.2),
|
||||
// the orchestrator itself doesn't append to completed.jsonl — the
|
||||
// runner does. The stopgap's partial entry stays unchanged here.
|
||||
const result = await v0_11_0.orchestrator(COMMON_OPTS);
|
||||
expect(['complete', 'partial']).toContain(result.status);
|
||||
|
||||
const completed = loadCompletedMigrations();
|
||||
const v0110 = completed.filter(e => e.version === '0.11.0');
|
||||
// Just the stopgap partial — orchestrator doesn't add its own entry.
|
||||
expect(v0110.length).toBe(1);
|
||||
expect(v0110[0].status).toBe('partial');
|
||||
expect(v0110[0].source).toBe('fix-v0.11.0.sh');
|
||||
} finally {
|
||||
restoreHomePath();
|
||||
}
|
||||
const completed = loadCompletedMigrations();
|
||||
const v0110 = completed.filter(e => e.version === '0.11.0');
|
||||
// Just the stopgap partial — orchestrator doesn't add its own entry.
|
||||
expect(v0110.length).toBe(1);
|
||||
expect(v0110[0].status).toBe('partial');
|
||||
expect(v0110[0].source).toBe('fix-v0.11.0.sh');
|
||||
}, 90_000);
|
||||
});
|
||||
|
||||
@@ -94,7 +94,7 @@ describeE2E('E2E: Minions resilience (OpenClaw real-world patterns)', () => {
|
||||
}, 30_000);
|
||||
|
||||
// --- 2. Runaway handler: ignores AbortSignal, dead-lettered by handleTimeouts ---
|
||||
test('runaway handler: ignores AbortSignal, handleTimeouts dead-letters it', async () => {
|
||||
test('runaway handler: ignores AbortSignal, handleTimeouts dead-letters in <2s', async () => {
|
||||
const { a, b } = await makeEngines();
|
||||
try {
|
||||
const queue = new MinionQueue(a);
|
||||
@@ -133,14 +133,8 @@ describeE2E('E2E: Minions resilience (OpenClaw real-world patterns)', () => {
|
||||
worker.stop();
|
||||
await startP;
|
||||
|
||||
// Correctness gate: the job MUST be dead-lettered with the timeout reason.
|
||||
// We intentionally do NOT assert a wall-clock upper bound (deadAt - started):
|
||||
// on a loaded CI runner the stall/timeout sweep cadence varies, and the only
|
||||
// thing that matters is that the runaway job terminates as 'dead'. The 3s poll
|
||||
// deadline above is the real timeout — if the sweep is too slow, finalStatus
|
||||
// stays '' and this toBe('dead') fails loudly.
|
||||
expect(finalStatus).toBe('dead');
|
||||
void deadAt; // retained for debugging; no timing assertion (flake-prone)
|
||||
expect(deadAt - started).toBeLessThan(2000);
|
||||
|
||||
const final = await queue.getJob(job.id);
|
||||
expect(final?.error_text).toMatch(/timeout exceeded/i);
|
||||
@@ -310,7 +304,7 @@ describeE2E('E2E: Minions resilience (OpenClaw real-world patterns)', () => {
|
||||
}, 60_000);
|
||||
|
||||
// --- 5. Cascade kill under load: cancelJob aborts all live descendants ---
|
||||
test('cascade kill: cancelJob on parent aborts 10 live children', async () => {
|
||||
test('cascade kill: cancelJob on parent aborts 10 live children within 2s', async () => {
|
||||
const { a, b } = await makeEngines();
|
||||
try {
|
||||
const queue = new MinionQueue(a);
|
||||
@@ -380,12 +374,8 @@ describeE2E('E2E: Minions resilience (OpenClaw real-world patterns)', () => {
|
||||
worker.stop();
|
||||
await startP;
|
||||
|
||||
// Correctness gate: all 10 cooperative handlers observed the abort and the
|
||||
// DB shows every descendant + root cancelled. We do NOT assert a wall-clock
|
||||
// upper bound on cancelElapsed — the 3s abort poll deadline above already
|
||||
// bounds the wait, and asserting a tighter time flakes on shared runners.
|
||||
expect(abortedChildren.size).toBe(10);
|
||||
void cancelElapsed; // retained for debugging; no timing assertion (flake-prone)
|
||||
expect(cancelElapsed).toBeLessThan(3000);
|
||||
|
||||
// DB truth: every descendant + root is 'cancelled'
|
||||
const conn = getConn();
|
||||
|
||||
@@ -78,11 +78,7 @@ describeE2E('v0.18.0 multi-source — Postgres schema shape (fresh install)', ()
|
||||
);
|
||||
expect(rows.length).toBe(1);
|
||||
expect(rows[0].is_nullable).toBe('NO');
|
||||
// Postgres renders a TEXT DEFAULT 'default' literal as `'default'::text`.
|
||||
// Assert the exact stored expression rather than a loose substring so a
|
||||
// drift in the schema DEFAULT (e.g. a different sentinel source id) fails
|
||||
// here instead of silently passing.
|
||||
expect(String(rows[0].column_default)).toBe("'default'::text");
|
||||
expect(String(rows[0].column_default)).toContain('default');
|
||||
});
|
||||
|
||||
test('composite UNIQUE pages(source_id, slug) replaces global UNIQUE(slug)', async () => {
|
||||
@@ -296,18 +292,6 @@ describeE2E('v0.18.0 multi-source — cascade delete covers every dependent row'
|
||||
`INSERT INTO files (source_id, page_id, filename, storage_path, content_hash)
|
||||
VALUES ('cascadetest', ${aliceId}, 'alice.pdf', 'cascadetest/people/alice/alice.pdf', 'fh1')`,
|
||||
);
|
||||
const aliceFile = await conn.unsafe(
|
||||
`SELECT id FROM files WHERE source_id = 'cascadetest' AND storage_path = 'cascadetest/people/alice/alice.pdf'`,
|
||||
);
|
||||
const aliceFileId = aliceFile[0].id as number;
|
||||
|
||||
// file_migration_ledger row keyed on the file (FK file_id ON DELETE
|
||||
// CASCADE). Removing the source cascades sources → files → ledger.
|
||||
await conn.unsafe(
|
||||
`INSERT INTO file_migration_ledger (file_id, storage_path_old, storage_path_new, status)
|
||||
VALUES (${aliceFileId}, 'cascadetest/people/alice/alice.pdf', 'cascadetest/people/alice/alice.pdf', 'pending')
|
||||
ON CONFLICT (file_id) DO NOTHING`,
|
||||
);
|
||||
|
||||
// Sanity: everything exists
|
||||
expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM pages WHERE source_id = 'cascadetest'`))[0].n).toBe(2);
|
||||
@@ -315,7 +299,6 @@ describeE2E('v0.18.0 multi-source — cascade delete covers every dependent row'
|
||||
expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM timeline_entries WHERE page_id = ${aliceId}`))[0].n).toBe(1);
|
||||
expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM links WHERE from_page_id = ${aliceId}`))[0].n).toBe(1);
|
||||
expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM files WHERE source_id = 'cascadetest'`))[0].n).toBe(1);
|
||||
expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM file_migration_ledger WHERE file_id = ${aliceFileId}`))[0].n).toBe(1);
|
||||
|
||||
// Remove the source.
|
||||
// v0.26.5: populated sources require --confirm-destructive; --yes alone is rejected.
|
||||
@@ -327,7 +310,6 @@ describeE2E('v0.18.0 multi-source — cascade delete covers every dependent row'
|
||||
expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM timeline_entries WHERE page_id = ${aliceId}`))[0].n).toBe(0);
|
||||
expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM links WHERE from_page_id = ${aliceId}`))[0].n).toBe(0);
|
||||
expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM files WHERE source_id = 'cascadetest'`))[0].n).toBe(0);
|
||||
expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM file_migration_ledger WHERE file_id = ${aliceFileId}`))[0].n).toBe(0);
|
||||
|
||||
// The sources row itself is gone.
|
||||
const src = await conn.unsafe(`SELECT id FROM sources WHERE id = 'cascadetest'`);
|
||||
@@ -396,10 +378,8 @@ describeE2E('v0.18.0 multi-source — sync --source routes through sources table
|
||||
|
||||
test('performSync with no sourceId falls back to global sync.repo_path', async () => {
|
||||
const engine = getEngine();
|
||||
// Self-contained: set the global config this test depends on directly
|
||||
// instead of inheriting the side effect of the previous test. Without
|
||||
// --source, performSync must read this global key.
|
||||
await engine.setConfig('sync.repo_path', '/some/other/default/path');
|
||||
// Global config is still '/some/other/default/path' from the
|
||||
// previous test. Without --source, performSync uses it.
|
||||
let err: Error | null = null;
|
||||
try {
|
||||
await performSync(engine, {});
|
||||
|
||||
@@ -112,27 +112,4 @@ describe('v0.29 E2E — getRecentSalience (Garry test)', () => {
|
||||
const rows = await engine.getRecentSalience({ days: 7, slugPrefix: 'nope/does-not-exist/' });
|
||||
expect(rows).toEqual([]);
|
||||
});
|
||||
|
||||
// TIM-37: the daily briefing writes to the vault and re-ingests as
|
||||
// `briefings/<date>`. Without this filter the briefing itself would top
|
||||
// every subsequent Brain Pulse — self-reference with no signal.
|
||||
describe('TIM-37 — briefings excluded from their own Brain Pulse', () => {
|
||||
test('default query hides briefings/* slugs', async () => {
|
||||
await engine.putPage('briefings/2026-05-19', {
|
||||
type: 'note',
|
||||
title: 'Daily Briefing — 2026-05-19',
|
||||
compiled_truth: 'Auto-generated cron briefing.',
|
||||
});
|
||||
const rows = await engine.getRecentSalience({ days: 7, limit: 50 });
|
||||
expect(rows.some(r => r.slug.startsWith('briefings/'))).toBe(false);
|
||||
});
|
||||
|
||||
test('explicit slugPrefix=briefings/ still returns them', async () => {
|
||||
const rows = await engine.getRecentSalience({ days: 7, slugPrefix: 'briefings/' });
|
||||
expect(rows.length).toBeGreaterThan(0);
|
||||
for (const r of rows) {
|
||||
expect(r.slug.startsWith('briefings/')).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -128,17 +128,6 @@ describe('SearchResult fields', () => {
|
||||
expect(r.chunk_index).toBeDefined();
|
||||
expect(typeof r.chunk_index).toBe('number');
|
||||
});
|
||||
|
||||
test('empty keyword query returns a defined array without throwing', async () => {
|
||||
const results = await engine.searchKeyword('');
|
||||
expect(Array.isArray(results)).toBe(true);
|
||||
});
|
||||
|
||||
test('zero vector search returns a defined array without throwing', async () => {
|
||||
const zeroVector = new Float32Array(1536);
|
||||
const results = await engine.searchVector(zeroVector);
|
||||
expect(Array.isArray(results)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('detail parameter', () => {
|
||||
@@ -156,11 +145,9 @@ describe('detail parameter', () => {
|
||||
});
|
||||
|
||||
test('detail=low on vector search filters to compiled_truth', async () => {
|
||||
// Use a timeline-direction embedding — detail=low filters to compiled_truth.
|
||||
// Vector search returns every chunk with an embedding (ordered by distance),
|
||||
// so the seeded compiled_truth chunks are non-empty and ALL compiled_truth.
|
||||
// Use a timeline-direction embedding — with detail=low, should get no results
|
||||
// or only compiled_truth results
|
||||
const results = await engine.searchVector(basisEmbedding(1), { detail: 'low' });
|
||||
expect(results.length).toBeGreaterThan(0);
|
||||
for (const r of results) {
|
||||
expect(r.chunk_source).toBe('compiled_truth');
|
||||
}
|
||||
|
||||
@@ -39,7 +39,6 @@ import { runSkillOpt } from '../../src/core/skillopt/orchestrator.ts';
|
||||
import {
|
||||
bestPath,
|
||||
loadHistory,
|
||||
proposedPath,
|
||||
skillPath,
|
||||
} from '../../src/core/skillopt/version-store.ts';
|
||||
import { loadRejectedBuffer } from '../../src/core/skillopt/rejected-buffer.ts';
|
||||
@@ -742,7 +741,7 @@ describe('skillopt T3 — F11 held-out gate, ablation opts, no-DB-pollution', ()
|
||||
} finally { fixture.cleanup(); }
|
||||
});
|
||||
|
||||
test('--no-mutate writes proposed.md and best.md, leaves SKILL.md untouched', async () => {
|
||||
test('--no-mutate writes proposed.md (best.md), leaves SKILL.md untouched', async () => {
|
||||
const fixture = setupFixture(SKILL_PEOPLE_ONLY, CITATIONS_BENCHMARK);
|
||||
try {
|
||||
installStub({
|
||||
@@ -754,9 +753,10 @@ describe('skillopt T3 — F11 held-out gate, ablation opts, no-DB-pollution', ()
|
||||
const result = await runOnce(fixture, { noMutate: true });
|
||||
expect(result.outcome).toBe('accepted');
|
||||
expect(result.mutatedSkillFile).toBe(false);
|
||||
expect(result.proposedPath).toBe(proposedPath(fixture.skillsDir, SKILL));
|
||||
expect(result.proposedPath).toBeDefined();
|
||||
// proposed.md (best.md) exists and carries the improvement.
|
||||
expect(fs.existsSync(result.proposedPath!)).toBe(true);
|
||||
expect(fs.readFileSync(result.proposedPath!, 'utf8')).toContain('## Citations');
|
||||
expect(fs.readFileSync(bestPath(fixture.skillsDir, SKILL), 'utf8')).toContain('## Citations');
|
||||
// SKILL.md on disk is UNCHANGED (still People-only).
|
||||
const skill = fs.readFileSync(skillPath(fixture.skillsDir, SKILL), 'utf8');
|
||||
expect(skill).not.toContain('## Citations');
|
||||
|
||||
@@ -73,7 +73,7 @@ describeE2E('E2E: Check-Update', () => {
|
||||
expect(stdout).toContain('--json');
|
||||
});
|
||||
|
||||
test('check-update --json contract holds regardless of real release state', async () => {
|
||||
test('handles no-releases gracefully (current repo state)', async () => {
|
||||
const proc = Bun.spawn(['bun', 'run', 'src/cli.ts', 'check-update', '--json'], {
|
||||
cwd: new URL('../..', import.meta.url).pathname,
|
||||
stdout: 'pipe',
|
||||
@@ -84,16 +84,8 @@ describeE2E('E2E: Check-Update', () => {
|
||||
|
||||
expect(exitCode).toBe(0);
|
||||
const output = JSON.parse(stdout);
|
||||
// Don't pin update_available to a literal value — the repo may or may not
|
||||
// have a published release. Assert the JSON shape instead.
|
||||
expect(typeof output.update_available).toBe('boolean');
|
||||
expect(output.current_version).toBe(VERSION);
|
||||
if (output.latest_version != null) {
|
||||
expect(typeof output.latest_version).toBe('string');
|
||||
}
|
||||
if (output.release_url != null) {
|
||||
expect(typeof output.release_url).toBe('string');
|
||||
}
|
||||
// With no releases, should return false and an error
|
||||
expect(output.update_available).toBe(false);
|
||||
});
|
||||
|
||||
test('version comparison wiring works end-to-end', () => {
|
||||
|
||||
@@ -803,107 +803,3 @@ describe('embedAllStale --source threading (D7)', () => {
|
||||
expect((firstCallOpts as { sourceId?: string }).sourceId).toBe('media-corpus');
|
||||
});
|
||||
});
|
||||
|
||||
// ────────────────────────────────────────────────────────────────
|
||||
// Code metadata preservation across re-embed (regression for #769)
|
||||
// ────────────────────────────────────────────────────────────────
|
||||
//
|
||||
// gbrain v0.30.1 and earlier silently clobbered code-chunk metadata
|
||||
// (language, symbol_name, symbol_type, start_line, end_line,
|
||||
// parent_symbol_path, doc_comment, symbol_name_qualified) on every
|
||||
// re-embed pass. The chunker populated those columns at import time,
|
||||
// but embed.ts loaded chunks via getChunks then mapped them to a
|
||||
// stripped ChunkInput carrying only 5 fields. upsertChunks then
|
||||
// OVERWROTE (not COALESCEd) the metadata columns from EXCLUDED, so
|
||||
// re-embed wiped them to NULL. End result on a real brain: 4875 code
|
||||
// pages, 47866 chunks, all with NULL language/symbol_name/symbol_type;
|
||||
// code-def returned 0 hits across every indexed repo.
|
||||
//
|
||||
// All three runEmbed paths (--stale autopilot, --all, --slugs) must
|
||||
// thread metadata through the re-upsert. Tests below assert that the
|
||||
// engine.upsertChunks call carries the same metadata it loaded.
|
||||
|
||||
describe('runEmbed preserves code-chunk metadata across re-embed (regression for #769)', () => {
|
||||
const fullCodeChunk = {
|
||||
chunk_index: 0,
|
||||
chunk_text: '[Java] foo/Bar.java:10-20 method baz',
|
||||
chunk_source: 'compiled_truth' as const,
|
||||
embedded_at: null,
|
||||
token_count: 12,
|
||||
language: 'java',
|
||||
symbol_name: 'baz',
|
||||
symbol_type: 'function',
|
||||
start_line: 10,
|
||||
end_line: 20,
|
||||
parent_symbol_path: ['Bar'],
|
||||
doc_comment: 'does the thing',
|
||||
symbol_name_qualified: 'Bar.baz',
|
||||
};
|
||||
|
||||
function metadataOf(chunk: any) {
|
||||
return {
|
||||
language: chunk.language,
|
||||
symbol_name: chunk.symbol_name,
|
||||
symbol_type: chunk.symbol_type,
|
||||
start_line: chunk.start_line,
|
||||
end_line: chunk.end_line,
|
||||
parent_symbol_path: chunk.parent_symbol_path,
|
||||
doc_comment: chunk.doc_comment,
|
||||
symbol_name_qualified: chunk.symbol_name_qualified,
|
||||
};
|
||||
}
|
||||
|
||||
test('--stale (autopilot path) carries code metadata into upsertChunks', async () => {
|
||||
const stale = [{
|
||||
slug: 'code-page',
|
||||
chunk_index: 0,
|
||||
chunk_text: fullCodeChunk.chunk_text,
|
||||
chunk_source: 'compiled_truth',
|
||||
model: null,
|
||||
token_count: 12,
|
||||
}];
|
||||
let upsertChunkArgs: any[] | null = null;
|
||||
const engine = mockEngine({
|
||||
countStaleChunks: async () => 1,
|
||||
listStaleChunks: async () => stale,
|
||||
getChunks: async () => [fullCodeChunk],
|
||||
upsertChunks: async (_slug: string, chunks: any[]) => { upsertChunkArgs = chunks; },
|
||||
});
|
||||
|
||||
await runEmbed(engine, ['--stale']);
|
||||
|
||||
expect(upsertChunkArgs).not.toBeNull();
|
||||
expect(upsertChunkArgs!).toHaveLength(1);
|
||||
expect(metadataOf(upsertChunkArgs![0])).toEqual(metadataOf(fullCodeChunk));
|
||||
});
|
||||
|
||||
test('--all (full re-embed) carries code metadata into upsertChunks', async () => {
|
||||
let upsertChunkArgs: any[] | null = null;
|
||||
const engine = mockEngine({
|
||||
listPages: async () => [{ slug: 'code-page' }],
|
||||
getChunks: async () => [fullCodeChunk],
|
||||
upsertChunks: async (_slug: string, chunks: any[]) => { upsertChunkArgs = chunks; },
|
||||
});
|
||||
|
||||
await runEmbed(engine, ['--all']);
|
||||
|
||||
expect(upsertChunkArgs).not.toBeNull();
|
||||
expect(upsertChunkArgs!).toHaveLength(1);
|
||||
expect(metadataOf(upsertChunkArgs![0])).toEqual(metadataOf(fullCodeChunk));
|
||||
});
|
||||
|
||||
test('--slugs (per-page embed) carries code metadata into upsertChunks', async () => {
|
||||
let upsertChunkArgs: any[] | null = null;
|
||||
const engine = mockEngine({
|
||||
getPage: async () => ({ slug: 'code-page', compiled_truth: 'x', timeline: '' }),
|
||||
getChunks: async () => [fullCodeChunk],
|
||||
upsertChunks: async (_slug: string, chunks: any[]) => { upsertChunkArgs = chunks; },
|
||||
});
|
||||
|
||||
await runEmbed(engine, ['--slugs', 'code-page']);
|
||||
|
||||
expect(upsertChunkArgs).not.toBeNull();
|
||||
expect(upsertChunkArgs!).toHaveLength(1);
|
||||
expect(metadataOf(upsertChunkArgs![0])).toEqual(metadataOf(fullCodeChunk));
|
||||
});
|
||||
});
|
||||
|
||||
@@ -351,106 +351,6 @@ describe('runExtractFacts — empty-fence guard (Codex R2-#7)', () => {
|
||||
expect(r.guardTriggered).toBe(false);
|
||||
expect(r.factsInserted).toBe(1);
|
||||
});
|
||||
|
||||
// ── #2484: structurally-unfenceable hot-memory rows ───────────
|
||||
// The inline facts writer (backstop.ts) keeps producing
|
||||
// `row_num IS NULL, entity_slug IS NOT NULL` rows AFTER the v0_32_2
|
||||
// migration completes: when a resolved slug has no fenceable page
|
||||
// (slugify-floor / stub-guard-blocked unprefixed slugs like
|
||||
// `wingman` or `people-jane-doe`), it falls through to a DB-only
|
||||
// insert with row_num NULL. The OLD guard predicate
|
||||
// (`row_num IS NULL AND entity_slug IS NOT NULL`) matched these and
|
||||
// jammed the phase forever (~16/day) — they can never be fenced (no
|
||||
// page to fence onto; the ledger-complete migration won't re-run).
|
||||
// The fix requires a LIVE backing page, so these rows no longer gate.
|
||||
test('#2484: unfenceable inline-writer rows (entity_slug set, NO backing page) do NOT trigger the guard', async () => {
|
||||
// Two unfenceable rows whose entity_slug has no page row at all.
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
await (engine as any).db.query(
|
||||
`INSERT INTO facts (source_id, entity_slug, fact, kind, visibility, notability,
|
||||
valid_from, source, confidence)
|
||||
VALUES
|
||||
('default', 'wingman', 'handoff note A', 'fact', 'private', 'medium', now(), 'mcp:extract_facts', 1.0),
|
||||
('default', 'people-jane-doe', 'handoff note B', 'fact', 'private', 'medium', now(), 'mcp:extract_facts', 1.0)`,
|
||||
);
|
||||
|
||||
// A real page with a fence that SHOULD reconcile (proves the phase
|
||||
// converges past the guard rather than early-returning).
|
||||
await putPage('people/alice', FACT_FENCE(
|
||||
`| 1 | real fenced fact | fact | 1.0 | world | high | 2026-01-01 | | s | |`,
|
||||
));
|
||||
|
||||
const r = await runExtractFacts(engine, { slugs: ['people/alice'] });
|
||||
|
||||
// Guard must NOT trip — the unfenceable rows are permanent by
|
||||
// construction, not a migration blocker.
|
||||
expect(r.guardTriggered).toBe(false);
|
||||
expect(r.legacyRowsPending).toBe(0);
|
||||
// The phase ran its reconcile pass (did not early-return).
|
||||
expect(r.factsInserted).toBe(1);
|
||||
|
||||
// The unfenceable rows survive untouched (still row_num NULL).
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const survivors = await (engine as any).db.query(
|
||||
`SELECT entity_slug FROM facts WHERE row_num IS NULL ORDER BY entity_slug`,
|
||||
);
|
||||
expect(survivors.rows.map((x: { entity_slug: string }) => x.entity_slug))
|
||||
.toEqual(['people-jane-doe', 'wingman']);
|
||||
});
|
||||
|
||||
test('#2484: a genuine legacy row WITH a backing page still triggers the guard (discriminator stays sharp)', async () => {
|
||||
// Same shape as the unfenceable row above (row_num NULL, entity_slug
|
||||
// set) — the ONLY difference is a live backing page exists, so the
|
||||
// migration's Phase B could fence it. This MUST still gate.
|
||||
await putPage('people/bob', FACT_FENCE(
|
||||
`| 1 | fence fact | fact | 1.0 | world | high | 2026-01-01 | | s | |`,
|
||||
));
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
await (engine as any).db.query(
|
||||
`INSERT INTO facts (source_id, entity_slug, fact, kind, visibility, notability,
|
||||
valid_from, source, confidence)
|
||||
VALUES ('default', 'people/bob', 'genuine legacy claim', 'fact', 'private', 'medium',
|
||||
now(), 'mcp:put_page', 1.0)`,
|
||||
);
|
||||
|
||||
const r = await runExtractFacts(engine, { slugs: ['people/bob'] });
|
||||
|
||||
expect(r.guardTriggered).toBe(true);
|
||||
expect(r.legacyRowsPending).toBe(1);
|
||||
expect(r.factsInserted).toBe(0);
|
||||
expect(r.factsDeleted).toBe(0);
|
||||
expect(r.warnings.some(w => w.includes('apply-migrations'))).toBe(true);
|
||||
});
|
||||
|
||||
test('#2484: a soft-deleted backing page makes its legacy row unfenceable (does NOT gate)', async () => {
|
||||
// Page exists then gets soft-deleted (deleted_at set). The migration
|
||||
// can't fence onto a deleted page, so the row must not gate.
|
||||
await putPage('people/carol', FACT_FENCE(
|
||||
`| 1 | live fact | fact | 1.0 | world | high | 2026-01-01 | | s | |`,
|
||||
));
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
await (engine as any).db.query(
|
||||
`INSERT INTO facts (source_id, entity_slug, fact, kind, visibility, notability,
|
||||
valid_from, source, confidence)
|
||||
VALUES ('default', 'people/carol', 'orphaned legacy claim', 'fact', 'private', 'medium',
|
||||
now(), 'mcp:put_page', 1.0)`,
|
||||
);
|
||||
// Soft-delete the page.
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
await (engine as any).db.query(
|
||||
`UPDATE pages SET deleted_at = now() WHERE slug = 'people/carol' AND source_id = 'default'`,
|
||||
);
|
||||
|
||||
// Reconcile a DIFFERENT live page so the phase has work to do.
|
||||
await putPage('people/dave', FACT_FENCE(
|
||||
`| 1 | dave fact | fact | 1.0 | world | high | 2026-01-01 | | s | |`,
|
||||
));
|
||||
|
||||
const r = await runExtractFacts(engine, { slugs: ['people/dave'] });
|
||||
expect(r.guardTriggered).toBe(false);
|
||||
expect(r.legacyRowsPending).toBe(0);
|
||||
expect(r.factsInserted).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('runExtractFacts — multi-source isolation', () => {
|
||||
|
||||
@@ -209,32 +209,6 @@ describe('gbrain extract --stale', () => {
|
||||
expect(usRows[0]?.eq).toBe(true);
|
||||
});
|
||||
|
||||
test('REGRESSION: page with updated_at BEFORE LINK_EXTRACTOR_VERSION_TS clears (no permanent-stale loop)', async () => {
|
||||
// The v112 watermark column ships with no backfill, so every pre-existing
|
||||
// page starts NULL-stale — and most pre-date the version bump. Pre-fix,
|
||||
// extractStaleFromDB stamped links_extracted_at = read updated_at; for a
|
||||
// page edited before LINK_EXTRACTOR_VERSION_TS the stamp landed BELOW the
|
||||
// version threshold, so the version arm (links_extracted_at < versionTs)
|
||||
// re-flagged it stale forever — an infinite re-extract loop that never
|
||||
// cleared the lag (observed: 97% of pages stuck permanently).
|
||||
await engine.putPage('people/alice', personPage('Alice'));
|
||||
await engine.putPage('companies/acme', companyPage('Acme', '[Alice](people/alice) leads [Acme](companies/acme).'));
|
||||
// Backdate every page to BEFORE the extractor version timestamp.
|
||||
await engine.executeRaw(`UPDATE pages SET updated_at = '2020-01-01T00:00:00Z'`);
|
||||
expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(2);
|
||||
|
||||
await runExtract(engine, ['--stale']);
|
||||
// Fixed: stamp = GREATEST(read updated_at, versionTs) → lifts old pages to
|
||||
// the threshold so the version arm clears, while a real future edit still
|
||||
// advances updated_at past the stamp (CDX-1 race protection preserved).
|
||||
expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(0);
|
||||
|
||||
// Second run must ALSO find 0 — the defining symptom of the bug was that it
|
||||
// never converged.
|
||||
await runExtract(engine, ['--stale']);
|
||||
expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(0);
|
||||
});
|
||||
|
||||
test('CDX-4 (D2): a link-flush throw aborts the sweep and leaves pages UNSTAMPED', async () => {
|
||||
await engine.putPage('people/alice', personPage('Alice'));
|
||||
await engine.putPage('companies/acme', companyPage('Acme', '[Alice](people/alice) founded [Acme](companies/acme).'));
|
||||
|
||||
+17
-49
@@ -8,11 +8,11 @@ import {
|
||||
|
||||
describe('extractMarkdownLinks', () => {
|
||||
it('extracts relative markdown links', () => {
|
||||
const content = 'Check [Alice](../people/alice-example.md) and [Acme](../../companies/acme-example.md).';
|
||||
const content = 'Check [Pedro](../people/pedro-franceschi.md) and [Brex](../../companies/brex.md).';
|
||||
const links = extractMarkdownLinks(content);
|
||||
expect(links).toHaveLength(2);
|
||||
expect(links[0].name).toBe('Alice');
|
||||
expect(links[0].relTarget).toBe('../people/alice-example.md');
|
||||
expect(links[0].name).toBe('Pedro');
|
||||
expect(links[0].relTarget).toBe('../people/pedro-franceschi.md');
|
||||
});
|
||||
|
||||
it('skips external URLs ending in .md', () => {
|
||||
@@ -34,12 +34,12 @@ describe('extractMarkdownLinks', () => {
|
||||
|
||||
describe('extractLinksFromFile', () => {
|
||||
it('resolves relative paths to slugs', async () => {
|
||||
const content = '---\ntitle: Test\n---\nSee [Alice](../people/alice.md).';
|
||||
const allSlugs = new Set(['people/alice', 'deals/test-deal']);
|
||||
const content = '---\ntitle: Test\n---\nSee [Pedro](../people/pedro.md).';
|
||||
const allSlugs = new Set(['people/pedro', 'deals/test-deal']);
|
||||
const links = await extractLinksFromFile(content, 'deals/test-deal.md', allSlugs);
|
||||
expect(links.length).toBeGreaterThanOrEqual(1);
|
||||
expect(links[0].from_slug).toBe('deals/test-deal');
|
||||
expect(links[0].to_slug).toBe('people/alice');
|
||||
expect(links[0].to_slug).toBe('people/pedro');
|
||||
});
|
||||
|
||||
it('skips links to non-existent pages', async () => {
|
||||
@@ -50,15 +50,15 @@ describe('extractLinksFromFile', () => {
|
||||
});
|
||||
|
||||
it('extracts frontmatter company links (v0.13, includeFrontmatter opt-in)', async () => {
|
||||
const content = '---\ncompany: acme-example\ntype: person\n---\nContent.';
|
||||
const content = '---\ncompany: brex\ntype: person\n---\nContent.';
|
||||
// v0.13 canonical: person page with company: X → person → company works_at (outgoing).
|
||||
// Resolver needs companies/acme-example to exist in allSlugs to emit the edge.
|
||||
const allSlugs = new Set(['people/test', 'companies/acme-example']);
|
||||
// Resolver needs companies/brex to exist in allSlugs to emit the edge.
|
||||
const allSlugs = new Set(['people/test', 'companies/brex']);
|
||||
const links = await extractLinksFromFile(content, 'people/test.md', allSlugs, { includeFrontmatter: true });
|
||||
const companyLinks = links.filter(l => l.link_type === 'works_at');
|
||||
expect(companyLinks.length).toBeGreaterThanOrEqual(1);
|
||||
expect(companyLinks[0].from_slug).toBe('people/test');
|
||||
expect(companyLinks[0].to_slug).toBe('companies/acme-example');
|
||||
expect(companyLinks[0].to_slug).toBe('companies/brex');
|
||||
});
|
||||
|
||||
it('extracts frontmatter investors array (v0.13: incoming direction)', async () => {
|
||||
@@ -79,22 +79,22 @@ describe('extractLinksFromFile', () => {
|
||||
it('frontmatter extraction is default OFF (back-compat)', async () => {
|
||||
// Without includeFrontmatter, fs-source no longer auto-extracts frontmatter.
|
||||
// Matches db-source behavior. User opts in with --include-frontmatter flag.
|
||||
const content = '---\ncompany: acme-example\ntype: person\n---\nContent.';
|
||||
const allSlugs = new Set(['people/test', 'companies/acme-example']);
|
||||
const content = '---\ncompany: brex\ntype: person\n---\nContent.';
|
||||
const allSlugs = new Set(['people/test', 'companies/brex']);
|
||||
const links = await extractLinksFromFile(content, 'people/test.md', allSlugs);
|
||||
expect(links).toEqual([]);
|
||||
});
|
||||
|
||||
it('infers link type from directory structure', async () => {
|
||||
const content = 'See [Acme](../companies/acme-example.md).';
|
||||
const allSlugs = new Set(['people/alice', 'companies/acme-example']);
|
||||
const links = await extractLinksFromFile(content, 'people/alice.md', allSlugs);
|
||||
const content = 'See [Brex](../companies/brex.md).';
|
||||
const allSlugs = new Set(['people/pedro', 'companies/brex']);
|
||||
const links = await extractLinksFromFile(content, 'people/pedro.md', allSlugs);
|
||||
expect(links[0].link_type).toBe('works_at');
|
||||
});
|
||||
|
||||
it('infers deal_for type for deals -> companies', async () => {
|
||||
const content = 'See [Acme](../companies/acme-example.md).';
|
||||
const allSlugs = new Set(['deals/seed', 'companies/acme-example']);
|
||||
const content = 'See [Brex](../companies/brex.md).';
|
||||
const allSlugs = new Set(['deals/seed', 'companies/brex']);
|
||||
const links = await extractLinksFromFile(content, 'deals/seed.md', allSlugs);
|
||||
expect(links[0].link_type).toBe('deal_for');
|
||||
});
|
||||
@@ -136,38 +136,6 @@ describe('extractTimelineFromContent', () => {
|
||||
expect(entries).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('does not split on hyphens inside markdown link targets', () => {
|
||||
const content = `- **2025-03-18** | Referenced in [Alice](../people/alice-example.md)`;
|
||||
const entries = extractTimelineFromContent(content, 'companies/acme-example');
|
||||
expect(entries).toHaveLength(1);
|
||||
expect(entries[0].source).toBe('markdown');
|
||||
expect(entries[0].summary).toBe('Referenced in [Alice](../people/alice-example.md)');
|
||||
});
|
||||
|
||||
it('does not split on spaced dashes inside link labels', () => {
|
||||
const content = `- **2025-03-18** | Referenced in [Deals — Q1 Review](../deals/q1-review.md)`;
|
||||
const entries = extractTimelineFromContent(content, 'companies/acme-example');
|
||||
expect(entries).toHaveLength(1);
|
||||
expect(entries[0].source).toBe('markdown');
|
||||
expect(entries[0].summary).toBe('Referenced in [Deals — Q1 Review](../deals/q1-review.md)');
|
||||
});
|
||||
|
||||
it('splits on the first spaced dash outside links', () => {
|
||||
const content = `- **2025-03-18** | [Board notes](../meetings/2025-03-18-board.md) — Approved the hire`;
|
||||
const entries = extractTimelineFromContent(content, 'test');
|
||||
expect(entries).toHaveLength(1);
|
||||
expect(entries[0].source).toBe('[Board notes](../meetings/2025-03-18-board.md)');
|
||||
expect(entries[0].summary).toBe('Approved the hire');
|
||||
});
|
||||
|
||||
it('keeps delimiterless bullet lines whole instead of dropping them', () => {
|
||||
const content = `- **2025-03-18** | Imported from legacy tracker`;
|
||||
const entries = extractTimelineFromContent(content, 'test');
|
||||
expect(entries).toHaveLength(1);
|
||||
expect(entries[0].source).toBe('markdown');
|
||||
expect(entries[0].summary).toBe('Imported from legacy tracker');
|
||||
});
|
||||
|
||||
it('extracts inline citation format entries', () => {
|
||||
const content = `Closed the seed round with fund-a leading. [Source: board meeting notes, 2025-04-02]`;
|
||||
const entries = extractTimelineFromContent(content, 'deals/acme-seed');
|
||||
|
||||
@@ -403,77 +403,6 @@ describe('extractPageLinks', () => {
|
||||
expect(candidates).toEqual([]);
|
||||
});
|
||||
|
||||
test('path-qualified wikilink outside DIR_PATTERN queries by final segment', async () => {
|
||||
// `[[notes/struktura]]` (dir not in DIR_PATTERN) falls to the generic
|
||||
// pass. The resolver's basename index is keyed by final path segments,
|
||||
// so the lookup must strip the dirname — mirroring the FS path
|
||||
// (resolveSlugAll). Regression: the raw literal was passed through,
|
||||
// which never matched, so these links silently dropped.
|
||||
const seen: string[] = [];
|
||||
const resolver: SlugResolver = {
|
||||
resolve: async () => null,
|
||||
resolveBasenameMatches: async (name) => {
|
||||
seen.push(name);
|
||||
return name === 'struktura' ? ['notes/struktura'] : [];
|
||||
},
|
||||
};
|
||||
const { candidates } = await extractPageLinks(
|
||||
'concepts/x', 'See [[notes/struktura]].',
|
||||
{}, 'concept', resolver, { globalBasename: true },
|
||||
);
|
||||
expect(seen).toContain('struktura');
|
||||
expect(seen).not.toContain('notes/struktura');
|
||||
expect(candidates.map(c => c.targetSlug)).toEqual(['notes/struktura']);
|
||||
expect(candidates[0].linkType).toBe('wikilink_basename');
|
||||
expect(candidates[0].linkSource).toBe('wikilink-resolved');
|
||||
});
|
||||
|
||||
test('path-qualified wikilink keeps only matches ending with the written path', async () => {
|
||||
// The written path disambiguates: `[[notes/struktura]]` must never
|
||||
// attach to `wiki/struktura` even though both share the basename.
|
||||
const resolver: SlugResolver = {
|
||||
resolve: async () => null,
|
||||
resolveBasenameMatches: async (name) =>
|
||||
name === 'struktura' ? ['notes/struktura', 'wiki/struktura'] : [],
|
||||
};
|
||||
const { candidates } = await extractPageLinks(
|
||||
'concepts/x', 'See [[notes/struktura]].',
|
||||
{}, 'concept', resolver, { globalBasename: true },
|
||||
);
|
||||
expect(candidates.map(c => c.targetSlug)).toEqual(['notes/struktura']);
|
||||
});
|
||||
|
||||
test('path-qualified wikilink matches a deeper real slug by path suffix', async () => {
|
||||
// The page lives at vault/notes/struktura; the author wrote the shorter
|
||||
// tail `[[notes/struktura]]`. Suffix matching connects them, while the
|
||||
// basename-only sibling `wiki/struktura` stays excluded.
|
||||
const resolver: SlugResolver = {
|
||||
resolve: async () => null,
|
||||
resolveBasenameMatches: async (name) =>
|
||||
name === 'struktura' ? ['vault/notes/struktura', 'wiki/struktura'] : [],
|
||||
};
|
||||
const { candidates } = await extractPageLinks(
|
||||
'concepts/x', 'See [[notes/struktura]].',
|
||||
{}, 'concept', resolver, { globalBasename: true },
|
||||
);
|
||||
expect(candidates.map(c => c.targetSlug)).toEqual(['vault/notes/struktura']);
|
||||
});
|
||||
|
||||
test('path-qualified self-link is dropped like the bare form', async () => {
|
||||
// `[[notes/struktura]]` written on notes/struktura itself must not
|
||||
// produce a self-loop (same guard as the bare `[[own-tail]]` case).
|
||||
const resolver: SlugResolver = {
|
||||
resolve: async () => null,
|
||||
resolveBasenameMatches: async (name) =>
|
||||
name === 'struktura' ? ['notes/struktura'] : [],
|
||||
};
|
||||
const { candidates } = await extractPageLinks(
|
||||
'notes/struktura', 'See [[notes/struktura]].',
|
||||
{}, 'concept', resolver, { globalBasename: true },
|
||||
);
|
||||
expect(candidates).toEqual([]);
|
||||
});
|
||||
|
||||
test('bare wikilink resolution does not interfere with DIR_PATTERN wikilinks', async () => {
|
||||
// 2b refs (people/alice) take the verb-inferred type;
|
||||
// 2c refs (struktura) take wikilink_basename. Same call.
|
||||
@@ -1236,43 +1165,6 @@ describe('makeResolver — fallback chain', () => {
|
||||
const out = await r.resolveBasenameMatches!('struktura');
|
||||
expect(out.sort()).toEqual(['notes/struktura', 'struktura']);
|
||||
});
|
||||
|
||||
test('opts.sourceId is forwarded to findByTitleFuzzy (twin of #1436 fix)', async () => {
|
||||
// Captures every (name, dirPrefix, minSimilarity, sourceId) call so we
|
||||
// can assert the resolver threads sourceId through. Without the wire-up,
|
||||
// findByTitleFuzzy would be called with sourceId=undefined and the SQL
|
||||
// could return cross-source slug suggestions that the FK filter
|
||||
// downstream silently drops.
|
||||
const calls: Array<{ name: string; dirPrefix?: string; minSimilarity?: number; sourceId?: string }> = [];
|
||||
const engine = {
|
||||
async getPage() { return null; },
|
||||
async findByTitleFuzzy(name: string, dirPrefix?: string, minSimilarity?: number, sourceId?: string) {
|
||||
calls.push({ name, dirPrefix, minSimilarity, sourceId });
|
||||
return null;
|
||||
},
|
||||
async searchKeyword() { return []; },
|
||||
} as unknown as BrainEngine;
|
||||
const r = makeResolver(engine, { mode: 'batch', sourceId: 'src-a' });
|
||||
await r.resolve('Alice Example', 'people');
|
||||
expect(calls.length).toBeGreaterThan(0);
|
||||
expect(calls.every(c => c.sourceId === 'src-a')).toBe(true);
|
||||
});
|
||||
|
||||
test('opts.sourceId omitted → findByTitleFuzzy receives undefined (back-compat)', async () => {
|
||||
const calls: Array<{ sourceId?: string }> = [];
|
||||
const engine = {
|
||||
async getPage() { return null; },
|
||||
async findByTitleFuzzy(_name: string, _dirPrefix?: string, _min?: number, sourceId?: string) {
|
||||
calls.push({ sourceId });
|
||||
return null;
|
||||
},
|
||||
async searchKeyword() { return []; },
|
||||
} as unknown as BrainEngine;
|
||||
const r = makeResolver(engine, { mode: 'batch' });
|
||||
await r.resolve('Alice Example', 'people');
|
||||
expect(calls.length).toBeGreaterThan(0);
|
||||
expect(calls.every(c => c.sourceId === undefined)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('FRONTMATTER_LINK_MAP integrity', () => {
|
||||
|
||||
@@ -32,29 +32,6 @@ describe('lintContent', () => {
|
||||
expect(issues.some(i => i.rule === 'code-fence-wrap')).toBe(true);
|
||||
});
|
||||
|
||||
test('no false positive: page CONTAINS an inner ```markdown code block', () => {
|
||||
// Real-world case: a docs/SKILL page that shows a markdown example inline.
|
||||
// Before this fix, the detector used the /m flag so ^/$ matched start/end
|
||||
// of any line, which fired on any file that simply contained a ```markdown
|
||||
// line. But fixContent's regex has no /m flag and can only strip whole-file
|
||||
// wrappers, so the issue was reported as "fixable: true" yet never fixed.
|
||||
const content =
|
||||
'---\ntitle: Skill\n---\n\n# Skill\n\nExample input shape:\n\n' +
|
||||
'```markdown\n# Inner page\nContent.\n```\n\nThat ends the example.\n';
|
||||
const issues = lintContent(content, 'test.md');
|
||||
expect(issues.filter(i => i.rule === 'code-fence-wrap')).toHaveLength(0);
|
||||
});
|
||||
|
||||
test('no false positive: multiple inner ```markdown blocks', () => {
|
||||
// Documentation pages frequently include several markdown examples.
|
||||
const content =
|
||||
'---\ntitle: Examples\n---\n\n# Examples\n\nFirst:\n\n' +
|
||||
'```markdown\nfoo\n```\n\nSecond:\n\n' +
|
||||
'```markdown\nbar\n```\n\nDone.\n';
|
||||
const issues = lintContent(content, 'test.md');
|
||||
expect(issues.filter(i => i.rule === 'code-fence-wrap')).toHaveLength(0);
|
||||
});
|
||||
|
||||
test('detects placeholder dates', () => {
|
||||
const content = '---\ntitle: Test\ntype: person\ncreated: YYYY-MM-DD\n---\n\n# Test';
|
||||
const issues = lintContent(content, 'test.md');
|
||||
|
||||
@@ -1,132 +0,0 @@
|
||||
/**
|
||||
* list_pages clamp local-trust + offset threading — op-level coverage.
|
||||
*
|
||||
* Pins (upstream draft "gbrain list silently clamps --limit to 100"):
|
||||
* - Local callers (ctx.remote === false) get an explicit limit above 100
|
||||
* honored — full enumeration is a legitimate local operation.
|
||||
* - Remote callers keep the 100-row DoS cap, and the clamp is now LOUD:
|
||||
* exactly one logger.warn (stderr, never stdout) naming both numbers.
|
||||
* - Defaults unchanged: no limit → 50 rows for both local and remote.
|
||||
* - `offset` threads through to the engine (PageFilters supported it all
|
||||
* along; the op layer dropped it, so `--offset` was silently ignored).
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { operationsByName } from '../src/core/operations.ts';
|
||||
import type { OperationContext } from '../src/core/operations.ts';
|
||||
|
||||
const SEED_COUNT = 120; // must exceed the remote cap (100) and the default (50)
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
for (let i = 0; i < SEED_COUNT; i++) {
|
||||
// Zero-padded slugs → sort:'slug' gives a deterministic order for the
|
||||
// offset assertions regardless of insert timestamps.
|
||||
await engine.putPage(`listclamp/page-${String(i).padStart(3, '0')}`, {
|
||||
type: 'note',
|
||||
title: `Page ${i}`,
|
||||
compiled_truth: 'body',
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (engine) await engine.disconnect();
|
||||
});
|
||||
|
||||
function mkCtx(overrides: Partial<OperationContext> = {}): {
|
||||
ctx: OperationContext;
|
||||
warnings: string[];
|
||||
} {
|
||||
const warnings: string[] = [];
|
||||
const ctx = {
|
||||
engine,
|
||||
config: {} as any,
|
||||
logger: {
|
||||
info: () => {},
|
||||
warn: (msg: string) => warnings.push(msg),
|
||||
error: () => {},
|
||||
} as any,
|
||||
dryRun: false,
|
||||
remote: false,
|
||||
...overrides,
|
||||
} as OperationContext;
|
||||
return { ctx, warnings };
|
||||
}
|
||||
|
||||
const op = () => operationsByName['list_pages'];
|
||||
|
||||
describe('list_pages — local callers escape the 100-row clamp', () => {
|
||||
test('remote=false with limit 100000 returns every page', async () => {
|
||||
const { ctx, warnings } = mkCtx({ remote: false });
|
||||
const rows = (await op().handler(ctx, { limit: 100000 })) as any[];
|
||||
expect(rows.length).toBe(SEED_COUNT);
|
||||
expect(warnings.length).toBe(0);
|
||||
});
|
||||
|
||||
test('remote=false default (no limit) is still 50 — default unchanged', async () => {
|
||||
const { ctx } = mkCtx({ remote: false });
|
||||
const rows = (await op().handler(ctx, {})) as any[];
|
||||
expect(rows.length).toBe(50);
|
||||
});
|
||||
});
|
||||
|
||||
describe('list_pages — remote callers keep the cap, loudly', () => {
|
||||
test('remote=true with limit 100000 returns 100 and warns once with both numbers', async () => {
|
||||
const { ctx, warnings } = mkCtx({ remote: true });
|
||||
const rows = (await op().handler(ctx, { limit: 100000 })) as any[];
|
||||
expect(rows.length).toBe(100);
|
||||
expect(warnings.length).toBe(1);
|
||||
expect(warnings[0]).toContain('list limit clamped from 100000 to 100');
|
||||
});
|
||||
|
||||
test('remote=true with limit <= 100 does not warn', async () => {
|
||||
const { ctx, warnings } = mkCtx({ remote: true });
|
||||
const rows = (await op().handler(ctx, { limit: 60 })) as any[];
|
||||
expect(rows.length).toBe(60);
|
||||
expect(warnings.length).toBe(0);
|
||||
});
|
||||
|
||||
test('anything not strictly remote===false is treated as remote (defense in depth)', async () => {
|
||||
// ctx.remote contract: consumers treat non-false as untrusted even if the
|
||||
// type is bypassed via cast.
|
||||
const { ctx, warnings } = mkCtx({ remote: undefined as any });
|
||||
const rows = (await op().handler(ctx, { limit: 100000 })) as any[];
|
||||
expect(rows.length).toBe(100);
|
||||
expect(warnings.length).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('list_pages — offset threads through (regression: was silently ignored)', () => {
|
||||
test('offset shifts the window under sort=slug', async () => {
|
||||
const { ctx } = mkCtx({ remote: false });
|
||||
const all = (await op().handler(ctx, { limit: 100000, sort: 'slug' })) as any[];
|
||||
const paged = (await op().handler(ctx, { limit: 10, offset: 5, sort: 'slug' })) as any[];
|
||||
expect(paged.length).toBe(10);
|
||||
expect(paged.map(r => r.slug)).toEqual(all.slice(5, 15).map(r => r.slug));
|
||||
});
|
||||
|
||||
test('offset near the end truncates the page', async () => {
|
||||
const { ctx } = mkCtx({ remote: false });
|
||||
const rows = (await op().handler(ctx, {
|
||||
limit: 100000,
|
||||
offset: SEED_COUNT - 7,
|
||||
sort: 'slug',
|
||||
})) as any[];
|
||||
expect(rows.length).toBe(7);
|
||||
});
|
||||
|
||||
test('garbage offset (negative / NaN) is ignored, not fatal', async () => {
|
||||
const { ctx } = mkCtx({ remote: false });
|
||||
const neg = (await op().handler(ctx, { limit: 10, offset: -5, sort: 'slug' })) as any[];
|
||||
const nan = (await op().handler(ctx, { limit: 10, offset: NaN, sort: 'slug' })) as any[];
|
||||
const base = (await op().handler(ctx, { limit: 10, sort: 'slug' })) as any[];
|
||||
expect(neg.map(r => r.slug)).toEqual(base.map(r => r.slug));
|
||||
expect(nan.map(r => r.slug)).toEqual(base.map(r => r.slug));
|
||||
});
|
||||
});
|
||||
@@ -9,7 +9,6 @@ import { loadConfigWithEngine, type GBrainConfig } from '../src/core/config.ts';
|
||||
|
||||
interface FakeEngine {
|
||||
getConfig(key: string): Promise<string | null | undefined>;
|
||||
listConfigKeys?(prefix: string): Promise<string[]>;
|
||||
}
|
||||
|
||||
function makeEngine(map: Record<string, string | null | undefined>): FakeEngine {
|
||||
@@ -17,9 +16,6 @@ function makeEngine(map: Record<string, string | null | undefined>): FakeEngine
|
||||
async getConfig(key: string) {
|
||||
return map[key];
|
||||
},
|
||||
async listConfigKeys(prefix: string) {
|
||||
return Object.keys(map).filter(key => key.startsWith(prefix));
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -96,31 +92,6 @@ describe('loadConfigWithEngine (Phase 4 / F3)', () => {
|
||||
expect(merged?.embedding_image_ocr).toBe(true);
|
||||
});
|
||||
|
||||
test('DB provider_base_urls.<provider> fills the gateway base URL map', async () => {
|
||||
const base: GBrainConfig = { engine: 'pglite' };
|
||||
const engine = makeEngine({
|
||||
'provider_base_urls.llama-server-reranker': 'http://127.0.0.1:8091/v1',
|
||||
});
|
||||
const merged = await loadConfigWithEngine(engine, base);
|
||||
expect(merged?.provider_base_urls?.['llama-server-reranker']).toBe('http://127.0.0.1:8091/v1');
|
||||
});
|
||||
|
||||
test('provider_base_urls merge is per-provider: file value wins and DB fills siblings', async () => {
|
||||
const base: GBrainConfig = {
|
||||
engine: 'pglite',
|
||||
provider_base_urls: {
|
||||
'llama-server-reranker': 'http://file.example/v1',
|
||||
},
|
||||
};
|
||||
const engine = makeEngine({
|
||||
'provider_base_urls.llama-server-reranker': 'http://db.example/v1',
|
||||
'provider_base_urls.openrouter': 'http://openrouter.example/v1',
|
||||
});
|
||||
const merged = await loadConfigWithEngine(engine, base);
|
||||
expect(merged?.provider_base_urls?.['llama-server-reranker']).toBe('http://file.example/v1');
|
||||
expect(merged?.provider_base_urls?.openrouter).toBe('http://openrouter.example/v1');
|
||||
});
|
||||
|
||||
test('engine.getConfig throwing is non-fatal — file/env config still returned', async () => {
|
||||
const base: GBrainConfig = {
|
||||
engine: 'pglite',
|
||||
|
||||
@@ -1,62 +0,0 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import {
|
||||
extractCycleFreshnessSourceIds,
|
||||
parseMaintainArgs,
|
||||
} from '../src/commands/maintain.ts';
|
||||
import type { Check } from '../src/commands/doctor.ts';
|
||||
|
||||
describe('maintain args', () => {
|
||||
test('defaults to dry-run unless --safe is explicit', () => {
|
||||
expect(parseMaintainArgs([])).toMatchObject({
|
||||
safe: false,
|
||||
dryRun: true,
|
||||
json: false,
|
||||
});
|
||||
});
|
||||
|
||||
test('--safe enables mutating safe mode', () => {
|
||||
expect(parseMaintainArgs(['--safe', '--json'])).toMatchObject({
|
||||
safe: true,
|
||||
dryRun: false,
|
||||
json: true,
|
||||
});
|
||||
});
|
||||
|
||||
test('--dry-run wins over --safe', () => {
|
||||
expect(parseMaintainArgs(['--safe', '--dry-run'])).toMatchObject({
|
||||
safe: true,
|
||||
dryRun: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('cycle freshness source extraction', () => {
|
||||
test('extracts stale source ids from doctor messages', () => {
|
||||
const checks: Check[] = [
|
||||
{
|
||||
name: 'cycle_freshness',
|
||||
status: 'fail',
|
||||
message: "Source 'brain-sync-remote-teffur' last cycled 40h ago. Run `gbrain dream --source <id>`.",
|
||||
},
|
||||
{
|
||||
name: 'cycle_freshness',
|
||||
status: 'fail',
|
||||
message: "Source 'wiki' last cycled 25h ago. Source 'wiki' last cycled 25h ago.",
|
||||
},
|
||||
];
|
||||
|
||||
expect(extractCycleFreshnessSourceIds(checks)).toEqual([
|
||||
'brain-sync-remote-teffur',
|
||||
'wiki',
|
||||
]);
|
||||
});
|
||||
|
||||
test('ignores ok and unrelated checks', () => {
|
||||
const checks: Check[] = [
|
||||
{ name: 'cycle_freshness', status: 'ok', message: "Source 'fresh' last cycled recently." },
|
||||
{ name: 'frontmatter_integrity', status: 'warn', message: "Source 'wiki' has frontmatter issues." },
|
||||
];
|
||||
|
||||
expect(extractCycleFreshnessSourceIds(checks)).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -343,47 +343,3 @@ describe('issue #1939 — non-string frontmatter coercion', () => {
|
||||
expect(parsed.title).toBe('A Normal Title');
|
||||
});
|
||||
});
|
||||
|
||||
// issue #2446 — when frontmatter has no `title:`, prefer the body's first H1
|
||||
// over the slug/filename-humanized fallback. Slug-based imports (contacts,
|
||||
// calendar) carry a correct `# Heading` but no frontmatter title; humanizing
|
||||
// the slug leaks date/id tokens and loses casing (`Defalco` vs `DeFalco`).
|
||||
describe('issue #2446 — body H1 fallback for missing frontmatter title', () => {
|
||||
test('no frontmatter title uses the body H1, not the slug-humanized junk', () => {
|
||||
const md = '---\ntype: person\n---\n\n# John DeFalco\n\nNotes about John.\n';
|
||||
const parsed = parseMarkdown(md, 'people/contact-20170928-5-john-defalco.md');
|
||||
expect(parsed.title).toBe('John DeFalco');
|
||||
// The slug-derived junk title must NOT win.
|
||||
expect(parsed.title).not.toBe('Contact 20170928 5 John Defalco');
|
||||
});
|
||||
|
||||
test('no frontmatter title and no H1 falls back to the inferred slug title', () => {
|
||||
const md = '---\ntype: note\n---\n\njust body prose, no heading\n';
|
||||
const parsed = parseMarkdown(md, 'people/alice-example.md');
|
||||
expect(parsed.title).toBe('Alice Example');
|
||||
});
|
||||
|
||||
test('frontmatter title wins over a body H1 (no regression)', () => {
|
||||
const md = '---\ntitle: Frontmatter Wins\n---\n\n# Body Heading\n\nbody\n';
|
||||
const parsed = parseMarkdown(md, 'people/some-slug.md');
|
||||
expect(parsed.title).toBe('Frontmatter Wins');
|
||||
});
|
||||
|
||||
test('h2 is not treated as the title; first real H1 is used', () => {
|
||||
const md = '---\ntype: note\n---\n\n## Subsection First\n\n# The Real Title\n\nbody\n';
|
||||
const parsed = parseMarkdown(md, 'notes/x.md');
|
||||
expect(parsed.title).toBe('The Real Title');
|
||||
});
|
||||
|
||||
test('a # inside a fenced code block is not mistaken for the title', () => {
|
||||
const md = '---\ntype: note\n---\n\n```sh\n# this is a shell comment, not a heading\n```\n\n# Actual Heading\n';
|
||||
const parsed = parseMarkdown(md, 'notes/x.md');
|
||||
expect(parsed.title).toBe('Actual Heading');
|
||||
});
|
||||
|
||||
test('trailing closing hashes are stripped from the H1', () => {
|
||||
const md = '---\ntype: note\n---\n\n# Closed ATX Heading #\n\nbody\n';
|
||||
const parsed = parseMarkdown(md, 'notes/x.md');
|
||||
expect(parsed.title).toBe('Closed ATX Heading');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -354,13 +354,6 @@ describe('MinionQueue: #1737 per-handler default timeout', () => {
|
||||
expect(sub.timeout_ms).toBe(30 * 60 * 1000);
|
||||
});
|
||||
|
||||
test('contextual per-chunk reindex gets the 60-min default', async () => {
|
||||
const job = await queue.add('contextual_reindex_per_chunk', { page_slug: 'large-transcript' }, undefined, {
|
||||
allowProtectedSubmit: true,
|
||||
});
|
||||
expect(job.timeout_ms).toBe(60 * 60 * 1000);
|
||||
});
|
||||
|
||||
test('explicit timeout_ms always wins over the default', async () => {
|
||||
const job = await queue.add('embed-backfill', { sourceId: 'x' }, { timeout_ms: 5000 });
|
||||
expect(job.timeout_ms).toBe(5000);
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
import { describe, expect, it } from 'bun:test';
|
||||
import { readFileSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
|
||||
describe('root OpenClaw plugin manifest', () => {
|
||||
it('declares the id required by OpenClaw plugin installs', () => {
|
||||
const manifest = JSON.parse(readFileSync(join(import.meta.dir, '..', 'openclaw.plugin.json'), 'utf8'));
|
||||
const entrySource = readFileSync(join(import.meta.dir, '..', 'src', 'openclaw-context-engine.ts'), 'utf8');
|
||||
const entryId = entrySource.match(/id:\s*'([^']+)'/)?.[1];
|
||||
|
||||
expect(manifest.id).toBe(entryId);
|
||||
expect(manifest.configSchema).toBeDefined();
|
||||
expect(typeof manifest.configSchema).toBe('object');
|
||||
expect(manifest.contracts?.contextEngines).toContain('gbrain-context');
|
||||
expect(entrySource).toContain('export function register');
|
||||
});
|
||||
});
|
||||
@@ -186,67 +186,11 @@ describe('shouldExclude — orphan filter regression (preserve curation)', () =>
|
||||
expect(shouldExclude('entities/anonymous')).toBe(true);
|
||||
expect(shouldExclude('atoms/fact-123')).toBe(true);
|
||||
expect(shouldExclude('skills/gbrain-operations')).toBe(true);
|
||||
expect(shouldExclude('dreaming/light/2026-07-20')).toBe(true);
|
||||
expect(shouldExclude('daily/2026-07-20')).toBe(true);
|
||||
expect(shouldExclude('agent-openclaw/daily/2026-07-20')).toBe(true);
|
||||
});
|
||||
|
||||
test('workspace convention slugs are excluded', () => {
|
||||
expect(shouldExclude('_brain-conventions')).toBe(true);
|
||||
expect(shouldExclude('_templates/decision')).toBe(true);
|
||||
expect(shouldExclude('extracts/2026-06-30/takes.proposed/round-single')).toBe(true);
|
||||
expect(shouldExclude('2026-07-20')).toBe(true);
|
||||
expect(shouldExclude('2026-07-20-qa-sweep')).toBe(true);
|
||||
expect(shouldExclude('agents/arya/identity')).toBe(true);
|
||||
expect(shouldExclude('agents/arya/memory/dreaming/deep/2026-07-20')).toBe(true);
|
||||
});
|
||||
|
||||
test('regular slugs are NOT excluded', () => {
|
||||
expect(shouldExclude('people/alice')).toBe(false);
|
||||
expect(shouldExclude('companies/acme')).toBe(false);
|
||||
expect(shouldExclude('writing/post-1')).toBe(false);
|
||||
expect(shouldExclude('agents/arya/qa-reports/launch-review')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getHealth orphan_pages uses shared exclusion policy', () => {
|
||||
test('excluded convention islands do not count against health', async () => {
|
||||
await engine.putPage('_templates/decision', {
|
||||
type: 'template', title: 'Decision', compiled_truth: 'template', timeline: '', frontmatter: {},
|
||||
});
|
||||
await engine.putPage('skills/arya/source-check', {
|
||||
type: 'concept', title: 'Skill', compiled_truth: 'skill', timeline: '', frontmatter: {},
|
||||
});
|
||||
await engine.putPage('agents/arya/identity', {
|
||||
type: 'note', title: 'Identity', compiled_truth: 'identity', timeline: '', frontmatter: {},
|
||||
});
|
||||
await engine.putPage('people/alice', {
|
||||
type: 'person', title: 'Alice', compiled_truth: 'real island', timeline: '', frontmatter: {},
|
||||
});
|
||||
|
||||
const health = await engine.getHealth();
|
||||
|
||||
expect(health.orphan_pages).toBe(1);
|
||||
});
|
||||
|
||||
test('per-brain config overrides (orphans.exclude_*) also apply to health', async () => {
|
||||
await engine.putPage('my-private-folder/secret-ref', {
|
||||
type: 'note', title: 'Ref', compiled_truth: 'ref', timeline: '', frontmatter: {},
|
||||
});
|
||||
await engine.putPage('one-off-fixture-page', {
|
||||
type: 'note', title: 'Fixture', compiled_truth: 'fixture', timeline: '', frontmatter: {},
|
||||
});
|
||||
await engine.putPage('people/alice', {
|
||||
type: 'person', title: 'Alice', compiled_truth: 'real island', timeline: '', frontmatter: {},
|
||||
});
|
||||
|
||||
expect((await engine.getHealth()).orphan_pages).toBe(3);
|
||||
|
||||
await engine.setConfig('orphans.exclude_prefixes', 'my-private-folder/');
|
||||
await engine.setConfig('orphans.exclude_slugs', 'one-off-fixture-page');
|
||||
expect((await engine.getHealth()).orphan_pages).toBe(1);
|
||||
|
||||
await engine.unsetConfig('orphans.exclude_prefixes');
|
||||
await engine.unsetConfig('orphans.exclude_slugs');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -66,10 +66,6 @@ describe('shouldExclude', () => {
|
||||
expect(shouldExclude('templates/meeting-note')).toBe(true);
|
||||
});
|
||||
|
||||
test('excludes deny-prefix: _templates/', () => {
|
||||
expect(shouldExclude('_templates/meeting-note')).toBe(true);
|
||||
});
|
||||
|
||||
test('excludes deny-prefix: openclaw/config/', () => {
|
||||
expect(shouldExclude('openclaw/config/agent')).toBe(true);
|
||||
});
|
||||
@@ -90,44 +86,10 @@ describe('shouldExclude', () => {
|
||||
expect(shouldExclude('entities/product-hunt')).toBe(true);
|
||||
});
|
||||
|
||||
test('excludes first-segment: skills, dreaming, and daily', () => {
|
||||
expect(shouldExclude('skills/arya/source-check')).toBe(true);
|
||||
expect(shouldExclude('dreaming/light/2026-07-20')).toBe(true);
|
||||
expect(shouldExclude('daily/2026-07-20')).toBe(true);
|
||||
expect(shouldExclude('agent-openclaw/daily/2026-07-20')).toBe(true);
|
||||
});
|
||||
|
||||
test('excludes root date logs and agent workspace conventions', () => {
|
||||
expect(shouldExclude('_brain-conventions')).toBe(true);
|
||||
expect(shouldExclude('2026-07-20')).toBe(true);
|
||||
expect(shouldExclude('2026-07-20-qa-sweep')).toBe(true);
|
||||
expect(shouldExclude('agents/arya/identity')).toBe(true);
|
||||
expect(shouldExclude('agents/arya/memory/dreaming/deep/2026-07-20')).toBe(true);
|
||||
});
|
||||
|
||||
test('excludes generated extracts', () => {
|
||||
expect(shouldExclude('extracts/2026-06-30/takes.proposed/round-single')).toBe(true);
|
||||
});
|
||||
|
||||
test('brain-specific exclusions come from config overrides, not global defaults', () => {
|
||||
// No baked-in defaults for these:
|
||||
expect(shouldExclude('my-private-folder/some-secret-ref.md')).toBe(false);
|
||||
expect(shouldExclude('one-off-fixture-page')).toBe(false);
|
||||
// The per-brain config plane (orphans.exclude_prefixes / exclude_slugs):
|
||||
const overrides = {
|
||||
excludePrefixes: ['my-private-folder/'],
|
||||
excludeSlugs: ['one-off-fixture-page'],
|
||||
};
|
||||
expect(shouldExclude('my-private-folder/some-secret-ref.md', overrides)).toBe(true);
|
||||
expect(shouldExclude('one-off-fixture-page', overrides)).toBe(true);
|
||||
expect(shouldExclude('people/jane-doe', overrides)).toBe(false);
|
||||
});
|
||||
|
||||
test('does NOT exclude a normal content page', () => {
|
||||
expect(shouldExclude('companies/acme')).toBe(false);
|
||||
expect(shouldExclude('people/jane-doe')).toBe(false);
|
||||
expect(shouldExclude('projects/gbrain')).toBe(false);
|
||||
expect(shouldExclude('agents/arya/qa-reports/launch-review')).toBe(false);
|
||||
});
|
||||
|
||||
test('does NOT exclude a page ending with log-like text that is not /log', () => {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user