mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-16 18:02:30 +00:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ba6b5d4690 | ||
|
|
827c1619ac |
@@ -3,7 +3,11 @@
|
||||
*
|
||||
* Usage:
|
||||
* gbrain migrate --to supabase [--url <connection_string>]
|
||||
* (--url is persisted to config.json, mode 0600, so the migrated brain
|
||||
* works without env — #1271)
|
||||
* gbrain migrate --to pglite [--path <db_path>]
|
||||
* (an explicit --path destination is bootstrapped with its own
|
||||
* <path>/.gbrain/config.json so GBRAIN_HOME=<path> just works — #1271)
|
||||
* gbrain migrate --to <engine> --force (overwrite non-empty target)
|
||||
*/
|
||||
|
||||
@@ -11,9 +15,9 @@ import { createEngine } from '../core/engine-factory.ts';
|
||||
import { loadConfig, saveConfig, toEngineConfig, gbrainPath, effectiveEnvDatabaseUrl, type GBrainConfig } from '../core/config.ts';
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import type { EngineConfig } from '../core/types.ts';
|
||||
import { writeFileSync, readFileSync, existsSync, unlinkSync } from 'fs';
|
||||
import { writeFileSync, readFileSync, existsSync, unlinkSync, mkdirSync, chmodSync } from 'fs';
|
||||
import { createHash } from 'crypto';
|
||||
import { resolve } from 'path';
|
||||
import { resolve, join } from 'path';
|
||||
import { createProgress } from '../core/progress.ts';
|
||||
import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts';
|
||||
|
||||
@@ -59,6 +63,31 @@ export interface MigrateManifest {
|
||||
started_at: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* #1271 Finding 1: make an explicit `--to pglite --path P` destination usable
|
||||
* as a standalone brain. Writes `P/.gbrain/config.json` (mode 0600, plus a
|
||||
* `*` .gitignore) so `GBRAIN_HOME=P` resolves without a manual `gbrain init`.
|
||||
* Never clobbers an existing config at the destination. Returns the written
|
||||
* config path, or null when skipped.
|
||||
*/
|
||||
export function bootstrapDestinationConfig(dbPath: string): string | null {
|
||||
const abs = resolve(dbPath);
|
||||
const dir = join(abs, '.gbrain');
|
||||
const file = join(dir, 'config.json');
|
||||
if (existsSync(file)) return null;
|
||||
mkdirSync(dir, { recursive: true });
|
||||
const cfg: GBrainConfig = { engine: 'pglite', database_path: abs };
|
||||
writeFileSync(file, JSON.stringify(cfg, null, 2) + '\n', { mode: 0o600 });
|
||||
try { chmodSync(file, 0o600); } catch { /* platform-specific */ }
|
||||
// Same worktree-safety pattern as saveConfig()'s ensureGitignore, scoped
|
||||
// to the destination home. Don't clobber a user-customized .gitignore.
|
||||
const gitignore = join(dir, '.gitignore');
|
||||
if (!existsSync(gitignore)) {
|
||||
writeFileSync(gitignore, '*\n', { mode: 0o600 });
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
export function migrationTargetId(config: EngineConfig): string {
|
||||
const locator = config.engine === 'postgres'
|
||||
? config.database_url ?? ''
|
||||
@@ -352,6 +381,25 @@ export async function runMigrateEngine(sourceEngine: BrainEngine, args: string[]
|
||||
};
|
||||
saveConfig(newConfig);
|
||||
|
||||
// #1271 Finding 2 (by design, but say it out loud): the connection string
|
||||
// is persisted so the migrated brain works without env. Mode 0600.
|
||||
if (opts.targetEngine === 'postgres' && opts.targetUrl) {
|
||||
console.error('Note: the --url connection string (including credentials) is persisted to config.json (mode 0600).');
|
||||
}
|
||||
|
||||
// #1271 Finding 1: an explicit --path destination doubles as a standalone
|
||||
// GBRAIN_HOME. Best-effort — never fail a completed migration over it.
|
||||
if (opts.targetEngine === 'pglite' && opts.targetPath) {
|
||||
try {
|
||||
const written = bootstrapDestinationConfig(opts.targetPath);
|
||||
if (written) {
|
||||
console.log(`Destination bootstrapped: ${written} (usable via GBRAIN_HOME=${resolve(opts.targetPath)})`);
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn(` WARN could not bootstrap destination config: ${e instanceof Error ? e.message : String(e)}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up
|
||||
clearManifest();
|
||||
|
||||
|
||||
@@ -1,100 +1,8 @@
|
||||
import type { Recipe } from '../types.ts';
|
||||
|
||||
/**
|
||||
* MiniMax transport shim (#1977). MiniMax's `/v1/embeddings` endpoint is NOT
|
||||
* OpenAI-compatible at the wire level despite the recipe's
|
||||
* `implementation: 'openai-compatible'`:
|
||||
* - Request: requires `texts` (the AI SDK sends `input`) plus an optional
|
||||
* `type: 'db' | 'query'` asymmetric-retrieval field, and rejects OpenAI's
|
||||
* `encoding_format`.
|
||||
* - Response: returns `{vectors: number[][], total_tokens}` where the AI
|
||||
* SDK's Zod schema expects `{data: [{embedding, index}], usage}`.
|
||||
*
|
||||
* Chat (`/chat/completions`) IS OpenAI-compatible, and this same fetch is
|
||||
* applied to every openai-compatible touchpoint by `applyOpenAICompatConfig`,
|
||||
* so everything outside the embeddings path passes through untouched — and
|
||||
* the response rewrite parses via `resp.clone()` only (never consume the
|
||||
* body of a response we return as-is; the DeepSeek shim rule). Fail-open:
|
||||
* any rewrite error returns the original request/response.
|
||||
*
|
||||
* @internal exported for tests.
|
||||
*/
|
||||
// Cast through `unknown` because Bun's `typeof fetch` carries a `preconnect`
|
||||
// member the arrow function does not implement (matches deepseek.ts).
|
||||
export const minimaxCompatFetch = (async (
|
||||
input: RequestInfo | URL,
|
||||
init?: RequestInit,
|
||||
): Promise<Response> => {
|
||||
const url =
|
||||
typeof input === 'string' ? input : input instanceof URL ? input.href : input.url;
|
||||
const isEmbeddings = url.includes('/embeddings');
|
||||
|
||||
// OUTBOUND (embeddings only): `input` → `texts`, default `type: 'db'`
|
||||
// (the recipe's documented symmetric default — the AI SDK adapter strips
|
||||
// the `type` threaded via providerOptions before it reaches the wire,
|
||||
// same class as #1400), and drop `encoding_format` (not a MiniMax param).
|
||||
if (isEmbeddings && init?.body && typeof init.body === 'string') {
|
||||
try {
|
||||
const parsed = JSON.parse(init.body);
|
||||
if (
|
||||
parsed && typeof parsed === 'object' &&
|
||||
parsed.input !== undefined && parsed.texts === undefined
|
||||
) {
|
||||
parsed.texts = Array.isArray(parsed.input) ? parsed.input : [parsed.input];
|
||||
delete parsed.input;
|
||||
delete parsed.encoding_format;
|
||||
if (parsed.type === undefined) parsed.type = 'db';
|
||||
// Drop Content-Length so fetch recomputes from the new body.
|
||||
const headers = new Headers(init.headers ?? {});
|
||||
headers.delete('content-length');
|
||||
init = { ...init, body: JSON.stringify(parsed), headers };
|
||||
}
|
||||
} catch {
|
||||
// Body wasn't JSON — pass through untouched.
|
||||
}
|
||||
}
|
||||
|
||||
const res = await fetch(input as any, init as any);
|
||||
|
||||
// INBOUND (embeddings only): `{vectors: [[...]]}` → `{data: [{embedding}]}`.
|
||||
// Anything else (chat completions, MiniMax base_resp errors, non-JSON)
|
||||
// returns the ORIGINAL response with its body unread.
|
||||
if (!isEmbeddings || !res.ok) return res;
|
||||
const ctype = res.headers.get('content-type') ?? '';
|
||||
if (!ctype.toLowerCase().includes('application/json')) return res;
|
||||
try {
|
||||
const json = await res.clone().json();
|
||||
if (!json || typeof json !== 'object' || !Array.isArray(json.vectors)) return res;
|
||||
const totalTokens = typeof json.total_tokens === 'number' ? json.total_tokens : 0;
|
||||
const rewritten = {
|
||||
object: 'list',
|
||||
data: (json.vectors as number[][]).map((embedding, index) => ({
|
||||
object: 'embedding',
|
||||
embedding,
|
||||
index,
|
||||
})),
|
||||
model: typeof json.model === 'string' ? json.model : 'embo-01',
|
||||
usage: { prompt_tokens: totalTokens, total_tokens: totalTokens },
|
||||
};
|
||||
// Fresh header set: the body changed, so upstream content-length /
|
||||
// content-encoding would now be wrong.
|
||||
const headers = new Headers(res.headers);
|
||||
headers.delete('content-length');
|
||||
headers.delete('content-encoding');
|
||||
return new Response(JSON.stringify(rewritten), {
|
||||
status: res.status,
|
||||
statusText: res.statusText,
|
||||
headers,
|
||||
});
|
||||
} catch {
|
||||
return res;
|
||||
}
|
||||
}) as unknown as typeof fetch;
|
||||
|
||||
/**
|
||||
* MiniMax (海螺AI). `/embeddings` endpoint at api.minimaxi.com (wire shape
|
||||
* normalized by `minimaxCompatFetch` above); OpenAI-compatible
|
||||
* `/chat/completions`. The flagship embedding model is `embo-01` (1536 dims).
|
||||
* MiniMax (海螺AI). OpenAI-compatible /embeddings endpoint at
|
||||
* api.minimax.chat. The flagship embedding model is `embo-01` (1536 dims).
|
||||
*
|
||||
* MiniMax's API takes an extra `type: 'db' | 'query'` field for asymmetric
|
||||
* retrieval. gbrain currently has no notion of "this is a document vs a
|
||||
@@ -130,25 +38,7 @@ export const minimax: Recipe = {
|
||||
// halving in the gateway catches token-limit errors at runtime.
|
||||
max_batch_tokens: 4096,
|
||||
},
|
||||
chat: {
|
||||
// Model list from MiniMax's /v1/models (#1977). Chat is genuinely
|
||||
// OpenAI-compatible — no wire rewrite needed (minimaxCompatFetch
|
||||
// passes non-embedding requests through untouched).
|
||||
models: [
|
||||
'MiniMax-M3',
|
||||
'MiniMax-M2.7',
|
||||
'MiniMax-M2.7-highspeed',
|
||||
'MiniMax-M2.5',
|
||||
'MiniMax-M2.5-highspeed',
|
||||
'MiniMax-M2.1',
|
||||
'MiniMax-M2.1-highspeed',
|
||||
'MiniMax-M2',
|
||||
],
|
||||
supports_tools: false,
|
||||
supports_subagent_loop: false,
|
||||
},
|
||||
},
|
||||
setup_hint:
|
||||
'Get an API key at https://www.minimaxi.com, then `export MINIMAX_API_KEY=...`',
|
||||
compat: { fetch: minimaxCompatFetch },
|
||||
};
|
||||
|
||||
@@ -6,14 +6,10 @@
|
||||
* - default auth: MINIMAX_API_KEY → "Bearer <key>"; missing → AIConfigError
|
||||
* - dimsProviderOptions threads `type: 'db'` for embo-01 (the asymmetric
|
||||
* retrieval field default) — pins the v1 indexing-only behavior
|
||||
* - #1977: chat touchpoint declared; minimaxCompatFetch rewrites the
|
||||
* embedding wire shape both directions, passes chat through with the
|
||||
* response body UNREAD (the consumed-body regression), fail-open.
|
||||
*/
|
||||
|
||||
import { afterEach, describe, expect, test } from 'bun:test';
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { getRecipe } from '../../src/core/ai/recipes/index.ts';
|
||||
import { minimaxCompatFetch } from '../../src/core/ai/recipes/minimax.ts';
|
||||
import { defaultResolveAuth } from '../../src/core/ai/gateway.ts';
|
||||
import { dimsProviderOptions } from '../../src/core/ai/dims.ts';
|
||||
import { AIConfigError } from '../../src/core/ai/errors.ts';
|
||||
@@ -60,106 +56,4 @@ describe('recipe: minimax', () => {
|
||||
expect(dimsProviderOptions('openai-compatible', 'voyage-3-lite', 512)).toBeUndefined();
|
||||
expect(dimsProviderOptions('openai-compatible', 'nomic-embed-text', 768)).toBeUndefined();
|
||||
});
|
||||
|
||||
test('chat touchpoint declared (#1977) so assertTouchpoint permits gbrain think', () => {
|
||||
const r = getRecipe('minimax')!;
|
||||
expect(r.touchpoints.chat).toBeDefined();
|
||||
expect(r.touchpoints.chat!.models).toContain('MiniMax-M3');
|
||||
expect(r.touchpoints.chat!.supports_tools).toBe(false);
|
||||
expect(r.touchpoints.chat!.supports_subagent_loop).toBe(false);
|
||||
});
|
||||
|
||||
test('recipe ships minimaxCompatFetch via compat.fetch (no env-templated base URL)', () => {
|
||||
const r = getRecipe('minimax')!;
|
||||
expect(r.compat?.fetch).toBe(minimaxCompatFetch);
|
||||
// base_urls config override must keep working: no resolveOpenAICompatConfig.
|
||||
expect(r.resolveOpenAICompatConfig).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('minimaxCompatFetch (#1977)', () => {
|
||||
const realFetch = globalThis.fetch;
|
||||
afterEach(() => { globalThis.fetch = realFetch; });
|
||||
|
||||
function stubFetch(body: unknown, init?: { status?: number; contentType?: string }) {
|
||||
const calls: { url: string; init?: RequestInit }[] = [];
|
||||
globalThis.fetch = (async (input: any, i?: RequestInit) => {
|
||||
calls.push({ url: String(input), init: i });
|
||||
return new Response(typeof body === 'string' ? body : JSON.stringify(body), {
|
||||
status: init?.status ?? 200,
|
||||
headers: { 'content-type': init?.contentType ?? 'application/json' },
|
||||
});
|
||||
}) as unknown as typeof fetch;
|
||||
return calls;
|
||||
}
|
||||
|
||||
const EMBED_URL = 'https://api.minimaxi.com/v1/embeddings';
|
||||
const CHAT_URL = 'https://api.minimaxi.com/v1/chat/completions';
|
||||
|
||||
test('embedding request: input → texts, type:db injected, encoding_format dropped', async () => {
|
||||
const calls = stubFetch({ vectors: [[0.1, 0.2]] });
|
||||
await minimaxCompatFetch(EMBED_URL, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json', 'content-length': '99' },
|
||||
body: JSON.stringify({ model: 'embo-01', input: ['hello', 'world'], encoding_format: 'float' }),
|
||||
});
|
||||
const wire = JSON.parse(calls[0]!.init!.body as string);
|
||||
expect(wire.texts).toEqual(['hello', 'world']);
|
||||
expect(wire.input).toBeUndefined();
|
||||
expect(wire.encoding_format).toBeUndefined();
|
||||
expect(wire.type).toBe('db');
|
||||
expect(new Headers(calls[0]!.init!.headers).get('content-length')).toBeNull();
|
||||
});
|
||||
|
||||
test('embedding response: {vectors} rewritten to OpenAI {data:[{embedding}]}', async () => {
|
||||
stubFetch({ vectors: [[0.1, 0.2], [0.3, 0.4]], total_tokens: 7 });
|
||||
const res = await minimaxCompatFetch(EMBED_URL, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ model: 'embo-01', input: ['a', 'b'] }),
|
||||
});
|
||||
const json = await res.json();
|
||||
expect(json.data).toEqual([
|
||||
{ object: 'embedding', embedding: [0.1, 0.2], index: 0 },
|
||||
{ object: 'embedding', embedding: [0.3, 0.4], index: 1 },
|
||||
]);
|
||||
expect(json.usage).toEqual({ prompt_tokens: 7, total_tokens: 7 });
|
||||
});
|
||||
|
||||
test('chat completion passes through with body UNREAD (consumed-body regression)', async () => {
|
||||
stubFetch({ choices: [{ message: { role: 'assistant', content: 'hi' } }] });
|
||||
const res = await minimaxCompatFetch(CHAT_URL, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ model: 'MiniMax-M3', messages: [{ role: 'user', content: 'say hi' }] }),
|
||||
});
|
||||
expect(res.bodyUsed).toBe(false); // the broken PR #2882 wrapper consumed this
|
||||
const json = await res.json(); // must NOT throw "Body already used"
|
||||
expect(json.choices[0].message.content).toBe('hi');
|
||||
});
|
||||
|
||||
test('chat request body is never rewritten (messages untouched, no type injected)', async () => {
|
||||
const calls = stubFetch({ choices: [] });
|
||||
const body = JSON.stringify({ model: 'MiniMax-M3', messages: [{ role: 'user', content: 'x' }] });
|
||||
await minimaxCompatFetch(CHAT_URL, { method: 'POST', body });
|
||||
expect(calls[0]!.init!.body).toBe(body);
|
||||
});
|
||||
|
||||
test('embedding error response ({vectors:null, base_resp}) passes through re-readable', async () => {
|
||||
stubFetch({ vectors: null, base_resp: { status_code: 2013, status_msg: 'invalid params' } });
|
||||
const res = await minimaxCompatFetch(EMBED_URL, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ model: 'embo-01', input: ['a'] }),
|
||||
});
|
||||
expect(res.bodyUsed).toBe(false);
|
||||
const json = await res.json();
|
||||
expect(json.base_resp.status_code).toBe(2013);
|
||||
});
|
||||
|
||||
test('fail-open: non-JSON response body passes through untouched', async () => {
|
||||
stubFetch('not json', { contentType: 'application/json' });
|
||||
const res = await minimaxCompatFetch(EMBED_URL, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ model: 'embo-01', input: ['a'] }),
|
||||
});
|
||||
expect(await res.text()).toBe('not json');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { mkdtempSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'fs';
|
||||
import { tmpdir } from 'os';
|
||||
import { join, resolve } from 'path';
|
||||
import { bootstrapDestinationConfig } from '../src/commands/migrate-engine.ts';
|
||||
import { loadConfigFileOnly } from '../src/core/config.ts';
|
||||
import { withEnv } from './helpers/with-env.ts';
|
||||
|
||||
describe('migrate --to pglite destination bootstrap (#1271)', () => {
|
||||
test('writes <path>/.gbrain/config.json so GBRAIN_HOME=<path> resolves a brain', async () => {
|
||||
const dest = mkdtempSync(join(tmpdir(), 'gbrain-dest-'));
|
||||
const written = bootstrapDestinationConfig(dest);
|
||||
const file = join(dest, '.gbrain', 'config.json');
|
||||
expect(written).toBe(file);
|
||||
|
||||
const cfg = JSON.parse(readFileSync(file, 'utf-8'));
|
||||
expect(cfg.engine).toBe('pglite');
|
||||
expect(cfg.database_path).toBe(resolve(dest));
|
||||
expect(statSync(file).mode & 0o777).toBe(0o600);
|
||||
// worktree safety: destination home is git-ignored like saveConfig()'s home
|
||||
expect(readFileSync(join(dest, '.gbrain', '.gitignore'), 'utf-8')).toBe('*\n');
|
||||
|
||||
// The exact failure mode from #1271: config resolution under
|
||||
// GBRAIN_HOME=<path> used to find nothing ("No brain configured").
|
||||
await withEnv({ GBRAIN_HOME: dest }, () => {
|
||||
const loaded = loadConfigFileOnly();
|
||||
expect(loaded?.engine).toBe('pglite');
|
||||
expect(loaded?.database_path).toBe(resolve(dest));
|
||||
});
|
||||
});
|
||||
|
||||
test('never clobbers an existing destination config', () => {
|
||||
const dest = mkdtempSync(join(tmpdir(), 'gbrain-dest-'));
|
||||
mkdirSync(join(dest, '.gbrain'), { recursive: true });
|
||||
writeFileSync(join(dest, '.gbrain', 'config.json'), '{"engine":"postgres"}\n');
|
||||
|
||||
expect(bootstrapDestinationConfig(dest)).toBe(null);
|
||||
expect(JSON.parse(readFileSync(join(dest, '.gbrain', 'config.json'), 'utf-8')).engine).toBe('postgres');
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user