From 76f22a0e8e173de5361739b2fed04842abb87fe4 Mon Sep 17 00:00:00 2001 From: paisley <8197966+su8su@users.noreply.github.com> Date: Thu, 30 Jul 2026 18:40:45 +0800 Subject: [PATCH] feat: Enable FTS-only memory search when no OpenAI embedding key is configured. (#1205) --- electron/utils/openclaw-auth.ts | 47 ++++++++--- electron/utils/openclaw-memory-search.ts | 48 ++++++++---- electron/utils/store.ts | 2 + harness/specs/rules/active-config-guards.md | 2 + .../specs/tasks/acp-attachment-open-with.md | 2 +- .../tasks/enable-fts-memory-search-default.md | 38 +++++++++ .../unit/attachment-open-with-native.test.ts | 11 ++- tests/unit/openclaw-auth.test.ts | 78 ++++++++++++++++++- tests/unit/openclaw-memory-search.test.ts | 57 +++++++++++--- 9 files changed, 243 insertions(+), 42 deletions(-) create mode 100644 harness/specs/tasks/enable-fts-memory-search-default.md diff --git a/electron/utils/openclaw-auth.ts b/electron/utils/openclaw-auth.ts index 67d64b34..8e8c428c 100644 --- a/electron/utils/openclaw-auth.ts +++ b/electron/utils/openclaw-auth.ts @@ -29,9 +29,13 @@ import { } from './provider-keys'; import { normalizePiAiModelCost, type PiAiModelCostRates } from '../shared/pi-ai-model-cost'; import { withConfigLock } from './config-mutex'; -import { ensureMemorySearchDisabledDefault, hasUserMemorySearchConfig } from './openclaw-memory-search'; +import { + ensureMemorySearchFtsDefault, + hasUserMemorySearchConfig, + MEMORY_SEARCH_FTS_MIGRATION_VERSION, +} from './openclaw-memory-search'; import { PORTS } from './config'; -import { getSetting } from './store'; +import { getSetting, setSetting } from './store'; import { assertValidApiProtocol, normalizeOpenClawApiProtocol, @@ -2713,16 +2717,31 @@ export async function batchSyncConfigFields(token: string): Promise { } // ── Memory search default ── - // OpenClaw defaults to the openai embedding provider; without a key that - // yields doctor errors and a broken memory_search tool. Seed enabled=false - // only when the user has no memorySearch config anywhere AND no OpenAI key - // (i.e. the default embedding model is unusable). Existing user config is - // never modified. - if (!hasUserMemorySearchConfig(config) - && !(await getProviderApiKeyFromOpenClaw('openai')) - && ensureMemorySearchDisabledDefault(config)) { + // OpenClaw 2026.7.1 supports provider=none as an explicit FTS-only mode. + // Migrate ClawX's exact legacy disabled default once, and otherwise seed + // FTS only when the user has no memorySearch config or OpenAI embedding key. + const memorySearchMigrationVersion = Number( + await getSetting('memorySearchFtsMigrationVersion'), + ) || 0; + const shouldMigrateLegacyMemorySearch = + memorySearchMigrationVersion < MEMORY_SEARCH_FTS_MIGRATION_VERSION; + let memorySearchDefaultResult = shouldMigrateLegacyMemorySearch + && hasUserMemorySearchConfig(config) + ? ensureMemorySearchFtsDefault(config, true) + : 'unchanged'; + + if (memorySearchDefaultResult === 'unchanged' + && !hasUserMemorySearchConfig(config) + && !(await getProviderApiKeyFromOpenClaw('openai'))) { + memorySearchDefaultResult = ensureMemorySearchFtsDefault(config); + } + + if (memorySearchDefaultResult !== 'unchanged') { modified = true; - console.log('[batch-sync] Seeded agents.defaults.memorySearch.enabled=false (no embedding provider configured)'); + console.log( + `[batch-sync] ${memorySearchDefaultResult === 'migrated' ? 'Migrated' : 'Seeded'} ` + + 'agents.defaults.memorySearch to FTS-only mode', + ); } // ── Custom provider contextWindow backfill ── @@ -2736,6 +2755,12 @@ export async function batchSyncConfigFields(token: string): Promise { await writeOpenClawJson(config); console.log('Synced gateway token, browser config, web_fetch SSRF policy, and session idle to openclaw.json'); } + if (shouldMigrateLegacyMemorySearch) { + await setSetting( + 'memorySearchFtsMigrationVersion', + MEMORY_SEARCH_FTS_MIGRATION_VERSION, + ); + } }); } diff --git a/electron/utils/openclaw-memory-search.ts b/electron/utils/openclaw-memory-search.ts index 793145c5..cc66de7e 100644 --- a/electron/utils/openclaw-memory-search.ts +++ b/electron/utils/openclaw-memory-search.ts @@ -1,14 +1,15 @@ /** * Memory search default seeding for openclaw.json. * - * OpenClaw enables semantic memory search by default with the `openai` - * embedding provider, so a user without an OpenAI key gets doctor errors and - * a broken memory_search tool. ClawX seeds `agents.defaults.memorySearch = - * { enabled: false }` at Gateway prelaunch — but only when the user has no - * memorySearch config anywhere (global defaults or per-agent overrides). - * Existing user config is never modified. + * OpenClaw defaults to the `openai` embedding provider. When no OpenAI key is + * available, ClawX explicitly selects OpenClaw's keyword-only FTS provider so + * memory_search remains useful without making an embedding request. */ +export const MEMORY_SEARCH_FTS_MIGRATION_VERSION = 1; + +export type MemorySearchDefaultResult = 'unchanged' | 'seeded' | 'migrated'; + function isRecord(value: unknown): value is Record { return Boolean(value) && typeof value === 'object' && !Array.isArray(value); } @@ -29,18 +30,35 @@ export function hasUserMemorySearchConfig(config: Record): bool } /** - * Seed `agents.defaults.memorySearch = { enabled: false }` when the user has - * no memorySearch config at all. Mutates `config` in place and returns true - * when a change was made. Never touches existing memorySearch objects. + * Seed OpenClaw's explicit FTS-only mode when no memorySearch config exists. + * When requested, also migrate the exact legacy ClawX-managed disabled + * default. Objects with any additional fields and per-agent overrides remain + * user-owned. */ -export function ensureMemorySearchDisabledDefault(config: Record): boolean { - if (hasUserMemorySearchConfig(config)) return false; - +export function ensureMemorySearchFtsDefault( + config: Record, + migrateLegacyDisabledDefault = false, +): MemorySearchDefaultResult { const agents = (isRecord(config.agents) ? config.agents : {}) as Record; - const defaults = (isRecord(agents.defaults) ? agents.defaults : {}) as Record; + const list = Array.isArray(agents.list) ? agents.list : []; + if (list.some((entry) => isRecord(entry) && entry.memorySearch !== undefined)) { + return 'unchanged'; + } - defaults.memorySearch = { enabled: false }; + const defaults = (isRecord(agents.defaults) ? agents.defaults : {}) as Record; + const memorySearch = defaults.memorySearch; + + if (memorySearch !== undefined) { + const isLegacyDisabledDefault = isRecord(memorySearch) + && Object.keys(memorySearch).length === 1 + && memorySearch.enabled === false; + if (!migrateLegacyDisabledDefault || !isLegacyDisabledDefault) { + return 'unchanged'; + } + } + + defaults.memorySearch = { enabled: true, provider: 'none' }; agents.defaults = defaults; config.agents = agents; - return true; + return memorySearch === undefined ? 'seeded' : 'migrated'; } diff --git a/electron/utils/store.ts b/electron/utils/store.ts index 419f79d2..49ded0c3 100644 --- a/electron/utils/store.ts +++ b/electron/utils/store.ts @@ -42,6 +42,7 @@ export interface AppSettings { proxyHttpsServer: string; proxyAllServer: string; proxyBypassRules: string; + memorySearchFtsMigrationVersion: number; // Update updateChannel: 'stable' | 'beta' | 'dev'; @@ -96,6 +97,7 @@ function createDefaultSettings(): AppSettings { proxyHttpsServer: '', proxyAllServer: '', proxyBypassRules: ';localhost;127.0.0.1;::1', + memorySearchFtsMigrationVersion: 0, // Update updateChannel: 'stable', diff --git a/harness/specs/rules/active-config-guards.md b/harness/specs/rules/active-config-guards.md index a655eba8..455063f4 100644 --- a/harness/specs/rules/active-config-guards.md +++ b/harness/specs/rules/active-config-guards.md @@ -15,4 +15,6 @@ Rules: - allowlists and entries must agree about which package owns a single-owner capability - disabling a bundled plugin is required when removing it from an allowlist is not sufficient to stop runtime loading - stale plugin registrations for unconfigured capabilities must be removed during sanitize or recovery paths +- when no embedding credentials or user-owned memory-search config exist, preserve `memory_search` through OpenClaw's explicit FTS-only provider instead of disabling the tool +- migrations may replace only the exact legacy ClawX-managed memory-search default, must run at most once, and must preserve later user opt-outs - tests for config rewrites should assert the final active config, not only intermediate helper output diff --git a/harness/specs/tasks/acp-attachment-open-with.md b/harness/specs/tasks/acp-attachment-open-with.md index 8b622b3b..4b07439e 100644 --- a/harness/specs/tasks/acp-attachment-open-with.md +++ b/harness/specs/tasks/acp-attachment-open-with.md @@ -104,7 +104,7 @@ The authoritative durable requirements are `harness/reference/acp-attachment-acc | Acceptance behavior | Test or durable rule | | --- | --- | | Deterministic handler normalization, presentation-only caching, 256/512/4096 and process/protocol bounds, icon degradation, sanitized environment, static JXA, SHA-256 Windows IDs, Main-owned association input, and post-ready invocation | `tests/unit/attachment-open-with.test.ts`, `attachment-access-safety` | -| Real macOS and Windows native bridge validity, static bundled helper resolution, and packaged-resource identity | `tests/unit/attachment-open-with-native.test.ts`, `.github/workflows/check.yml`, `.github/workflows/release.yml` | +| Real macOS and Windows native bridge validity, static bundled helper resolution, and packaged-resource identity; native CI smoke allows cold PowerShell compilation overhead while mocked service tests enforce the production process timeout | `tests/unit/attachment-open-with-native.test.ts`, `tests/unit/attachment-open-with.test.ts`, `.github/workflows/check.yml`, `.github/workflows/release.yml` | | Per-operation attachment authorization, generation revalidation, forged-handler rejection, scoped reveal, and sensitive diagnostic-payload exclusion | `tests/unit/attachment-access.test.ts`, `attachment-access-safety` | | Shared `AcpFileCard` sibling controls, exact attachment eligibility, lazy/repeated discovery, stale-result rejection, sorting, icon fallback, silent failure, localization, and keyboard interaction | `tests/unit/acp-chat-components.test.tsx`, `ui-i18n-design-tokens` | | End-to-end click routing, typed host requests, platform menu behavior, and failure isolation | `tests/e2e/chat-acp-attachments.spec.ts` | diff --git a/harness/specs/tasks/enable-fts-memory-search-default.md b/harness/specs/tasks/enable-fts-memory-search-default.md new file mode 100644 index 00000000..77684a1e --- /dev/null +++ b/harness/specs/tasks/enable-fts-memory-search-default.md @@ -0,0 +1,38 @@ +--- +id: enable-fts-memory-search-default +title: Enable keyword-only memory search when embeddings are unavailable +scenario: gateway-backend-communication +taskType: runtime-bridge +intent: Keep OpenClaw memory_search usable without an OpenAI embedding key by selecting its explicit FTS-only provider. +touchedAreas: + - electron/utils/openclaw-memory-search.ts + - electron/utils/openclaw-auth.ts + - electron/utils/store.ts + - tests/unit/openclaw-memory-search.test.ts + - tests/unit/openclaw-auth.test.ts + - harness/specs/rules/active-config-guards.md + - harness/specs/tasks/enable-fts-memory-search-default.md +expectedUserBehavior: + - A user without memory-search configuration or an OpenAI embedding key gets keyword-only memory search instead of a disabled memory_search tool. + - A user with an OpenAI embedding key and no memory-search configuration retains OpenClaw's default embedding-backed behavior. + - Existing global or per-agent memory-search configuration remains user-owned. + - The exact legacy ClawX-managed disabled default is migrated to FTS-only once, after which an explicit user opt-out remains respected. +requiredProfiles: + - fast + - comms +requiredTests: + - tests/unit/openclaw-memory-search.test.ts + - tests/unit/openclaw-auth.test.ts +acceptance: + - ClawX seeds agents.defaults.memorySearch with enabled true and provider none only when no memory-search configuration and no OpenAI embedding key exist. + - The exact legacy agents.defaults.memorySearch shape with only enabled false migrates to the FTS-only default at most once. + - Any memory-search object with additional fields and all per-agent overrides remain unchanged. + - The migration marker is persisted outside openclaw.json so OpenClaw schema validation is unaffected. + - Targeted unit tests, type checks, communication regression checks, and harness validation pass. +docs: + required: false +--- + +OpenClaw 2026.7.1 supports deliberate keyword-only recall through +`agents.defaults.memorySearch.provider: "none"`. Use that mode as ClawX's +safe no-key default instead of disabling memory search. diff --git a/tests/unit/attachment-open-with-native.test.ts b/tests/unit/attachment-open-with-native.test.ts index 12aff235..f02a0d3e 100644 --- a/tests/unit/attachment-open-with-native.test.ts +++ b/tests/unit/attachment-open-with-native.test.ts @@ -27,12 +27,15 @@ import { HANDLER_NAME_MAX_LENGTH, NATIVE_PATH_MAX_LENGTH, PROCESS_MAX_BUFFER_BYTES, - PROCESS_TIMEOUT_MS, createAttachmentOpenWithService, type AttachmentOpenWithDependencies, } from '@electron/services/attachment-open-with'; const temporaryDirectories: string[] = []; +// Native smoke tests include cold PowerShell startup, Add-Type compilation, +// and Windows Defender scanning on a fresh CI runner. Production still keeps +// its separate five-second process bound, covered by mocked service tests. +const NATIVE_SMOKE_TIMEOUT_MS = 30_000; function hasControlCharacters(value: string): boolean { return [...value].some((character) => { @@ -117,7 +120,7 @@ async function runWindowsHelper(...args: string[]): Promise<{ const timeout = setTimeout(() => { child.kill(); finish(new Error('native Windows helper timeout')); - }, PROCESS_TIMEOUT_MS); + }, NATIVE_SMOKE_TIMEOUT_MS); const finish = (error?: Error, code: number | null = null) => { if (settled) return; settled = true; @@ -214,7 +217,7 @@ describe('attachment open-with native bridges', () => { const timeout = setTimeout(() => { child.kill(); finish(new Error('native matched prepare-open timeout')); - }, PROCESS_TIMEOUT_MS); + }, NATIVE_SMOKE_TIMEOUT_MS); const finish = (error?: Error, code: number | null = null) => { if (settled) return; settled = true; @@ -266,7 +269,7 @@ describe('attachment open-with native bridges', () => { const nonmatchingResult = await runWindowsHelper('prepare-open', filePath, '0'.repeat(64)); expect(nonmatchingResult.code).not.toBe(0); expect(nonmatchingResult.stdout).not.toContain('{"ready":true}'); - }, PROCESS_TIMEOUT_MS * 4); + }, NATIVE_SMOKE_TIMEOUT_MS * 4); it('resolves and executes the exact helper staged in a packaged resources tree', async () => { const root = await mkdtemp(join(tmpdir(), 'clawx-packaged-open-with-')); diff --git a/tests/unit/openclaw-auth.test.ts b/tests/unit/openclaw-auth.test.ts index e58b5af9..335ced1d 100644 --- a/tests/unit/openclaw-auth.test.ts +++ b/tests/unit/openclaw-auth.test.ts @@ -4,12 +4,13 @@ import { mkdir, readFile, rm, writeFile } from 'fs/promises'; import { join } from 'path'; import { beforeEach, describe, expect, it, vi } from 'vitest'; -const { testHome, testUserData, getSettingMock } = vi.hoisted(() => { +const { testHome, testUserData, getSettingMock, setSettingMock } = vi.hoisted(() => { const suffix = Math.random().toString(36).slice(2); return { testHome: `/tmp/clawx-openclaw-auth-${suffix}`, testUserData: `/tmp/clawx-openclaw-auth-user-data-${suffix}`, getSettingMock: vi.fn(), + setSettingMock: vi.fn(), }; }); @@ -35,6 +36,7 @@ vi.mock('electron', () => ({ vi.mock('@electron/utils/store', () => ({ getSetting: getSettingMock, + setSetting: setSettingMock, })); vi.mock('@electron/utils/paths', async () => { @@ -2292,10 +2294,14 @@ describe('batchSyncConfigFields', () => { beforeEach(async () => { vi.resetModules(); vi.restoreAllMocks(); + getSettingMock.mockReset(); + setSettingMock.mockReset(); getSettingMock.mockImplementation(async (key: string) => { if (key === 'gatewayPort') return 18789; + if (key === 'memorySearchFtsMigrationVersion') return 0; return undefined; }); + setSettingMock.mockResolvedValue(undefined); await rm(testHome, { recursive: true, force: true }); await rm(testUserData, { recursive: true, force: true }); }); @@ -2391,6 +2397,76 @@ describe('batchSyncConfigFields', () => { expect(defaults.compaction).toEqual({ mode: 'default', reserveTokensFloor: 30000 }); }); + it('seeds FTS-only memory search when no OpenAI embedding key exists', async () => { + await writeOpenClawJson({ gateway: { auth: { mode: 'token', token: 'old' } } }); + + const { batchSyncConfigFields } = await import('@electron/utils/openclaw-auth'); + await batchSyncConfigFields('new-token'); + + const config = await readOpenClawJson(); + const defaults = ((config.agents as Record).defaults as Record); + expect(defaults.memorySearch).toEqual({ enabled: true, provider: 'none' }); + expect(setSettingMock).toHaveBeenCalledWith('memorySearchFtsMigrationVersion', 1); + }); + + it('keeps OpenClaw defaults when an OpenAI embedding key exists', async () => { + await writeOpenClawJson({ gateway: { auth: { mode: 'token', token: 'old' } } }); + await writeAgentAuthProfiles('main', { + version: 1, + profiles: { + 'openai:default': { + type: 'api_key', + provider: 'openai', + key: 'sk-openai-test', + }, + }, + order: { openai: ['openai:default'] }, + }); + + const { batchSyncConfigFields } = await import('@electron/utils/openclaw-auth'); + await batchSyncConfigFields('new-token'); + + const config = await readOpenClawJson(); + const defaults = ((config.agents as Record).defaults as Record); + expect(defaults.memorySearch).toBeUndefined(); + expect(setSettingMock).toHaveBeenCalledWith('memorySearchFtsMigrationVersion', 1); + }); + + it('migrates the exact legacy disabled memory-search default once', async () => { + await writeOpenClawJson({ + gateway: { auth: { mode: 'token', token: 'old' } }, + agents: { defaults: { memorySearch: { enabled: false } } }, + }); + + const { batchSyncConfigFields } = await import('@electron/utils/openclaw-auth'); + await batchSyncConfigFields('new-token'); + + const config = await readOpenClawJson(); + const defaults = ((config.agents as Record).defaults as Record); + expect(defaults.memorySearch).toEqual({ enabled: true, provider: 'none' }); + expect(setSettingMock).toHaveBeenCalledWith('memorySearchFtsMigrationVersion', 1); + }); + + it('respects an explicit memory-search opt-out after the migration completed', async () => { + getSettingMock.mockImplementation(async (key: string) => { + if (key === 'gatewayPort') return 18789; + if (key === 'memorySearchFtsMigrationVersion') return 1; + return undefined; + }); + await writeOpenClawJson({ + gateway: { auth: { mode: 'token', token: 'old' } }, + agents: { defaults: { memorySearch: { enabled: false } } }, + }); + + const { batchSyncConfigFields } = await import('@electron/utils/openclaw-auth'); + await batchSyncConfigFields('new-token'); + + const config = await readOpenClawJson(); + const defaults = ((config.agents as Record).defaults as Record); + expect(defaults.memorySearch).toEqual({ enabled: false }); + expect(setSettingMock).not.toHaveBeenCalled(); + }); + it('backfills contextWindow on custom provider model rows that lack one', async () => { await writeOpenClawJson({ gateway: { auth: { mode: 'token', token: 'old' } }, diff --git a/tests/unit/openclaw-memory-search.test.ts b/tests/unit/openclaw-memory-search.test.ts index 46fb04fa..988ad2a9 100644 --- a/tests/unit/openclaw-memory-search.test.ts +++ b/tests/unit/openclaw-memory-search.test.ts @@ -1,20 +1,20 @@ import { describe, expect, it } from 'vitest'; import { - ensureMemorySearchDisabledDefault, + ensureMemorySearchFtsDefault, hasUserMemorySearchConfig, } from '@electron/utils/openclaw-memory-search'; describe('openclaw-memory-search', () => { - it('seeds memorySearch.enabled=false when no memorySearch config exists', () => { + it('seeds FTS-only memory search when no memorySearch config exists', () => { const config: Record = { agents: { defaults: { model: { primary: 'custom-customfc/gpt-5.5' } } }, }; - expect(ensureMemorySearchDisabledDefault(config)).toBe(true); + expect(ensureMemorySearchFtsDefault(config)).toBe('seeded'); expect(config).toEqual({ agents: { defaults: { model: { primary: 'custom-customfc/gpt-5.5' }, - memorySearch: { enabled: false }, + memorySearch: { enabled: true, provider: 'none' }, }, }, }); @@ -22,9 +22,11 @@ describe('openclaw-memory-search', () => { it('seeds on a completely empty config', () => { const config: Record = {}; - expect(ensureMemorySearchDisabledDefault(config)).toBe(true); + expect(ensureMemorySearchFtsDefault(config)).toBe('seeded'); expect(config).toEqual({ - agents: { defaults: { memorySearch: { enabled: false } } }, + agents: { + defaults: { memorySearch: { enabled: true, provider: 'none' } }, + }, }); }); @@ -42,7 +44,7 @@ describe('openclaw-memory-search', () => { }, }; const before = JSON.parse(JSON.stringify(config)); - expect(ensureMemorySearchDisabledDefault(config)).toBe(false); + expect(ensureMemorySearchFtsDefault(config, true)).toBe('unchanged'); expect(config).toEqual(before); }); @@ -57,7 +59,7 @@ describe('openclaw-memory-search', () => { }, }; const before = JSON.parse(JSON.stringify(config)); - expect(ensureMemorySearchDisabledDefault(config)).toBe(false); + expect(ensureMemorySearchFtsDefault(config, true)).toBe('unchanged'); expect(config).toEqual(before); }); @@ -66,7 +68,7 @@ describe('openclaw-memory-search', () => { agents: { defaults: { memorySearch: { enabled: true } } }, }; expect(hasUserMemorySearchConfig(config)).toBe(true); - expect(ensureMemorySearchDisabledDefault(config)).toBe(false); + expect(ensureMemorySearchFtsDefault(config, true)).toBe('unchanged'); expect((config.agents as { defaults: { memorySearch: { enabled: boolean } } }).defaults.memorySearch.enabled).toBe(true); }); @@ -75,6 +77,41 @@ describe('openclaw-memory-search', () => { agents: { defaults: { memorySearch: {} } }, }; expect(hasUserMemorySearchConfig(config)).toBe(true); - expect(ensureMemorySearchDisabledDefault(config)).toBe(false); + expect(ensureMemorySearchFtsDefault(config, true)).toBe('unchanged'); + }); + + it('migrates only the exact legacy disabled default when requested', () => { + const config: Record = { + agents: { defaults: { memorySearch: { enabled: false } } }, + }; + expect(ensureMemorySearchFtsDefault(config, true)).toBe('migrated'); + expect(config).toEqual({ + agents: { + defaults: { memorySearch: { enabled: true, provider: 'none' } }, + }, + }); + }); + + it('preserves the exact legacy disabled shape when migration is complete', () => { + const config: Record = { + agents: { defaults: { memorySearch: { enabled: false } } }, + }; + expect(ensureMemorySearchFtsDefault(config)).toBe('unchanged'); + expect(config).toEqual({ + agents: { defaults: { memorySearch: { enabled: false } } }, + }); + }); + + it('preserves disabled configs with additional user-owned fields', () => { + const config: Record = { + agents: { + defaults: { + memorySearch: { enabled: false, provider: 'none' }, + }, + }, + }; + const before = JSON.parse(JSON.stringify(config)); + expect(ensureMemorySearchFtsDefault(config, true)).toBe('unchanged'); + expect(config).toEqual(before); }); });