diff --git a/src/core/search/hybrid.ts b/src/core/search/hybrid.ts index ecbba2b0d..d6d7b1590 100644 --- a/src/core/search/hybrid.ts +++ b/src/core/search/hybrid.ts @@ -1132,7 +1132,29 @@ export async function hybridSearch( // provider (Voyage, ZE) works fine. const { isAvailable } = await import('../ai/gateway.ts'); const providerProbe = resolvedCol.embeddingModel || undefined; - if (!isAvailable('embedding', providerProbe)) { + // Image/both/unified routing embeds via the MULTIMODAL provider, not the + // text provider — so a multimodal-only install (text provider absent) must + // still reach the multimodal branch below. Probe the multimodal provider + // explicitly and only short-circuit when neither the text provider nor (for + // multimodal-routed queries) the multimodal provider is reachable. Without + // this guard a multimodal-only install would fall to keyword-only here and + // never run the image/unified vector path. + const multimodalProviderProbe = + cfgForColumn?.embedding_multimodal_model ?? 'voyage:voyage-multimodal-3'; + // The LLM intent tie-break (below) can escalate a regex-'text' query to + // 'image'/'both'; account for that possibility so an ambiguous query on a + // multimodal-only install still reaches the multimodal branch. + const mayEscalateToMultimodal = + earlyModality === 'text' && + resolvedMode.cross_modal_llm_intent && + isAmbiguousModalityQuery(query); + const willTryMultimodal = + (resolvedMode.unified_multimodal === true || + earlyModality === 'image' || + earlyModality === 'both' || + mayEscalateToMultimodal) && + isAvailable('embedding', multimodalProviderProbe); + if (!isAvailable('embedding', providerProbe) && !willTryMultimodal) { // v0.43 — fuse the relational arm with keyword so typed-edge answers // survive on the no-embedding-provider path (the relational win is most // valuable exactly when vector is unavailable). The title arm fuses here @@ -1267,7 +1289,10 @@ export async function hybridSearch( if (unifiedRouting) { try { const { isAvailable: aiIsAvailable, embedQueryMultimodal } = await import('../ai/gateway.ts'); - if (!aiIsAvailable('embedding')) { + // Probe the MULTIMODAL provider, not the global default — on a + // multimodal-only install the global default (text) is absent but the + // multimodal provider is configured, and unified routing embeds via it. + if (!aiIsAvailable('embedding', multimodalProviderProbe)) { throw new Error('gateway not configured for embedding — unified multimodal would also fail'); } const unifiedEmbedding = await embedQueryMultimodal(query); @@ -1302,7 +1327,10 @@ export async function hybridSearch( // OR the embed throws, log a structured warning and fall through to text. try { const { isAvailable: aiIsAvailable, embedQueryMultimodal } = await import('../ai/gateway.ts'); - if (!aiIsAvailable('embedding')) { + // Probe the MULTIMODAL provider, not the global default — the image side + // embeds via the multimodal model, which may be configured even when the + // text/global-default embedding provider is absent (multimodal-only). + if (!aiIsAvailable('embedding', multimodalProviderProbe)) { throw new Error('gateway not configured for embedding — multimodal would also fail'); } const imageEmbedding = await embedQueryMultimodal(query); diff --git a/test/search-multimodal-no-embed.serial.test.ts b/test/search-multimodal-no-embed.serial.test.ts new file mode 100644 index 000000000..ae031c36f --- /dev/null +++ b/test/search-multimodal-no-embed.serial.test.ts @@ -0,0 +1,120 @@ +// Regression: no-embedding-provider early-return must be multimodal-aware. +// +// On a multimodal-only install (text embedding provider ABSENT, a multimodal +// provider such as Voyage multimodal-3 PRESENT), hybridSearch's +// no-embedding-provider short-circuit used to probe ONLY the text column's +// provider. Since that provider is unreachable, search returned to the +// keyword-only path (vector_enabled:false) BEFORE the image/unified vector +// routing below ever ran — so image and unified queries silently degraded to +// keyword search even though a usable multimodal vector path existed. +// +// The fix adds a `willTryMultimodal` guard that also probes the multimodal +// provider so the early-return does not fire when multimodal vectoring is +// still possible. These tests assert that the multimodal (Voyage) embedding +// endpoint is actually reached on a text-provider-absent install. + +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from 'bun:test'; +import { PGLiteEngine } from '../src/core/pglite-engine.ts'; +import { resetPgliteState } from './helpers/reset-pglite.ts'; +import { configureGateway, resetGateway } from '../src/core/ai/gateway.ts'; +import { hybridSearch } from '../src/core/search/hybrid.ts'; + +let engine: PGLiteEngine; +let fetchHandler: ((url: string, init: RequestInit) => Promise) | null = null; +const origFetch = globalThis.fetch; + +beforeAll(async () => { + engine = new PGLiteEngine(); + await engine.connect({}); + await engine.initSchema(); +}); + +afterAll(async () => { + await engine.disconnect(); +}); + +beforeEach(async () => { + await resetPgliteState(engine); + globalThis.fetch = (async (url: string | URL | Request, init?: RequestInit) => { + if (!fetchHandler) throw new Error('no fetch handler'); + return fetchHandler(typeof url === 'string' ? url : url.toString(), init ?? {}); + }) as typeof fetch; + + // Multimodal-only install: a text embedding model is *configured* but its + // required auth env (OPENAI_API_KEY) is ABSENT, so the text provider is + // unreachable. The multimodal provider (Voyage) IS reachable (VOYAGE_API_KEY + // present). This is exactly the install shape the fix targets. + configureGateway({ + embedding_model: 'openai:text-embedding-3-large', + embedding_dimensions: 1536, + embedding_multimodal_model: 'voyage:voyage-multimodal-3', + env: { VOYAGE_API_KEY: 'test' }, + }); +}); + +afterEach(() => { + globalThis.fetch = origFetch; + resetGateway(); + fetchHandler = null; +}); + +describe('multimodal-only install: no-embedding early-return is multimodal-aware', () => { + test('image query still reaches the multimodal vector path (does not short-circuit to keyword)', async () => { + let voyageCalled = 0; + let openaiCalled = 0; + fetchHandler = async (url) => { + if (url.includes('multimodalembeddings')) { + voyageCalled++; + return new Response(JSON.stringify({ + data: [{ embedding: Array.from({ length: 1024 }, () => 0.1), index: 0 }], + }), { status: 200 }); + } + if (url.includes('api.openai.com') && url.includes('embeddings')) { + openaiCalled++; + } + return new Response(JSON.stringify({ + data: [{ embedding: Array.from({ length: 1536 }, () => 0.1), index: 0 }], + }), { status: 200 }); + }; + + const results = await hybridSearch(engine, 'a photo of a red bicycle', { + limit: 5, + crossModal: 'image', + }); + + // Pre-fix: the text-provider probe failed → early-return → Voyage never + // called. Post-fix: the image branch runs and embeds via the multimodal + // (Voyage) provider. + expect(voyageCalled).toBeGreaterThanOrEqual(1); + // The unreachable text provider must never have been dialed. + expect(openaiCalled).toBe(0); + expect(Array.isArray(results)).toBe(true); + }); + + test('unified_multimodal routing reaches the multimodal vector path on a text-provider-absent install', async () => { + await engine.setConfig('search.unified_multimodal', 'true'); + let voyageCalled = 0; + let openaiCalled = 0; + fetchHandler = async (url) => { + if (url.includes('multimodalembeddings')) { + voyageCalled++; + return new Response(JSON.stringify({ + data: [{ embedding: Array.from({ length: 1024 }, () => 0.1), index: 0 }], + }), { status: 200 }); + } + if (url.includes('api.openai.com') && url.includes('embeddings')) { + openaiCalled++; + } + return new Response(JSON.stringify({ + data: [{ embedding: Array.from({ length: 1536 }, () => 0.1), index: 0 }], + }), { status: 200 }); + }; + + await hybridSearch(engine, 'totally text query', { limit: 5 }); + + // Unified routing forces the multimodal endpoint even for a text-shaped + // query; pre-fix the early-return fired first and Voyage was never called. + expect(voyageCalled).toBeGreaterThanOrEqual(1); + expect(openaiCalled).toBe(0); + }); +});