diff --git a/lib/multimodal.test.ts b/lib/multimodal.test.ts index d6e2f6e..46ece4b 100644 --- a/lib/multimodal.test.ts +++ b/lib/multimodal.test.ts @@ -165,26 +165,23 @@ describe('buildApiContent — files', () => { // --- audio attachments --- describe('buildApiContent — audio', () => { - it('skips audio binary, keeps text content (transcript from Whisper)', () => { + it('returns plain string for audio-only message (transcript from Whisper)', () => { const result = buildApiContent(msg({ content: 'Hello this is my transcribed message', media: [audioAttachment()], })) - const parts = result as Array<{ type: string; text?: string }> - // Should only have the text part, no audio binary - expect(parts).toHaveLength(1) - expect(parts[0]).toEqual({ type: 'text', text: 'Hello this is my transcribed message' }) + // Audio is skipped and no other non-text parts exist, so return plain string + // to avoid wrapping in ContentPart[] which the gateway may not handle + expect(result).toBe('Hello this is my transcribed message') }) - it('returns text content string when audio is the only attachment and content exists', () => { + it('returns plain string when audio is the only attachment', () => { const result = buildApiContent(msg({ content: 'transcribed words', media: [audioAttachment()], })) - // With audio-only + text content, should return a parts array with just text - const parts = result as Array<{ type: string }> - expect(parts).toHaveLength(1) - expect(parts[0].type).toBe('text') + expect(typeof result).toBe('string') + expect(result).toBe('transcribed words') }) }) diff --git a/lib/multimodal.ts b/lib/multimodal.ts index 0bc202e..b3b6ab7 100644 --- a/lib/multimodal.ts +++ b/lib/multimodal.ts @@ -14,6 +14,7 @@ export function buildApiContent(msg: Message): MessageContent { if (!media || media.length === 0) return msg.content const parts: ContentPart[] = [] + let attachmentAdded = false if (msg.content) { parts.push({ type: 'text', text: msg.content }) @@ -22,6 +23,7 @@ export function buildApiContent(msg: Message): MessageContent { for (const attachment of media) { if (attachment.type === 'image') { parts.push({ type: 'image_url', image_url: { url: attachment.url } }) + attachmentAdded = true } else if (attachment.type === 'file') { const label = attachment.name || 'unknown' const sizeNote = attachment.size ? ` (${Math.round(attachment.size / 1024)} KB)` : '' @@ -32,10 +34,16 @@ export function buildApiContent(msg: Message): MessageContent { } else { parts.push({ type: 'text', text: `[Attached file: ${label}${sizeNote}]` }) } + attachmentAdded = true } // Audio: transcript already in msg.content via Whisper — skip binary } + // If no attachment actually contributed to parts (e.g., audio-only message + // where the transcript is already in msg.content), return a plain string + // so the gateway doesn't receive an unnecessary ContentPart[] wrapper. + if (!attachmentAdded) return msg.content + return parts.length > 0 ? parts : msg.content }