From c96b2336c620436a4439a04ce11cf7aac83eaa48 Mon Sep 17 00:00:00 2001 From: paisley <8197966+su8su@users.noreply.github.com> Date: Mon, 20 Jul 2026 10:50:41 +0800 Subject: [PATCH] fix(chat): preserve image generation thinking state across sessions (#1180) --- .../acp-image-generation-compatibility.md | 8 ++ src/pages/Chat/ChatInput.tsx | 29 ++++- src/pages/Chat/index.tsx | 4 + src/stores/acp-chat-session.ts | 109 ++++++++++++++++-- tests/e2e/chat-run-state-events.spec.ts | 10 ++ tests/unit/acp-chat-store.test.ts | 24 +++- tests/unit/chat-acp-page.test.tsx | 1 + tests/unit/chat-input.test.tsx | 32 +++++ 8 files changed, 208 insertions(+), 9 deletions(-) diff --git a/harness/specs/tasks/acp-image-generation-compatibility.md b/harness/specs/tasks/acp-image-generation-compatibility.md index d1068f6d..05e493dd 100644 --- a/harness/specs/tasks/acp-image-generation-compatibility.md +++ b/harness/specs/tasks/acp-image-generation-compatibility.md @@ -14,9 +14,13 @@ touchedAreas: - src/lib/acp/reducer.ts - src/lib/acp/timeline-types.ts - src/stores/acp-chat-session.ts + - src/pages/Chat/index.tsx + - src/pages/Chat/ChatInput.tsx - tests/unit/acp-image-generation-compat.test.ts - tests/unit/acp-reducer.test.ts - tests/unit/acp-chat-store.test.ts + - tests/unit/chat-acp-page.test.tsx + - tests/unit/chat-input.test.tsx - tests/e2e/chat-run-state-events.spec.ts - shared/i18n/locales/en/chat.json - shared/i18n/locales/zh/chat.json @@ -27,6 +31,7 @@ touchedAreas: - README.ja-JP.md expectedUserBehavior: - ACP Chat first shows the image_generate background task start tool result. + - After the normal thinking state ends, the composer shows a distinct image-generation indicator until the generated image or a failure reply is rendered, including after switching away from and back to the conversation; users may edit a draft while another send is prevented. - When OpenClaw later exposes a trusted internal-UI source reply through ACP or Gateway host events, ClawX preserves its exact user-facing text instead of replacing it with a generic caption. - Successful replies include the hydrated image preview, while text-only generation failures remain visible as assistant replies. - Arbitrary local paths and generic MEDIA: prose without approved image-generation context are not rendered as images. @@ -57,6 +62,9 @@ acceptance: - Internal-UI sourceReply text is authoritative for both successful media replies and text-only failure replies. - ClawX hydrates previews through hostApi.media.thumbnails before rendering images. - Duplicate completion records do not create duplicate assistant image replies. + - Live background image generation shows its dedicated generating label until its success or failure completion is projected, without changing the existing sending/thinking behavior. + - Switching conversations preserves each live image-generation pending state and restores its indicator on return. + - Completion evidence received while the image conversation is inactive is deferred to that conversation, and a second prompt cannot be sent until the image task settles. - Stale preview resolution does not append to a different active session or generation. docs: required: true diff --git a/src/pages/Chat/ChatInput.tsx b/src/pages/Chat/ChatInput.tsx index 9c2a3875..4dda91f7 100644 --- a/src/pages/Chat/ChatInput.tsx +++ b/src/pages/Chat/ChatInput.tsx @@ -47,6 +47,7 @@ interface ChatInputProps { onStop?: () => void; disabled?: boolean; sending?: boolean; + imageGenerating?: boolean; workspaceLabel?: string; workspacePath?: string; workspaceReadOnly?: boolean; @@ -201,6 +202,7 @@ export function ChatInput({ onStop, disabled = false, sending = false, + imageGenerating = false, workspaceLabel, workspacePath, workspaceReadOnly = false, @@ -678,7 +680,11 @@ export function ChatInput({ const allReady = attachments.length === 0 || attachments.every(a => a.status === 'ready'); const hasFailedAttachments = attachments.some((a) => a.status === 'error'); - const canSend = (input.trim() || attachments.length > 0) && allReady && !inputDisabled && !sending; + const canSend = (input.trim() || attachments.length > 0) + && allReady + && !inputDisabled + && !sending + && !imageGenerating; const canStop = sending && !inputDisabled && !!onStop; const handleSend = useCallback(async () => { @@ -887,6 +893,27 @@ export function ChatInput({ )} + {!sending && imageGenerating && ( +
+
+ )} + {/* Attachment Previews */} {attachments.length > 0 && (
diff --git a/src/pages/Chat/index.tsx b/src/pages/Chat/index.tsx index b5f54013..96f6d4bc 100644 --- a/src/pages/Chat/index.tsx +++ b/src/pages/Chat/index.tsx @@ -164,6 +164,9 @@ export function Chat() { const acpTimeline = useAcpChatSessionStore((s) => s.timeline); const acpLoading = useAcpChatSessionStore((s) => s.loading); const acpSending = useAcpChatSessionStore((s) => s.sending); + const imageGenerationPending = useAcpChatSessionStore( + (s) => Boolean(s.pendingImageGenerationTaskIds?.length), + ); const acpCancelling = useAcpChatSessionStore((s) => s.cancelling); const acpError = useAcpChatSessionStore((s) => s.error); const acpActiveSessionKey = useAcpChatSessionStore((s) => s.activeSessionKey); @@ -443,6 +446,7 @@ export function Chat() { onStop={() => void cancelAcp()} disabled={acpLoading || acpCancelling || !cwd} sending={composerBusy} + imageGenerating={imageGenerationPending} workspaceLabel={workspaceLabel} workspacePath={cwd} workspaceReadOnly={effectiveWorkspace.readOnly} diff --git a/src/stores/acp-chat-session.ts b/src/stores/acp-chat-session.ts index 8b62ccf2..541b6f56 100644 --- a/src/stores/acp-chat-session.ts +++ b/src/stores/acp-chat-session.ts @@ -71,8 +71,14 @@ type LiveSessionSnapshot = { workspaceRoot: string | null; cwd: string | null; generation: number; + sending: boolean; + pendingImageGenerationTaskIds: string[]; timeline: AcpTimelineSnapshot; deferredImageUpdates: Array<{ key: string; event: AcpSessionUpdateEnvelope }>; + deferredImageCompletions: Array<{ + key: string; + evidence: ImageGenerationCompletionEvidence; + }>; }; const liveSessionSnapshots = new Map(); let loadRequestSeq = 0; @@ -130,6 +136,7 @@ export type AcpChatSessionState = { generation: number; loading: boolean; sending: boolean; + pendingImageGenerationTaskIds: string[]; cancelling: boolean; error: string | null; timeline: AcpTimelineSnapshot; @@ -193,15 +200,21 @@ function applyPermissionRequestToTimeline( } function captureLiveSession(state: AcpChatSessionState): void { - if (!state.sending || !state.activeSessionKey) return; + if ( + (!state.sending && state.pendingImageGenerationTaskIds.length === 0) + || !state.activeSessionKey + ) return; const existing = liveSessionSnapshots.get(state.activeSessionKey); liveSessionSnapshots.set(state.activeSessionKey, { sessionKey: state.activeSessionKey, workspaceRoot: state.workspaceRoot, cwd: state.cwd, generation: state.generation, + sending: state.sending, + pendingImageGenerationTaskIds: state.pendingImageGenerationTaskIds, timeline: state.timeline, deferredImageUpdates: existing?.deferredImageUpdates ?? [], + deferredImageCompletions: existing?.deferredImageCompletions ?? [], }); } @@ -331,6 +344,28 @@ function imageGenerationTaskIdFromSessionKey(sessionKey: string | undefined): st return match?.[1] ?? null; } +function deferInactiveImageGenerationCompletion( + activeSessionKey: string | null, + evidence: ImageGenerationCompletionEvidence, +): boolean { + const taskId = evidence.taskId ?? imageGenerationTaskIdFromSessionKey(evidence.sessionKey); + if (!taskId) return false; + for (const [sessionKey, snapshot] of liveSessionSnapshots) { + if ( + sessionKey === activeSessionKey + || !snapshot.pendingImageGenerationTaskIds.includes(taskId) + ) continue; + const key = imageGenerationEvidenceKey(evidence); + const deferredImageCompletions = snapshot.deferredImageCompletions.filter( + (entry) => entry.key !== key, + ); + deferredImageCompletions.push({ key, evidence }); + liveSessionSnapshots.set(sessionKey, { ...snapshot, deferredImageCompletions }); + return true; + } + return false; +} + function resolveImageGenerationProjectionSession( state: AcpChatSessionState, evidence: ImageGenerationCompletionEvidence, @@ -918,6 +953,7 @@ export const useAcpChatSessionStore = create((set, get) => generation: 0, loading: false, sending: false, + pendingImageGenerationTaskIds: [], cancelling: false, error: null, timeline: createEmptyAcpTimeline(EMPTY_SESSION_ID, 0), @@ -936,6 +972,7 @@ export const useAcpChatSessionStore = create((set, get) => generation, loading: false, sending: false, + pendingImageGenerationTaskIds: [], cancelling: false, error: null, timeline: createEmptyAcpTimeline(input.sessionKey, generation), @@ -950,14 +987,17 @@ export const useAcpChatSessionStore = create((set, get) => const generation = get().generation; const liveSnapshot = liveSessionSnapshots.get(input.sessionKey); invalidateTranscriptSupplement(); - resetImageGenerationCompatSession(input.sessionKey); + if (!liveSnapshot?.pendingImageGenerationTaskIds.length) { + resetImageGenerationCompatSession(input.sessionKey); + } set({ activeSessionKey: input.sessionKey, workspaceRoot: input.workspaceRoot, cwd: input.cwd, generation, loading: true, - sending: !!liveSnapshot, + sending: liveSnapshot?.sending ?? false, + pendingImageGenerationTaskIds: liveSnapshot?.pendingImageGenerationTaskIds ?? [], cancelling: false, error: null, timeline: liveSnapshot?.timeline ?? createEmptyAcpTimeline(input.sessionKey, generation), @@ -1018,6 +1058,16 @@ export const useAcpChatSessionStore = create((set, get) => const currentResumedSnapshot = result.resumedActivePrompt ? liveSessionSnapshots.get(input.sessionKey) : undefined; + const currentBackgroundSnapshot = !result.resumedActivePrompt + ? liveSessionSnapshots.get(input.sessionKey) + : undefined; + const restorableBackgroundSnapshot = currentBackgroundSnapshot + && currentBackgroundSnapshot.pendingImageGenerationTaskIds.length > 0 + ? currentBackgroundSnapshot + : undefined; + if (currentBackgroundSnapshot && !restorableBackgroundSnapshot) { + resetImageGenerationCompatSession(input.sessionKey); + } let timeline = currentResumedSnapshot?.generation === generation ? currentResumedSnapshot.timeline : createEmptyAcpTimeline(input.sessionKey, generation); @@ -1030,7 +1080,11 @@ export const useAcpChatSessionStore = create((set, get) => ); set({ loading: false, - sending: !!currentResumedSnapshot, + sending: currentResumedSnapshot?.sending ?? false, + pendingImageGenerationTaskIds: + currentResumedSnapshot?.pendingImageGenerationTaskIds + ?? restorableBackgroundSnapshot?.pendingImageGenerationTaskIds + ?? [], error: null, generation, timeline, @@ -1040,16 +1094,21 @@ export const useAcpChatSessionStore = create((set, get) => ...currentResumedSnapshot, timeline, deferredImageUpdates: [], + deferredImageCompletions: [], }); } else { liveSessionSnapshots.delete(input.sessionKey); } resolvePendingAttachments(input.sessionKey, generation, pendingAttachments); - for (const { event } of currentResumedSnapshot?.deferredImageUpdates ?? []) { + const restoredSnapshot = currentResumedSnapshot ?? restorableBackgroundSnapshot; + for (const { event } of restoredSnapshot?.deferredImageUpdates ?? []) { get().recordImageGenerationStart(event); const evidence = extractImageGenerationCompletionFromAcpEnvelope(event); if (evidence) void get().projectImageGenerationCompletion(evidence); } + for (const { evidence } of restoredSnapshot?.deferredImageCompletions ?? []) { + void get().projectImageGenerationCompletion(evidence); + } for (const event of sessionUpdates) { get().recordImageGenerationStart(event); const evidence = extractImageGenerationCompletionFromAcpEnvelope(event); @@ -1250,6 +1309,18 @@ export const useAcpChatSessionStore = create((set, get) => session.taskStartedAt = Date.now(); session.taskIds.add(start.taskId); recordImageGenerationStartAnchor(session, start, false); + set((current) => ( + current.activeSessionKey === start.sessionKey + && current.generation === event.generation + && !current.pendingImageGenerationTaskIds.includes(start.taskId) + ? { + pendingImageGenerationTaskIds: [ + ...current.pendingImageGenerationTaskIds, + start.taskId, + ], + } + : {} + )); const operation = activeTranscriptSupplement; if ( operation?.liveUserMessageId @@ -1328,6 +1399,14 @@ export const useAcpChatSessionStore = create((set, get) => const correlatedTaskId = evidence.taskId ?? imageGenerationTaskIdFromSessionKey(evidence.sessionKey) ?? (usesReplayImageGenerationContext(evidence) ? compat.lastReplayTaskId : compat.lastTaskId); + const settlePendingTask = (current: AcpChatSessionState): string[] => { + if (!correlatedTaskId) { + return usesReplayImageGenerationContext(evidence) + ? current.pendingImageGenerationTaskIds + : []; + } + return current.pendingImageGenerationTaskIds.filter((taskId) => taskId !== correlatedTaskId); + }; const key = imageGenerationEvidenceKey({ ...evidence, sessionKey, @@ -1354,6 +1433,9 @@ export const useAcpChatSessionStore = create((set, get) => details: projectionTraceDetails(evidence), }); if (compat.delivered.has(key)) { + set((current) => ({ + pendingImageGenerationTaskIds: settlePendingTask(current), + })); stopLiveTranscriptSupplementRetry(sessionKey, generation, correlatedTaskId); } return; @@ -1504,8 +1586,12 @@ export const useAcpChatSessionStore = create((set, get) => captions?.set(existingKey, currentCaption); set((current) => ({ timeline: replaceSyntheticImageCaptionAtItem(current.timeline, duplicateItemId, currentCaption.text), + pendingImageGenerationTaskIds: settlePendingTask(current), })); } + set((current) => ({ + pendingImageGenerationTaskIds: settlePendingTask(current), + })); if (missingCount === 0) commitDelivery(sessionKey, key, reservationOwner); else releaseDelivery(sessionKey, key, reservationOwner); recordProjectionTrace({ @@ -1529,6 +1615,7 @@ export const useAcpChatSessionStore = create((set, get) => parts, afterItemId, }), + pendingImageGenerationTaskIds: settlePendingTask(current), }; }); if (missingCount === 0) commitDelivery(sessionKey, key, reservationOwner); @@ -1632,10 +1719,18 @@ export function ensureAcpChatSubscriptions(): void { }); hostEvents.onGatewayChatMessage((event) => { const evidence = extractImageGenerationCompletionFromGatewayChatMessage(event); - if (evidence) void useAcpChatSessionStore.getState().projectImageGenerationCompletion(evidence); + const state = useAcpChatSessionStore.getState(); + if ( + evidence + && !deferInactiveImageGenerationCompletion(state.activeSessionKey, evidence) + ) void state.projectImageGenerationCompletion(evidence); }); hostEvents.onChatRuntimeEvent((event) => { const evidence = extractImageGenerationCompletionFromRuntimeEvent(event); - if (evidence) void useAcpChatSessionStore.getState().projectImageGenerationCompletion(evidence); + const state = useAcpChatSessionStore.getState(); + if ( + evidence + && !deferInactiveImageGenerationCompletion(state.activeSessionKey, evidence) + ) void state.projectImageGenerationCompletion(evidence); }); } diff --git a/tests/e2e/chat-run-state-events.spec.ts b/tests/e2e/chat-run-state-events.spec.ts index 469a54a3..bb94d637 100644 --- a/tests/e2e/chat-run-state-events.spec.ts +++ b/tests/e2e/chat-run-state-events.spec.ts @@ -309,6 +309,13 @@ test.describe('ClawX chat run state events', () => { const timeline = page.getByTestId('acp-chat-timeline'); await expect(timeline).toBeVisible({ timeout: 30_000 }); await expect(page.getByTestId('acp-tool-call-card')).toContainText('Background task started for image generation'); + await expect(page.getByTestId('chat-composer-image-generation-indicator')).toBeVisible(); + await expect(page.getByTestId('chat-composer-image-generation-indicator')).toContainText('Generating image'); + await page.getByTestId('sidebar-new-chat').click(); + await expect(page.getByTestId('chat-composer-image-generation-indicator')).toHaveCount(0); + await page.getByTestId(`sidebar-session-${MAIN_SESSION_KEY}`).click(); + await expect(page.getByTestId('chat-composer-image-generation-indicator')).toBeVisible(); + await expect(page.getByTestId('chat-composer-image-generation-indicator')).toContainText('Generating image'); await emitGatewayChatMessage(app, { message: { @@ -330,6 +337,7 @@ test.describe('ClawX chat run state events', () => { const image = timeline.getByRole('img', { name: 'Image' }); await expect(image).toBeVisible(); await expect(image).toHaveAttribute('src', ONE_PIXEL_PNG_DATA_URL); + await expect(page.getByTestId('chat-composer-image-generation-indicator')).toHaveCount(0); await expect(page.getByTestId('image-preview-unavailable')).toHaveCount(0); await expect(page.getByTestId('chat-execution-graph')).toHaveCount(0); } finally { @@ -358,6 +366,7 @@ test.describe('ClawX chat run state events', () => { }], locations: [], }]); + await expect(page.getByTestId('chat-composer-image-generation-indicator')).toBeVisible(); await emitGatewayChatMessage(app, { message: { @@ -375,6 +384,7 @@ test.describe('ClawX chat run state events', () => { }); await expect(page.getByText('Image generation failed because no image model is available.')).toBeVisible(); + await expect(page.getByTestId('chat-composer-image-generation-indicator')).toHaveCount(0); await expect(page.getByTestId('acp-image-part')).toHaveCount(0); } finally { await closeElectronApp(app); diff --git a/tests/unit/acp-chat-store.test.ts b/tests/unit/acp-chat-store.test.ts index 0efabc11..4ec1bfb7 100644 --- a/tests/unit/acp-chat-store.test.ts +++ b/tests/unit/acp-chat-store.test.ts @@ -1453,6 +1453,10 @@ describe('ACP Chat store', () => { }); it('projects trusted image-generation Gateway media into the ACP timeline', async () => { + hostApiMock.loadAcpSession + .mockResolvedValueOnce({ success: true, generation: 1 }) + .mockResolvedValueOnce({ success: true, generation: 2 }) + .mockResolvedValueOnce({ success: true, generation: 3 }); const { ensureAcpChatSubscriptions, useAcpChatSessionStore } = await importStore(); ensureAcpChatSubscriptions(); await useAcpChatSessionStore.getState().loadSession({ sessionKey: 'agent:pi:s1', workspaceRoot: '/repo', cwd: '/repo' }); @@ -1475,6 +1479,15 @@ describe('ACP Chat store', () => { }, }, }); + expect(useAcpChatSessionStore.getState().pendingImageGenerationTaskIds).toEqual([ + '32aa3a12-a05b-4074-af4e-246cc4a9a303', + ]); + await useAcpChatSessionStore.getState().loadSession({ + sessionKey: 'agent:pi:s2', + workspaceRoot: '/repo-2', + cwd: '/repo-2', + }); + expect(useAcpChatSessionStore.getState().pendingImageGenerationTaskIds).toEqual([]); hostApiMock.mediaThumbnails.mockResolvedValueOnce({ '/tmp/sky.png': { preview: 'data:image/png;base64,abc123', fileSize: 67 }, }); @@ -1482,7 +1495,7 @@ describe('ACP Chat store', () => { hostEventsMock.gatewayChatMessageListener?.({ message: { sessionKey: 'agent:pi:s1', - runId: 'run-1', + runId: 'image_generate:32aa3a12-a05b-4074-af4e-246cc4a9a303:ok', message: { role: 'toolresult', toolName: 'message', @@ -1490,6 +1503,12 @@ describe('ACP Chat store', () => { }, }, }); + expect(hostApiMock.mediaThumbnails).not.toHaveBeenCalled(); + await useAcpChatSessionStore.getState().loadSession({ + sessionKey: 'agent:pi:s1', + workspaceRoot: '/repo', + cwd: '/repo', + }); await new Promise((resolve) => setTimeout(resolve, 0)); expect(hostApiMock.mediaThumbnails).toHaveBeenCalledWith({ @@ -1511,6 +1530,7 @@ describe('ACP Chat store', () => { { kind: 'image', source: 'data:image/png;base64,abc123', mimeType: 'image/png', alt: 'Image' }, ], }); + expect(useAcpChatSessionStore.getState().pendingImageGenerationTaskIds).toEqual([]); }); it('records image-generation start detection trace entries', async () => { @@ -1996,6 +2016,7 @@ describe('ACP Chat store', () => { }, }, }); + expect(useAcpChatSessionStore.getState().pendingImageGenerationTaskIds).toEqual([taskId]); hostEventsMock.runtimeEventListener?.({ type: 'tool.completed', @@ -2029,6 +2050,7 @@ describe('ACP Chat store', () => { text: 'Image generation failed because no image model is available.', }], }); + expect(useAcpChatSessionStore.getState().pendingImageGenerationTaskIds).toEqual([]); }); it('upgrades a generic image caption when authoritative source-reply text arrives later', async () => { diff --git a/tests/unit/chat-acp-page.test.tsx b/tests/unit/chat-acp-page.test.tsx index c0ca86aa..148a009b 100644 --- a/tests/unit/chat-acp-page.test.tsx +++ b/tests/unit/chat-acp-page.test.tsx @@ -42,6 +42,7 @@ const { acpState, agentsState, artifactPanelState, artifactPanelProps, chatState } as AcpTimelineSnapshot, loading: false, sending: false, + pendingImageGenerationTaskIds: [] as string[], cancelling: false, error: null as string | null, activeSessionKey: 'agent:main:main' as string | null, diff --git a/tests/unit/chat-input.test.tsx b/tests/unit/chat-input.test.tsx index f32c4dfd..b7f498e2 100644 --- a/tests/unit/chat-input.test.tsx +++ b/tests/unit/chat-input.test.tsx @@ -112,6 +112,8 @@ function translate(key: string, vars?: Record): string { return 'Stop'; case 'composer.thinking': return 'Thinking…'; + case 'imageGeneration.generating': + return 'Generating image, please wait…'; case 'composer.gatewayConnected': return 'connected'; case 'composer.gatewayStarting': @@ -250,6 +252,36 @@ describe('ChatInput agent targeting', () => { expect(screen.queryByTestId('chat-composer-zoomies')).not.toBeInTheDocument(); }); + it('shows an image-generation indicator without locking the composer for background work', () => { + render( + + + , + ); + + const indicator = screen.getByRole('status', { name: 'Generating image, please wait…' }); + expect(indicator).toHaveAttribute('data-testid', 'chat-composer-image-generation-indicator'); + expect(screen.queryByTestId('chat-composer-working-indicator')).not.toBeInTheDocument(); + const input = screen.getByTestId('chat-composer-input'); + expect(input).not.toBeDisabled(); + fireEvent.change(input, { target: { value: 'Queue this after the image' } }); + expect(screen.getByTestId('chat-composer-send')).toBeDisabled(); + }); + + it('keeps the existing thinking indicator while sending even when image generation has started', () => { + render( + + + , + ); + + expect(screen.getByRole('status', { name: 'Thinking…' })).toHaveAttribute( + 'data-testid', + 'chat-composer-working-indicator', + ); + expect(screen.queryByTestId('chat-composer-image-generation-indicator')).not.toBeInTheDocument(); + }); + it('waits for the provider snapshot before clearing an unavailable model override', async () => { let resolveSnapshot!: () => void; agentsState.updateAgentModel.mockResolvedValue(undefined);