fix(chat): preserve image generation thinking state across sessions (#1180)

This commit is contained in:
paisley
2026-07-20 10:50:41 +08:00
committed by GitHub
parent a42a7e4256
commit c96b2336c6
8 changed files with 208 additions and 9 deletions
@@ -14,9 +14,13 @@ touchedAreas:
- src/lib/acp/reducer.ts
- src/lib/acp/timeline-types.ts
- src/stores/acp-chat-session.ts
- src/pages/Chat/index.tsx
- src/pages/Chat/ChatInput.tsx
- tests/unit/acp-image-generation-compat.test.ts
- tests/unit/acp-reducer.test.ts
- tests/unit/acp-chat-store.test.ts
- tests/unit/chat-acp-page.test.tsx
- tests/unit/chat-input.test.tsx
- tests/e2e/chat-run-state-events.spec.ts
- shared/i18n/locales/en/chat.json
- shared/i18n/locales/zh/chat.json
@@ -27,6 +31,7 @@ touchedAreas:
- README.ja-JP.md
expectedUserBehavior:
- ACP Chat first shows the image_generate background task start tool result.
- After the normal thinking state ends, the composer shows a distinct image-generation indicator until the generated image or a failure reply is rendered, including after switching away from and back to the conversation; users may edit a draft while another send is prevented.
- When OpenClaw later exposes a trusted internal-UI source reply through ACP or Gateway host events, ClawX preserves its exact user-facing text instead of replacing it with a generic caption.
- Successful replies include the hydrated image preview, while text-only generation failures remain visible as assistant replies.
- Arbitrary local paths and generic MEDIA: prose without approved image-generation context are not rendered as images.
@@ -57,6 +62,9 @@ acceptance:
- Internal-UI sourceReply text is authoritative for both successful media replies and text-only failure replies.
- ClawX hydrates previews through hostApi.media.thumbnails before rendering images.
- Duplicate completion records do not create duplicate assistant image replies.
- Live background image generation shows its dedicated generating label until its success or failure completion is projected, without changing the existing sending/thinking behavior.
- Switching conversations preserves each live image-generation pending state and restores its indicator on return.
- Completion evidence received while the image conversation is inactive is deferred to that conversation, and a second prompt cannot be sent until the image task settles.
- Stale preview resolution does not append to a different active session or generation.
docs:
required: true
+28 -1
View File
@@ -47,6 +47,7 @@ interface ChatInputProps {
onStop?: () => void;
disabled?: boolean;
sending?: boolean;
imageGenerating?: boolean;
workspaceLabel?: string;
workspacePath?: string;
workspaceReadOnly?: boolean;
@@ -201,6 +202,7 @@ export function ChatInput({
onStop,
disabled = false,
sending = false,
imageGenerating = false,
workspaceLabel,
workspacePath,
workspaceReadOnly = false,
@@ -678,7 +680,11 @@ export function ChatInput({
const allReady = attachments.length === 0 || attachments.every(a => a.status === 'ready');
const hasFailedAttachments = attachments.some((a) => a.status === 'error');
const canSend = (input.trim() || attachments.length > 0) && allReady && !inputDisabled && !sending;
const canSend = (input.trim() || attachments.length > 0)
&& allReady
&& !inputDisabled
&& !sending
&& !imageGenerating;
const canStop = sending && !inputDisabled && !!onStop;
const handleSend = useCallback(async () => {
@@ -887,6 +893,27 @@ export function ChatInput({
</div>
)}
{!sending && imageGenerating && (
<div
data-testid="chat-composer-image-generation-indicator"
role="status"
aria-live="polite"
aria-label={t('imageGeneration.generating')}
className="mb-2 flex h-5 items-center gap-2 text-sm text-muted-foreground"
>
<span
data-testid="chat-composer-image-generation-dot-pulse"
aria-hidden="true"
className="clawx-chat-thinking-dot-pulse"
>
<span className="clawx-chat-thinking-dot-pulse-inner">
<span className="clawx-chat-thinking-dot-pulse-dot" />
</span>
</span>
<span>{t('imageGeneration.generating')}</span>
</div>
)}
{/* Attachment Previews */}
{attachments.length > 0 && (
<div className="flex gap-2 mb-3 flex-wrap">
+4
View File
@@ -164,6 +164,9 @@ export function Chat() {
const acpTimeline = useAcpChatSessionStore((s) => s.timeline);
const acpLoading = useAcpChatSessionStore((s) => s.loading);
const acpSending = useAcpChatSessionStore((s) => s.sending);
const imageGenerationPending = useAcpChatSessionStore(
(s) => Boolean(s.pendingImageGenerationTaskIds?.length),
);
const acpCancelling = useAcpChatSessionStore((s) => s.cancelling);
const acpError = useAcpChatSessionStore((s) => s.error);
const acpActiveSessionKey = useAcpChatSessionStore((s) => s.activeSessionKey);
@@ -443,6 +446,7 @@ export function Chat() {
onStop={() => void cancelAcp()}
disabled={acpLoading || acpCancelling || !cwd}
sending={composerBusy}
imageGenerating={imageGenerationPending}
workspaceLabel={workspaceLabel}
workspacePath={cwd}
workspaceReadOnly={effectiveWorkspace.readOnly}
+102 -7
View File
@@ -71,8 +71,14 @@ type LiveSessionSnapshot = {
workspaceRoot: string | null;
cwd: string | null;
generation: number;
sending: boolean;
pendingImageGenerationTaskIds: string[];
timeline: AcpTimelineSnapshot;
deferredImageUpdates: Array<{ key: string; event: AcpSessionUpdateEnvelope }>;
deferredImageCompletions: Array<{
key: string;
evidence: ImageGenerationCompletionEvidence;
}>;
};
const liveSessionSnapshots = new Map<string, LiveSessionSnapshot>();
let loadRequestSeq = 0;
@@ -130,6 +136,7 @@ export type AcpChatSessionState = {
generation: number;
loading: boolean;
sending: boolean;
pendingImageGenerationTaskIds: string[];
cancelling: boolean;
error: string | null;
timeline: AcpTimelineSnapshot;
@@ -193,15 +200,21 @@ function applyPermissionRequestToTimeline(
}
function captureLiveSession(state: AcpChatSessionState): void {
if (!state.sending || !state.activeSessionKey) return;
if (
(!state.sending && state.pendingImageGenerationTaskIds.length === 0)
|| !state.activeSessionKey
) return;
const existing = liveSessionSnapshots.get(state.activeSessionKey);
liveSessionSnapshots.set(state.activeSessionKey, {
sessionKey: state.activeSessionKey,
workspaceRoot: state.workspaceRoot,
cwd: state.cwd,
generation: state.generation,
sending: state.sending,
pendingImageGenerationTaskIds: state.pendingImageGenerationTaskIds,
timeline: state.timeline,
deferredImageUpdates: existing?.deferredImageUpdates ?? [],
deferredImageCompletions: existing?.deferredImageCompletions ?? [],
});
}
@@ -331,6 +344,28 @@ function imageGenerationTaskIdFromSessionKey(sessionKey: string | undefined): st
return match?.[1] ?? null;
}
function deferInactiveImageGenerationCompletion(
activeSessionKey: string | null,
evidence: ImageGenerationCompletionEvidence,
): boolean {
const taskId = evidence.taskId ?? imageGenerationTaskIdFromSessionKey(evidence.sessionKey);
if (!taskId) return false;
for (const [sessionKey, snapshot] of liveSessionSnapshots) {
if (
sessionKey === activeSessionKey
|| !snapshot.pendingImageGenerationTaskIds.includes(taskId)
) continue;
const key = imageGenerationEvidenceKey(evidence);
const deferredImageCompletions = snapshot.deferredImageCompletions.filter(
(entry) => entry.key !== key,
);
deferredImageCompletions.push({ key, evidence });
liveSessionSnapshots.set(sessionKey, { ...snapshot, deferredImageCompletions });
return true;
}
return false;
}
function resolveImageGenerationProjectionSession(
state: AcpChatSessionState,
evidence: ImageGenerationCompletionEvidence,
@@ -918,6 +953,7 @@ export const useAcpChatSessionStore = create<AcpChatSessionState>((set, get) =>
generation: 0,
loading: false,
sending: false,
pendingImageGenerationTaskIds: [],
cancelling: false,
error: null,
timeline: createEmptyAcpTimeline(EMPTY_SESSION_ID, 0),
@@ -936,6 +972,7 @@ export const useAcpChatSessionStore = create<AcpChatSessionState>((set, get) =>
generation,
loading: false,
sending: false,
pendingImageGenerationTaskIds: [],
cancelling: false,
error: null,
timeline: createEmptyAcpTimeline(input.sessionKey, generation),
@@ -950,14 +987,17 @@ export const useAcpChatSessionStore = create<AcpChatSessionState>((set, get) =>
const generation = get().generation;
const liveSnapshot = liveSessionSnapshots.get(input.sessionKey);
invalidateTranscriptSupplement();
resetImageGenerationCompatSession(input.sessionKey);
if (!liveSnapshot?.pendingImageGenerationTaskIds.length) {
resetImageGenerationCompatSession(input.sessionKey);
}
set({
activeSessionKey: input.sessionKey,
workspaceRoot: input.workspaceRoot,
cwd: input.cwd,
generation,
loading: true,
sending: !!liveSnapshot,
sending: liveSnapshot?.sending ?? false,
pendingImageGenerationTaskIds: liveSnapshot?.pendingImageGenerationTaskIds ?? [],
cancelling: false,
error: null,
timeline: liveSnapshot?.timeline ?? createEmptyAcpTimeline(input.sessionKey, generation),
@@ -1018,6 +1058,16 @@ export const useAcpChatSessionStore = create<AcpChatSessionState>((set, get) =>
const currentResumedSnapshot = result.resumedActivePrompt
? liveSessionSnapshots.get(input.sessionKey)
: undefined;
const currentBackgroundSnapshot = !result.resumedActivePrompt
? liveSessionSnapshots.get(input.sessionKey)
: undefined;
const restorableBackgroundSnapshot = currentBackgroundSnapshot
&& currentBackgroundSnapshot.pendingImageGenerationTaskIds.length > 0
? currentBackgroundSnapshot
: undefined;
if (currentBackgroundSnapshot && !restorableBackgroundSnapshot) {
resetImageGenerationCompatSession(input.sessionKey);
}
let timeline = currentResumedSnapshot?.generation === generation
? currentResumedSnapshot.timeline
: createEmptyAcpTimeline(input.sessionKey, generation);
@@ -1030,7 +1080,11 @@ export const useAcpChatSessionStore = create<AcpChatSessionState>((set, get) =>
);
set({
loading: false,
sending: !!currentResumedSnapshot,
sending: currentResumedSnapshot?.sending ?? false,
pendingImageGenerationTaskIds:
currentResumedSnapshot?.pendingImageGenerationTaskIds
?? restorableBackgroundSnapshot?.pendingImageGenerationTaskIds
?? [],
error: null,
generation,
timeline,
@@ -1040,16 +1094,21 @@ export const useAcpChatSessionStore = create<AcpChatSessionState>((set, get) =>
...currentResumedSnapshot,
timeline,
deferredImageUpdates: [],
deferredImageCompletions: [],
});
} else {
liveSessionSnapshots.delete(input.sessionKey);
}
resolvePendingAttachments(input.sessionKey, generation, pendingAttachments);
for (const { event } of currentResumedSnapshot?.deferredImageUpdates ?? []) {
const restoredSnapshot = currentResumedSnapshot ?? restorableBackgroundSnapshot;
for (const { event } of restoredSnapshot?.deferredImageUpdates ?? []) {
get().recordImageGenerationStart(event);
const evidence = extractImageGenerationCompletionFromAcpEnvelope(event);
if (evidence) void get().projectImageGenerationCompletion(evidence);
}
for (const { evidence } of restoredSnapshot?.deferredImageCompletions ?? []) {
void get().projectImageGenerationCompletion(evidence);
}
for (const event of sessionUpdates) {
get().recordImageGenerationStart(event);
const evidence = extractImageGenerationCompletionFromAcpEnvelope(event);
@@ -1250,6 +1309,18 @@ export const useAcpChatSessionStore = create<AcpChatSessionState>((set, get) =>
session.taskStartedAt = Date.now();
session.taskIds.add(start.taskId);
recordImageGenerationStartAnchor(session, start, false);
set((current) => (
current.activeSessionKey === start.sessionKey
&& current.generation === event.generation
&& !current.pendingImageGenerationTaskIds.includes(start.taskId)
? {
pendingImageGenerationTaskIds: [
...current.pendingImageGenerationTaskIds,
start.taskId,
],
}
: {}
));
const operation = activeTranscriptSupplement;
if (
operation?.liveUserMessageId
@@ -1328,6 +1399,14 @@ export const useAcpChatSessionStore = create<AcpChatSessionState>((set, get) =>
const correlatedTaskId = evidence.taskId
?? imageGenerationTaskIdFromSessionKey(evidence.sessionKey)
?? (usesReplayImageGenerationContext(evidence) ? compat.lastReplayTaskId : compat.lastTaskId);
const settlePendingTask = (current: AcpChatSessionState): string[] => {
if (!correlatedTaskId) {
return usesReplayImageGenerationContext(evidence)
? current.pendingImageGenerationTaskIds
: [];
}
return current.pendingImageGenerationTaskIds.filter((taskId) => taskId !== correlatedTaskId);
};
const key = imageGenerationEvidenceKey({
...evidence,
sessionKey,
@@ -1354,6 +1433,9 @@ export const useAcpChatSessionStore = create<AcpChatSessionState>((set, get) =>
details: projectionTraceDetails(evidence),
});
if (compat.delivered.has(key)) {
set((current) => ({
pendingImageGenerationTaskIds: settlePendingTask(current),
}));
stopLiveTranscriptSupplementRetry(sessionKey, generation, correlatedTaskId);
}
return;
@@ -1504,8 +1586,12 @@ export const useAcpChatSessionStore = create<AcpChatSessionState>((set, get) =>
captions?.set(existingKey, currentCaption);
set((current) => ({
timeline: replaceSyntheticImageCaptionAtItem(current.timeline, duplicateItemId, currentCaption.text),
pendingImageGenerationTaskIds: settlePendingTask(current),
}));
}
set((current) => ({
pendingImageGenerationTaskIds: settlePendingTask(current),
}));
if (missingCount === 0) commitDelivery(sessionKey, key, reservationOwner);
else releaseDelivery(sessionKey, key, reservationOwner);
recordProjectionTrace({
@@ -1529,6 +1615,7 @@ export const useAcpChatSessionStore = create<AcpChatSessionState>((set, get) =>
parts,
afterItemId,
}),
pendingImageGenerationTaskIds: settlePendingTask(current),
};
});
if (missingCount === 0) commitDelivery(sessionKey, key, reservationOwner);
@@ -1632,10 +1719,18 @@ export function ensureAcpChatSubscriptions(): void {
});
hostEvents.onGatewayChatMessage((event) => {
const evidence = extractImageGenerationCompletionFromGatewayChatMessage(event);
if (evidence) void useAcpChatSessionStore.getState().projectImageGenerationCompletion(evidence);
const state = useAcpChatSessionStore.getState();
if (
evidence
&& !deferInactiveImageGenerationCompletion(state.activeSessionKey, evidence)
) void state.projectImageGenerationCompletion(evidence);
});
hostEvents.onChatRuntimeEvent((event) => {
const evidence = extractImageGenerationCompletionFromRuntimeEvent(event);
if (evidence) void useAcpChatSessionStore.getState().projectImageGenerationCompletion(evidence);
const state = useAcpChatSessionStore.getState();
if (
evidence
&& !deferInactiveImageGenerationCompletion(state.activeSessionKey, evidence)
) void state.projectImageGenerationCompletion(evidence);
});
}
+10
View File
@@ -309,6 +309,13 @@ test.describe('ClawX chat run state events', () => {
const timeline = page.getByTestId('acp-chat-timeline');
await expect(timeline).toBeVisible({ timeout: 30_000 });
await expect(page.getByTestId('acp-tool-call-card')).toContainText('Background task started for image generation');
await expect(page.getByTestId('chat-composer-image-generation-indicator')).toBeVisible();
await expect(page.getByTestId('chat-composer-image-generation-indicator')).toContainText('Generating image');
await page.getByTestId('sidebar-new-chat').click();
await expect(page.getByTestId('chat-composer-image-generation-indicator')).toHaveCount(0);
await page.getByTestId(`sidebar-session-${MAIN_SESSION_KEY}`).click();
await expect(page.getByTestId('chat-composer-image-generation-indicator')).toBeVisible();
await expect(page.getByTestId('chat-composer-image-generation-indicator')).toContainText('Generating image');
await emitGatewayChatMessage(app, {
message: {
@@ -330,6 +337,7 @@ test.describe('ClawX chat run state events', () => {
const image = timeline.getByRole('img', { name: 'Image' });
await expect(image).toBeVisible();
await expect(image).toHaveAttribute('src', ONE_PIXEL_PNG_DATA_URL);
await expect(page.getByTestId('chat-composer-image-generation-indicator')).toHaveCount(0);
await expect(page.getByTestId('image-preview-unavailable')).toHaveCount(0);
await expect(page.getByTestId('chat-execution-graph')).toHaveCount(0);
} finally {
@@ -358,6 +366,7 @@ test.describe('ClawX chat run state events', () => {
}],
locations: [],
}]);
await expect(page.getByTestId('chat-composer-image-generation-indicator')).toBeVisible();
await emitGatewayChatMessage(app, {
message: {
@@ -375,6 +384,7 @@ test.describe('ClawX chat run state events', () => {
});
await expect(page.getByText('Image generation failed because no image model is available.')).toBeVisible();
await expect(page.getByTestId('chat-composer-image-generation-indicator')).toHaveCount(0);
await expect(page.getByTestId('acp-image-part')).toHaveCount(0);
} finally {
await closeElectronApp(app);
+23 -1
View File
@@ -1453,6 +1453,10 @@ describe('ACP Chat store', () => {
});
it('projects trusted image-generation Gateway media into the ACP timeline', async () => {
hostApiMock.loadAcpSession
.mockResolvedValueOnce({ success: true, generation: 1 })
.mockResolvedValueOnce({ success: true, generation: 2 })
.mockResolvedValueOnce({ success: true, generation: 3 });
const { ensureAcpChatSubscriptions, useAcpChatSessionStore } = await importStore();
ensureAcpChatSubscriptions();
await useAcpChatSessionStore.getState().loadSession({ sessionKey: 'agent:pi:s1', workspaceRoot: '/repo', cwd: '/repo' });
@@ -1475,6 +1479,15 @@ describe('ACP Chat store', () => {
},
},
});
expect(useAcpChatSessionStore.getState().pendingImageGenerationTaskIds).toEqual([
'32aa3a12-a05b-4074-af4e-246cc4a9a303',
]);
await useAcpChatSessionStore.getState().loadSession({
sessionKey: 'agent:pi:s2',
workspaceRoot: '/repo-2',
cwd: '/repo-2',
});
expect(useAcpChatSessionStore.getState().pendingImageGenerationTaskIds).toEqual([]);
hostApiMock.mediaThumbnails.mockResolvedValueOnce({
'/tmp/sky.png': { preview: 'data:image/png;base64,abc123', fileSize: 67 },
});
@@ -1482,7 +1495,7 @@ describe('ACP Chat store', () => {
hostEventsMock.gatewayChatMessageListener?.({
message: {
sessionKey: 'agent:pi:s1',
runId: 'run-1',
runId: 'image_generate:32aa3a12-a05b-4074-af4e-246cc4a9a303:ok',
message: {
role: 'toolresult',
toolName: 'message',
@@ -1490,6 +1503,12 @@ describe('ACP Chat store', () => {
},
},
});
expect(hostApiMock.mediaThumbnails).not.toHaveBeenCalled();
await useAcpChatSessionStore.getState().loadSession({
sessionKey: 'agent:pi:s1',
workspaceRoot: '/repo',
cwd: '/repo',
});
await new Promise((resolve) => setTimeout(resolve, 0));
expect(hostApiMock.mediaThumbnails).toHaveBeenCalledWith({
@@ -1511,6 +1530,7 @@ describe('ACP Chat store', () => {
{ kind: 'image', source: 'data:image/png;base64,abc123', mimeType: 'image/png', alt: 'Image' },
],
});
expect(useAcpChatSessionStore.getState().pendingImageGenerationTaskIds).toEqual([]);
});
it('records image-generation start detection trace entries', async () => {
@@ -1996,6 +2016,7 @@ describe('ACP Chat store', () => {
},
},
});
expect(useAcpChatSessionStore.getState().pendingImageGenerationTaskIds).toEqual([taskId]);
hostEventsMock.runtimeEventListener?.({
type: 'tool.completed',
@@ -2029,6 +2050,7 @@ describe('ACP Chat store', () => {
text: 'Image generation failed because no image model is available.',
}],
});
expect(useAcpChatSessionStore.getState().pendingImageGenerationTaskIds).toEqual([]);
});
it('upgrades a generic image caption when authoritative source-reply text arrives later', async () => {
+1
View File
@@ -42,6 +42,7 @@ const { acpState, agentsState, artifactPanelState, artifactPanelProps, chatState
} as AcpTimelineSnapshot,
loading: false,
sending: false,
pendingImageGenerationTaskIds: [] as string[],
cancelling: false,
error: null as string | null,
activeSessionKey: 'agent:main:main' as string | null,
+32
View File
@@ -112,6 +112,8 @@ function translate(key: string, vars?: Record<string, unknown>): string {
return 'Stop';
case 'composer.thinking':
return 'Thinking…';
case 'imageGeneration.generating':
return 'Generating image, please wait…';
case 'composer.gatewayConnected':
return 'connected';
case 'composer.gatewayStarting':
@@ -250,6 +252,36 @@ describe('ChatInput agent targeting', () => {
expect(screen.queryByTestId('chat-composer-zoomies')).not.toBeInTheDocument();
});
it('shows an image-generation indicator without locking the composer for background work', () => {
render(
<TooltipProvider>
<ChatInput onSend={vi.fn()} imageGenerating />
</TooltipProvider>,
);
const indicator = screen.getByRole('status', { name: 'Generating image, please wait…' });
expect(indicator).toHaveAttribute('data-testid', 'chat-composer-image-generation-indicator');
expect(screen.queryByTestId('chat-composer-working-indicator')).not.toBeInTheDocument();
const input = screen.getByTestId('chat-composer-input');
expect(input).not.toBeDisabled();
fireEvent.change(input, { target: { value: 'Queue this after the image' } });
expect(screen.getByTestId('chat-composer-send')).toBeDisabled();
});
it('keeps the existing thinking indicator while sending even when image generation has started', () => {
render(
<TooltipProvider>
<ChatInput onSend={vi.fn()} sending imageGenerating />
</TooltipProvider>,
);
expect(screen.getByRole('status', { name: 'Thinking…' })).toHaveAttribute(
'data-testid',
'chat-composer-working-indicator',
);
expect(screen.queryByTestId('chat-composer-image-generation-indicator')).not.toBeInTheDocument();
});
it('waits for the provider snapshot before clearing an unavailable model override', async () => {
let resolveSnapshot!: () => void;
agentsState.updateAgentModel.mockResolvedValue(undefined);