From 461ba012fc20449fe2c81230387289abf2e6f0e6 Mon Sep 17 00:00:00 2001 From: Won Park Date: Thu, 19 Mar 2026 22:57:16 -0700 Subject: [PATCH] Feat/restore image generation history (#15223) Restore image generation items in resumed thread history --- .../schema/json/ServerNotification.json | 6 ++ .../codex_app_server_protocol.schemas.json | 6 ++ .../codex_app_server_protocol.v2.schemas.json | 6 ++ .../json/v2/ItemCompletedNotification.json | 6 ++ .../json/v2/ItemStartedNotification.json | 6 ++ .../schema/json/v2/ReviewStartResponse.json | 6 ++ .../schema/json/v2/ThreadForkResponse.json | 6 ++ .../schema/json/v2/ThreadListResponse.json | 6 ++ .../json/v2/ThreadMetadataUpdateResponse.json | 6 ++ .../schema/json/v2/ThreadReadResponse.json | 6 ++ .../schema/json/v2/ThreadResumeResponse.json | 6 ++ .../json/v2/ThreadRollbackResponse.json | 6 ++ .../schema/json/v2/ThreadStartResponse.json | 6 ++ .../json/v2/ThreadStartedNotification.json | 6 ++ .../json/v2/ThreadUnarchiveResponse.json | 6 ++ .../json/v2/TurnCompletedNotification.json | 6 ++ .../schema/json/v2/TurnStartResponse.json | 6 ++ .../json/v2/TurnStartedNotification.json | 6 ++ .../schema/typescript/v2/ThreadItem.ts | 2 +- .../src/protocol/thread_history.rs | 57 +++++++++++++++++++ .../app-server-protocol/src/protocol/v2.rs | 4 ++ codex-rs/core/src/codex_tests.rs | 7 ++- codex-rs/core/src/rollout/policy.rs | 28 ++++++++- codex-rs/core/src/stream_events_utils.rs | 10 ++-- .../src/app/app_server_adapter.rs | 4 +- codex-rs/tui_app_server/src/chatwidget.rs | 3 +- 26 files changed, 213 insertions(+), 10 deletions(-) diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index f9cbe76e2..045301e09 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -2817,6 +2817,12 @@ "null" ] }, + "savedPath": { + "type": [ + "string", + "null" + ] + }, "status": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index 68bf7477e..3d392be1a 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -12540,6 +12540,12 @@ "null" ] }, + "savedPath": { + "type": [ + "string", + "null" + ] + }, "status": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index 772eb6f47..e06b5d1a1 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -10300,6 +10300,12 @@ "null" ] }, + "savedPath": { + "type": [ + "string", + "null" + ] + }, "status": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json index 3b9746620..396410786 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json @@ -1026,6 +1026,12 @@ "null" ] }, + "savedPath": { + "type": [ + "string", + "null" + ] + }, "status": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json index b77b34536..abb8aee5d 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json @@ -1026,6 +1026,12 @@ "null" ] }, + "savedPath": { + "type": [ + "string", + "null" + ] + }, "status": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json index 7f4a2b1f4..98b485b57 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json @@ -1140,6 +1140,12 @@ "null" ] }, + "savedPath": { + "type": [ + "string", + "null" + ] + }, "status": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json index 44734226d..8aee99f90 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json @@ -1633,6 +1633,12 @@ "null" ] }, + "savedPath": { + "type": [ + "string", + "null" + ] + }, "status": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json index 766fe48ce..05f3ae87c 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json @@ -1391,6 +1391,12 @@ "null" ] }, + "savedPath": { + "type": [ + "string", + "null" + ] + }, "status": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json index 1ef137f9e..214c25f54 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json @@ -1391,6 +1391,12 @@ "null" ] }, + "savedPath": { + "type": [ + "string", + "null" + ] + }, "status": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json index 3b7726c42..2a8fe06ec 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json @@ -1391,6 +1391,12 @@ "null" ] }, + "savedPath": { + "type": [ + "string", + "null" + ] + }, "status": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json index ba42df4ac..468325cef 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json @@ -1633,6 +1633,12 @@ "null" ] }, + "savedPath": { + "type": [ + "string", + "null" + ] + }, "status": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json index bb9dcbdd9..def818dcf 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json @@ -1391,6 +1391,12 @@ "null" ] }, + "savedPath": { + "type": [ + "string", + "null" + ] + }, "status": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json index ba7138320..c225b1c0f 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json @@ -1633,6 +1633,12 @@ "null" ] }, + "savedPath": { + "type": [ + "string", + "null" + ] + }, "status": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json index 53806b272..df7670cdb 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json @@ -1391,6 +1391,12 @@ "null" ] }, + "savedPath": { + "type": [ + "string", + "null" + ] + }, "status": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json index 3430d24e3..d95cd4dd8 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json @@ -1391,6 +1391,12 @@ "null" ] }, + "savedPath": { + "type": [ + "string", + "null" + ] + }, "status": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json index 40ce73e52..b0220247a 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json @@ -1140,6 +1140,12 @@ "null" ] }, + "savedPath": { + "type": [ + "string", + "null" + ] + }, "status": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json index 954321c16..cd9f63bb6 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json @@ -1140,6 +1140,12 @@ "null" ] }, + "savedPath": { + "type": [ + "string", + "null" + ] + }, "status": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json index 66ce68373..3cc16db92 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json @@ -1140,6 +1140,12 @@ "null" ] }, + "savedPath": { + "type": [ + "string", + "null" + ] + }, "status": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts index f1f864ae4..9202f3728 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts @@ -97,4 +97,4 @@ reasoningEffort: ReasoningEffort | null, /** * Last known status of the target agents, when available. */ -agentsStates: { [key in string]?: CollabAgentState }, } | { "type": "webSearch", id: string, query: string, action: WebSearchAction | null, } | { "type": "imageView", id: string, path: string, } | { "type": "imageGeneration", id: string, status: string, revisedPrompt: string | null, result: string, } | { "type": "enteredReviewMode", id: string, review: string, } | { "type": "exitedReviewMode", id: string, review: string, } | { "type": "contextCompaction", id: string, }; +agentsStates: { [key in string]?: CollabAgentState }, } | { "type": "webSearch", id: string, query: string, action: WebSearchAction | null, } | { "type": "imageView", id: string, path: string, } | { "type": "imageGeneration", id: string, status: string, revisedPrompt: string | null, result: string, savedPath?: string, } | { "type": "enteredReviewMode", id: string, review: string, } | { "type": "exitedReviewMode", id: string, review: string, } | { "type": "contextCompaction", id: string, }; diff --git a/codex-rs/app-server-protocol/src/protocol/thread_history.rs b/codex-rs/app-server-protocol/src/protocol/thread_history.rs index d7482b10c..11dfe2976 100644 --- a/codex-rs/app-server-protocol/src/protocol/thread_history.rs +++ b/codex-rs/app-server-protocol/src/protocol/thread_history.rs @@ -569,6 +569,7 @@ impl ThreadHistoryBuilder { status: String::new(), revised_prompt: None, result: String::new(), + saved_path: None, }; self.upsert_item_in_current_turn(item); } @@ -579,6 +580,7 @@ impl ThreadHistoryBuilder { status: payload.status.clone(), revised_prompt: payload.revised_prompt.clone(), result: payload.result.clone(), + saved_path: payload.saved_path.clone(), }; self.upsert_item_in_current_turn(item); } @@ -1385,6 +1387,61 @@ mod tests { ); } + #[test] + fn replays_image_generation_end_events_into_turn_history() { + let items = vec![ + RolloutItem::EventMsg(EventMsg::TurnStarted(TurnStartedEvent { + turn_id: "turn-image".into(), + model_context_window: None, + collaboration_mode_kind: Default::default(), + })), + RolloutItem::EventMsg(EventMsg::UserMessage(UserMessageEvent { + message: "generate an image".into(), + images: None, + text_elements: Vec::new(), + local_images: Vec::new(), + })), + RolloutItem::EventMsg(EventMsg::ImageGenerationEnd(ImageGenerationEndEvent { + call_id: "ig_123".into(), + status: "completed".into(), + revised_prompt: Some("final prompt".into()), + result: "Zm9v".into(), + saved_path: Some("/tmp/ig_123.png".into()), + })), + RolloutItem::EventMsg(EventMsg::TurnComplete(TurnCompleteEvent { + turn_id: "turn-image".into(), + last_agent_message: None, + })), + ]; + + let turns = build_turns_from_rollout_items(&items); + assert_eq!(turns.len(), 1); + assert_eq!( + turns[0], + Turn { + id: "turn-image".into(), + status: TurnStatus::Completed, + error: None, + items: vec![ + ThreadItem::UserMessage { + id: "item-1".into(), + content: vec![UserInput::Text { + text: "generate an image".into(), + text_elements: Vec::new(), + }], + }, + ThreadItem::ImageGeneration { + id: "ig_123".into(), + status: "completed".into(), + revised_prompt: Some("final prompt".into()), + result: "Zm9v".into(), + saved_path: Some("/tmp/ig_123.png".into()), + }, + ], + } + ); + } + #[test] fn splits_reasoning_when_interleaved() { let events = vec![ diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index 1c8903a14..d43581aaf 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -4256,6 +4256,9 @@ pub enum ThreadItem { status: String, revised_prompt: Option, result: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + saved_path: Option, }, #[serde(rename_all = "camelCase")] #[ts(rename_all = "camelCase")] @@ -4432,6 +4435,7 @@ impl From for ThreadItem { status: image.status, revised_prompt: image.revised_prompt, result: image.result, + saved_path: image.saved_path, }, CoreTurnItem::ContextCompaction(compaction) => { ThreadItem::ContextCompaction { id: compaction.id } diff --git a/codex-rs/core/src/codex_tests.rs b/codex-rs/core/src/codex_tests.rs index a814eab95..a5412eff2 100644 --- a/codex-rs/core/src/codex_tests.rs +++ b/codex-rs/core/src/codex_tests.rs @@ -3751,7 +3751,12 @@ async fn handle_output_item_done_records_image_save_history_message() { image_output_path.display(), )) .into(); - assert_eq!(history.raw_items(), &[save_message, item]); + let copy_message: ResponseItem = DeveloperInstructions::new( + "If you need to use a generated image at another path, copy it and leave the original in place unless the user explicitly asks you to delete it." + .to_string(), + ) + .into(); + assert_eq!(history.raw_items(), &[save_message, copy_message, item]); assert_eq!( std::fs::read(&expected_saved_path).expect("saved file"), b"foo" diff --git a/codex-rs/core/src/rollout/policy.rs b/codex-rs/core/src/rollout/policy.rs index 4600431c6..8b1f94dbd 100644 --- a/codex-rs/core/src/rollout/policy.rs +++ b/codex-rs/core/src/rollout/policy.rs @@ -105,7 +105,8 @@ fn event_msg_persistence_mode(ev: &EventMsg) -> Option { | EventMsg::UndoCompleted(_) | EventMsg::TurnAborted(_) | EventMsg::TurnStarted(_) - | EventMsg::TurnComplete(_) => Some(EventPersistenceMode::Limited), + | EventMsg::TurnComplete(_) + | EventMsg::ImageGenerationEnd(_) => Some(EventPersistenceMode::Limited), EventMsg::ItemCompleted(event) => { // Plan items are derived from streaming tags and are not part of the // raw ResponseItem history, so we persist their completion to replay @@ -123,7 +124,6 @@ fn event_msg_persistence_mode(ev: &EventMsg) -> Option { | EventMsg::PatchApplyEnd(_) | EventMsg::McpToolCallEnd(_) | EventMsg::ViewImageToolCall(_) - | EventMsg::ImageGenerationEnd(_) | EventMsg::CollabAgentSpawnEnd(_) | EventMsg::CollabAgentInteractionEnd(_) | EventMsg::CollabWaitingEnd(_) @@ -183,3 +183,27 @@ fn event_msg_persistence_mode(ev: &EventMsg) -> Option { | EventMsg::ImageGenerationBegin(_) => None, } } + +#[cfg(test)] +mod tests { + use super::EventPersistenceMode; + use super::should_persist_event_msg; + use codex_protocol::protocol::EventMsg; + use codex_protocol::protocol::ImageGenerationEndEvent; + + #[test] + fn persists_image_generation_end_events_in_limited_mode() { + let event = EventMsg::ImageGenerationEnd(ImageGenerationEndEvent { + call_id: "ig_123".into(), + status: "completed".into(), + revised_prompt: Some("final prompt".into()), + result: "Zm9v".into(), + saved_path: None, + }); + + assert!(should_persist_event_msg( + &event, + EventPersistenceMode::Limited + )); + } +} diff --git a/codex-rs/core/src/stream_events_utils.rs b/codex-rs/core/src/stream_events_utils.rs index 01b74f3a7..cd77f1d5a 100644 --- a/codex-rs/core/src/stream_events_utils.rs +++ b/codex-rs/core/src/stream_events_utils.rs @@ -372,11 +372,13 @@ pub(crate) async fn handle_non_tool_response_item( image_output_path.display(), )) .into(); - sess.record_conversation_items( - turn_context, - std::slice::from_ref(&message), + let copy_message: ResponseItem = DeveloperInstructions::new( + "If you need to use a generated image at another path, copy it and leave the original in place unless the user explicitly asks you to delete it." + .to_string(), ) - .await; + .into(); + sess.record_conversation_items(turn_context, &[message, copy_message]) + .await; } Err(err) => { let output_path = image_generation_artifact_path( diff --git a/codex-rs/tui_app_server/src/app/app_server_adapter.rs b/codex-rs/tui_app_server/src/app/app_server_adapter.rs index d9cd97a4f..0d2112853 100644 --- a/codex-rs/tui_app_server/src/app/app_server_adapter.rs +++ b/codex-rs/tui_app_server/src/app/app_server_adapter.rs @@ -995,12 +995,13 @@ fn thread_item_to_core(item: &ThreadItem) -> Option { status, revised_prompt, result, + saved_path, } => Some(TurnItem::ImageGeneration(ImageGenerationItem { id: id.clone(), status: status.clone(), revised_prompt: revised_prompt.clone(), result: result.clone(), - saved_path: None, + saved_path: saved_path.clone(), })), ThreadItem::ContextCompaction { id } => { Some(TurnItem::ContextCompaction(ContextCompactionItem { @@ -1850,6 +1851,7 @@ mod tests { status: "completed".to_string(), revised_prompt: Some("diagram".to_string()), result: "image.png".to_string(), + saved_path: None, }, ThreadItem::ContextCompaction { id: "compact-1".to_string(), diff --git a/codex-rs/tui_app_server/src/chatwidget.rs b/codex-rs/tui_app_server/src/chatwidget.rs index 4faa8b40e..5e0cff03c 100644 --- a/codex-rs/tui_app_server/src/chatwidget.rs +++ b/codex-rs/tui_app_server/src/chatwidget.rs @@ -5725,13 +5725,14 @@ impl ChatWidget { status, revised_prompt, result, + saved_path, } => { self.on_image_generation_end(ImageGenerationEndEvent { call_id: id, result, revised_prompt, status, - saved_path: None, + saved_path, }); } ThreadItem::EnteredReviewMode { review, .. } => {