core-agent-ide/codex-rs/protocol/src/items.rs
charley-oai 1fa8350ae7
Add text element metadata to protocol, app server, and core (#9331)
The second part of breaking up PR
https://github.com/openai/codex/pull/9116

Summary:

- Add `TextElement` / `ByteRange` to protocol user inputs and user
message events with defaults.
- Thread `text_elements` through app-server v1/v2 request handling and
history rebuild.
- Preserve UI metadata only in user input/events (not `ContentItem`)
while keeping local image attachments in user events for rehydration.

Details:

- Protocol: `UserInput::Text` carries `text_elements`;
`UserMessageEvent` carries `text_elements` + `local_images`.
Serialization includes empty vectors for backward compatibility.
- app-server-protocol: v1 defines `V1TextElement` / `V1ByteRange` in
camelCase with conversions; v2 uses its own camelCase wrapper.
- app-server: v1/v2 input mapping includes `text_elements`; thread
history rebuilds include them.
- Core: user event emission preserves UI metadata while model history
stays clean; history replay round-trips the metadata.
2026-01-15 17:26:41 -08:00

205 lines
6 KiB
Rust

use crate::protocol::AgentMessageEvent;
use crate::protocol::AgentReasoningEvent;
use crate::protocol::AgentReasoningRawContentEvent;
use crate::protocol::EventMsg;
use crate::protocol::UserMessageEvent;
use crate::protocol::WebSearchEndEvent;
use crate::user_input::ByteRange;
use crate::user_input::TextElement;
use crate::user_input::UserInput;
use schemars::JsonSchema;
use serde::Deserialize;
use serde::Serialize;
use ts_rs::TS;
#[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema)]
#[serde(tag = "type")]
#[ts(tag = "type")]
pub enum TurnItem {
UserMessage(UserMessageItem),
AgentMessage(AgentMessageItem),
Reasoning(ReasoningItem),
WebSearch(WebSearchItem),
}
#[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema)]
pub struct UserMessageItem {
pub id: String,
pub content: Vec<UserInput>,
}
#[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema)]
#[serde(tag = "type")]
#[ts(tag = "type")]
pub enum AgentMessageContent {
Text { text: String },
}
#[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema)]
pub struct AgentMessageItem {
pub id: String,
pub content: Vec<AgentMessageContent>,
}
#[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema)]
pub struct ReasoningItem {
pub id: String,
pub summary_text: Vec<String>,
#[serde(default)]
pub raw_content: Vec<String>,
}
#[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema)]
pub struct WebSearchItem {
pub id: String,
pub query: String,
}
impl UserMessageItem {
pub fn new(content: &[UserInput]) -> Self {
Self {
id: uuid::Uuid::new_v4().to_string(),
content: content.to_vec(),
}
}
pub fn as_legacy_event(&self) -> EventMsg {
// Legacy user-message events flatten only text inputs into `message` and
// rebase text element ranges onto that concatenated text.
EventMsg::UserMessage(UserMessageEvent {
message: self.message(),
images: Some(self.image_urls()),
local_images: self.local_image_paths(),
text_elements: self.text_elements(),
})
}
pub fn message(&self) -> String {
self.content
.iter()
.map(|c| match c {
UserInput::Text { text, .. } => text.clone(),
_ => String::new(),
})
.collect::<Vec<String>>()
.join("")
}
pub fn text_elements(&self) -> Vec<TextElement> {
let mut out = Vec::new();
let mut offset = 0usize;
for input in &self.content {
if let UserInput::Text {
text,
text_elements,
} = input
{
// Text element ranges are relative to each text chunk; offset them so they align
// with the concatenated message returned by `message()`.
for elem in text_elements {
out.push(TextElement {
byte_range: ByteRange {
start: offset + elem.byte_range.start,
end: offset + elem.byte_range.end,
},
placeholder: elem.placeholder.clone(),
});
}
offset += text.len();
}
}
out
}
pub fn image_urls(&self) -> Vec<String> {
self.content
.iter()
.filter_map(|c| match c {
UserInput::Image { image_url } => Some(image_url.clone()),
_ => None,
})
.collect()
}
pub fn local_image_paths(&self) -> Vec<std::path::PathBuf> {
self.content
.iter()
.filter_map(|c| match c {
UserInput::LocalImage { path } => Some(path.clone()),
_ => None,
})
.collect()
}
}
impl AgentMessageItem {
pub fn new(content: &[AgentMessageContent]) -> Self {
Self {
id: uuid::Uuid::new_v4().to_string(),
content: content.to_vec(),
}
}
pub fn as_legacy_events(&self) -> Vec<EventMsg> {
self.content
.iter()
.map(|c| match c {
AgentMessageContent::Text { text } => EventMsg::AgentMessage(AgentMessageEvent {
message: text.clone(),
}),
})
.collect()
}
}
impl ReasoningItem {
pub fn as_legacy_events(&self, show_raw_agent_reasoning: bool) -> Vec<EventMsg> {
let mut events = Vec::new();
for summary in &self.summary_text {
events.push(EventMsg::AgentReasoning(AgentReasoningEvent {
text: summary.clone(),
}));
}
if show_raw_agent_reasoning {
for entry in &self.raw_content {
events.push(EventMsg::AgentReasoningRawContent(
AgentReasoningRawContentEvent {
text: entry.clone(),
},
));
}
}
events
}
}
impl WebSearchItem {
pub fn as_legacy_event(&self) -> EventMsg {
EventMsg::WebSearchEnd(WebSearchEndEvent {
call_id: self.id.clone(),
query: self.query.clone(),
})
}
}
impl TurnItem {
pub fn id(&self) -> String {
match self {
TurnItem::UserMessage(item) => item.id.clone(),
TurnItem::AgentMessage(item) => item.id.clone(),
TurnItem::Reasoning(item) => item.id.clone(),
TurnItem::WebSearch(item) => item.id.clone(),
}
}
pub fn as_legacy_events(&self, show_raw_agent_reasoning: bool) -> Vec<EventMsg> {
match self {
TurnItem::UserMessage(item) => vec![item.as_legacy_event()],
TurnItem::AgentMessage(item) => item.as_legacy_events(),
TurnItem::WebSearch(item) => vec![item.as_legacy_event()],
TurnItem::Reasoning(item) => item.as_legacy_events(show_raw_agent_reasoning),
}
}
}