diff --git a/codex-rs/core/src/features.rs b/codex-rs/core/src/features.rs index 5015e5f72..8cece9593 100644 --- a/codex-rs/core/src/features.rs +++ b/codex-rs/core/src/features.rs @@ -91,6 +91,8 @@ pub enum Feature { Tui2, /// Enable discovery and injection of skills. Skills, + /// Enforce UTF8 output in Powershell. + PowershellUtf8, } impl Feature { @@ -386,6 +388,12 @@ pub const FEATURES: &[FeatureSpec] = &[ stage: Stage::Experimental, default_enabled: true, }, + FeatureSpec { + id: Feature::PowershellUtf8, + key: "powershell_utf8", + stage: Stage::Experimental, + default_enabled: false, + }, FeatureSpec { id: Feature::Tui2, key: "tui2", diff --git a/codex-rs/core/src/powershell.rs b/codex-rs/core/src/powershell.rs index 3863ff401..148605b4b 100644 --- a/codex-rs/core/src/powershell.rs +++ b/codex-rs/core/src/powershell.rs @@ -8,6 +8,30 @@ use crate::shell::detect_shell_type; const POWERSHELL_FLAGS: &[&str] = &["-nologo", "-noprofile", "-command", "-c"]; +/// Prefixed command for powershell shell calls to force UTF-8 console output. +pub(crate) const UTF8_OUTPUT_PREFIX: &str = + "[Console]::OutputEncoding=[System.Text.Encoding]::UTF8;\n"; + +pub(crate) fn prefix_powershell_script_with_utf8(command: &[String]) -> Vec { + let Some((_, script)) = extract_powershell_command(command) else { + return command.to_vec(); + }; + + let trimmed = script.trim_start(); + let script = if trimmed.starts_with(UTF8_OUTPUT_PREFIX) { + script.to_string() + } else { + format!("{UTF8_OUTPUT_PREFIX}{script}") + }; + + let mut command: Vec = command[..(command.len() - 1)] + .iter() + .map(std::string::ToString::to_string) + .collect(); + command.push(script); + command +} + /// Extract the PowerShell script body from an invocation such as: /// /// - ["pwsh", "-NoProfile", "-Command", "Get-ChildItem -Recurse | Select-String foo"] @@ -22,7 +46,10 @@ pub fn extract_powershell_command(command: &[String]) -> Option<(&str, &str)> { } let shell = &command[0]; - if detect_shell_type(&PathBuf::from(shell)) != Some(ShellType::PowerShell) { + if !matches!( + detect_shell_type(&PathBuf::from(shell)), + Some(ShellType::PowerShell) + ) { return None; } @@ -36,7 +63,7 @@ pub fn extract_powershell_command(command: &[String]) -> Option<(&str, &str)> { } if flag.eq_ignore_ascii_case("-Command") || flag.eq_ignore_ascii_case("-c") { let script = &command[i + 1]; - return Some((shell, script.as_str())); + return Some((shell, script)); } i += 1; } diff --git a/codex-rs/core/src/tools/runtimes/shell.rs b/codex-rs/core/src/tools/runtimes/shell.rs index 2c6c6cd29..528efad74 100644 --- a/codex-rs/core/src/tools/runtimes/shell.rs +++ b/codex-rs/core/src/tools/runtimes/shell.rs @@ -5,8 +5,11 @@ Executes shell requests under the orchestrator: asks for approval when needed, builds a CommandSpec, and runs it under the current SandboxAttempt. */ use crate::exec::ExecToolCallOutput; +use crate::features::Feature; +use crate::powershell::prefix_powershell_script_with_utf8; use crate::sandboxing::SandboxPermissions; use crate::sandboxing::execute_env; +use crate::shell::ShellType; use crate::tools::runtimes::build_command_spec; use crate::tools::runtimes::maybe_wrap_shell_lc_with_snapshot; use crate::tools::sandboxing::Approvable; @@ -144,6 +147,13 @@ impl ToolRuntime for ShellRuntime { let base_command = &req.command; let session_shell = ctx.session.user_shell(); let command = maybe_wrap_shell_lc_with_snapshot(base_command, session_shell.as_ref()); + let command = if matches!(session_shell.shell_type, ShellType::PowerShell) + && ctx.session.features().enabled(Feature::PowershellUtf8) + { + prefix_powershell_script_with_utf8(&command) + } else { + command + }; let spec = build_command_spec( &command, diff --git a/codex-rs/core/src/tools/runtimes/unified_exec.rs b/codex-rs/core/src/tools/runtimes/unified_exec.rs index a38ac9bc3..da1f518e8 100644 --- a/codex-rs/core/src/tools/runtimes/unified_exec.rs +++ b/codex-rs/core/src/tools/runtimes/unified_exec.rs @@ -7,7 +7,10 @@ the session manager to spawn PTYs once an ExecEnv is prepared. use crate::error::CodexErr; use crate::error::SandboxErr; use crate::exec::ExecExpiration; +use crate::features::Feature; +use crate::powershell::prefix_powershell_script_with_utf8; use crate::sandboxing::SandboxPermissions; +use crate::shell::ShellType; use crate::tools::runtimes::build_command_spec; use crate::tools::runtimes::maybe_wrap_shell_lc_with_snapshot; use crate::tools::sandboxing::Approvable; @@ -165,6 +168,13 @@ impl<'a> ToolRuntime for UnifiedExecRunt let base_command = &req.command; let session_shell = ctx.session.user_shell(); let command = maybe_wrap_shell_lc_with_snapshot(base_command, session_shell.as_ref()); + let command = if matches!(session_shell.shell_type, ShellType::PowerShell) + && ctx.session.features().enabled(Feature::PowershellUtf8) + { + prefix_powershell_script_with_utf8(&command) + } else { + command + }; let spec = build_command_spec( &command, diff --git a/codex-rs/core/tests/suite/apply_patch_cli.rs b/codex-rs/core/tests/suite/apply_patch_cli.rs index 880e74d95..1f99c846a 100644 --- a/codex-rs/core/tests/suite/apply_patch_cli.rs +++ b/codex-rs/core/tests/suite/apply_patch_cli.rs @@ -2,10 +2,13 @@ use anyhow::Result; use core_test_support::responses::ev_apply_patch_call; +use core_test_support::responses::ev_apply_patch_custom_tool_call; use core_test_support::responses::ev_shell_command_call; use core_test_support::test_codex::ApplyPatchModelOutput; use pretty_assertions::assert_eq; use std::fs; +use std::sync::atomic::AtomicI32; +use std::sync::atomic::Ordering; use codex_core::features::Feature; use codex_core::protocol::AskForApproval; @@ -29,6 +32,11 @@ use core_test_support::test_codex::test_codex; use core_test_support::wait_for_event; use serde_json::json; use test_case::test_case; +use wiremock::Mock; +use wiremock::Respond; +use wiremock::ResponseTemplate; +use wiremock::matchers::method; +use wiremock::matchers::path_regex; pub async fn apply_patch_harness() -> Result { apply_patch_harness_with(|builder| builder).await @@ -718,6 +726,132 @@ async fn apply_patch_shell_command_heredoc_with_cd_updates_relative_workdir() -> Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn apply_patch_cli_can_use_shell_command_output_as_patch_input() -> Result<()> { + skip_if_no_network!(Ok(())); + + let harness = apply_patch_harness_with(|builder| builder.with_model("gpt-5.1")).await?; + + let source_contents = "line1\nnaïve café\nline3\n"; + let source_path = harness.path("source.txt"); + fs::write(&source_path, source_contents)?; + + let read_call_id = "read-source"; + let apply_call_id = "apply-from-read"; + + fn stdout_from_shell_output(output: &str) -> String { + let normalized = output.replace("\r\n", "\n").replace('\r', "\n"); + normalized + .split_once("Output:\n") + .map(|x| x.1) + .unwrap_or("") + .trim_end_matches('\n') + .to_string() + } + + fn function_call_output_text(body: &serde_json::Value, call_id: &str) -> String { + body.get("input") + .and_then(serde_json::Value::as_array) + .and_then(|items| { + items.iter().find(|item| { + item.get("type").and_then(serde_json::Value::as_str) + == Some("function_call_output") + && item.get("call_id").and_then(serde_json::Value::as_str) == Some(call_id) + }) + }) + .and_then(|item| item.get("output").and_then(serde_json::Value::as_str)) + .expect("function_call_output output string") + .to_string() + } + + struct DynamicApplyFromRead { + num_calls: AtomicI32, + read_call_id: String, + apply_call_id: String, + } + + impl Respond for DynamicApplyFromRead { + fn respond(&self, request: &wiremock::Request) -> ResponseTemplate { + let call_num = self.num_calls.fetch_add(1, Ordering::SeqCst); + match call_num { + 0 => { + let command = if cfg!(windows) { + "Get-Content -Encoding utf8 source.txt" + } else { + "cat source.txt" + }; + let body = sse(vec![ + ev_response_created("resp-1"), + ev_shell_command_call(&self.read_call_id, command), + ev_completed("resp-1"), + ]); + ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_string(body) + } + 1 => { + let body_json: serde_json::Value = + request.body_json().expect("request body should be json"); + let read_output = function_call_output_text(&body_json, &self.read_call_id); + eprintln!("read_output: \n{read_output}"); + let stdout = stdout_from_shell_output(&read_output); + eprintln!("stdout: \n{stdout}"); + let patch_lines = stdout + .lines() + .map(|line| format!("+{line}")) + .collect::>() + .join("\n"); + let patch = format!( + "*** Begin Patch\n*** Add File: target.txt\n{patch_lines}\n*** End Patch" + ); + + eprintln!("patch: \n{patch}"); + + let body = sse(vec![ + ev_response_created("resp-2"), + ev_apply_patch_custom_tool_call(&self.apply_call_id, &patch), + ev_completed("resp-2"), + ]); + ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_string(body) + } + 2 => { + let body = sse(vec![ + ev_assistant_message("msg-1", "ok"), + ev_completed("resp-3"), + ]); + ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_string(body) + } + _ => panic!("no response for call {call_num}"), + } + } + } + + let responder = DynamicApplyFromRead { + num_calls: AtomicI32::new(0), + read_call_id: read_call_id.to_string(), + apply_call_id: apply_call_id.to_string(), + }; + Mock::given(method("POST")) + .and(path_regex(".*/responses$")) + .respond_with(responder) + .expect(3) + .mount(harness.server()) + .await; + + harness + .submit("read source.txt, then apply it to target.txt") + .await?; + + let target_contents = fs::read_to_string(harness.path("target.txt"))?; + assert_eq!(target_contents, source_contents); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn apply_patch_shell_command_heredoc_with_cd_emits_turn_diff() -> Result<()> { skip_if_no_network!(Ok(())); diff --git a/codex-rs/core/tests/suite/shell_command.rs b/codex-rs/core/tests/suite/shell_command.rs index 34124b35d..6aa0da2d3 100644 --- a/codex-rs/core/tests/suite/shell_command.rs +++ b/codex-rs/core/tests/suite/shell_command.rs @@ -1,4 +1,5 @@ use anyhow::Result; +use codex_core::features::Feature; use core_test_support::assert_regex_match; use core_test_support::responses::ev_assistant_message; use core_test_support::responses::ev_completed; @@ -12,6 +13,7 @@ use core_test_support::test_codex::TestCodexBuilder; use core_test_support::test_codex::TestCodexHarness; use core_test_support::test_codex::test_codex; use serde_json::json; +use test_case::test_case; fn shell_responses_with_timeout( call_id: &str, @@ -201,7 +203,6 @@ async fn shell_command_times_out_with_timeout_ms() -> anyhow::Result<()> { skip_if_no_network!(Ok(())); let harness = shell_command_harness_with(|builder| builder.with_model("gpt-5.1")).await?; - let call_id = "shell-command-timeout"; let command = if cfg!(windows) { "timeout /t 5" @@ -224,3 +225,61 @@ async fn shell_command_times_out_with_timeout_ms() -> anyhow::Result<()> { Ok(()) } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[test_case(true ; "with_login")] +#[test_case(false ; "without_login")] +async fn unicode_output(login: bool) -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let harness = shell_command_harness_with(|builder| { + builder.with_model("gpt-5.2").with_config(|config| { + config.features.enable(Feature::PowershellUtf8); + }) + }) + .await?; + + let call_id = "unicode_output"; + mount_shell_responses( + &harness, + call_id, + "git -c alias.say='!printf \"%s\" \"naïve_café\"' say", + Some(login), + ) + .await; + harness.submit("run the command without login").await?; + + let output = harness.function_call_stdout(call_id).await; + assert_shell_command_output(&output, "naïve_café")?; + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[test_case(true ; "with_login")] +#[test_case(false ; "without_login")] +async fn unicode_output_with_newlines(login: bool) -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let harness = shell_command_harness_with(|builder| { + builder.with_model("gpt-5.2").with_config(|config| { + config.features.enable(Feature::PowershellUtf8); + }) + }) + .await?; + + let call_id = "unicode_output"; + mount_shell_responses( + &harness, + call_id, + "echo 'line1\nnaïve café\nline3'", + Some(login), + ) + .await; + harness.submit("run the command without login").await?; + + let output = harness.function_call_stdout(call_id).await; + assert_shell_command_output(&output, "line1\\nnaïve café\\nline3")?; + + Ok(()) +}