diff --git a/.codex/skills/remote-tests/SKILL.md b/.codex/skills/remote-tests/SKILL.md new file mode 100644 index 000000000..ee35fc2b2 --- /dev/null +++ b/.codex/skills/remote-tests/SKILL.md @@ -0,0 +1,16 @@ +--- +name: remote-tests +description: How to run tests using remote executor. +--- + +Some codex integration tests support a running against a remote executor. +This means that when CODEX_TEST_REMOTE_ENV environment variable is set they will attempt to start an executor process in a docker container CODEX_TEST_REMOTE_ENV points to and use it in tests. + +Docker container is built and initialized via ./scripts/test-remote-env.sh + +Currently running remote tests is only supported on Linux, so you need to use a devbox to run them + +You can list devboxes via `applied_devbox ls`, pick the one with `codex` in the name. +Connect to devbox via `ssh `. +Reuse the same checkout of codex in `~/code/codex`. Reset files if needed. Multiple checkouts take longer to build and take up more space. +Check whether the SHA and modified files are in sync between remote and local. diff --git a/codex-rs/core/tests/suite/view_image.rs b/codex-rs/core/tests/suite/view_image.rs index f0cc6a989..efc2e5332 100644 --- a/codex-rs/core/tests/suite/view_image.rs +++ b/codex-rs/core/tests/suite/view_image.rs @@ -83,26 +83,73 @@ fn absolute_path(path: &Path) -> anyhow::Result anyhow::Result> { + let image = ImageBuffer::from_pixel(width, height, Rgba(rgba)); + let mut cursor = Cursor::new(Vec::new()); + DynamicImage::ImageRgba8(image).write_to(&mut cursor, image::ImageFormat::Png)?; + Ok(cursor.into_inner()) +} + +async fn create_workspace_directory(test: &TestCodex, rel_path: &str) -> anyhow::Result { + let abs_path = test.config.cwd.join(rel_path); + test.fs() + .create_directory( + &absolute_path(&abs_path)?, + CreateDirectoryOptions { recursive: true }, + ) + .await?; + Ok(abs_path) +} + +async fn write_workspace_file( + test: &TestCodex, + rel_path: &str, + contents: Vec, +) -> anyhow::Result { + let abs_path = test.config.cwd.join(rel_path); + if let Some(parent) = abs_path.parent() { + test.fs() + .create_directory( + &absolute_path(parent)?, + CreateDirectoryOptions { recursive: true }, + ) + .await?; + } + test.fs() + .write_file(&absolute_path(&abs_path)?, contents) + .await?; + Ok(abs_path) +} + +async fn write_workspace_png( + test: &TestCodex, + rel_path: &str, + width: u32, + height: u32, + rgba: [u8; 4], +) -> anyhow::Result { + write_workspace_file(test, rel_path, png_bytes(width, height, rgba)?).await +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn user_turn_with_local_image_attaches_image() -> anyhow::Result<()> { skip_if_no_network!(Ok(())); let server = start_mock_server().await; + let mut builder = test_codex(); + let test = builder.build_remote_aware(&server).await?; let TestCodex { codex, - cwd, + config, session_configured, .. - } = test_codex().build(&server).await?; + } = &test; - let rel_path = "user-turn/example.png"; - let abs_path = cwd.path().join(rel_path); - if let Some(parent) = abs_path.parent() { - std::fs::create_dir_all(parent)?; - } let original_width = 2304; let original_height = 864; + let local_image_dir = tempfile::tempdir()?; + let abs_path = local_image_dir.path().join("example.png"); let image = ImageBuffer::from_pixel(original_width, original_height, Rgba([20u8, 40, 60, 255])); image.save(&abs_path)?; @@ -121,7 +168,7 @@ async fn user_turn_with_local_image_attaches_image() -> anyhow::Result<()> { path: abs_path.clone(), }], final_output_json_schema: None, - cwd: cwd.path().to_path_buf(), + cwd: config.cwd.clone(), approval_policy: AskForApproval::Never, sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, @@ -134,7 +181,7 @@ async fn user_turn_with_local_image_attaches_image() -> anyhow::Result<()> { .await?; wait_for_event_with_timeout( - &codex, + codex, |event| matches!(event, EventMsg::TurnComplete(_)), // Empirically, image attachment can be slow under Bazel/RBE. Duration::from_secs(10), @@ -191,27 +238,18 @@ async fn view_image_tool_attaches_local_image() -> anyhow::Result<()> { } = &test; let cwd = config.cwd.clone(); - let rel_path = PathBuf::from("assets/example.png"); - let abs_path = cwd.join(&rel_path); - let abs_path_absolute = absolute_path(&abs_path)?; - let assets_dir = cwd.join("assets"); - - let file_system = test.fs(); - + let rel_path = "assets/example.png"; + let abs_path = cwd.join(rel_path); let original_width = 2304; let original_height = 864; - let image = ImageBuffer::from_pixel(original_width, original_height, Rgba([255u8, 0, 0, 255])); - let mut cursor = Cursor::new(Vec::new()); - DynamicImage::ImageRgba8(image).write_to(&mut cursor, image::ImageFormat::Png)?; - file_system - .create_directory( - &absolute_path(&assets_dir)?, - CreateDirectoryOptions { recursive: true }, - ) - .await?; - file_system - .write_file(&abs_path_absolute, cursor.into_inner()) - .await?; + write_workspace_png( + &test, + rel_path, + original_width, + original_height, + [255u8, 0, 0, 255], + ) + .await?; let call_id = "view-image-call"; let arguments = serde_json::json!({ "path": rel_path }).to_string(); @@ -333,22 +371,25 @@ async fn view_image_tool_can_preserve_original_resolution_when_requested_on_gpt5 .enable(Feature::ImageDetailOriginal) .expect("test config should allow feature update"); }); + let test = builder.build_remote_aware(&server).await?; let TestCodex { codex, - cwd, + config, session_configured, .. - } = builder.build(&server).await?; + } = &test; let rel_path = "assets/original-example.png"; - let abs_path = cwd.path().join(rel_path); - if let Some(parent) = abs_path.parent() { - std::fs::create_dir_all(parent)?; - } let original_width = 2304; let original_height = 864; - let image = ImageBuffer::from_pixel(original_width, original_height, Rgba([0u8, 80, 255, 255])); - image.save(&abs_path)?; + write_workspace_png( + &test, + rel_path, + original_width, + original_height, + [0u8, 80, 255, 255], + ) + .await?; let call_id = "view-image-original"; let arguments = serde_json::json!({ "path": rel_path, "detail": "original" }).to_string(); @@ -375,7 +416,7 @@ async fn view_image_tool_can_preserve_original_resolution_when_requested_on_gpt5 text_elements: Vec::new(), }], final_output_json_schema: None, - cwd: cwd.path().to_path_buf(), + cwd: config.cwd.clone(), approval_policy: AskForApproval::Never, sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, @@ -388,7 +429,7 @@ async fn view_image_tool_can_preserve_original_resolution_when_requested_on_gpt5 .await?; wait_for_event_with_timeout( - &codex, + codex, |event| matches!(event, EventMsg::TurnComplete(_)), Duration::from_secs(10), ) @@ -437,20 +478,16 @@ async fn view_image_tool_errors_clearly_for_unsupported_detail_values() -> anyho .enable(Feature::ImageDetailOriginal) .expect("test config should allow feature update"); }); + let test = builder.build_remote_aware(&server).await?; let TestCodex { codex, - cwd, + config, session_configured, .. - } = builder.build(&server).await?; + } = &test; let rel_path = "assets/unsupported-detail.png"; - let abs_path = cwd.path().join(rel_path); - if let Some(parent) = abs_path.parent() { - std::fs::create_dir_all(parent)?; - } - let image = ImageBuffer::from_pixel(256, 128, Rgba([0u8, 80, 255, 255])); - image.save(&abs_path)?; + write_workspace_png(&test, rel_path, 256, 128, [0u8, 80, 255, 255]).await?; let call_id = "view-image-unsupported-detail"; let arguments = serde_json::json!({ "path": rel_path, "detail": "low" }).to_string(); @@ -477,7 +514,7 @@ async fn view_image_tool_errors_clearly_for_unsupported_detail_values() -> anyho text_elements: Vec::new(), }], final_output_json_schema: None, - cwd: cwd.path().to_path_buf(), + cwd: config.cwd.clone(), approval_policy: AskForApproval::Never, sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, @@ -489,7 +526,7 @@ async fn view_image_tool_errors_clearly_for_unsupported_detail_values() -> anyho }) .await?; - wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; let req = mock.single_request(); let body_with_tool_output = req.body_json(); @@ -523,22 +560,25 @@ async fn view_image_tool_treats_null_detail_as_omitted() -> anyhow::Result<()> { .enable(Feature::ImageDetailOriginal) .expect("test config should allow feature update"); }); + let test = builder.build_remote_aware(&server).await?; let TestCodex { codex, - cwd, + config, session_configured, .. - } = builder.build(&server).await?; + } = &test; let rel_path = "assets/null-detail.png"; - let abs_path = cwd.path().join(rel_path); - if let Some(parent) = abs_path.parent() { - std::fs::create_dir_all(parent)?; - } let original_width = 2304; let original_height = 864; - let image = ImageBuffer::from_pixel(original_width, original_height, Rgba([0u8, 80, 255, 255])); - image.save(&abs_path)?; + write_workspace_png( + &test, + rel_path, + original_width, + original_height, + [0u8, 80, 255, 255], + ) + .await?; let call_id = "view-image-null-detail"; let arguments = serde_json::json!({ "path": rel_path, "detail": null }).to_string(); @@ -565,7 +605,7 @@ async fn view_image_tool_treats_null_detail_as_omitted() -> anyhow::Result<()> { text_elements: Vec::new(), }], final_output_json_schema: None, - cwd: cwd.path().to_path_buf(), + cwd: config.cwd.clone(), approval_policy: AskForApproval::Never, sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, @@ -577,7 +617,7 @@ async fn view_image_tool_treats_null_detail_as_omitted() -> anyhow::Result<()> { }) .await?; - wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; let req = mock.single_request(); let function_output = req.function_call_output(call_id); @@ -619,22 +659,25 @@ async fn view_image_tool_resizes_when_model_lacks_original_detail_support() -> a .enable(Feature::ImageDetailOriginal) .expect("test config should allow feature update"); }); + let test = builder.build_remote_aware(&server).await?; let TestCodex { codex, - cwd, + config, session_configured, .. - } = builder.build(&server).await?; + } = &test; let rel_path = "assets/original-example-lower-model.png"; - let abs_path = cwd.path().join(rel_path); - if let Some(parent) = abs_path.parent() { - std::fs::create_dir_all(parent)?; - } let original_width = 2304; let original_height = 864; - let image = ImageBuffer::from_pixel(original_width, original_height, Rgba([0u8, 80, 255, 255])); - image.save(&abs_path)?; + write_workspace_png( + &test, + rel_path, + original_width, + original_height, + [0u8, 80, 255, 255], + ) + .await?; let call_id = "view-image-original-lower-model"; let arguments = serde_json::json!({ "path": rel_path }).to_string(); @@ -661,7 +704,7 @@ async fn view_image_tool_resizes_when_model_lacks_original_detail_support() -> a text_elements: Vec::new(), }], final_output_json_schema: None, - cwd: cwd.path().to_path_buf(), + cwd: config.cwd.clone(), approval_policy: AskForApproval::Never, sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, @@ -674,7 +717,7 @@ async fn view_image_tool_resizes_when_model_lacks_original_detail_support() -> a .await?; wait_for_event_with_timeout( - &codex, + codex, |event| matches!(event, EventMsg::TurnComplete(_)), Duration::from_secs(10), ) @@ -726,22 +769,25 @@ async fn view_image_tool_does_not_force_original_resolution_with_capability_feat .enable(Feature::ImageDetailOriginal) .expect("test config should allow feature update"); }); + let test = builder.build_remote_aware(&server).await?; let TestCodex { codex, - cwd, + config, session_configured, .. - } = builder.build(&server).await?; + } = &test; let rel_path = "assets/original-example-capability-only.png"; - let abs_path = cwd.path().join(rel_path); - if let Some(parent) = abs_path.parent() { - std::fs::create_dir_all(parent)?; - } let original_width = 2304; let original_height = 864; - let image = ImageBuffer::from_pixel(original_width, original_height, Rgba([0u8, 80, 255, 255])); - image.save(&abs_path)?; + write_workspace_png( + &test, + rel_path, + original_width, + original_height, + [0u8, 80, 255, 255], + ) + .await?; let call_id = "view-image-capability-only"; let arguments = serde_json::json!({ "path": rel_path }).to_string(); @@ -768,7 +814,7 @@ async fn view_image_tool_does_not_force_original_resolution_with_capability_feat text_elements: Vec::new(), }], final_output_json_schema: None, - cwd: cwd.path().to_path_buf(), + cwd: config.cwd.clone(), approval_policy: AskForApproval::Never, sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, @@ -781,7 +827,7 @@ async fn view_image_tool_does_not_force_original_resolution_with_capability_feat .await?; wait_for_event_with_timeout( - &codex, + codex, |event| matches!(event, EventMsg::TurnComplete(_)), Duration::from_secs(10), ) @@ -1043,16 +1089,17 @@ async fn view_image_tool_errors_when_path_is_directory() -> anyhow::Result<()> { let server = start_mock_server().await; + let mut builder = test_codex(); + let test = builder.build_remote_aware(&server).await?; let TestCodex { codex, - cwd, + config, session_configured, .. - } = test_codex().build(&server).await?; + } = &test; let rel_path = "assets"; - let abs_path = cwd.path().join(rel_path); - std::fs::create_dir_all(&abs_path)?; + let abs_path = create_workspace_directory(&test, rel_path).await?; let call_id = "view-image-directory"; let arguments = serde_json::json!({ "path": rel_path }).to_string(); @@ -1079,7 +1126,7 @@ async fn view_image_tool_errors_when_path_is_directory() -> anyhow::Result<()> { text_elements: Vec::new(), }], final_output_json_schema: None, - cwd: cwd.path().to_path_buf(), + cwd: config.cwd.clone(), approval_policy: AskForApproval::Never, sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, @@ -1091,7 +1138,7 @@ async fn view_image_tool_errors_when_path_is_directory() -> anyhow::Result<()> { }) .await?; - wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; let req = mock.single_request(); let body_with_tool_output = req.body_json(); @@ -1116,19 +1163,18 @@ async fn view_image_tool_errors_for_non_image_files() -> anyhow::Result<()> { let server = start_mock_server().await; + let mut builder = test_codex(); + let test = builder.build_remote_aware(&server).await?; let TestCodex { codex, - cwd, + config, session_configured, .. - } = test_codex().build(&server).await?; + } = &test; let rel_path = "assets/example.json"; - let abs_path = cwd.path().join(rel_path); - if let Some(parent) = abs_path.parent() { - std::fs::create_dir_all(parent)?; - } - std::fs::write(&abs_path, br#"{ "message": "hello" }"#)?; + let abs_path = + write_workspace_file(&test, rel_path, br#"{ "message": "hello" }"#.to_vec()).await?; let call_id = "view-image-non-image"; let arguments = serde_json::json!({ "path": rel_path }).to_string(); @@ -1155,7 +1201,7 @@ async fn view_image_tool_errors_for_non_image_files() -> anyhow::Result<()> { text_elements: Vec::new(), }], final_output_json_schema: None, - cwd: cwd.path().to_path_buf(), + cwd: config.cwd.clone(), approval_policy: AskForApproval::Never, sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, @@ -1167,7 +1213,7 @@ async fn view_image_tool_errors_for_non_image_files() -> anyhow::Result<()> { }) .await?; - wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; let request = mock.single_request(); assert!( @@ -1198,15 +1244,17 @@ async fn view_image_tool_errors_when_file_missing() -> anyhow::Result<()> { let server = start_mock_server().await; + let mut builder = test_codex(); + let test = builder.build_remote_aware(&server).await?; let TestCodex { codex, - cwd, + config, session_configured, .. - } = test_codex().build(&server).await?; + } = &test; let rel_path = "missing/example.png"; - let abs_path = cwd.path().join(rel_path); + let abs_path = config.cwd.join(rel_path); let call_id = "view-image-missing"; let arguments = serde_json::json!({ "path": rel_path }).to_string(); @@ -1233,7 +1281,7 @@ async fn view_image_tool_errors_when_file_missing() -> anyhow::Result<()> { text_elements: Vec::new(), }], final_output_json_schema: None, - cwd: cwd.path().to_path_buf(), + cwd: config.cwd.clone(), approval_policy: AskForApproval::Never, sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, @@ -1245,7 +1293,7 @@ async fn view_image_tool_errors_when_file_missing() -> anyhow::Result<()> { }) .await?; - wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; let req = mock.single_request(); let body_with_tool_output = req.body_json(); @@ -1322,21 +1370,16 @@ async fn view_image_tool_returns_unsupported_message_for_text_only_model() -> an ) .await; - let TestCodex { codex, cwd, .. } = test_codex() + let mut builder = test_codex() .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) .with_config(|config| { config.model = Some(model_slug.to_string()); - }) - .build(&server) - .await?; + }); + let test = builder.build_remote_aware(&server).await?; + let TestCodex { codex, config, .. } = &test; let rel_path = "assets/example.png"; - let abs_path = cwd.path().join(rel_path); - if let Some(parent) = abs_path.parent() { - std::fs::create_dir_all(parent)?; - } - let image = ImageBuffer::from_pixel(20, 20, Rgba([255u8, 0, 0, 255])); - image.save(&abs_path)?; + write_workspace_png(&test, rel_path, 20, 20, [255u8, 0, 0, 255]).await?; let call_id = "view-image-unsupported-model"; let arguments = serde_json::json!({ "path": rel_path }).to_string(); @@ -1360,7 +1403,7 @@ async fn view_image_tool_returns_unsupported_message_for_text_only_model() -> an text_elements: Vec::new(), }], final_output_json_schema: None, - cwd: cwd.path().to_path_buf(), + cwd: config.cwd.clone(), approval_policy: AskForApproval::Never, sandbox_policy: SandboxPolicy::DangerFullAccess, model: model_slug.to_string(), @@ -1372,7 +1415,7 @@ async fn view_image_tool_returns_unsupported_message_for_text_only_model() -> an }) .await?; - wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; let output_text = mock .single_request() @@ -1414,20 +1457,17 @@ async fn replaces_invalid_local_image_after_bad_request() -> anyhow::Result<()> let completion_mock = responses::mount_sse_once(&server, success_response).await; + let mut builder = test_codex(); + let test = builder.build_remote_aware(&server).await?; let TestCodex { codex, - cwd, + config, session_configured, .. - } = test_codex().build(&server).await?; + } = &test; let rel_path = "assets/poisoned.png"; - let abs_path = cwd.path().join(rel_path); - if let Some(parent) = abs_path.parent() { - std::fs::create_dir_all(parent)?; - } - let image = ImageBuffer::from_pixel(1024, 512, Rgba([10u8, 20, 30, 255])); - image.save(&abs_path)?; + let abs_path = write_workspace_png(&test, rel_path, 1024, 512, [10u8, 20, 30, 255]).await?; let session_model = session_configured.model.clone(); @@ -1437,7 +1477,7 @@ async fn replaces_invalid_local_image_after_bad_request() -> anyhow::Result<()> path: abs_path.clone(), }], final_output_json_schema: None, - cwd: cwd.path().to_path_buf(), + cwd: config.cwd.clone(), approval_policy: AskForApproval::Never, sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model,