Add remote test skill (#15324)

Teach codex to run remote tests.

.codex/skills/remote-tests/SKILL.md

@@ -0,0 +1,16 @@
+---
+name: remote-tests
+description: How to run tests using remote executor.
+---
+
+Some codex integration tests support a running against a remote executor.
+This means that when CODEX_TEST_REMOTE_ENV environment variable is set they will attempt to start an executor process in a docker container CODEX_TEST_REMOTE_ENV points to and use it in tests.
+
+Docker container is built and initialized via ./scripts/test-remote-env.sh
+
+Currently running remote tests is only supported on Linux, so you need to use a devbox to run them
+
+You can list devboxes via `applied_devbox ls`, pick the one with `codex` in the name.
+Connect to devbox via `ssh <devbox_name>`.
+Reuse the same checkout of codex in `~/code/codex`. Reset files if needed. Multiple checkouts take longer to build and take up more space.
+Check whether the SHA and modified files are in sync between remote and local.

codex-rs/core/tests/suite/view_image.rs

@@ -83,26 +83,73 @@ fn absolute_path(path: &Path) -> anyhow::Result<codex_utils_absolute_path::Absol
         .map_err(|err| anyhow::anyhow!("invalid absolute path {}: {err}", path.display()))
 }
 
+fn png_bytes(width: u32, height: u32, rgba: [u8; 4]) -> anyhow::Result<Vec<u8>> {
+    let image = ImageBuffer::from_pixel(width, height, Rgba(rgba));
+    let mut cursor = Cursor::new(Vec::new());
+    DynamicImage::ImageRgba8(image).write_to(&mut cursor, image::ImageFormat::Png)?;
+    Ok(cursor.into_inner())
+}
+
+async fn create_workspace_directory(test: &TestCodex, rel_path: &str) -> anyhow::Result<PathBuf> {
+    let abs_path = test.config.cwd.join(rel_path);
+    test.fs()
+        .create_directory(
+            &absolute_path(&abs_path)?,
+            CreateDirectoryOptions { recursive: true },
+        )
+        .await?;
+    Ok(abs_path)
+}
+
+async fn write_workspace_file(
+    test: &TestCodex,
+    rel_path: &str,
+    contents: Vec<u8>,
+) -> anyhow::Result<PathBuf> {
+    let abs_path = test.config.cwd.join(rel_path);
+    if let Some(parent) = abs_path.parent() {
+        test.fs()
+            .create_directory(
+                &absolute_path(parent)?,
+                CreateDirectoryOptions { recursive: true },
+            )
+            .await?;
+    }
+    test.fs()
+        .write_file(&absolute_path(&abs_path)?, contents)
+        .await?;
+    Ok(abs_path)
+}
+
+async fn write_workspace_png(
+    test: &TestCodex,
+    rel_path: &str,
+    width: u32,
+    height: u32,
+    rgba: [u8; 4],
+) -> anyhow::Result<PathBuf> {
+    write_workspace_file(test, rel_path, png_bytes(width, height, rgba)?).await
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn user_turn_with_local_image_attaches_image() -> anyhow::Result<()> {
     skip_if_no_network!(Ok(()));
 
     let server = start_mock_server().await;
 
+    let mut builder = test_codex();
+    let test = builder.build_remote_aware(&server).await?;
     let TestCodex {
         codex,
-        cwd,
+        config,
         session_configured,
         ..
-    } = test_codex().build(&server).await?;
+    } = &test;
 
-    let rel_path = "user-turn/example.png";
-    let abs_path = cwd.path().join(rel_path);
-    if let Some(parent) = abs_path.parent() {
-        std::fs::create_dir_all(parent)?;
-    }
     let original_width = 2304;
     let original_height = 864;
+    let local_image_dir = tempfile::tempdir()?;
+    let abs_path = local_image_dir.path().join("example.png");
     let image = ImageBuffer::from_pixel(original_width, original_height, Rgba([20u8, 40, 60, 255]));
     image.save(&abs_path)?;
 
@@ -121,7 +168,7 @@ async fn user_turn_with_local_image_attaches_image() -> anyhow::Result<()> {
                 path: abs_path.clone(),
             }],
             final_output_json_schema: None,
-            cwd: cwd.path().to_path_buf(),
+            cwd: config.cwd.clone(),
             approval_policy: AskForApproval::Never,
             sandbox_policy: SandboxPolicy::DangerFullAccess,
             model: session_model,
@@ -134,7 +181,7 @@ async fn user_turn_with_local_image_attaches_image() -> anyhow::Result<()> {
         .await?;
 
     wait_for_event_with_timeout(
-        &codex,
+        codex,
         |event| matches!(event, EventMsg::TurnComplete(_)),
         // Empirically, image attachment can be slow under Bazel/RBE.
         Duration::from_secs(10),
@@ -191,27 +238,18 @@ async fn view_image_tool_attaches_local_image() -> anyhow::Result<()> {
     } = &test;
     let cwd = config.cwd.clone();
 
-    let rel_path = PathBuf::from("assets/example.png");
-    let abs_path = cwd.join(&rel_path);
-    let abs_path_absolute = absolute_path(&abs_path)?;
-    let assets_dir = cwd.join("assets");
-
-    let file_system = test.fs();
-
+    let rel_path = "assets/example.png";
+    let abs_path = cwd.join(rel_path);
     let original_width = 2304;
     let original_height = 864;
-    let image = ImageBuffer::from_pixel(original_width, original_height, Rgba([255u8, 0, 0, 255]));
-    let mut cursor = Cursor::new(Vec::new());
-    DynamicImage::ImageRgba8(image).write_to(&mut cursor, image::ImageFormat::Png)?;
-    file_system
-        .create_directory(
-            &absolute_path(&assets_dir)?,
-            CreateDirectoryOptions { recursive: true },
-        )
-        .await?;
-    file_system
-        .write_file(&abs_path_absolute, cursor.into_inner())
-        .await?;
+    write_workspace_png(
+        &test,
+        rel_path,
+        original_width,
+        original_height,
+        [255u8, 0, 0, 255],
+    )
+    .await?;
 
     let call_id = "view-image-call";
     let arguments = serde_json::json!({ "path": rel_path }).to_string();
@@ -333,22 +371,25 @@ async fn view_image_tool_can_preserve_original_resolution_when_requested_on_gpt5
                 .enable(Feature::ImageDetailOriginal)
                 .expect("test config should allow feature update");
         });
+    let test = builder.build_remote_aware(&server).await?;
     let TestCodex {
         codex,
-        cwd,
+        config,
         session_configured,
         ..
-    } = builder.build(&server).await?;
+    } = &test;
 
     let rel_path = "assets/original-example.png";
-    let abs_path = cwd.path().join(rel_path);
-    if let Some(parent) = abs_path.parent() {
-        std::fs::create_dir_all(parent)?;
-    }
     let original_width = 2304;
     let original_height = 864;
-    let image = ImageBuffer::from_pixel(original_width, original_height, Rgba([0u8, 80, 255, 255]));
-    image.save(&abs_path)?;
+    write_workspace_png(
+        &test,
+        rel_path,
+        original_width,
+        original_height,
+        [0u8, 80, 255, 255],
+    )
+    .await?;
 
     let call_id = "view-image-original";
     let arguments = serde_json::json!({ "path": rel_path, "detail": "original" }).to_string();
@@ -375,7 +416,7 @@ async fn view_image_tool_can_preserve_original_resolution_when_requested_on_gpt5
                 text_elements: Vec::new(),
             }],
             final_output_json_schema: None,
-            cwd: cwd.path().to_path_buf(),
+            cwd: config.cwd.clone(),
             approval_policy: AskForApproval::Never,
             sandbox_policy: SandboxPolicy::DangerFullAccess,
             model: session_model,
@@ -388,7 +429,7 @@ async fn view_image_tool_can_preserve_original_resolution_when_requested_on_gpt5
         .await?;
 
     wait_for_event_with_timeout(
-        &codex,
+        codex,
         |event| matches!(event, EventMsg::TurnComplete(_)),
         Duration::from_secs(10),
     )
@@ -437,20 +478,16 @@ async fn view_image_tool_errors_clearly_for_unsupported_detail_values() -> anyho
                 .enable(Feature::ImageDetailOriginal)
                 .expect("test config should allow feature update");
         });
+    let test = builder.build_remote_aware(&server).await?;
     let TestCodex {
         codex,
-        cwd,
+        config,
         session_configured,
         ..
-    } = builder.build(&server).await?;
+    } = &test;
 
     let rel_path = "assets/unsupported-detail.png";
-    let abs_path = cwd.path().join(rel_path);
-    if let Some(parent) = abs_path.parent() {
-        std::fs::create_dir_all(parent)?;
-    }
-    let image = ImageBuffer::from_pixel(256, 128, Rgba([0u8, 80, 255, 255]));
-    image.save(&abs_path)?;
+    write_workspace_png(&test, rel_path, 256, 128, [0u8, 80, 255, 255]).await?;
 
     let call_id = "view-image-unsupported-detail";
     let arguments = serde_json::json!({ "path": rel_path, "detail": "low" }).to_string();
@@ -477,7 +514,7 @@ async fn view_image_tool_errors_clearly_for_unsupported_detail_values() -> anyho
                 text_elements: Vec::new(),
             }],
             final_output_json_schema: None,
-            cwd: cwd.path().to_path_buf(),
+            cwd: config.cwd.clone(),
             approval_policy: AskForApproval::Never,
             sandbox_policy: SandboxPolicy::DangerFullAccess,
             model: session_model,
@@ -489,7 +526,7 @@ async fn view_image_tool_errors_clearly_for_unsupported_detail_values() -> anyho
         })
         .await?;
 
-    wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await;
+    wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await;
 
     let req = mock.single_request();
     let body_with_tool_output = req.body_json();
@@ -523,22 +560,25 @@ async fn view_image_tool_treats_null_detail_as_omitted() -> anyhow::Result<()> {
                 .enable(Feature::ImageDetailOriginal)
                 .expect("test config should allow feature update");
         });
+    let test = builder.build_remote_aware(&server).await?;
     let TestCodex {
         codex,
-        cwd,
+        config,
         session_configured,
         ..
-    } = builder.build(&server).await?;
+    } = &test;
 
     let rel_path = "assets/null-detail.png";
-    let abs_path = cwd.path().join(rel_path);
-    if let Some(parent) = abs_path.parent() {
-        std::fs::create_dir_all(parent)?;
-    }
     let original_width = 2304;
     let original_height = 864;
-    let image = ImageBuffer::from_pixel(original_width, original_height, Rgba([0u8, 80, 255, 255]));
-    image.save(&abs_path)?;
+    write_workspace_png(
+        &test,
+        rel_path,
+        original_width,
+        original_height,
+        [0u8, 80, 255, 255],
+    )
+    .await?;
 
     let call_id = "view-image-null-detail";
     let arguments = serde_json::json!({ "path": rel_path, "detail": null }).to_string();
@@ -565,7 +605,7 @@ async fn view_image_tool_treats_null_detail_as_omitted() -> anyhow::Result<()> {
                 text_elements: Vec::new(),
             }],
             final_output_json_schema: None,
-            cwd: cwd.path().to_path_buf(),
+            cwd: config.cwd.clone(),
             approval_policy: AskForApproval::Never,
             sandbox_policy: SandboxPolicy::DangerFullAccess,
             model: session_model,
@@ -577,7 +617,7 @@ async fn view_image_tool_treats_null_detail_as_omitted() -> anyhow::Result<()> {
         })
         .await?;
 
-    wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await;
+    wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await;
 
     let req = mock.single_request();
     let function_output = req.function_call_output(call_id);
@@ -619,22 +659,25 @@ async fn view_image_tool_resizes_when_model_lacks_original_detail_support() -> a
             .enable(Feature::ImageDetailOriginal)
             .expect("test config should allow feature update");
     });
+    let test = builder.build_remote_aware(&server).await?;
     let TestCodex {
         codex,
-        cwd,
+        config,
         session_configured,
         ..
-    } = builder.build(&server).await?;
+    } = &test;
 
     let rel_path = "assets/original-example-lower-model.png";
-    let abs_path = cwd.path().join(rel_path);
-    if let Some(parent) = abs_path.parent() {
-        std::fs::create_dir_all(parent)?;
-    }
     let original_width = 2304;
     let original_height = 864;
-    let image = ImageBuffer::from_pixel(original_width, original_height, Rgba([0u8, 80, 255, 255]));
-    image.save(&abs_path)?;
+    write_workspace_png(
+        &test,
+        rel_path,
+        original_width,
+        original_height,
+        [0u8, 80, 255, 255],
+    )
+    .await?;
 
     let call_id = "view-image-original-lower-model";
     let arguments = serde_json::json!({ "path": rel_path }).to_string();
@@ -661,7 +704,7 @@ async fn view_image_tool_resizes_when_model_lacks_original_detail_support() -> a
                 text_elements: Vec::new(),
             }],
             final_output_json_schema: None,
-            cwd: cwd.path().to_path_buf(),
+            cwd: config.cwd.clone(),
             approval_policy: AskForApproval::Never,
             sandbox_policy: SandboxPolicy::DangerFullAccess,
             model: session_model,
@@ -674,7 +717,7 @@ async fn view_image_tool_resizes_when_model_lacks_original_detail_support() -> a
         .await?;
 
     wait_for_event_with_timeout(
-        &codex,
+        codex,
         |event| matches!(event, EventMsg::TurnComplete(_)),
         Duration::from_secs(10),
     )
@@ -726,22 +769,25 @@ async fn view_image_tool_does_not_force_original_resolution_with_capability_feat
                 .enable(Feature::ImageDetailOriginal)
                 .expect("test config should allow feature update");
         });
+    let test = builder.build_remote_aware(&server).await?;
     let TestCodex {
         codex,
-        cwd,
+        config,
         session_configured,
         ..
-    } = builder.build(&server).await?;
+    } = &test;
 
     let rel_path = "assets/original-example-capability-only.png";
-    let abs_path = cwd.path().join(rel_path);
-    if let Some(parent) = abs_path.parent() {
-        std::fs::create_dir_all(parent)?;
-    }
     let original_width = 2304;
     let original_height = 864;
-    let image = ImageBuffer::from_pixel(original_width, original_height, Rgba([0u8, 80, 255, 255]));
-    image.save(&abs_path)?;
+    write_workspace_png(
+        &test,
+        rel_path,
+        original_width,
+        original_height,
+        [0u8, 80, 255, 255],
+    )
+    .await?;
 
     let call_id = "view-image-capability-only";
     let arguments = serde_json::json!({ "path": rel_path }).to_string();
@@ -768,7 +814,7 @@ async fn view_image_tool_does_not_force_original_resolution_with_capability_feat
                 text_elements: Vec::new(),
             }],
             final_output_json_schema: None,
-            cwd: cwd.path().to_path_buf(),
+            cwd: config.cwd.clone(),
             approval_policy: AskForApproval::Never,
             sandbox_policy: SandboxPolicy::DangerFullAccess,
             model: session_model,
@@ -781,7 +827,7 @@ async fn view_image_tool_does_not_force_original_resolution_with_capability_feat
         .await?;
 
     wait_for_event_with_timeout(
-        &codex,
+        codex,
         |event| matches!(event, EventMsg::TurnComplete(_)),
         Duration::from_secs(10),
     )
@@ -1043,16 +1089,17 @@ async fn view_image_tool_errors_when_path_is_directory() -> anyhow::Result<()> {
 
     let server = start_mock_server().await;
 
+    let mut builder = test_codex();
+    let test = builder.build_remote_aware(&server).await?;
     let TestCodex {
         codex,
-        cwd,
+        config,
         session_configured,
         ..
-    } = test_codex().build(&server).await?;
+    } = &test;
 
     let rel_path = "assets";
-    let abs_path = cwd.path().join(rel_path);
-    std::fs::create_dir_all(&abs_path)?;
+    let abs_path = create_workspace_directory(&test, rel_path).await?;
 
     let call_id = "view-image-directory";
     let arguments = serde_json::json!({ "path": rel_path }).to_string();
@@ -1079,7 +1126,7 @@ async fn view_image_tool_errors_when_path_is_directory() -> anyhow::Result<()> {
                 text_elements: Vec::new(),
             }],
             final_output_json_schema: None,
-            cwd: cwd.path().to_path_buf(),
+            cwd: config.cwd.clone(),
             approval_policy: AskForApproval::Never,
             sandbox_policy: SandboxPolicy::DangerFullAccess,
             model: session_model,
@@ -1091,7 +1138,7 @@ async fn view_image_tool_errors_when_path_is_directory() -> anyhow::Result<()> {
         })
         .await?;
 
-    wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await;
+    wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await;
 
     let req = mock.single_request();
     let body_with_tool_output = req.body_json();
@@ -1116,19 +1163,18 @@ async fn view_image_tool_errors_for_non_image_files() -> anyhow::Result<()> {
 
     let server = start_mock_server().await;
 
+    let mut builder = test_codex();
+    let test = builder.build_remote_aware(&server).await?;
     let TestCodex {
         codex,
-        cwd,
+        config,
         session_configured,
         ..
-    } = test_codex().build(&server).await?;
+    } = &test;
 
     let rel_path = "assets/example.json";
-    let abs_path = cwd.path().join(rel_path);
-    if let Some(parent) = abs_path.parent() {
-        std::fs::create_dir_all(parent)?;
-    }
-    std::fs::write(&abs_path, br#"{ "message": "hello" }"#)?;
+    let abs_path =
+        write_workspace_file(&test, rel_path, br#"{ "message": "hello" }"#.to_vec()).await?;
 
     let call_id = "view-image-non-image";
     let arguments = serde_json::json!({ "path": rel_path }).to_string();
@@ -1155,7 +1201,7 @@ async fn view_image_tool_errors_for_non_image_files() -> anyhow::Result<()> {
                 text_elements: Vec::new(),
             }],
             final_output_json_schema: None,
-            cwd: cwd.path().to_path_buf(),
+            cwd: config.cwd.clone(),
             approval_policy: AskForApproval::Never,
             sandbox_policy: SandboxPolicy::DangerFullAccess,
             model: session_model,
@@ -1167,7 +1213,7 @@ async fn view_image_tool_errors_for_non_image_files() -> anyhow::Result<()> {
         })
         .await?;
 
-    wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await;
+    wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await;
 
     let request = mock.single_request();
     assert!(
@@ -1198,15 +1244,17 @@ async fn view_image_tool_errors_when_file_missing() -> anyhow::Result<()> {
 
     let server = start_mock_server().await;
 
+    let mut builder = test_codex();
+    let test = builder.build_remote_aware(&server).await?;
     let TestCodex {
         codex,
-        cwd,
+        config,
         session_configured,
         ..
-    } = test_codex().build(&server).await?;
+    } = &test;
 
     let rel_path = "missing/example.png";
-    let abs_path = cwd.path().join(rel_path);
+    let abs_path = config.cwd.join(rel_path);
 
     let call_id = "view-image-missing";
     let arguments = serde_json::json!({ "path": rel_path }).to_string();
@@ -1233,7 +1281,7 @@ async fn view_image_tool_errors_when_file_missing() -> anyhow::Result<()> {
                 text_elements: Vec::new(),
             }],
             final_output_json_schema: None,
-            cwd: cwd.path().to_path_buf(),
+            cwd: config.cwd.clone(),
             approval_policy: AskForApproval::Never,
             sandbox_policy: SandboxPolicy::DangerFullAccess,
             model: session_model,
@@ -1245,7 +1293,7 @@ async fn view_image_tool_errors_when_file_missing() -> anyhow::Result<()> {
         })
         .await?;
 
-    wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await;
+    wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await;
 
     let req = mock.single_request();
     let body_with_tool_output = req.body_json();
@@ -1322,21 +1370,16 @@ async fn view_image_tool_returns_unsupported_message_for_text_only_model() -> an
     )
     .await;
 
-    let TestCodex { codex, cwd, .. } = test_codex()
+    let mut builder = test_codex()
         .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing())
         .with_config(|config| {
             config.model = Some(model_slug.to_string());
-        })
-        .build(&server)
-        .await?;
+        });
+    let test = builder.build_remote_aware(&server).await?;
+    let TestCodex { codex, config, .. } = &test;
 
     let rel_path = "assets/example.png";
-    let abs_path = cwd.path().join(rel_path);
-    if let Some(parent) = abs_path.parent() {
-        std::fs::create_dir_all(parent)?;
-    }
-    let image = ImageBuffer::from_pixel(20, 20, Rgba([255u8, 0, 0, 255]));
-    image.save(&abs_path)?;
+    write_workspace_png(&test, rel_path, 20, 20, [255u8, 0, 0, 255]).await?;
 
     let call_id = "view-image-unsupported-model";
     let arguments = serde_json::json!({ "path": rel_path }).to_string();
@@ -1360,7 +1403,7 @@ async fn view_image_tool_returns_unsupported_message_for_text_only_model() -> an
                 text_elements: Vec::new(),
             }],
             final_output_json_schema: None,
-            cwd: cwd.path().to_path_buf(),
+            cwd: config.cwd.clone(),
             approval_policy: AskForApproval::Never,
             sandbox_policy: SandboxPolicy::DangerFullAccess,
             model: model_slug.to_string(),
@@ -1372,7 +1415,7 @@ async fn view_image_tool_returns_unsupported_message_for_text_only_model() -> an
         })
         .await?;
 
-    wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await;
+    wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await;
 
     let output_text = mock
         .single_request()
@@ -1414,20 +1457,17 @@ async fn replaces_invalid_local_image_after_bad_request() -> anyhow::Result<()>
 
     let completion_mock = responses::mount_sse_once(&server, success_response).await;
 
+    let mut builder = test_codex();
+    let test = builder.build_remote_aware(&server).await?;
     let TestCodex {
         codex,
-        cwd,
+        config,
         session_configured,
         ..
-    } = test_codex().build(&server).await?;
+    } = &test;
 
     let rel_path = "assets/poisoned.png";
-    let abs_path = cwd.path().join(rel_path);
-    if let Some(parent) = abs_path.parent() {
-        std::fs::create_dir_all(parent)?;
-    }
-    let image = ImageBuffer::from_pixel(1024, 512, Rgba([10u8, 20, 30, 255]));
-    image.save(&abs_path)?;
+    let abs_path = write_workspace_png(&test, rel_path, 1024, 512, [10u8, 20, 30, 255]).await?;
 
     let session_model = session_configured.model.clone();
 
@@ -1437,7 +1477,7 @@ async fn replaces_invalid_local_image_after_bad_request() -> anyhow::Result<()>
                 path: abs_path.clone(),
             }],
             final_output_json_schema: None,
-            cwd: cwd.path().to_path_buf(),
+            cwd: config.cwd.clone(),
             approval_policy: AskForApproval::Never,
             sandbox_policy: SandboxPolicy::DangerFullAccess,
             model: session_model,