From 08a000866f8757c35483bebd44848c94dd1d63b9 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 12 Feb 2026 00:08:32 -0800 Subject: [PATCH] Fix linux-musl release link failures caused by glibc-only libcap artifacts (#11556) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem: The `aarch64-unknown-linux-musl` release build was failing at link time with `/usr/bin/ld: cannot find -lcap` while building binaries that transitively pull in `codex-linux-sandbox`. Why this is the right fix: `codex-linux-sandbox` compiles vendored bubblewrap and links `libcap`. In the musl jobs, we were installing distro `libcap-dev`, which provides host/glibc artifacts. That is not a valid source of target-compatible static libcap for musl cross-linking, so the fix is to produce a target-compatible libcap inside the musl tool bootstrap and point pkg-config at it. This also closes the CI coverage gap that allowed this to slip through: the `rust-ci.yml` matrix did not exercise `aarch64-unknown-linux-musl` in `release` mode. Adding that target/profile combination to CI is the right regression barrier for this class of failure. What changed: - Updated `.github/scripts/install-musl-build-tools.sh` to install tooling needed to fetch/build libcap sources (`curl`, `xz-utils`, certs). - Added deterministic libcap bootstrap in the musl tool root: - download `libcap-2.75` from kernel.org - verify SHA256 - build with the target musl compiler (`*-linux-musl-gcc`) - stage `libcap.a` and headers under the target tool root - generate a target-scoped `libcap.pc` - Exported target `PKG_CONFIG_PATH` so builds resolve the staged musl libcap instead of host pkg-config/lib paths. - Updated `.github/workflows/rust-ci.yml` to add a `release` matrix entry for `aarch64-unknown-linux-musl` on the ARM runner. - Updated `.github/workflows/rust-ci.yml` to set `CARGO_PROFILE_RELEASE_LTO=thin` for `release` matrix entries (and keep `fat` for non-release entries), matching the release-build tradeoff already used in `rust-release.yml` while reducing CI runtime. Verification: - Reproduced the original failure in CI-like containers: - `aarch64-unknown-linux-musl` failed with `cannot find -lcap`. - Verified the underlying mismatch by forcing host libcap into the link: - link then failed with glibc-specific unresolved symbols (`__isoc23_*`, `__*_chk`), confirming host libcap was unsuitable. - Verified the fix in CI-like containers after this change: - `cargo build -p codex-linux-sandbox --target aarch64-unknown-linux-musl --release` -> pass - `cargo build -p codex-linux-sandbox --target x86_64-unknown-linux-musl --release` -> pass - Triggered `rust-ci` on this branch and confirmed the new job appears: - `Lint/Build — ubuntu-24.04-arm - aarch64-unknown-linux-musl (release)` --- .github/scripts/install-musl-build-tools.sh | 52 ++++++++++++++++++++- .github/workflows/rust-ci.yml | 8 ++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/.github/scripts/install-musl-build-tools.sh b/.github/scripts/install-musl-build-tools.sh index bd392b854..e4c6683d0 100644 --- a/.github/scripts/install-musl-build-tools.sh +++ b/.github/scripts/install-musl-build-tools.sh @@ -17,7 +17,7 @@ if [[ -n "${APT_INSTALL_ARGS:-}" ]]; then fi sudo apt-get update "${apt_update_args[@]}" -sudo apt-get install -y "${apt_install_args[@]}" musl-tools pkg-config libcap-dev g++ clang libc++-dev libc++abi-dev lld +sudo apt-get install -y "${apt_install_args[@]}" ca-certificates curl musl-tools pkg-config libcap-dev g++ clang libc++-dev libc++abi-dev lld xz-utils case "${TARGET}" in x86_64-unknown-linux-musl) @@ -32,6 +32,11 @@ case "${TARGET}" in ;; esac +libcap_version="2.75" +libcap_sha256="de4e7e064c9ba451d5234dd46e897d7c71c96a9ebf9a0c445bc04f4742d83632" +libcap_tarball_name="libcap-${libcap_version}.tar.xz" +libcap_download_url="https://mirrors.edge.kernel.org/pub/linux/libs/security/linux-privs/libcap2/${libcap_tarball_name}" + # Use the musl toolchain as the Rust linker to avoid Zig injecting its own CRT. if command -v "${arch}-linux-musl-gcc" >/dev/null; then musl_linker="$(command -v "${arch}-linux-musl-gcc")" @@ -47,6 +52,43 @@ runner_temp="${RUNNER_TEMP:-/tmp}" tool_root="${runner_temp}/codex-musl-tools-${TARGET}" mkdir -p "${tool_root}" +libcap_root="${tool_root}/libcap-${libcap_version}" +libcap_src_root="${libcap_root}/src" +libcap_prefix="${libcap_root}/prefix" +libcap_pkgconfig_dir="${libcap_prefix}/lib/pkgconfig" + +if [[ ! -f "${libcap_prefix}/lib/libcap.a" ]]; then + mkdir -p "${libcap_src_root}" "${libcap_prefix}/lib" "${libcap_prefix}/include/sys" "${libcap_prefix}/include/linux" "${libcap_pkgconfig_dir}" + libcap_tarball="${libcap_root}/${libcap_tarball_name}" + + curl -fsSL "${libcap_download_url}" -o "${libcap_tarball}" + echo "${libcap_sha256} ${libcap_tarball}" | sha256sum -c - + + tar -xJf "${libcap_tarball}" -C "${libcap_src_root}" + libcap_source_dir="${libcap_src_root}/libcap-${libcap_version}" + make -C "${libcap_source_dir}/libcap" -j"$(nproc)" \ + CC="${musl_linker}" \ + AR=ar \ + RANLIB=ranlib + + cp "${libcap_source_dir}/libcap/libcap.a" "${libcap_prefix}/lib/libcap.a" + cp "${libcap_source_dir}/libcap/include/uapi/linux/capability.h" "${libcap_prefix}/include/linux/capability.h" + cp "${libcap_source_dir}/libcap/../libcap/include/sys/capability.h" "${libcap_prefix}/include/sys/capability.h" + + cat > "${libcap_pkgconfig_dir}/libcap.pc" </dev/null; then zig_bin="$(command -v zig)" @@ -220,6 +262,14 @@ echo "CMAKE_ARGS=-DCMAKE_HAVE_THREADS_LIBRARY=1 -DCMAKE_USE_PTHREADS_INIT=1 -DCM # Allow pkg-config resolution during cross-compilation. echo "PKG_CONFIG_ALLOW_CROSS=1" >> "$GITHUB_ENV" +pkg_config_path="${libcap_pkgconfig_dir}" +if [[ -n "${PKG_CONFIG_PATH:-}" ]]; then + pkg_config_path="${pkg_config_path}:${PKG_CONFIG_PATH}" +fi +echo "PKG_CONFIG_PATH=${pkg_config_path}" >> "$GITHUB_ENV" +pkg_config_path_var="PKG_CONFIG_PATH_${TARGET}" +pkg_config_path_var="${pkg_config_path_var//-/_}" +echo "${pkg_config_path_var}=${libcap_pkgconfig_dir}" >> "$GITHUB_ENV" if [[ -n "${sysroot}" && "${sysroot}" != "/" ]]; then echo "PKG_CONFIG_SYSROOT_DIR=${sysroot}" >> "$GITHUB_ENV" diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml index f9c866e57..c52013149 100644 --- a/.github/workflows/rust-ci.yml +++ b/.github/workflows/rust-ci.yml @@ -99,6 +99,8 @@ jobs: USE_SCCACHE: ${{ startsWith(matrix.runner, 'windows') && 'false' || 'true' }} CARGO_INCREMENTAL: "0" SCCACHE_CACHE_SIZE: 10G + # In rust-ci, representative release-profile checks use thin LTO for faster feedback. + CARGO_PROFILE_RELEASE_LTO: ${{ matrix.profile == 'release' && 'thin' || 'fat' }} strategy: fail-fast: false @@ -160,6 +162,12 @@ jobs: runs_on: group: codex-runners labels: codex-linux-x64 + - runner: ubuntu-24.04-arm + target: aarch64-unknown-linux-musl + profile: release + runs_on: + group: codex-runners + labels: codex-linux-arm64 - runner: windows-x64 target: x86_64-pc-windows-msvc profile: release