3b6c50d925
## Summary
Improve Bazel CI failure diagnostics by printing the tail of each failed
target’s test.log directly in the GitHub Actions output.
Today, when a large Bazel test target fails (for example tests of
`codex-core`), the workflow often only shows a target-level Exit 101
plus a path to Bazel’s test.log. That makes it hard to see the actual
failing Rust test and panic without digging into artifacts or
reproducing locally.
This change makes the workflow automatically surface that information
inline.
## What Changed
In .github/workflows/bazel.yml:
- Capture Bazel console output via tee
- Preserve the Bazel exit code when piping (PIPESTATUS[0])
- On failure:
- Parse failed Bazel test targets from FAIL: //... lines
- Resolve Bazel test log directory via bazel info bazel-testlogs
- Print tail -n 200 for each failed target’s test.log
- Group each target’s output in GitHub Actions logs (::group::)
## Bonus
Disable `experimental_remote_repo_contents_cache` to prevent "Permission
Denied"
206 lines
8.6 KiB
YAML
206 lines
8.6 KiB
YAML
name: Bazel (experimental)
|
|
|
|
# Note this workflow was originally derived from:
|
|
# https://github.com/cerisier/toolchains_llvm_bootstrapped/blob/main/.github/workflows/ci.yaml
|
|
|
|
on:
|
|
pull_request: {}
|
|
push:
|
|
branches:
|
|
- main
|
|
workflow_dispatch:
|
|
|
|
concurrency:
|
|
# Cancel previous actions from the same PR or branch except 'main' branch.
|
|
# See https://docs.github.com/en/actions/using-jobs/using-concurrency and https://docs.github.com/en/actions/learn-github-actions/contexts for more info.
|
|
group: concurrency-group::${{ github.workflow }}::${{ github.event.pull_request.number > 0 && format('pr-{0}', github.event.pull_request.number) || github.ref_name }}${{ github.ref_name == 'main' && format('::{0}', github.run_id) || ''}}
|
|
cancel-in-progress: ${{ github.ref_name != 'main' }}
|
|
jobs:
|
|
test:
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
# macOS
|
|
- os: macos-15-xlarge
|
|
target: aarch64-apple-darwin
|
|
- os: macos-15-xlarge
|
|
target: x86_64-apple-darwin
|
|
|
|
# Linux
|
|
- os: ubuntu-24.04-arm
|
|
target: aarch64-unknown-linux-gnu
|
|
- os: ubuntu-24.04
|
|
target: x86_64-unknown-linux-gnu
|
|
- os: ubuntu-24.04-arm
|
|
target: aarch64-unknown-linux-musl
|
|
- os: ubuntu-24.04
|
|
target: x86_64-unknown-linux-musl
|
|
# TODO: Enable Windows once we fix the toolchain issues there.
|
|
#- os: windows-latest
|
|
# target: x86_64-pc-windows-gnullvm
|
|
runs-on: ${{ matrix.os }}
|
|
|
|
# Configure a human readable name for each job
|
|
name: Local Bazel build on ${{ matrix.os }} for ${{ matrix.target }}
|
|
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
|
|
# Some integration tests rely on DotSlash being installed.
|
|
# See https://github.com/openai/codex/pull/7617.
|
|
- name: Install DotSlash
|
|
uses: facebook/install-dotslash@v2
|
|
|
|
- name: Make DotSlash available in PATH (Unix)
|
|
if: runner.os != 'Windows'
|
|
run: cp "$(which dotslash)" /usr/local/bin
|
|
|
|
- name: Make DotSlash available in PATH (Windows)
|
|
if: runner.os == 'Windows'
|
|
shell: pwsh
|
|
run: Copy-Item (Get-Command dotslash).Source -Destination "$env:LOCALAPPDATA\Microsoft\WindowsApps\dotslash.exe"
|
|
|
|
# Install Bazel via Bazelisk
|
|
- name: Set up Bazel
|
|
uses: bazelbuild/setup-bazelisk@v3
|
|
|
|
- name: Check MODULE.bazel.lock is up to date
|
|
if: matrix.os == 'ubuntu-24.04' && matrix.target == 'x86_64-unknown-linux-gnu'
|
|
shell: bash
|
|
run: ./scripts/check-module-bazel-lock.sh
|
|
|
|
# TODO(mbolin): Bring this back once we have caching working. Currently,
|
|
# we never seem to get a cache hit but we still end up paying the cost of
|
|
# uploading at the end of the build, which takes over a minute!
|
|
#
|
|
# Cache build and external artifacts so that the next ci build is incremental.
|
|
# Because github action caches cannot be updated after a build, we need to
|
|
# store the contents of each build in a unique cache key, then fall back to loading
|
|
# it on the next ci run. We use hashFiles(...) in the key and restore-keys- with
|
|
# the prefix to load the most recent cache for the branch on a cache miss. You
|
|
# should customize the contents of hashFiles to capture any bazel input sources,
|
|
# although this doesn't need to be perfect. If none of the input sources change
|
|
# then a cache hit will load an existing cache and bazel won't have to do any work.
|
|
# In the case of a cache miss, you want the fallback cache to contain most of the
|
|
# previously built artifacts to minimize build time. The more precise you are with
|
|
# hashFiles sources the less work bazel will have to do.
|
|
# - name: Mount bazel caches
|
|
# uses: actions/cache@v5
|
|
# with:
|
|
# path: |
|
|
# ~/.cache/bazel-repo-cache
|
|
# ~/.cache/bazel-repo-contents-cache
|
|
# key: bazel-cache-${{ matrix.os }}-${{ hashFiles('**/BUILD.bazel', '**/*.bzl', 'MODULE.bazel') }}
|
|
# restore-keys: |
|
|
# bazel-cache-${{ matrix.os }}
|
|
|
|
- name: Configure Bazel startup args (Windows)
|
|
if: runner.os == 'Windows'
|
|
shell: pwsh
|
|
run: |
|
|
# Use a very short path to reduce argv/path length issues.
|
|
"BAZEL_STARTUP_ARGS=--output_user_root=C:\" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
|
|
|
- name: bazel test //...
|
|
env:
|
|
BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }}
|
|
shell: bash
|
|
run: |
|
|
set -o pipefail
|
|
|
|
bazel_console_log="$(mktemp)"
|
|
|
|
print_failed_bazel_test_logs() {
|
|
local console_log="$1"
|
|
local testlogs_dir
|
|
|
|
testlogs_dir="$(bazel $BAZEL_STARTUP_ARGS info bazel-testlogs 2>/dev/null || echo bazel-testlogs)"
|
|
|
|
local failed_targets=()
|
|
while IFS= read -r target; do
|
|
failed_targets+=("$target")
|
|
done < <(
|
|
grep -E '^FAIL: //' "$console_log" \
|
|
| sed -E 's#^FAIL: (//[^ ]+).*#\1#' \
|
|
| sort -u
|
|
)
|
|
|
|
if [[ ${#failed_targets[@]} -eq 0 ]]; then
|
|
echo "No failed Bazel test targets were found in console output."
|
|
return
|
|
fi
|
|
|
|
for target in "${failed_targets[@]}"; do
|
|
local rel_path="${target#//}"
|
|
rel_path="${rel_path/:/\/}"
|
|
local test_log="${testlogs_dir}/${rel_path}/test.log"
|
|
|
|
echo "::group::Bazel test log tail for ${target}"
|
|
if [[ -f "$test_log" ]]; then
|
|
tail -n 200 "$test_log"
|
|
else
|
|
echo "Missing test log: $test_log"
|
|
fi
|
|
echo "::endgroup::"
|
|
done
|
|
}
|
|
|
|
bazel_args=(
|
|
test
|
|
//...
|
|
--test_verbose_timeout_warnings
|
|
--build_metadata=REPO_URL=https://github.com/openai/codex.git
|
|
--build_metadata=COMMIT_SHA=$(git rev-parse HEAD)
|
|
--build_metadata=ROLE=CI
|
|
--build_metadata=VISIBILITY=PUBLIC
|
|
)
|
|
|
|
if [[ -n "${BUILDBUDDY_API_KEY:-}" ]]; then
|
|
echo "BuildBuddy API key is available; using remote Bazel configuration."
|
|
# Work around Bazel 9 remote repo contents cache / overlay materialization failures
|
|
# seen in CI (for example "is not a symlink" or permission errors while
|
|
# materializing external repos such as rules_perl). We still use BuildBuddy for
|
|
# remote execution/cache; this only disables the startup-level repo contents cache.
|
|
set +e
|
|
bazel $BAZEL_STARTUP_ARGS \
|
|
--noexperimental_remote_repo_contents_cache \
|
|
--bazelrc=.github/workflows/ci.bazelrc \
|
|
"${bazel_args[@]}" \
|
|
"--remote_header=x-buildbuddy-api-key=$BUILDBUDDY_API_KEY" \
|
|
2>&1 | tee "$bazel_console_log"
|
|
bazel_status=${PIPESTATUS[0]}
|
|
set -e
|
|
else
|
|
echo "BuildBuddy API key is not available; using local Bazel configuration."
|
|
# Keep fork/community PRs on Bazel but disable remote services that are
|
|
# configured in .bazelrc and require auth.
|
|
#
|
|
# Flag docs:
|
|
# - Command-line reference: https://bazel.build/reference/command-line-reference
|
|
# - Remote caching overview: https://bazel.build/remote/caching
|
|
# - Remote execution overview: https://bazel.build/remote/rbe
|
|
# - Build Event Protocol overview: https://bazel.build/remote/bep
|
|
#
|
|
# --noexperimental_remote_repo_contents_cache:
|
|
# disable remote repo contents cache enabled in .bazelrc startup options.
|
|
# https://bazel.build/reference/command-line-reference#startup_options-flag--experimental_remote_repo_contents_cache
|
|
# --remote_cache= and --remote_executor=:
|
|
# clear remote cache/execution endpoints configured in .bazelrc.
|
|
# https://bazel.build/reference/command-line-reference#common_options-flag--remote_cache
|
|
# https://bazel.build/reference/command-line-reference#common_options-flag--remote_executor
|
|
set +e
|
|
bazel $BAZEL_STARTUP_ARGS \
|
|
--noexperimental_remote_repo_contents_cache \
|
|
"${bazel_args[@]}" \
|
|
--remote_cache= \
|
|
--remote_executor= \
|
|
2>&1 | tee "$bazel_console_log"
|
|
bazel_status=${PIPESTATUS[0]}
|
|
set -e
|
|
fi
|
|
|
|
if [[ ${bazel_status:-0} -ne 0 ]]; then
|
|
print_failed_bazel_test_logs "$bazel_console_log"
|
|
exit "$bazel_status"
|
|
fi
|