|
| 1 | +#!/bin/bash |
| 2 | +# Copyright 2025 The JAX Authors. |
| 3 | + |
| 4 | +# |
| 5 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | +# you may not use this file except in compliance with the License. |
| 7 | +# You may obtain a copy of the License at |
| 8 | +# |
| 9 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +# |
| 11 | +# Unless required by applicable law or agreed to in writing, software |
| 12 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | +# See the License for the specific language governing permissions and |
| 15 | +# limitations under the License. |
| 16 | +# ============================================================================== |
| 17 | + |
| 18 | +# Script that can be run at the end of jobs to copy the logs or pytest results to a transient CI bucket |
| 19 | + |
| 20 | +GCS_BUCKET="gs://general-ml-ci-transient/jax-github-actions/logs/$GITHUB_REPOSITORY/$GITHUB_RUN_ID/$GITHUB_RUN_ATTEMPT" |
| 21 | + |
| 22 | +# Check if we are in a Bazel workspace. |
| 23 | +if [ ! -f "WORKSPACE" ] && [ ! -f "WORKSPACE.bazel" ]; then |
| 24 | + echo "ERROR: No WORKSPACE file found. Please run this script from the root of your Bazel workspace." |
| 25 | + exit 1 |
| 26 | +fi |
| 27 | + |
| 28 | + |
| 29 | +# --- Main Logic --- |
| 30 | + |
| 31 | +echo "🚀 Starting Bazel test log upload to GCS..." |
| 32 | +echo "Destination Bucket: $GCS_BUCKET" |
| 33 | + |
| 34 | +# Get the path to the bazel-out directory. We use XDG_CACHE_HOME if set as it doesn't require starting bazel and is much quicker |
| 35 | +if [ -n "$XDG_CACHE_HOME" ]; then |
| 36 | + MD5=($(echo -n $(pwd) | md5sum)) |
| 37 | + BAZEL_OUT_DIR="$XDG_CACHE_HOME/bazel/_bazel_root/$MD5/execroot/__main__/bazel-out" |
| 38 | +else |
| 39 | + BAZEL_OUT_DIR=$(bazel info output_path) |
| 40 | +fi |
| 41 | + |
| 42 | +if [ ! -d "$BAZEL_OUT_DIR" ]; then |
| 43 | + echo "ERROR: Could not find the Bazel output directory at '$BAZEL_OUT_DIR'." |
| 44 | + echo "Have you built or tested any targets yet?" |
| 45 | + exit 1 |
| 46 | +fi |
| 47 | + |
| 48 | +echo "Searching for 'testlogs' directories under: $BAZEL_OUT_DIR" |
| 49 | + |
| 50 | +# Use 'find' to locate all directories named 'testlogs'. |
| 51 | +found_logs=0 |
| 52 | +find "$BAZEL_OUT_DIR" -type d -name "testlogs" | while read -r testlogs_path; do |
| 53 | + found_logs=1 |
| 54 | + echo "======================================================================" |
| 55 | + echo "Found testlogs directory: $testlogs_path" |
| 56 | + |
| 57 | + # To avoid naming collisions in GCS, we create a descriptive path from the |
| 58 | + # log file's location relative to the bazel-out directory. |
| 59 | + # e.g., 'k8-fastbuild/testlogs' becomes 'k8-fastbuild-testlogs' |
| 60 | + relative_path=${testlogs_path#"$BAZEL_OUT_DIR/"} |
| 61 | + gcs_prefix=$(echo "$relative_path" | tr '/' '-') |
| 62 | + |
| 63 | + # Define the final destination path in the GCS bucket. |
| 64 | + GCS_DESTINATION_PATH="${GCS_BUCKET}/${gcs_prefix}/" |
| 65 | + |
| 66 | + echo "Uploading contents to: $GCS_DESTINATION_PATH" |
| 67 | + |
| 68 | + # Use gsutil to copy the entire directory's contents recursively. |
| 69 | + # The '-m' flag enables parallel (multi-threaded/multi-processing) uploads. |
| 70 | + # The '-r' flag copies directories recursively. |
| 71 | + # The trailing '*' ensures the *contents* of the directory are copied. |
| 72 | + # gsutil -m cp -r "${testlogs_path}/*" "$GCS_DESTINATION_PATH" |
| 73 | + |
| 74 | + echo "Upload complete for $testlogs_path" |
| 75 | +done |
| 76 | + |
| 77 | +echo "Found logs $found_logs" |
| 78 | +# --- Final Check --- |
| 79 | +if [[ "$found_logs" -eq 0 ]]; then |
| 80 | + echo "Log: No 'testlogs' directories were found." |
| 81 | +fi |
| 82 | + |
| 83 | +echo "======================================================================" |
| 84 | +echo "Script finished." |
| 85 | + |
0 commit comments