Add a facility for running script tests (#161)

Script tests are shell scripts that take a Bazel binary and create their
own workspace directories, set up files, etc.

These are similar to the existing integration tests in
https://github.com/bazelbuild/bazel/tree/master/src/test/shell/integration.

---------

Co-authored-by: Chuck Grindel <chuck.grindel@gmail.com>

Author: katre

.bazelrc

@@ -3,8 +3,8 @@
 # This lets us glob() up all the files inside the examples to make them inputs to tests
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, run tools/update_deleted_packages.sh
-build --deleted_packages=examples/custom_test_runner,examples/custom_test_runner/Sources/CustomTestRunner,examples/custom_test_runner/Sources/CustomTestRunnerLib,examples/custom_test_runner/Tests/CustomTestRunnerTests,examples/custom_test_runner/integration_tests,examples/custom_test_runner/integration_tests/workspace,examples/env_var_with_rootpath/sample_workspace,examples/simple,examples/simple/mockascript,examples/simple/mockascript/private,examples/simple/tests,tests/params_tests/workspace
-query --deleted_packages=examples/custom_test_runner,examples/custom_test_runner/Sources/CustomTestRunner,examples/custom_test_runner/Sources/CustomTestRunnerLib,examples/custom_test_runner/Tests/CustomTestRunnerTests,examples/custom_test_runner/integration_tests,examples/custom_test_runner/integration_tests/workspace,examples/env_var_with_rootpath/sample_workspace,examples/simple,examples/simple/mockascript,examples/simple/mockascript/private,examples/simple/tests,tests/params_tests/workspace
+build --deleted_packages=examples/custom_test_runner,examples/custom_test_runner/integration_tests,examples/custom_test_runner/integration_tests/workspace,examples/custom_test_runner/Sources/CustomTestRunner,examples/custom_test_runner/Sources/CustomTestRunnerLib,examples/custom_test_runner/Tests/CustomTestRunnerTests,examples/env_var_with_rootpath/sample_workspace,examples/simple,examples/simple/mockascript,examples/simple/mockascript/private,examples/simple/tests,tests/params_tests/workspace
+query --deleted_packages=examples/custom_test_runner,examples/custom_test_runner/integration_tests,examples/custom_test_runner/integration_tests/workspace,examples/custom_test_runner/Sources/CustomTestRunner,examples/custom_test_runner/Sources/CustomTestRunnerLib,examples/custom_test_runner/Tests/CustomTestRunnerTests,examples/env_var_with_rootpath/sample_workspace,examples/simple,examples/simple/mockascript,examples/simple/mockascript/private,examples/simple/tests,tests/params_tests/workspace
 
 # Import Shared settings
 import %workspace%/shared.bazelrc

bazel_integration_test/defs.bzl

@@ -17,6 +17,10 @@ load(
     "//bazel_integration_test/private:integration_test_utils.bzl",
     _integration_test_utils = "integration_test_utils",
 )
+load(
+    "//bazel_integration_test/private:script_test.bzl",
+    _script_test = "script_test",
+)
 
 # Macros
 bazel_integration_test = _bazel_integration_test
@@ -28,3 +32,6 @@ bazel_binaries = _bazel_binaries
 
 # APIs
 integration_test_utils = _integration_test_utils
+
+# Script tests
+script_test = _script_test

bazel_integration_test/private/script_test.bzl

@@ -0,0 +1,114 @@
+"A facility for script tests that create workspaces and call Bazel."
+
+load(
+    "//bazel_integration_test/private:bazel_integration_test.bzl",
+    "bazel_integration_test",
+)
+
+_RUNNER_SCRIPT = """
+#!/bin/bash
+
+script="${PWD}/%s"
+
+working_dir="${BIT_WORKSPACE_DIR:-}"
+cd "${working_dir}"
+
+# Run the actual tests
+# Bazel is available as ${BIT_BAZEL_BINARY}.
+if "${script}"; then
+  echo "test passed"
+else
+  echo "test failed"
+  exit 1
+fi
+
+exit 0
+"""
+
+def _write_runner_impl(ctx):
+    script = ctx.file.script
+    runner = ctx.outputs.out
+
+    ctx.actions.write(
+        output = runner,
+        content = _RUNNER_SCRIPT % script.path,
+    )
+
+    return [DefaultInfo(
+        executable = runner,
+        runfiles = ctx.runfiles(files = ctx.files.script),
+    )]
+
+_write_runner = rule(
+    implementation = _write_runner_impl,
+    attrs = {
+        "out": attr.output(mandatory = True),
+        "script": attr.label(allow_single_file = True, mandatory = True),
+    },
+    executable = True,
+)
+
+def script_test(
+        name,
+        srcs,
+        deps,
+        bazel_binaries,
+        bazel_version):
+    """
+    Macro that allows writing script tests which create a Bazel workspace and call the binary directly.
+
+    These are intended to be like the Bazel integration tests found at https://github.com/bazelbuild/bazel/tree/master/src/test/shell/integration.
+
+    Tests will find the Bazel binary via the `${BIT_BAZEL_BINARY}` environment variable.
+
+    Args:
+        name: name of the resulting test
+        srcs: The file containing the shell script. This attribute must be a
+            singleton list, whose only element is the shell script. This script
+            must be executable, and may be a source file or a generated file. All
+            other files required at runtime (whether scripts or data) belong in the
+            deps attribute.
+        deps: The list of library targets to be used in this test.
+        bazel_binaries: Optional for WORKSPACE loaded repositories. Required
+            for repositories that enable bzlmod. The value for this parameter
+            is loaded by adding
+            `load("@bazel_binaries//:defs.bzl", "bazel_binaries")` to your
+            build file.
+        bazel_version: Optional. A `string` value representing the semantic
+                       version of Bazel to use for the integration test. If a
+                       version is not specified, then the `bazel_binary` must
+                       be specified.
+    """
+    runner_script = "%s_runner.sh" % name
+    runner_target = "%s_runner" % name
+
+    if len(srcs) != 1:
+        fail("script_test.srcs must be a singleton list, got %s" % srcs)
+
+    src = srcs[0]
+
+    # Generate the test runner script that calls inner.
+    _write_runner(
+        name = "generate_%s" % runner_target,
+        script = src,
+        out = runner_script,
+        testonly = True,
+    )
+
+    # Create the test runner that sets up and calls the test.
+    native.sh_binary(
+        name = runner_target,
+        testonly = True,
+        srcs = [runner_script],
+        data = [src],
+        deps = deps,
+        tags = ["manual"],
+    )
+
+    # The actual test that is invoked.
+    bazel_integration_test(
+        name = name,
+        bazel_binaries = bazel_binaries,
+        bazel_version = bazel_version,
+        test_runner = ":%s" % runner_target,
+    )

examples/BUILD.bazel

@@ -6,6 +6,7 @@ load(
     "bazel_integration_tests",
     "default_test_runner",
     "integration_test_utils",
+    "script_test",
 )
 
 bzlformat_pkg(name = "bzlformat")
@@ -119,6 +120,19 @@ bazel_integration_test(
     test_runner = ":dynamic_workspace_test_runner",
 )
 
+# MARK: - script_test sample
+
+script_test(
+    name = "sample_script_test",
+    srcs = ["sample_script_test.sh"],
+    bazel_binaries = bazel_binaries,
+    bazel_version = bazel_binaries.versions.current,
+    deps = [
+        "@bazel_tools//tools/bash/runfiles",
+        "@cgrindel_bazel_starlib//shlib/lib:assertions",
+    ],
+)
+
 # MARK: - Test Suite
 
 # By default, the integration test targets are tagged as `manual`. This
@@ -133,6 +147,7 @@ test_suite(
     tests = [
         ":custom_test_runner_test",
         ":dynamic_workspace_test",
+        ":sample_script_test",
         ":simple_test",
         ":use_create_scratch_dir_test",
     ] + integration_test_utils.bazel_integration_test_names(

examples/sample_script_test.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# A sample test script which wants to execute Bazel commands and check the
+# result with assertions. script tests are responsible for creating the needed
+# files in the workspace and calling `${bazel}` as needed.
+
+# --- begin runfiles.bash initialization v2 ---
+# Copy-pasted from the Bazel Bash runfiles library v2.
+set -uo pipefail; f=bazel_tools/tools/bash/runfiles/runfiles.bash
+source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \
+  source "$0.runfiles/$f" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \
+  { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e
+# --- end runfiles.bash initialization v2 ---
+
+# Use shlib assertions for testing.
+assertions_sh_location=cgrindel_bazel_starlib/shlib/lib/assertions.sh
+assertions_sh="$(rlocation "${assertions_sh_location}")" || \
+  (echo >&2 "Failed to locate ${assertions_sh_location}" && exit 1)
+# shellcheck disable=SC1090 # external source
+source "${assertions_sh}"
+
+logfile="${TEST_TMPDIR}/test.log"
+
+# Verify the Bazel binary by checking the release number.
+touch WORKSPACE
+${BIT_BAZEL_BINARY} info release > "${logfile}"
+
+if ! grep -E -q "^release [0-9]+\.[0-9]+\.[0-9]$" "${logfile}"; then
+  fail "Did not find release in output of $(bazel info release)"
+fi