SERVER-95382 Introduce python module to find the mongo toolchain (#31229)

GitOrigin-RevId: cd137fb

Author: ryanberryhill

bazel/mongo_script_rules.bzl

@@ -0,0 +1,89 @@
+"""Common mongo-specific bazel build rules intended to be used for buildscripts.
+"""
+
+load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain")
+
+MONGO_TOOLCHAIN_V4_PATH = "/opt/mongodbtoolchain/v4"
+MONGO_TOOLCHAIN_V5_PATH = "external/mongo_toolchain_v5/v5"
+
+def _py_cxx_wrapper(*, python_path, toolchain_path, python_interpreter, main_py):
+    return "\n".join([
+        "export PYTHONPATH={}".format(python_path),
+        "export MONGO_TOOLCHAIN_PATH={}".format(toolchain_path),
+        "{} {}".format(python_interpreter, main_py),
+    ])
+
+def _py_cxx_test_impl(ctx):
+    python = ctx.toolchains["@bazel_tools//tools/python:toolchain_type"].py3_runtime
+
+    python_path = []
+    for dep in ctx.attr.deps:
+        for path in dep[PyInfo].imports.to_list():
+            if path not in python_path:
+                python_path.append(
+                    ctx.expand_make_variables(
+                        "python_library_imports",
+                        "$${RUNFILES_DIR}/" + path,
+                        ctx.var,
+                    ),
+                )
+    python_path_str = ctx.configuration.host_path_separator.join(python_path)
+
+    cc_toolchain = find_cpp_toolchain(ctx)
+    runfiles = ctx.runfiles(
+        files = (
+            ctx.files.srcs +
+            ctx.files.data +
+            ctx.files.deps +
+            ctx.files.main +
+            cc_toolchain.all_files.to_list()
+        ),
+    )
+    transitive_runfiles = []
+    for runfiles_attr in (
+        [ctx.attr.main],
+        ctx.attr.srcs,
+        ctx.attr.deps,
+        ctx.attr.data,
+    ):
+        for target in runfiles_attr:
+            transitive_runfiles.append(target[DefaultInfo].default_runfiles)
+    runfiles = runfiles.merge_all(transitive_runfiles)
+
+    main_py = ctx.attr.main.files.to_list()[0].path
+    script = _py_cxx_wrapper(
+        python_path = python_path_str,
+        toolchain_path = ctx.attr.toolchain_path,
+        python_interpreter = python.interpreter.path,
+        main_py = main_py,
+    )
+    ctx.actions.write(
+        output = ctx.outputs.executable,
+        content = script,
+    )
+
+    return DefaultInfo(files = depset([ctx.outputs.executable]), runfiles = runfiles)
+
+py_cxx_test = rule(
+    implementation = _py_cxx_test_impl,
+    attrs = {
+        "main": attr.label(allow_single_file = True, mandatory = True),
+        "srcs": attr.label_list(allow_files = [".py"]),
+        "deps": attr.label_list(),
+        "data": attr.label_list(),
+        "toolchain_path": attr.string(mandatory = True),
+    },
+    toolchains = ["@bazel_tools//tools/cpp:toolchain_type", "@bazel_tools//tools/python:toolchain_type"],
+    executable = True,
+    test = True,
+)
+
+def mongo_toolchain_py_cxx_test(**kwargs):
+    py_cxx_test(
+        toolchain_path = select({
+            "//bazel/config:mongo_toolchain_v5": MONGO_TOOLCHAIN_V5_PATH,
+            "//conditions:default": MONGO_TOOLCHAIN_V4_PATH,
+        }),
+        target_compatible_with = ["@//bazel/platforms:use_mongo_toolchain"],
+        **kwargs
+    )

buildscripts/BUILD.bazel

@@ -68,6 +68,10 @@ py_binary(
             "click",
             group = "evergreen",
         ),
+        dependency(
+            "typing-extensions",
+            group = "core",
+        ),
     ],
 )
 
@@ -194,3 +198,31 @@ sh_binary(
     },
     visibility = ["//visibility:public"],
 )
+
+py_library(
+    name = "mongo_toolchain",
+    srcs = [
+        "mongo_toolchain.py",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        dependency(
+            "typer",
+            group = "core",
+        ),
+    ],
+)
+
+py_library(
+    name = "clang_tidy_lib",
+    srcs = [
+        "apply_clang_tidy_fixes.py",
+        "clang_tidy.py",
+        "clang_tidy_vscode.py",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        "mongo_toolchain",
+        "simple_report",
+    ],
+)

buildscripts/clang_format.py

@@ -32,6 +32,7 @@
 
 from buildscripts.linter import git, parallel
 from buildscripts.linter.filediff import gather_changed_files_for_lint
+from buildscripts.mongo_toolchain import get_mongo_toolchain
 
 ##############################################################################
 #
@@ -47,12 +48,14 @@
 # Name of clang-format as a binary
 CLANG_FORMAT_PROGNAME = "clang-format"
 
+TOOLCHAIN_VERSION = "v4"
+
 CLANG_FORMAT_HTTP_DARWIN_CACHE = (
     "http://mongodbtoolchain.build.10gen.cc/toolchain/osx/clang-format-12.0.1"
 )
 
-# TODO: Move clang format to the v4 toolchain
-CLANG_FORMAT_TOOLCHAIN_PATH = "/opt/mongodbtoolchain/v4/bin/clang-format"
+toolchain = get_mongo_toolchain(version=TOOLCHAIN_VERSION)
+CLANG_FORMAT_TOOLCHAIN_PATH = toolchain.get_tool_path(CLANG_FORMAT_PROGNAME)
 
 
 ##############################################################################

buildscripts/clang_tidy.py

@@ -18,7 +18,12 @@
 from typing import Any, Dict, List, Optional, Tuple
 
 import yaml
+
+# Get relative imports to work when the package is not installed on the PYTHONPATH.
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
+
 from clang_tidy_vscode import CHECKS_SO
+from mongo_toolchain import get_mongo_toolchain
 from simple_report import make_report, put_report, try_combine_reports
 
 checks_so = ""
@@ -163,7 +168,8 @@ def __dedup_errors(clang_tidy_errors_threads: List[str]) -> str:
 
 
 def _run_tidy(args, parser_defaults):
-    clang_tidy_binary = f"/opt/mongodbtoolchain/{args.clang_tidy_toolchain}/bin/clang-tidy"
+    toolchain = get_mongo_toolchain(version=args.clang_tidy_toolchain)
+    clang_tidy_binary = toolchain.get_tool_path("clang-tidy")
 
     if os.path.exists(args.check_module):
         mongo_tidy_check_module = args.check_module
@@ -366,8 +372,7 @@ def main():
         default=False,
         help="if this is a test evaluating clang tidy itself.",
     )
-    # TODO: Is there someway to get this without hardcoding this much
-    parser.add_argument("-y", "--clang-tidy-toolchain", type=str, default="v4")
+    parser.add_argument("-y", "--clang-tidy-toolchain", type=str, default=None)
     parser.add_argument("-f", "--clang-tidy-cfg", type=str, default=config_file)
     args = parser.parse_args()
 

buildscripts/clang_tidy_vscode.py

@@ -28,6 +28,11 @@
 import subprocess
 import sys
 
+# Get relative imports to work when the package is not installed on the PYTHONPATH.
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
+
+from mongo_toolchain import get_mongo_toolchain
+
 CHECKS_SO = [
     "build/install/lib/libmongo_tidy_checks.so",
 ]
@@ -38,7 +43,8 @@
 
 
 def main():
-    clang_tidy_args = ["/opt/mongodbtoolchain/v4/bin/clang-tidy"]
+    toolchain = get_mongo_toolchain()
+    clang_tidy_args = [toolchain.get_tool_path("clang-tidy")]
     for check_lib in CHECKS_SO:
         if os.path.isfile(check_lib):
             clang_tidy_args += [f"-load={check_lib}"]