Refactor and separate concerns of external python package handling code (#953)

Author: groodt

.pre-commit-config.yaml

@@ -3,7 +3,7 @@
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
   - repo: https://github.com/keith/pre-commit-buildifier
-    rev: 5.1.0.1
+    rev: 6.0.0
     hooks:
       - id: buildifier
         args: &args 

MODULE.bazel

@@ -1,15 +1,13 @@
 module(
     name = "rules_python",
-    compatibility_level = 1,
     version = "0.0.0",
+    compatibility_level = 1,
 )
 
 bazel_dep(name = "platforms", version = "0.0.4")
 
 internal_deps = use_extension("@rules_python//python:extensions.bzl", "internal_deps")
-
 internal_deps.install()
-
 use_repo(
     internal_deps,
     "pypi__build",

examples/bzlmod/MODULE.bazel

@@ -1,35 +1,30 @@
 module(
     name = "example_bzlmod",
-    compatibility_level = 1,
     version = "0.0.0",
+    compatibility_level = 1,
 )
 
 bazel_dep(name = "rules_python", version = "0.0.0")
-
 local_path_override(
     module_name = "rules_python",
     path = "../..",
 )
 
 python = use_extension("@rules_python//python:extensions.bzl", "python")
-
 python.toolchain(
     name = "python3_9",
     python_version = "3.9",
 )
-
 use_repo(python, "python3_9_toolchains")
 
 register_toolchains(
     "@python3_9_toolchains//:all",
 )
 
 pip = use_extension("@rules_python//python:extensions.bzl", "pip")
-
 pip.parse(
     name = "pip",
     requirements_lock = "//:requirements_lock.txt",
     requirements_windows = "//:requirements_windows.txt",
 )
-
 use_repo(pip, "pip")

python/extensions.bzl

@@ -78,7 +78,7 @@ def _pip_impl(module_ctx):
                     environment = attr.environment,
                 )
 
-# Keep in sync with python/pip_install/extract_wheels/bazel.py
+# Keep in sync with python/pip_install/tools/bazel.py
 def _sanitize_name(name):
     return name.replace("-", "_").replace(".", "_").lower()
 

python/pip_install/BUILD.bazel

@@ -1,11 +1,11 @@
-exports_files(["pip_compile.py"])
-
 filegroup(
     name = "distribution",
     srcs = glob(["*.bzl"]) + [
         "BUILD.bazel",
-        "pip_compile.py",
-        "//python/pip_install/extract_wheels:distribution",
+        "//python/pip_install/tools/dependency_resolver:distribution",
+        "//python/pip_install/tools/lib:distribution",
+        "//python/pip_install/tools/lock_file_generator:distribution",
+        "//python/pip_install/tools/wheel_installer:distribution",
         "//python/pip_install/private:distribution",
     ],
     visibility = ["//:__pkg__"],
@@ -22,7 +22,10 @@ filegroup(
 filegroup(
     name = "py_srcs",
     srcs = [
-        "//python/pip_install/extract_wheels:py_srcs",
+        "//python/pip_install/tools/dependency_resolver:py_srcs",
+        "//python/pip_install/tools/lib:py_srcs",
+        "//python/pip_install/tools/lock_file_generator:py_srcs",
+        "//python/pip_install/tools/wheel_installer:py_srcs",
     ],
     visibility = ["//python/pip_install/private:__pkg__"],
 )

python/pip_install/pip_repository.bzl

@@ -307,7 +307,7 @@ def _pip_repository_impl(rctx):
     args = [
         python_interpreter,
         "-m",
-        "python.pip_install.extract_wheels.parse_requirements_to_bzl",
+        "python.pip_install.tools.lock_file_generator.lock_file_generator",
         "--requirements_lock",
         rctx.path(requirements_txt),
         "--requirements_lock_label",
@@ -524,7 +524,7 @@ def _whl_library_impl(rctx):
     args = [
         python_interpreter,
         "-m",
-        "python.pip_install.extract_wheels.wheel_installer",
+        "python.pip_install.tools.wheel_installer.wheel_installer",
         "--requirement",
         rctx.attr.requirement,
         "--repo",

python/pip_install/private/pip_install_utils.bzl

@@ -1,9 +1,9 @@
 """Utilities for `rules_python` pip rules"""
 
 _SRCS_TEMPLATE = """\
-\"\"\"A generate file containing all source files used for `@rules_python//python/pip_install:pip_repository.bzl` rules
+\"\"\"A generated file containing all source files used for `@rules_python//python/pip_install:pip_repository.bzl` rules
 
-This file is auto-generated from the `@rules_python//python/pip_install/private:srcs_module.install` target. Please
+This file is auto-generated from the `@rules_python//python/pip_install/private:srcs_module.update` target. Please
 `bazel run` this target to apply any updates. Note that doing so will discard any local modifications.
 "\"\"
 

python/pip_install/private/srcs.bzl

@@ -1,18 +1,21 @@
-"""A generate file containing all source files used for `@rules_python//python/pip_install:pip_repository.bzl` rules
+"""A generated file containing all source files used for `@rules_python//python/pip_install:pip_repository.bzl` rules
 
-This file is auto-generated from the `@rules_python//python/pip_install/private:srcs_module.install` target. Please
+This file is auto-generated from the `@rules_python//python/pip_install/private:srcs_module.update` target. Please
 `bazel run` this target to apply any updates. Note that doing so will discard any local modifications.
 """
 
 # Each source file is tracked as a target so `pip_repository` rules will know to automatically rebuild if any of the
 # sources changed.
 PIP_INSTALL_PY_SRCS = [
-    "@rules_python//python/pip_install/extract_wheels:__init__.py",
-    "@rules_python//python/pip_install/extract_wheels:annotation.py",
-    "@rules_python//python/pip_install/extract_wheels:arguments.py",
-    "@rules_python//python/pip_install/extract_wheels:bazel.py",
-    "@rules_python//python/pip_install/extract_wheels:namespace_pkgs.py",
-    "@rules_python//python/pip_install/extract_wheels:parse_requirements_to_bzl.py",
-    "@rules_python//python/pip_install/extract_wheels:wheel.py",
-    "@rules_python//python/pip_install/extract_wheels:wheel_installer.py",
+    "@rules_python//python/pip_install/tools/dependency_resolver:__init__.py",
+    "@rules_python//python/pip_install/tools/dependency_resolver:dependency_resolver.py",
+    "@rules_python//python/pip_install/tools/lib:__init__.py",
+    "@rules_python//python/pip_install/tools/lib:annotation.py",
+    "@rules_python//python/pip_install/tools/lib:arguments.py",
+    "@rules_python//python/pip_install/tools/lib:bazel.py",
+    "@rules_python//python/pip_install/tools/lock_file_generator:__init__.py",
+    "@rules_python//python/pip_install/tools/lock_file_generator:lock_file_generator.py",
+    "@rules_python//python/pip_install/tools/wheel_installer:namespace_pkgs.py",
+    "@rules_python//python/pip_install/tools/wheel_installer:wheel.py",
+    "@rules_python//python/pip_install/tools/wheel_installer:wheel_installer.py",
 ]

python/pip_install/repositories.bzl

@@ -89,7 +89,7 @@ py_library(
     srcs = glob(["**/*.py"]),
     data = glob(["**/*"], exclude=[
         # These entries include those put into user-installed dependencies by
-        # data_exclude in /python/pip_install/extract_wheels/bazel.py
+        # data_exclude in /python/pip_install/tools/bazel.py
         # to avoid non-determinism following pip install's behavior.
         "**/*.py",
         "**/*.pyc",

python/pip_install/requirements.bzl

@@ -57,7 +57,7 @@ def compile_pip_requirements(
 
     # Use the Label constructor so this is expanded in the context of the file
     # where it appears, which is to say, in @rules_python
-    pip_compile = Label("//python/pip_install:pip_compile.py")
+    pip_compile = Label("//python/pip_install/tools/dependency_resolver:dependency_resolver.py")
 
     loc = "$(rootpath {})"
 

python/pip_install/tools/dependency_resolver/BUILD.bazel

@@ -0,0 +1,19 @@
+exports_files(["dependency_resolver.py"])
+
+filegroup(
+    name = "distribution",
+    srcs = glob(
+        ["*"],
+        exclude = ["*_test.py"],
+    ),
+    visibility = ["//python/pip_install:__subpackages__"],
+)
+
+filegroup(
+    name = "py_srcs",
+    srcs = glob(
+        include = ["**/*.py"],
+        exclude = ["**/*_test.py"],
+    ),
+    visibility = ["//python/pip_install:__subpackages__"],
+)

python/pip_install/extract_wheels/__init__.py renamed to python/pip_install/tools/dependency_resolver/__init__.py

@@ -1,5 +1,4 @@
-load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
-load("//python/pip_install:repositories.bzl", "requirement")
+load("//python:defs.bzl", "py_library", "py_test")
 load(":annotations_test_helpers.bzl", "package_annotation", "package_annotations_file")
 
 py_library(
@@ -8,31 +7,8 @@ py_library(
         "annotation.py",
         "arguments.py",
         "bazel.py",
-        "namespace_pkgs.py",
-        "parse_requirements_to_bzl.py",
-        "wheel.py",
-        "wheel_installer.py",
     ],
-    deps = [
-        requirement("installer"),
-        requirement("setuptools"),
-    ],
-)
-
-py_binary(
-    name = "wheel_installer",
-    srcs = [
-        "wheel_installer.py",
-    ],
-    deps = [":lib"],
-)
-
-py_binary(
-    name = "parse_requirements_to_bzl",
-    srcs = [
-        "parse_requirements_to_bzl.py",
-    ],
-    deps = [":lib"],
+    visibility = ["//python/pip_install:__subpackages__"],
 )
 
 package_annotations_file(
@@ -70,57 +46,18 @@ py_test(
     srcs = ["annotations_test.py"],
     data = [":mock_annotations"],
     env = {"MOCK_ANNOTATIONS": "$(rootpath :mock_annotations)"},
-    tags = ["unit"],
     deps = [
         ":lib",
         "//python/runfiles",
     ],
 )
 
-py_test(
-    name = "namespace_pkgs_test",
-    size = "small",
-    srcs = [
-        "namespace_pkgs_test.py",
-    ],
-    tags = ["unit"],
-    deps = [
-        ":lib",
-    ],
-)
-
-py_test(
-    name = "wheel_installer_test",
-    size = "small",
-    srcs = [
-        "wheel_installer_test.py",
-    ],
-    data = ["//examples/wheel:minimal_with_py_package"],
-    tags = ["unit"],
-    deps = [
-        ":lib",
-    ],
-)
-
 py_test(
     name = "arguments_test",
     size = "small",
     srcs = [
         "arguments_test.py",
     ],
-    tags = ["unit"],
-    deps = [
-        ":lib",
-    ],
-)
-
-py_test(
-    name = "parse_requirements_to_bzl_test",
-    size = "small",
-    srcs = [
-        "parse_requirements_to_bzl_test.py",
-    ],
-    tags = ["unit"],
     deps = [
         ":lib",
     ],

python/pip_install/pip_compile.py renamed to python/pip_install/tools/dependency_resolver/dependency_resolver.py

@@ -5,7 +5,7 @@
 import unittest
 from pathlib import Path
 
-from python.pip_install.extract_wheels.annotation import Annotation, AnnotationsMap
+from python.pip_install.tools.lib.annotation import Annotation, AnnotationsMap
 from python.runfiles import runfiles
 
 

python/pip_install/extract_wheels/BUILD.bazel renamed to python/pip_install/tools/lib/BUILD.bazel

@@ -1,4 +1,4 @@
-"""Helper macros and rules for testing the `annotations` module of `extract_wheels`"""
+"""Helper macros and rules for testing the `annotations` module of `tools`"""
 
 load("//python:pip.bzl", _package_annotation = "package_annotation")
 

python/pip_install/tools/lib/__init__.py

@@ -2,7 +2,7 @@
 import json
 import unittest
 
-from python.pip_install.extract_wheels import arguments
+from python.pip_install.tools.lib import arguments
 
 
 class ArgumentsTestCase(unittest.TestCase):

python/pip_install/extract_wheels/annotation.py renamed to python/pip_install/tools/lib/annotation.py

@@ -0,0 +1,50 @@
+load("//python:defs.bzl", "py_binary", "py_library", "py_test")
+load("//python/pip_install:repositories.bzl", "requirement")
+
+py_library(
+    name = "lib",
+    srcs = [
+        "lock_file_generator.py",
+    ],
+    deps = [
+        "//python/pip_install/tools/lib",
+        requirement("pip"),
+    ],
+)
+
+py_binary(
+    name = "lock_file_generator",
+    srcs = [
+        "lock_file_generator.py",
+    ],
+    deps = [":lib"],
+)
+
+py_test(
+    name = "lock_file_generator_test",
+    size = "small",
+    srcs = [
+        "lock_file_generator_test.py",
+    ],
+    deps = [
+        ":lib",
+    ],
+)
+
+filegroup(
+    name = "distribution",
+    srcs = glob(
+        ["*"],
+        exclude = ["*_test.py"],
+    ),
+    visibility = ["//python/pip_install:__subpackages__"],
+)
+
+filegroup(
+    name = "py_srcs",
+    srcs = glob(
+        include = ["**/*.py"],
+        exclude = ["**/*_test.py"],
+    ),
+    visibility = ["//python/pip_install:__subpackages__"],
+)

python/pip_install/extract_wheels/annotations_test.py renamed to python/pip_install/tools/lib/annotations_test.py

@@ -16,7 +16,7 @@
 )
 from pip._internal.req.req_install import InstallRequirement
 
-from python.pip_install.extract_wheels import annotation, arguments, bazel
+from python.pip_install.tools.lib import annotation, arguments, bazel
 
 
 def parse_install_requirements(