Merge remainder of pyproject support (#41676)

* conglomerated improvements to support building and testing pure pyproject.toml packages

---------

Co-authored-by: Laurent Mazuel <laurent.mazuel@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: 3 people

eng/ci_tools.txt

@@ -15,7 +15,7 @@ typing-extensions==4.12.2
 pyproject-api==1.8.0
 cibuildwheel==2.16.5
 importlib-metadata==8.5.0
-build==1.2.2
+build==1.2.2.post1
 
 # requirements leveraged for testing
 pytest==8.3.5

eng/scripts/get_package_properties.py

@@ -47,7 +47,7 @@
     # Use abspath for the os.walk because if setup parsing fails it often changes cwd which throws off the relative walk
     for root, dirs, files in os.walk(os.path.abspath(args.search_path)):
         if re.search(r"sdk[\\/][^\\/]+[\\/][^\\/]+$", root):
-            if "setup.py" in files:
+            if "setup.py" in files or "pyproject.toml" in files:
                 try:
                     parsed = ParsedSetup.from_path(root)
 

eng/test_tools.txt

@@ -7,6 +7,7 @@ pytest-xdist==3.2.1
 coverage==7.6.1
 bandit==1.6.2
 pyproject-api==1.8.0
+build==1.2.2.post1
 
 # locking packages defined as deps from azure-sdk-tools
 Jinja2==3.1.6

eng/tox/install_depend_packages.py

@@ -313,7 +313,7 @@ def check_req_against_exclusion(req, req_to_exclude):
 
 
 def filter_dev_requirements(
-    setup_py_path,
+    package_directory,
     released_packages,
     temp_dir,
     additional_filter_fn: Optional[Callable[[str, List[str], List[Requirement]], List[str]]] = None,
@@ -326,7 +326,7 @@ def filter_dev_requirements(
     by the package).
     """
     # This method returns list of requirements from dev_requirements by filtering out packages in given list
-    dev_req_path = os.path.join(os.path.dirname(setup_py_path), DEV_REQ_FILE)
+    dev_req_path = os.path.join(package_directory, DEV_REQ_FILE)
     requirements = []
     with open(dev_req_path, "r") as dev_req_file:
         requirements = dev_req_file.readlines()
@@ -350,7 +350,7 @@ def filter_dev_requirements(
 
     if additional_filter_fn:
         # this filter function handles the case where a dev requirement is incompatible with the current set of targeted packages
-        filtered_req = additional_filter_fn(setup_py_path, filtered_req, released_packages)
+        filtered_req = additional_filter_fn(package_directory, filtered_req, released_packages)
 
     logging.info("Filtered dev requirements: %s", filtered_req)
 
@@ -408,10 +408,12 @@ def install_packages(packages, req_file):
     )
 
     args = parser.parse_args()
-    setup_path = os.path.join(os.path.abspath(args.target_package), "setup.py")
+
+    setup_path = os.path.join(os.path.abspath(args.target_package))
 
     if not (os.path.exists(setup_path) and os.path.exists(args.work_dir)):
         logging.error("Invalid arguments. Please make sure target directory and working directory are valid path")
         sys.exit(1)
 
+
     install_dependent_packages(setup_path, args.dependency_type, args.work_dir)

eng/tox/tox.ini

@@ -10,7 +10,7 @@
 # You can control which pip implementation is used by setting the TOX_PIP_IMPL environment variable:
 # - TOX_PIP_IMPL=pip (default) - Uses standard pip for package installation
 # - TOX_PIP_IMPL=uv - Uses uv pip for faster package installation
-# 
+#
 # Example usage:
 #   tox -e pylint                                                    # Use standard pip (default)
 #   TOX_PIP_IMPL=pip tox -e pylint                                  # Use standard pip explicitly
@@ -34,7 +34,6 @@ pip_command = {[tox]pip_impl}
 deps =
   -r {repository_root}/eng/test_tools.txt
 
-
 [coverage:paths]
 source =
     azure
@@ -60,6 +59,7 @@ pkgs =
     packaging==24.2
     urllib3==2.2.3
     tomli==2.2.1
+    build==1.2.2.post1
 
 [pytest]
 ignore_args=--ignore=.tox --ignore=build --ignore=.eggs --ignore=samples
@@ -430,7 +430,7 @@ deps =
     {[packaging]pkgs}
 commands =
     python -m {[tox]pip_command} install {repository_root}/tools/azure-sdk-tools --no-deps
-    python {tox_root}/setup.py --q sdist -d {envtmpdir}
+    sdk_build_package --package_folder {tox_root} -d {envtmpdir} --package_type sdist
     python {repository_root}/eng/tox/verify_sdist.py -d {envtmpdir} -t {tox_root}
 
 

eng/tox/verify_sdist.py

@@ -45,7 +45,7 @@ def get_root_directories_in_sdist(dist_dir: str, version: str) -> List[str]:
     """
     # find sdist zip file
     # extract sdist and find list of directories in sdist
-    path_to_zip = glob.glob(os.path.join(dist_dir, "*{}*.tar.gz".format(version)))[0]
+    path_to_zip = glob.glob(os.path.join(dist_dir, "**", "*{}*.tar.gz".format(version)), recursive=True)[0]
     extract_location = os.path.join(dist_dir, "unzipped")
     # Cleanup any files in unzipped
     cleanup(extract_location)

tools/azure-sdk-tools/ci_tools/build.py

@@ -10,6 +10,57 @@
 from ci_tools.versioning.version_set_dev import get_dev_version, format_build_id
 
 
+def build_package() -> None:
+    parser = argparse.ArgumentParser(
+        description="""This is a secondary entrypoint for the "build" action. This command is used to install dependencies and build a specific package within the azure-sdk-for-python repository.""",
+    )
+
+    parser.add_argument(
+        "-d",
+        "--distribution-directory",
+        dest="distribution_directory",
+        help="The path to the distribution directory. Should be passed $(Build.ArtifactStagingDirectory) from the devops yaml definition."
+        + "If that is not provided, will default to env variable SDK_ARTIFACT_DIRECTORY -> <calculated_repo_root>/.artifacts.",
+    )
+
+    parser.add_argument(
+        "--package_folder",
+        dest="package_folder",
+        required=True,
+        help="The target that should be assembled",
+    )
+
+    parser.add_argument(
+        "--package_type",
+        dest="package_type",
+        choices=["sdist", "whl", "all"],
+        default="all",
+        help="The type of package to build: sdist (source distribution only), whl (wheel only), or all (both)",
+    )
+
+    args = parser.parse_args()
+
+    target_package = ParsedSetup.from_path(args.package_folder)
+    artifact_directory = get_artifact_directory(args.distribution_directory)
+
+    enable_sdist = True
+    enable_wheel = True
+
+    if args.package_type == "sdist":
+        enable_wheel = False
+
+    if args.package_type == "whl":
+        enable_sdist = False
+
+    build_packages(
+        [target_package.folder],
+        artifact_directory,
+        False,
+        DEFAULT_BUILD_ID,
+        enable_wheel,
+        enable_sdist
+    )
+
 def build() -> None:
     parser = argparse.ArgumentParser(
         description="""This is the primary entrypoint for the "build" action. This command is used to build any package within the azure-sdk-for-python repository.""",
@@ -125,6 +176,8 @@ def build() -> None:
         artifact_directory,
         str_to_bool(args.is_dev_build),
         build_id,
+        True,
+        True
     )
 
 
@@ -146,6 +199,8 @@ def build_packages(
     distribution_directory: Optional[str] = None,
     is_dev_build: bool = False,
     build_id: str = "",
+    enable_wheel: bool = True,
+    enable_sdist: bool = True
 ):
     logging.log(level=logging.INFO, msg=f"Generating {targeted_packages} using python{sys.version}")
 
@@ -168,7 +223,7 @@ def build_packages(
             set_version_py(setup_parsed.setup_filename, new_version)
             set_dev_classifier(setup_parsed.setup_filename, new_version)
 
-        create_package(package_root, dist_dir)
+        create_package(package_root, dist_dir, enable_wheel, enable_sdist)
 
 
 def create_package(
@@ -193,7 +248,7 @@ def create_package(
         pip_output = get_pip_list_output(sys.executable)
         necessary_install_requirements = [req for req in setup_parsed.requires if parse_require(req).key not in pip_output.keys()]
         run([sys.executable, "-m", "pip", "install", *necessary_install_requirements], cwd=setup_parsed.folder)
-        run([sys.executable, "-m", "build", f"-n{'s' if enable_sdist else ''}{'w' if enable_wheel else ''}", "-o", dist], cwd=setup_parsed.folder)
+        run([sys.executable, "-m", "build", f"-n{'s' if enable_sdist else ''}{'w' if enable_wheel else ''}", "-o", dist], cwd=setup_parsed.folder, check=True)
     else:
         if enable_wheel:
             if setup_parsed.ext_modules:

tools/azure-sdk-tools/ci_tools/functions.py

@@ -404,7 +404,7 @@ def process_requires(setup_py_path: str, is_dev_build: bool = False):
     else:
         logging.info("Packages not available on PyPI:{}".format(requirement_to_update))
         update_requires(setup_py_path, requirement_to_update)
-        logging.info("Package requirement is updated in setup.py")
+        logging.info(f"Package requirement is updated in {'pyproject.toml' if pkg_details.is_pyproject else 'setup.py'}.")
 
 
 def find_sdist(dist_dir: str, pkg_name: str, pkg_version: str) -> Optional[str]:

tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py

@@ -394,6 +394,7 @@ def parse_pyproject(
     python_requires = project_config.get("requires-python")
     requires = project_config.get("dependencies")
     is_new_sdk = name in NEW_REQ_PACKAGES or any(map(lambda x: (parse_require(x).key in NEW_REQ_PACKAGES), requires))
+
     name_space = name.replace("-", ".")
     package_data = get_value_from_dict(toml_dict, "tool.setuptools.package-data", None)
     include_package_data = get_value_from_dict(toml_dict, "tool.setuptools.include-package-data", True)

tools/azure-sdk-tools/ci_tools/scenario/generation.py

@@ -51,7 +51,6 @@ def create_package_and_install(
 
     commands_options = []
     built_pkg_path = ""
-    setup_py_path = os.path.join(target_setup, "setup.py")
     additional_downloaded_reqs = []
 
     if not os.path.exists(distribution_directory):
@@ -68,10 +67,10 @@ def create_package_and_install(
     if cache_dir:
         commands_options.extend(["--cache-dir", cache_dir])
 
-    target_package = ParsedSetup.from_path(setup_py_path)
+    target_package = ParsedSetup.from_path(target_setup)
 
     discovered_packages = discover_packages(
-        setup_py_path, distribution_directory, target_setup, package_type, force_create
+        target_setup, distribution_directory, target_setup, package_type, force_create
     )
 
     # ensure that discovered packages are always copied to the distribution directory regardless of other factors
@@ -104,7 +103,7 @@ def create_package_and_install(
                 logging.info("Installing {w} from fresh built package.".format(w=built_package))
 
             if not pre_download_disabled:
-                requirements = ParsedSetup.from_path(os.path.join(os.path.abspath(target_setup), "setup.py")).requires
+                requirements = ParsedSetup.from_path(os.path.join(os.path.abspath(target_setup))).requires
                 azure_requirements = [req.split(";")[0] for req in requirements if req.startswith("azure-")]
 
                 if azure_requirements: