Add Classifier Verification (#41610)

* enhance verify_whl tox environment to check that the classifier for the package matches the package version being released

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: McCoy Patiño <39780829+mccoyp@users.noreply.github.com>

Author: 3 people

doc/eng_sys_checks.md

@@ -5,6 +5,7 @@
   - [Skipping a tox test environment at build queue time](#skipping-a-tox-test-environment-at-build-queue-time)
   - [Skipping entire sections of builds](#skipping-entire-sections-of-builds)
   - [The pyproject.toml](#the-pyprojecttoml)
+    - [Coverage Enforcement](#coverage-enforcement)
   - [Environment variables important to CI](#environment-variables-important-to-ci)
     - [Atomic Overrides](#atomic-overrides)
     - [Enable test logging in CI pipelines](#enable-test-logging-in-ci-pipelines)
@@ -13,6 +14,7 @@
     - [Pyright](#pyright)
     - [Verifytypes](#verifytypes)
     - [Pylint](#pylint)
+    - [Sphinx and docstring checker](#sphinx-and-docstring-checker)
     - [Bandit](#bandit)
     - [ApiStubGen](#apistubgen)
     - [black](#black)

eng/tox/verify_sdist.py

@@ -20,13 +20,16 @@
 from typing import List, Mapping, Any
 
 from ci_tools.parsing import ParsedSetup
+from ci_tools.functions import verify_package_classifiers
 
 logging.getLogger().setLevel(logging.INFO)
 
 ALLOWED_ROOT_DIRECTORIES = ["azure", "tests", "samples", "examples"]
 
 EXCLUDED_PYTYPE_PACKAGES = ["azure-keyvault", "azure", "azure-common"]
 
+EXCLUDED_CLASSIFICATION_PACKAGES = []
+
 
 def get_root_directories_in_source(package_dir: str) -> List[str]:
     """
@@ -138,18 +141,34 @@ def verify_sdist_pytyped(
     pkg_dir = os.path.abspath(args.target_package)
     pkg_details = ParsedSetup.from_path(pkg_dir)
 
+    error_occurred = False
+
     if should_verify_package(pkg_details.name):
-        logging.info("Verifying sdist for package [%s]", pkg_details.name)
+        logging.info(f"Verifying sdist for package {pkg_details.name}")
         if verify_sdist(pkg_dir, args.dist_dir, pkg_details.version):
-            logging.info("Verified sdist for package [%s]", pkg_details.name)
+            logging.info(f"Verified sdist for package {pkg_details.name}")
         else:
-            logging.info("Failed to verify sdist for package [%s]", pkg_details.name)
-            exit(1)
+            logging.error(f"Failed to verify sdist for package {pkg_details.name}")
+            error_occurred = True
 
     if pkg_details.name not in EXCLUDED_PYTYPE_PACKAGES and "-nspkg" not in pkg_details.name and "-mgmt" not in pkg_details.name:
-        logging.info("Verifying presence of py.typed: [%s]", pkg_details.name)
+        logging.info(f"Verifying presence of py.typed: {pkg_details.name}")
         if verify_sdist_pytyped(pkg_dir, pkg_details.namespace, pkg_details.package_data, pkg_details.include_package_data):
-            logging.info("Py.typed setup.py kwargs are set properly: [%s]", pkg_details.name)
+            logging.info(f"Py.typed setup.py kwargs are set properly: {pkg_details.name}")
+        else:
+            logging.error(f"Py.typed verification failed for package {pkg_details.name}. Check messages above.")
+            error_occurred = True
+
+    if pkg_details.name not in EXCLUDED_CLASSIFICATION_PACKAGES and "-nspkg" not in pkg_details.name:
+        logging.info(f"Verifying package classifiers: {pkg_details.name}")
+
+        status, message = verify_package_classifiers(pkg_details.name, pkg_details.version, pkg_details.classifiers)
+        if status:
+            logging.info(f"Package classifiers are set properly: {pkg_details.name}")
         else:
-            logging.info("Verified py.typed [%s]. Check messages above.", pkg_details.name)
-            exit(1)
+            logging.error(f"{message}")
+            error_occurred = True
+
+    if error_occurred:
+        logging.error(f"{pkg_details.name} failed sdist verification. Check outputs above.")
+        exit(1)

sdk/textanalytics/azure-ai-textanalytics/LICENSE

@@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+SOFTWARE.

tools/azure-sdk-tools/ci_tools/functions.py

@@ -15,7 +15,7 @@
 from pypi_tools.pypi import PyPIClient
 
 import os, sys, platform, glob, re, logging
-from typing import List, Any, Optional
+from typing import List, Any, Optional, Tuple
 
 INACTIVE_CLASSIFIER = "Development Status :: 7 - Inactive"
 
@@ -42,10 +42,7 @@
     "azure-mgmt-core",
 ]
 
-TEST_COMPATIBILITY_MAP = {
-    "azure-ai-ml": ">=3.7",
-    "azure-ai-evaluation": ">=3.9, !=3.13.*"
-}
+TEST_COMPATIBILITY_MAP = {"azure-ai-ml": ">=3.7", "azure-ai-evaluation": ">=3.9, !=3.13.*"}
 TEST_PYTHON_DISTRO_INCOMPATIBILITY_MAP = {
     "azure-storage-blob": "pypy",
     "azure-storage-queue": "pypy",
@@ -697,7 +694,9 @@ def is_package_compatible(
     return True
 
 
-def get_total_coverage(coverage_file: str, coverage_config_file: str, package_name: str, repo_root: Optional[str] = None) -> Optional[float]:
+def get_total_coverage(
+    coverage_file: str, coverage_config_file: str, package_name: str, repo_root: Optional[str] = None
+) -> Optional[float]:
     try:
         import coverage
         from coverage.exceptions import NoDataError
@@ -717,7 +716,9 @@ def get_total_coverage(coverage_file: str, coverage_config_file: str, package_na
     try:
         if repo_root:
             os.chdir(repo_root)
-        logging.info(f"Running coverage report against \"{coverage_file}\" with \"{coverage_config_file}\" from \"{os.getcwd()}\".")
+        logging.info(
+            f'Running coverage report against "{coverage_file}" with "{coverage_config_file}" from "{os.getcwd()}".'
+        )
         report = cov.report()
     except NoDataError as e:
         logging.info(f"Package {package_name} did not generate any coverage output: {e}")
@@ -891,3 +892,38 @@ def handle_incompatible_minimum_dev_reqs(
                     cleansed_reqs.append(cleansed_dev_requirement_line)
 
     return cleansed_reqs
+
+
+def verify_package_classifiers(package_name: str, package_version: str, package_classifiers: List[str]) -> Tuple[bool, Optional[str]]:
+    """
+    Verify that the package classifiers match the expected classifiers.
+    :param str package_name: The name of the package being verified. Used for detail in the error response.
+    :param str package_version: The version of the package being verified.
+    :param List[str] package_classifiers: The classifiers of the package being verified.
+    :returns: A tuple, (x, y), where x is whether the package version matches its classifiers, and y is an error message or None.
+    """
+
+    dev_status = parse(package_version)
+
+    # gather all development‐status classifiers
+    dev_classifiers = [c for c in package_classifiers if c.startswith("Development Status ::")]
+
+    # beta releases: enforce that only development status 4 is present
+    if dev_status.is_prerelease:
+        for c in dev_classifiers:
+            if "4 - Beta" not in c:
+                return False, f"{package_name} has version {package_version} and is a beta release, but has development status '{c}'. Expected 'Development Status :: 4 - Beta' ONLY."
+        return True, None
+
+    # ga releases: all development statuses must be >= 5
+    for c in dev_classifiers:
+        try:
+            # "Development Status :: 5 - Production/Stable"
+            # or Development Status :: 6 - Mature
+            # or Development Status :: 7 - Inactive
+            num = int(c.split("::")[1].split("-")[0].strip())
+        except (IndexError, ValueError):
+            return False, f"{package_name} has version {package_version} and is a GA release, but failed to pull a status number from status '{c}'. Expecting format identical to 'Development Status :: 5 - Production/Stable'."
+        if num < 5:
+            return False, f"{package_name} has version {package_version} and is a GA release, but had development status '{c}'. Expecting a development classifier that is equal or greater than 'Development Status :: 5 - Production/Stable'."
+    return True, None

tools/azure-sdk-tools/tests/test_classifier_checks.py

@@ -0,0 +1,22 @@
+import pytest
+
+from ci_tools.functions import verify_package_classifiers
+
+
+@pytest.mark.parametrize(
+    "package_name, package_version, package_classifiers, expected_result",
+    [
+        ("a", "1.0.0", ["Development Status :: 4 - Beta"], False),
+        ("b", "1.0.0", ["Development Status :: 5 - Production/Stable"], True),
+        ("b", "1.0.0a1", ["Development Status :: 5 - Production/Stable"], False),
+        ("b", "1.0.0a1", ["Development Status :: 4 - Beta"], True),
+        ("c", "1.0.0b1", ["Development Status :: 4 - Beta"], True),
+        ("c", "1.0.0b1", ["Development Status :: 4 - Beta", "Development Status :: 5 - Production/Stable"], False),
+        ("c", "1.0.0", ["Development Status :: 4 - Beta", "Development Status :: 5 - Production/Stable"], False),
+        ("c", "1.0.1", ["Development Status :: 7 - Inactive"], True),
+        ("c", "1.0.1b1", ["Development Status :: 7 - Inactive"], False),
+    ],
+)
+def test_classifier_enforcement(package_name, package_version, package_classifiers, expected_result):
+    result = verify_package_classifiers(package_name, package_version, package_classifiers)
+    assert result[0] is expected_result