BLD Remove support for setuptools build (scikit-learn#29400)

Co-authored-by: Olivier Grisel <olivier.grisel@ensta.org>

Author: lesteve

.codecov.yml

@@ -30,5 +30,4 @@ ignore:
 - "sklearn/_build_utils"
 - "sklearn/__check_build"
 - "sklearn/_min_dependencies.py"
-- "**/setup.py"
 - "**/conftest.py"

.coveragerc

@@ -6,4 +6,3 @@ omit =
     */sklearn/externals/*
     */sklearn/_build_utils/*
     */benchmarks/*
-    **/setup.py

.github/workflows/wheels.yml

@@ -164,7 +164,6 @@ jobs:
           CIBW_PRERELEASE_PYTHONS: ${{ matrix.prerelease_pythons }}
           CIBW_FREE_THREADED_SUPPORT: ${{ matrix.free_threaded_support }}
           CIBW_ENVIRONMENT: SKLEARN_SKIP_NETWORK_TESTS=1
-            SKLEARN_BUILD_PARALLEL=3
           CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }}
           CIBW_ARCHS: all
           CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }}
@@ -216,8 +215,6 @@ jobs:
 
       - name: Build source distribution
         run: bash build_tools/github/build_source.sh
-        env:
-          SKLEARN_BUILD_PARALLEL: 3
 
       - name: Test source distribution
         run: bash build_tools/github/test_source.sh

Makefile

@@ -7,9 +7,7 @@ all:
 	@echo "Please use 'make <target>' where <target> is one of"
 	@echo "  dev                  build scikit-learn with Meson"
 	@echo "  clean                clean scikit-learn Meson build. Very rarely needed,"
-	@echo "                       one use case is when switching back to setuptools"
-	@echo "  dev-setuptools       build scikit-learn with setuptools (deprecated)"
-	@echo "  clean-setuptools     clean scikit-learn setuptools build (deprecated)"
+	@echo "                       since meson-python recompiles on import."
 
 .PHONY: all
 
@@ -23,14 +21,7 @@ clean: clean-meson
 clean-meson:
 	pip uninstall -y scikit-learn
 	# It seems in some cases removing the folder avoids weird compilation
-	# errors (e.g. when switching from numpy>=2 to numpy<2). For some 
+	# errors (e.g. when switching from numpy>=2 to numpy<2). For some
 	# reason ninja clean -C $(DEFAULT_MESON_BUILD_DIR) is not
-	# enough
+	# enough.
 	rm -rf $(DEFAULT_MESON_BUILD_DIR)
-
-dev-setuptools:
-	$(PYTHON) setup.py build_ext -i
-
-clean-setuptools:
-	$(PYTHON) setup.py clean
-	rm -rf dist

azure-pipelines.yml

@@ -208,7 +208,6 @@ jobs:
         SKLEARN_ENABLE_DEBUG_CYTHON_DIRECTIVES: '1'
         SKLEARN_RUN_FLOAT32_TESTS: '1'
         SKLEARN_TESTS_GLOBAL_RANDOM_SEED: '2'  # non-default seed
-        BUILD_WITH_SETUPTOOLS: 'true'
       # Linux environment to test the latest available dependencies.
       # It runs tests requiring lightgbm, pandas and PyAMG.
       pylatest_pip_openblas_pandas:
@@ -236,7 +235,7 @@ jobs:
       )
     matrix:
       debian_atlas_32bit:
-        DOCKER_CONTAINER: 'i386/debian:11.2'
+        DOCKER_CONTAINER: 'i386/debian:12'
         DISTRIB: 'debian-32'
         COVERAGE: "true"
         LOCK_FILE: './build_tools/azure/debian_atlas_32bit_lock.txt'

build_tools/azure/install.sh

@@ -107,10 +107,6 @@ scikit_learn_install() {
     setup_ccache
     show_installed_libraries
 
-    # Set parallelism to 3 to overlap IO bound tasks with CPU bound tasks on CI
-    # workers with 2 cores when building the compiled extensions of scikit-learn.
-    export SKLEARN_BUILD_PARALLEL=3
-
     if [[ "$UNAMESTR" == "Darwin" && "$SKLEARN_TEST_NO_OPENMP" == "true" ]]; then
         # Without openmp, we use the system clang. Here we use /usr/bin/ar
         # instead because llvm-ar errors
@@ -129,9 +125,7 @@ scikit_learn_install() {
         export LDFLAGS="$LDFLAGS -Wl,--sysroot=/"
     fi
 
-    if [[ "$BUILD_WITH_SETUPTOOLS" == "true" ]]; then
-        python setup.py develop
-    elif [[ "$PIP_BUILD_ISOLATION" == "true" ]]; then
+    if [[ "$PIP_BUILD_ISOLATION" == "true" ]]; then
         # Check that pip can automatically build scikit-learn with the build
         # dependencies specified in pyproject.toml using an isolated build
         # environment:
@@ -143,12 +137,6 @@ scikit_learn_install() {
            # toolchain
            ADDITIONAL_PIP_OPTIONS='-Csetup-args=--vsenv'
         fi
-        # TODO Always add --check-build-dependencies when all CI builds have
-        # pip >= 22.1.1. At the time of writing, two CI builds (debian32_atlas and
-        # ubuntu_atlas) have an older pip
-        if pip install --help | grep check-build-dependencies; then
-            ADDITIONAL_PIP_OPTIONS="$ADDITIONAL_PIP_OPTIONS --check-build-dependencies"
-        fi
         # Use the pre-installed build dependencies and build directly in the
         # current environment.
         pip install --verbose --no-build-isolation --editable . $ADDITIONAL_PIP_OPTIONS

build_tools/azure/pymin_conda_defaults_openblas_environment.yml

@@ -15,9 +15,11 @@ dependencies:
   - pytest
   - pytest-xdist
   - pillow
+  - ninja
   - pytest-cov
   - coverage
   - ccache
   - pip
   - pip:
     - threadpoolctl==3.1.0  # min
+    - meson-python==0.16.0  # min

build_tools/azure/pymin_conda_defaults_openblas_linux-64_conda.lock

@@ -1,6 +1,6 @@
 # Generated by conda-lock.
 # platform: linux-64
-# input_hash: 244b6a064d3785ea62baaf9436848821d153846b455c2976f5e811182e848c83
+# input_hash: e4db53ad2240ff5f57679dd93701c30b6712ac3a43ec04e18b74132f2948b4cd
 @EXPLICIT
 https://repo.anaconda.com/pkgs/main/linux-64/_libgcc_mutex-0.1-main.conda#c3473ff8bdb3d124ed5ff11ec380d6f9
 https://repo.anaconda.com/pkgs/main/linux-64/blas-1.0-openblas.conda#9ddfcaef10d79366c90128f5dc444be8
@@ -28,6 +28,7 @@ https://repo.anaconda.com/pkgs/main/linux-64/libwebp-base-1.3.2-h5eee18b_0.conda
 https://repo.anaconda.com/pkgs/main/linux-64/libxcb-1.15-h7f8727e_0.conda#ada518dcadd6aaee9aae47ba9a671553
 https://repo.anaconda.com/pkgs/main/linux-64/lz4-c-1.9.4-h6a678d5_1.conda#2ee58861f2b92b868ce761abb831819d
 https://repo.anaconda.com/pkgs/main/linux-64/ncurses-6.4-h6a678d5_0.conda#5558eec6e2191741a92f832ea826251c
+https://repo.anaconda.com/pkgs/main/linux-64/ninja-base-1.10.2-hd09550d_5.conda#09dcbad622d58caaeefe46cd399f0a76
 https://repo.anaconda.com/pkgs/main/linux-64/openssl-3.0.14-h5eee18b_0.conda#37b6dad6aa49000a4230a9f0cad172f6
 https://repo.anaconda.com/pkgs/main/linux-64/xz-5.4.6-h5eee18b_1.conda#1562802f843297ee776a50b9329597ed
 https://repo.anaconda.com/pkgs/main/linux-64/zlib-1.2.13-h5eee18b_1.conda#92e42d8310108b0a440fb2e60b2b2a25
@@ -56,7 +57,7 @@ https://repo.anaconda.com/pkgs/main/linux-64/libclang-14.0.6-default_hc6dbbc7_1.
 https://repo.anaconda.com/pkgs/main/linux-64/libpq-12.17-hdbd6064_0.conda#6bed363e25859faff66bf546a11c10e8
 https://repo.anaconda.com/pkgs/main/linux-64/openjpeg-2.4.0-h9ca470c_1.conda#dfd4b36eb8ddaffeca0ab412de63c3e2
 https://repo.anaconda.com/pkgs/main/linux-64/python-3.9.19-h955ad1f_1.conda#4b453281859c293c9d577271f3b18a0d
-https://repo.anaconda.com/pkgs/main/linux-64/certifi-2024.6.2-py39h06a4308_0.conda#738daf43271605d7291ecae0e8cac41c
+https://repo.anaconda.com/pkgs/main/linux-64/certifi-2024.7.4-py39h06a4308_0.conda#add87fa3b69a43e4e9ea1e619b267c4b
 https://repo.anaconda.com/pkgs/main/noarch/cycler-0.11.0-pyhd3eb1b0_0.conda#f5e365d2cdb66d547eb8c3ab93843aab
 https://repo.anaconda.com/pkgs/main/linux-64/cython-3.0.10-py39h5eee18b_0.conda#1419a658ed2b4d5c3ac1964f33143b64
 https://repo.anaconda.com/pkgs/main/linux-64/exceptiongroup-1.2.0-py39h06a4308_0.conda#960e2cb83ac5134df8e593a130aa11af
@@ -66,6 +67,7 @@ https://repo.anaconda.com/pkgs/main/noarch/iniconfig-1.1.1-pyhd3eb1b0_0.tar.bz2#
 https://repo.anaconda.com/pkgs/main/linux-64/joblib-1.2.0-py39h06a4308_0.conda#ac1f5687d70aa1128cbecb26bc9e559d
 https://repo.anaconda.com/pkgs/main/linux-64/kiwisolver-1.4.4-py39h6a678d5_0.conda#3d57aedbfbd054ce57fb3c1e4448828c
 https://repo.anaconda.com/pkgs/main/linux-64/mysql-5.7.24-h721c034_2.conda#dfc19ca2466d275c4c1f73b62c57f37b
+https://repo.anaconda.com/pkgs/main/linux-64/ninja-1.10.2-h06a4308_5.conda#6fc219bbc4c8dbb9060b5b7fe31ae83d
 https://repo.anaconda.com/pkgs/main/linux-64/numpy-base-1.21.6-py39h375b286_1.conda#0061d9193658774ab79fc85d143a94fc
 https://repo.anaconda.com/pkgs/main/linux-64/packaging-24.1-py39h06a4308_0.conda#e80d41ffc9450162ef10cbbb9b4ec7e9
 https://repo.anaconda.com/pkgs/main/linux-64/pillow-10.3.0-py39h5eee18b_0.conda#b346d6c71267c1553b6c18d3db5fdf6d
@@ -96,4 +98,7 @@ https://repo.anaconda.com/pkgs/main/linux-64/pyamg-4.2.3-py39h79cecc1_0.conda#af
 https://repo.anaconda.com/pkgs/main/linux-64/qt-main-5.15.2-h53bd1ea_10.conda#bd0c79e82df6323f638bdcb871891b61
 https://repo.anaconda.com/pkgs/main/linux-64/pyqt-5.15.10-py39h6a678d5_0.conda#52da5ff9b1144b078d2f41bab0b213f2
 https://repo.anaconda.com/pkgs/main/linux-64/matplotlib-3.3.4-py39h06a4308_0.conda#384fc5e01ebfcf30e7161119d3029b5a
+# pip meson @ https://files.pythonhosted.org/packages/44/b2/d4433391a7c5e94a39b50ca7295a8ceba736e7c72c455752a60122f52453/meson-1.4.1-py3-none-any.whl#sha256=d5acc3abae2dad3c70ddcbd10acac92b78b144d34d43f40f5b8ac31dfd8a826a
 # pip threadpoolctl @ https://files.pythonhosted.org/packages/61/cf/6e354304bcb9c6413c4e02a747b600061c21d38ba51e7e544ac7bc66aecc/threadpoolctl-3.1.0-py3-none-any.whl#sha256=8b99adda265feb6773280df41eece7b2e6561b772d21ffd52e372f999024907b
+# pip pyproject-metadata @ https://files.pythonhosted.org/packages/aa/5f/bb5970d3d04173b46c9037109f7f05fc8904ff5be073ee49bb6ff00301bc/pyproject_metadata-0.8.0-py3-none-any.whl#sha256=ad858d448e1d3a1fb408ac5bac9ea7743e7a8bbb472f2693aaa334d2db42f526
+# pip meson-python @ https://files.pythonhosted.org/packages/91/c0/104cb6244c83fe6bc3886f144cc433db0c0c78efac5dc00e409a5a08c87d/meson_python-0.16.0-py3-none-any.whl#sha256=842dc9f5dc29e55fc769ff1b6fe328412fe6c870220fc321060a1d2d395e69e8

build_tools/circle/build_doc.sh

@@ -176,9 +176,6 @@ conda activate $CONDA_ENV_NAME
 
 show_installed_libraries
 
-# Set parallelism to 3 to overlap IO bound tasks with CPU bound tasks on CI
-# workers with 2 cores when building the compiled extensions of scikit-learn.
-export SKLEARN_BUILD_PARALLEL=3
 pip install -e . --no-build-isolation
 
 echo "ccache build summary:"

build_tools/cirrus/arm_wheel.yml

@@ -8,7 +8,6 @@ linux_arm64_wheel_task:
     memory: 4G
   env:
     CIBW_ENVIRONMENT: SKLEARN_SKIP_NETWORK_TESTS=1
-                      SKLEARN_BUILD_PARALLEL=5
     CIBW_TEST_COMMAND: bash {project}/build_tools/wheels/test_wheels.sh
     CIBW_TEST_REQUIRES: pytest pandas threadpoolctl pytest-xdist
     CIBW_BUILD_VERBOSITY: 1