Merging main

Signed-off-by: Adam Li <adam2392@gmail.com>

Author: adam2392

.github/dependabot.yml

@@ -0,0 +1,21 @@
+version: 2
+updates:
+  # Maintain dependencies for GitHub Actions as recommended in SPEC8:
+  # https://github.com/scientific-python/specs/pull/325
+  # At the time of writing, release critical workflows such as
+  # pypa/gh-action-pypi-publish should use hash-based versioning for security
+  # reasons. This strategy may be generalized to all other github actions
+  # in the future.
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "monthly"
+    groups:
+      actions:
+        patterns:
+          - "*"
+    labels:
+      - "Build / CI"
+      - "dependencies"
+    reviewers:
+      - "scikit-learn/core-devs"

.github/workflows/cuda-gpu-ci.yml

@@ -0,0 +1,44 @@
+name: CUDA GPU
+on:
+  workflow_dispatch:
+    inputs:
+      pr_id:
+        description: Test the contents of this Pull Request
+        required: true
+
+permissions: read-all
+
+jobs:
+  tests:
+    runs-on:
+      group: cuda-gpu-runner-group
+    name: Run Array API unit tests
+    steps:
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.12'
+      - name: Checkout main repository
+        uses: actions/checkout@v4
+      - run: |
+          git fetch origin +refs/pull/${{ inputs.pr_id }}/head:pr-${{ inputs.pr_id }}
+          git checkout pr-${{ inputs.pr_id }}
+          echo "Checked out commit $(git rev-parse HEAD)"
+      - name: Cache conda environment
+        id: cache-conda
+        uses: actions/cache@v3
+        with:
+          path: ~/conda
+          key: ${{ runner.os }}-build-${{ hashFiles('build_tools/github/create_gpu_environment.sh') }}-${{ hashFiles('build_tools/github/pylatest_conda_forge_cuda_array-api_linux-64_conda.lock') }}
+      - name: Install miniforge
+        if: ${{ steps.cache-conda.outputs.cache-hit != 'true' }}
+        run: bash build_tools/github/create_gpu_environment.sh
+      - name: Install scikit-learn
+        run: |
+          source "${HOME}/conda/etc/profile.d/conda.sh"
+          conda activate sklearn
+          pip install --verbose --no-build-isolation --config-settings editable-verbose=true --editable .
+      - name: Run array API tests
+        run: |
+          source "${HOME}/conda/etc/profile.d/conda.sh"
+          conda activate sklearn
+          pytest -k 'array_api'

.github/workflows/publish_pypi.yml

@@ -39,10 +39,13 @@ jobs:
       run: |
         python build_tools/github/check_wheels.py
     - name: Publish package to TestPyPI
-      uses: pypa/gh-action-pypi-publish@v1.8.5
+      uses: pypa/gh-action-pypi-publish@81e9d935c883d0b210363ab89cf05f3894778450 # v1.8.14
       with:
-        repository_url: https://test.pypi.org/legacy/
+        repository-url: https://test.pypi.org/legacy/
+        print-hash: true
       if: ${{ github.event.inputs.pypi_repo == 'testpypi' }}
     - name: Publish package to PyPI
-      uses: pypa/gh-action-pypi-publish@v1.8.5
+      uses: pypa/gh-action-pypi-publish@81e9d935c883d0b210363ab89cf05f3894778450 # v1.8.14
       if: ${{ github.event.inputs.pypi_repo == 'pypi' }}
+      with:
+        print-hash: true

.github/workflows/update-lock-files.yml

@@ -6,6 +6,10 @@ on:
   schedule:
     - cron: '0 5 * * 1'
 
+# XXX Set the right permissions, per step??
+# Can we set read only at the global level here and then elevate to write for some steps?
+#permissions: read-all
+
 jobs:
   update_lock_files:
     if: github.repository == 'scikit-learn/scikit-learn'
@@ -25,9 +29,8 @@ jobs:
           - name: cirrus-arm
             update_script_args: "--select-tag arm"
             additional_commit_message: "[cirrus arm]"
-          - name: pypy
-            update_script_args: "--select-tag pypy"
-            additional_commit_message: "[pypy]"
+          - name: array-api
+            update_script_args: "--select-tag cuda"
 
     steps:
       - uses: actions/checkout@v4
@@ -59,6 +62,14 @@ jobs:
             ### Note
             If the CI tasks fail, create a new branch based on this PR and add the required fixes to that branch.
 
+      # The CUDA workflow needs to be triggered explicitly as it uses an expensive runner
+      - name: Trigger additional tests
+        if: steps.cpr.outputs.pull-request-number != '' && matrix.name == 'array-api'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh workflow run .github/workflows/cuda-gpu-ci.yml -f pr_id=${{steps.cpr.outputs.pull-request-number}}
+
       - name: Check Pull Request
         if: steps.cpr.outputs.pull-request-number != ''
         run: |

Makefile

@@ -5,17 +5,13 @@
 PYTHON ?= python
 CYTHON ?= cython
 PYTEST ?= pytest
-CTAGS ?= ctags
 
 # skip doctests on 32bit python
 BITS := $(shell python -c 'import struct; print(8 * struct.calcsize("P"))')
 
 all: clean inplace test
 
-clean-ctags:
-	rm -f tags
-
-clean: clean-ctags
+clean:
 	$(PYTHON) setup.py clean
 	rm -rf dist
 
@@ -24,7 +20,7 @@ inplace:
 	$(PYTHON) setup.py build_ext -i
 
 dev-meson:
-	pip install --verbose --no-build-isolation --editable . --config-settings editable-verbose=true
+	pip install --verbose --no-build-isolation --editable . --check-build-dependencies --config-settings editable-verbose=true
 
 clean-meson:
 	pip uninstall -y scikit-learn
@@ -49,25 +45,11 @@ test-coverage-parallel:
 
 test: test-code test-sphinxext test-doc
 
-trailing-spaces:
-	find sklearn -name "*.py" -exec perl -pi -e 's/[ \t]*$$//' {} \;
-
 cython:
 	python setup.py build_src
 
-ctags:
-	# make tags for symbol based navigation in emacs and vim
-	# Install with: sudo apt-get install exuberant-ctags
-	$(CTAGS) --python-kinds=-i -R sklearn
-
 doc: inplace
 	$(MAKE) -C doc html
 
 doc-noplot: inplace
 	$(MAKE) -C doc html-noplot
-
-code-analysis:
-	build_tools/linting.sh
-
-build-dev:
-	pip install --verbose --no-build-isolation --editable .

azure-pipelines.yml

@@ -98,37 +98,16 @@ jobs:
         LOCK_FILE: './build_tools/azure/python_nogil_lock.txt'
         COVERAGE: 'false'
 
-- template: build_tools/azure/posix-docker.yml
-  parameters:
-    name: Linux_Nightly_PyPy
-    vmImage: ubuntu-20.04
-    dependsOn: [linting, git_commit]
-    condition: |
-      and(
-        succeeded(),
-        not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')),
-        or(
-          eq(variables['Build.Reason'], 'Schedule'),
-          contains(dependencies['git_commit']['outputs']['commit.message'], '[pypy]')
-        )
-      )
-    matrix:
-      pypy3:
-        DOCKER_CONTAINER: 'condaforge/miniforge3:4.10.3-5'
-        DISTRIB: 'conda-pypy3'
-        LOCK_FILE: './build_tools/azure/pypy3_linux-64_conda.lock'
-
-
 - job: Linux_Nightly_Pyodide
   pool:
     vmImage: ubuntu-22.04
   variables:
     # Need to match Python version and Emscripten version for the correct
     # Pyodide version. For example, for Pyodide version 0.25.1, see
     # https://github.com/pyodide/pyodide/blob/0.25.1/Makefile.envs
-    PYODIDE_VERSION: '0.25.1'
-    EMSCRIPTEN_VERSION: '3.1.46'
-    PYTHON_VERSION: '3.11.3'
+    PYODIDE_VERSION: '0.26.0'
+    EMSCRIPTEN_VERSION: '3.1.58'
+    PYTHON_VERSION: '3.12.1'
 
   dependsOn: [git_commit, linting]
   condition: |

build_tools/azure/debian_atlas_32bit_lock.txt

@@ -6,15 +6,15 @@
 #
 attrs==23.2.0
     # via pytest
-coverage==7.5.1
+coverage==7.5.3
     # via pytest-cov
 cython==3.0.10
     # via -r build_tools/azure/debian_atlas_32bit_requirements.txt
 iniconfig==2.0.0
     # via pytest
 joblib==1.2.0
     # via -r build_tools/azure/debian_atlas_32bit_requirements.txt
-meson==1.4.0
+meson==1.4.1
     # via meson-python
 meson-python==0.16.0
     # via -r build_tools/azure/debian_atlas_32bit_requirements.txt

build_tools/azure/install.sh

@@ -38,13 +38,7 @@ pre_python_environment_install() {
         apt-get install -y python3-dev python3-numpy python3-scipy \
                 python3-matplotlib libatlas3-base libatlas-base-dev \
                 python3-virtualenv python3-pandas ccache git
-
-    elif [[ "$DISTRIB" == "conda-pypy3" ]]; then
-        # need compilers
-        apt-get -yq update
-        apt-get -yq install build-essential
     fi
-
 }
 
 check_packages_dev_version() {
@@ -59,12 +53,7 @@ check_packages_dev_version() {
 
 python_environment_install_and_activate() {
     if [[ "$DISTRIB" == "conda"* ]]; then
-        # Install/update conda with the libmamba solver because the legacy
-        # solver can be slow at installing a specific version of conda-lock.
-        conda install -n base conda conda-libmamba-solver -y
-        conda config --set solver libmamba
-        conda install -c conda-forge "$(get_dep conda-lock min)" -y
-        conda-lock install --name $VIRTUALENV $LOCK_FILE
+        create_conda_environment_from_lock_file $VIRTUALENV $LOCK_FILE
         source activate $VIRTUALENV
 
     elif [[ "$DISTRIB" == "ubuntu" || "$DISTRIB" == "debian-32" ]]; then
@@ -140,6 +129,12 @@ scikit_learn_install() {
            # toolchain
            ADDITIONAL_PIP_OPTIONS='-Csetup-args=--vsenv'
         fi
+        # TODO Always add --check-build-dependencies when all CI builds have
+        # pip >= 22.1.1. At the time of writing, two CI builds (debian32_atlas and
+        # ubuntu_atlas) have an older pip
+        if pip install --help | grep check-build-dependencies; then
+            ADDITIONAL_PIP_OPTIONS="$ADDITIONAL_PIP_OPTIONS --check-build-dependencies"
+        fi
         # Use the pre-installed build dependencies and build directly in the
         # current environment.
         pip install --verbose --no-build-isolation --editable . $ADDITIONAL_PIP_OPTIONS