merged back from submodulev3, overrode Partitioner and Splitter changes

Author: SamuelCarliles3

.circleci/config.yml

@@ -11,7 +11,7 @@ jobs:
           command: |
             source build_tools/shared.sh
             # Include pytest compatibility with mypy
-            pip install pytest ruff $(get_dep mypy min) $(get_dep black min) cython-lint
+            pip install pytest $(get_dep ruff min) $(get_dep mypy min) $(get_dep black min) cython-lint
       - run:
           name: linting
           command: ./build_tools/linting.sh

.codecov.yml

@@ -30,5 +30,4 @@ ignore:
 - "sklearn/_build_utils"
 - "sklearn/__check_build"
 - "sklearn/_min_dependencies.py"
-- "**/setup.py"
 - "**/conftest.py"

.coveragerc

@@ -6,4 +6,3 @@ omit =
     */sklearn/externals/*
     */sklearn/_build_utils/*
     */benchmarks/*
-    **/setup.py

.github/workflows/artifact-redirector.yml

@@ -15,7 +15,7 @@ jobs:
     name: Run CircleCI artifacts redirector
     steps:
       - name: GitHub Action step
-        uses: larsoner/circleci-artifacts-redirector-action@master
+        uses: scientific-python/circleci-artifacts-redirector-action@v1
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           api-token: ${{ secrets.CIRCLECI_TOKEN }}

.github/workflows/assign.yml

@@ -19,8 +19,11 @@ jobs:
        && !github.event.issue.assignee
     steps:
       - run: |
+          # Using REST API directly because assigning through gh has some severe limitations. For more details, see
+          # https://github.com/scikit-learn/scikit-learn/issues/29395#issuecomment-2206776963
           echo "Assigning issue ${{ github.event.issue.number }} to ${{ github.event.comment.user.login }}"
-          gh issue edit $ISSUE --add-assignee ${{ github.event.comment.user.login }}
+          curl -H "Authorization: token $GH_TOKEN" -d '{"assignees": ["${{ github.event.comment.user.login }}"]}' \
+              https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/assignees
           gh issue edit $ISSUE --remove-label "help wanted"
         env:
           GH_TOKEN: ${{ github.token }}

.github/workflows/check-changelog.yml

@@ -4,65 +4,30 @@ name: Check Changelog
 # To bypass this check, label the PR with "No Changelog Needed".
 on:
   pull_request:
-    types: [opened, edited, labeled, unlabeled, synchronize]
+    types: [opened, synchronize, labeled, unlabeled]
 
 jobs:
   check:
     name: A reviewer will let you know if it is required or can be bypassed
     runs-on: ubuntu-latest
-    if: ${{ contains(github.event.pull_request.labels.*.name, 'No Changelog Needed') == 0 }}
     steps:
-      - name: Get PR number and milestone
-        run: |
-          echo "PR_NUMBER=${{ github.event.pull_request.number }}" >> $GITHUB_ENV
-          echo "TAGGED_MILESTONE=${{ github.event.pull_request.milestone.title }}" >> $GITHUB_ENV
       - uses: actions/checkout@v4
         with:
           fetch-depth: '0'
-      - name: Check the changelog entry
+      - name: Check if tests have changed
+        id: tests_changed
         run: |
           set -xe
           changed_files=$(git diff --name-only origin/main)
           # Changelog should be updated only if tests have been modified
-          if [[ ! "$changed_files" =~ tests ]]
-          then
-            exit 0
-          fi
-          all_changelogs=$(cat ./doc/whats_new/v*.rst)
-          if [[ "$all_changelogs" =~ :pr:\`$PR_NUMBER\` ]]
+          if [[ "$changed_files" =~ tests ]]
           then
-            echo "Changelog has been updated."
-            # If the pull request is milestoned check the correspondent changelog
-            if exist -f ./doc/whats_new/v${TAGGED_MILESTONE:0:4}.rst
-            then
-              expected_changelog=$(cat ./doc/whats_new/v${TAGGED_MILESTONE:0:4}.rst)
-              if [[ "$expected_changelog" =~ :pr:\`$PR_NUMBER\` ]]
-              then
-                echo "Changelog and milestone correspond."
-              else
-                echo "Changelog and milestone do not correspond."
-                echo "If you see this error make sure that the tagged milestone for the PR"
-                echo "and the edited changelog filename properly match."
-                exit 1
-              fi
-            fi
-          else
-            echo "A Changelog entry is missing."
-            echo ""
-            echo "Please add an entry to the changelog at 'doc/whats_new/v*.rst'"
-            echo "to document your change assuming that the PR will be merged"
-            echo "in time for the next release of scikit-learn."
-            echo ""
-            echo "Look at other entries in that file for inspiration and please"
-            echo "reference this pull request using the ':pr:' directive and"
-            echo "credit yourself (and other contributors if applicable) with"
-            echo "the ':user:' directive."
-            echo ""
-            echo "If you see this error and there is already a changelog entry,"
-            echo "check that the PR number is correct."
-            echo ""
-            echo "If you believe that this PR does not warrant a changelog"
-            echo "entry, say so in a comment so that a maintainer will label"
-            echo "the PR with 'No Changelog Needed' to bypass this check."
-            exit 1
+            echo "check_changelog=true" >> $GITHUB_OUTPUT
           fi
+
+      - name: Check changelog entry
+        if: steps.tests_changed.outputs.check_changelog == 'true'
+        uses: scientific-python/action-towncrier-changelog@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          BOT_USERNAME: changelog-bot

.github/workflows/cuda-ci.yml

@@ -0,0 +1,76 @@
+name: CUDA GPU
+
+# Only run this workflow when a Pull Request is labeled with the
+# 'CUDA CI' label.
+on:
+  pull_request:
+    types:
+      - labeled
+
+jobs:
+  build_wheel:
+    if: contains(github.event.pull_request.labels.*.name, 'CUDA CI')
+    runs-on: "ubuntu-latest"
+    name: Build wheel for Pull Request
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v2.21.3
+        env:
+          CIBW_BUILD: cp312-manylinux_x86_64
+          CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
+          CIBW_BUILD_VERBOSITY: 1
+          CIBW_ARCHS: x86_64
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: cibw-wheels
+          path: ./wheelhouse/*.whl
+
+  tests:
+    if: contains(github.event.pull_request.labels.*.name, 'CUDA CI')
+    needs: [build_wheel]
+    runs-on:
+      group: cuda-gpu-runner-group
+    # Set this high enough so that the tests can comforatble run. We set a
+    # timeout to make abusing this workflow less attractive.
+    timeout-minutes: 20
+    name: Run Array API unit tests
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: cibw-wheels
+          path: ~/dist
+
+      - uses: actions/setup-python@v5
+        with:
+          # XXX: The 3.12.4 release of Python on GitHub Actions is corrupted:
+          # https://github.com/actions/setup-python/issues/886
+          python-version: '3.12.3'
+      - name: Checkout main repository
+        uses: actions/checkout@v4
+      - name: Cache conda environment
+        id: cache-conda
+        uses: actions/cache@v4
+        with:
+          path: ~/conda
+          key: ${{ runner.os }}-build-${{ hashFiles('build_tools/github/create_gpu_environment.sh') }}-${{ hashFiles('build_tools/github/pylatest_conda_forge_cuda_array-api_linux-64_conda.lock') }}
+      - name: Install miniforge
+        if: ${{ steps.cache-conda.outputs.cache-hit != 'true' }}
+        run: bash build_tools/github/create_gpu_environment.sh
+      - name: Install scikit-learn
+        run: |
+          source "${HOME}/conda/etc/profile.d/conda.sh"
+          conda activate sklearn
+          pip install ~/dist/cibw-wheels/$(ls ~/dist/cibw-wheels)
+
+      - name: Run array API tests
+        run: |
+          source "${HOME}/conda/etc/profile.d/conda.sh"
+          conda activate sklearn
+          python -c "import sklearn; sklearn.show_versions()"
+
+          SCIPY_ARRAY_API=1 pytest --pyargs sklearn -k 'array_api'
+        # Run in /home/runner to not load sklearn from the checkout repo
+        working-directory: /home/runner

.github/workflows/cuda-gpu-ci.yml

@@ -0,0 +1,23 @@
+name: Remove "CUDA CI" Label
+
+# This workflow removes the "CUDA CI" label that triggers the actual
+# CUDA CI. It is separate so that we can use the `pull_request_target`
+# trigger which has a API token with write access.
+on:
+  pull_request_target:
+    types:
+      - labeled
+
+# In order to remove the "CUDA CI" label we need to have write permissions for PRs
+permissions:
+  pull-requests: write
+
+jobs:
+  label-remover:
+    if: contains(github.event.pull_request.labels.*.name, 'CUDA CI')
+    name: Remove "CUDA CI" Label
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions-ecosystem/action-remove-labels@v1
+        with:
+          labels: CUDA CI

.github/workflows/cuda-label-remover.yml

@@ -33,7 +33,7 @@ jobs:
         run: |
           source build_tools/shared.sh
           # Include pytest compatibility with mypy
-          pip install pytest ruff $(get_dep mypy min) $(get_dep black min) cython-lint
+          pip install pytest $(get_dep ruff min) $(get_dep mypy min) $(get_dep black min) cython-lint
           # we save the versions of the linters to be used in the error message later.
           python -c "from importlib.metadata import version; print(f\"ruff={version('ruff')}\")" >> /tmp/versions.txt
           python -c "from importlib.metadata import version; print(f\"mypy={version('mypy')}\")" >> /tmp/versions.txt

.github/workflows/lint.yml

@@ -39,13 +39,13 @@ jobs:
       run: |
         python build_tools/github/check_wheels.py
     - name: Publish package to TestPyPI
-      uses: pypa/gh-action-pypi-publish@81e9d935c883d0b210363ab89cf05f3894778450 # v1.8.14
+      uses: pypa/gh-action-pypi-publish@fb13cb306901256ace3dab689990e13a5550ffaa # v1.11.0
       with:
         repository-url: https://test.pypi.org/legacy/
         print-hash: true
       if: ${{ github.event.inputs.pypi_repo == 'testpypi' }}
     - name: Publish package to PyPI
-      uses: pypa/gh-action-pypi-publish@81e9d935c883d0b210363ab89cf05f3894778450 # v1.8.14
+      uses: pypa/gh-action-pypi-publish@fb13cb306901256ace3dab689990e13a5550ffaa # v1.11.0
       if: ${{ github.event.inputs.pypi_repo == 'pypi' }}
       with:
         print-hash: true

.github/workflows/publish_pypi.yml

@@ -0,0 +1,93 @@
+# Workflow to update lock files in a PR, triggered by specific PR comments
+name: Update lock files in PR
+on:
+  issue_comment:
+    types: [created]
+
+permissions:
+  contents: write
+  statuses: write
+
+jobs:
+  update-lock-files:
+    if: >-
+      github.repository == 'scikit-learn/scikit-learn'
+      && github.event.issue.pull_request
+      && startsWith(github.event.comment.body, '@scikit-learn-bot update lock-files')
+    runs-on: ubuntu-latest
+
+    steps:
+      # There is no direct way to get the HEAD information directly from issue_comment
+      # event, so we use the GitHub CLI to get the PR head ref and repository
+      - name: Get pull request HEAD information
+        id: pr-head-info
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          pr_info=$(gh pr view ${{ github.event.issue.number }} --repo ${{ github.repository }} --json headRefName,headRefOid,headRepository,headRepositoryOwner)
+          pr_head_ref=$(echo "$pr_info" | jq -r '.headRefName')
+          pr_head_sha=$(echo "$pr_info" | jq -r '.headRefOid')
+          pr_head_repository=$(echo "$pr_info" | jq -r '.headRepositoryOwner.login + "/" + .headRepository.name')
+          echo "pr_head_ref=$pr_head_ref" >> $GITHUB_OUTPUT
+          echo "pr_head_sha=$pr_head_sha" >> $GITHUB_OUTPUT
+          echo "pr_head_repository=$pr_head_repository" >> $GITHUB_OUTPUT
+
+      # Set the status of the latest commit in the PR to indicate that the update is in progress
+      # https://docs.github.com/en/rest/commits/statuses?apiVersion=2022-11-28#create-a-commit-status
+      - name: Set pending status
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh api \
+            --method POST \
+            -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            /repos/${{ github.repository }}/statuses/${{ steps.pr-head-info.outputs.pr_head_sha }} \
+            -f "state=pending" \
+            -f "target_url=https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
+            -f "description=Updating lock files..." \
+            -f "context=update-lock-files-pr"
+
+      - name: Check out the PR branch
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ steps.pr-head-info.outputs.pr_head_ref }}
+          repository: ${{ steps.pr-head-info.outputs.pr_head_repository }}
+
+      # We overwrite all the scripts we are going to use in this workflow with their
+      # versions on main; since this workflow has the write permissions this is to avoid
+      # malicious changes to these scripts in PRs to be executed
+      - name: Download scripts from main
+        run: |
+          curl https://raw.githubusercontent.com/${{ github.repository }}/main/build_tools/shared.sh --retry 5 -o ./build_tools/shared.sh
+          curl https://raw.githubusercontent.com/${{ github.repository }}/main/build_tools/update_environments_and_lock_files.py --retry 5 -o ./build_tools/update_environments_and_lock_files.py
+          curl https://raw.githubusercontent.com/${{ github.repository }}/main/build_tools/on_pr_comment_update_environments_and_lock_files.py --retry 5 -o ./build_tools/on_pr_comment_update_environments_and_lock_files.py
+
+      - name: Update lock files
+        env:
+          COMMENT: ${{ github.event.comment.body }}
+        # We download the lock files update scripts from main, since this workflow is
+        # run from main itself
+        run: |
+          source build_tools/shared.sh
+          source $CONDA/bin/activate
+          conda install -n base conda conda-libmamba-solver -y
+          conda config --set solver libmamba
+          conda install -c conda-forge "$(get_dep conda-lock min)" -y
+
+          python build_tools/on_pr_comment_update_environments_and_lock_files.py
+
+      - name: Set completion status
+        if: ${{ always() }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh api \
+            --method POST \
+            -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            /repos/${{ github.repository }}/statuses/${{ steps.pr-head-info.outputs.pr_head_sha }} \
+            -f "state=${{ job.status == 'success' && 'success' || 'error' }}" \
+            -f "target_url=https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
+            -f "description=Lock files ${{ job.status == 'success' && 'updated' || 'failed to update' }}." \
+            -f "context=update-lock-files-pr"