Merge branch 'scikit-learn:main' into submodulev3

Author: adam2392

README.rst

@@ -37,7 +37,7 @@
 .. |SciPyMinVersion| replace:: 1.5.0
 .. |JoblibMinVersion| replace:: 1.1.1
 .. |ThreadpoolctlMinVersion| replace:: 2.0.0
-.. |MatplotlibMinVersion| replace:: 3.1.3
+.. |MatplotlibMinVersion| replace:: 3.3.4
 .. |Scikit-ImageMinVersion| replace:: 0.16.2
 .. |PandasMinVersion| replace:: 1.0.5
 .. |SeabornMinVersion| replace:: 0.9.0

azure-pipelines.yml

@@ -127,11 +127,11 @@ jobs:
     vmImage: ubuntu-22.04
   variables:
     # Need to match Python version and Emscripten version for the correct
-    # Pyodide version. For example, for Pyodide version 0.23.4, see
-    # https://github.com/pyodide/pyodide/blob/0.23.4/Makefile.envs
-    PYODIDE_VERSION: '0.23.4'
-    EMSCRIPTEN_VERSION: '3.1.32'
-    PYTHON_VERSION: '3.11.2'
+    # Pyodide version. For example, for Pyodide version 0.24.1, see
+    # https://github.com/pyodide/pyodide/blob/0.24.1/Makefile.envs
+    PYODIDE_VERSION: '0.24.1'
+    EMSCRIPTEN_VERSION: '3.1.45'
+    PYTHON_VERSION: '3.11.3'
 
   dependsOn: [git_commit, linting]
   condition: |
@@ -150,7 +150,7 @@ jobs:
         addToPath: true
 
     - bash: bash build_tools/azure/install_pyodide.sh
-      displayName: Build Pyodide wheel and install it in a Pyodide venv
+      displayName: Build Pyodide wheel
 
     - bash: bash build_tools/azure/test_script_pyodide.sh
       displayName: Test Pyodide wheel

build_tools/azure/install_pyodide.sh

@@ -15,8 +15,6 @@ pyodide build
 
 ls -ltrh dist
 
-pyodide venv pyodide-venv
-source pyodide-venv/bin/activate
-
-pip install dist/*.whl
-pip list
+# The Pyodide js library is needed by build_tools/azure/test_script_pyodide.sh
+# to run tests inside Pyodide
+npm install pyodide@$PYODIDE_VERSION

build_tools/azure/py38_conda_defaults_openblas_environment.yml

@@ -11,7 +11,7 @@ dependencies:
   - cython<3.0.0
   - joblib
   - threadpoolctl=2.2.0
-  - matplotlib=3.1.3  # min
+  - matplotlib=3.3.4  # min
   - pandas
   - pyamg
   - pytest

build_tools/azure/py38_conda_defaults_openblas_linux-64_conda.lock

@@ -1,10 +1,10 @@
 # Generated by conda-lock.
 # platform: linux-64
-# input_hash: cc5492b4677e6d5132ab4ab70eda13c942bdf5f6dd53af977e801c42d5f48132
+# input_hash: 8ab29d5bd10968567d77fd3563ef1c61a10907318ccab81f9e3703588aa4dfd6
 @EXPLICIT
 https://repo.anaconda.com/pkgs/main/linux-64/_libgcc_mutex-0.1-main.conda#c3473ff8bdb3d124ed5ff11ec380d6f9
 https://repo.anaconda.com/pkgs/main/linux-64/blas-1.0-openblas.conda#9ddfcaef10d79366c90128f5dc444be8
-https://repo.anaconda.com/pkgs/main/linux-64/ca-certificates-2023.05.30-h06a4308_0.conda#979be8dd2368decd342b13e01540d297
+https://repo.anaconda.com/pkgs/main/linux-64/ca-certificates-2023.08.22-h06a4308_0.conda#243d5065a09a3e85ab888c05f5b6445a
 https://repo.anaconda.com/pkgs/main/linux-64/ld_impl_linux-64-2.38-h1181459_1.conda#68eedfd9c06f2b0e6888d8db345b7f5b
 https://repo.anaconda.com/pkgs/main/linux-64/libgfortran4-7.5.0-ha8ba4b0_17.conda#e3883581cbf0a98672250c3e80d292bf
 https://repo.anaconda.com/pkgs/main/linux-64/libgfortran-ng-7.5.0-ha8ba4b0_17.conda#ecb35c8952579d5c8dc56c6e076ba948
@@ -21,12 +21,12 @@ https://repo.anaconda.com/pkgs/main/linux-64/libdeflate-1.17-h5eee18b_0.conda#b4
 https://repo.anaconda.com/pkgs/main/linux-64/libffi-3.4.4-h6a678d5_0.conda#06e288f9250abef59b9a367d151fc339
 https://repo.anaconda.com/pkgs/main/linux-64/libopenblas-0.3.18-hf726d26_0.conda#10422bb3b9b022e27798fc368cda69ba
 https://repo.anaconda.com/pkgs/main/linux-64/libuuid-1.41.5-h5eee18b_0.conda#4a6a2354414c9080327274aa514e5299
-https://repo.anaconda.com/pkgs/main/linux-64/libwebp-base-1.2.4-h5eee18b_1.conda#a65a20c48061ecf2a6f4f02eae9f2366
+https://repo.anaconda.com/pkgs/main/linux-64/libwebp-base-1.3.2-h5eee18b_0.conda#9179fc7baefa1e027f572edbc519d805
 https://repo.anaconda.com/pkgs/main/linux-64/libxcb-1.15-h7f8727e_0.conda#ada518dcadd6aaee9aae47ba9a671553
 https://repo.anaconda.com/pkgs/main/linux-64/lz4-c-1.9.4-h6a678d5_0.conda#53915e9402180a7f22ea619c41089520
 https://repo.anaconda.com/pkgs/main/linux-64/ncurses-6.4-h6a678d5_0.conda#5558eec6e2191741a92f832ea826251c
 https://repo.anaconda.com/pkgs/main/linux-64/nspr-4.35-h6a678d5_0.conda#208fff5d60133bcff6998a70c9f5203b
-https://repo.anaconda.com/pkgs/main/linux-64/openssl-3.0.10-h7f8727e_2.conda#066a828cc9dcd120af8c503381d6a1b8
+https://repo.anaconda.com/pkgs/main/linux-64/openssl-3.0.11-h7f8727e_2.conda#6cad6f2dcde73f8625d729c6db1272d0
 https://repo.anaconda.com/pkgs/main/linux-64/pcre-8.45-h295c915_0.conda#b32ccc24d1d9808618c1e898da60f68d
 https://repo.anaconda.com/pkgs/main/linux-64/xz-5.4.2-h5eee18b_0.conda#bcd31de48a0dcb44bc5b99675800c5cc
 https://repo.anaconda.com/pkgs/main/linux-64/zlib-1.2.13-h5eee18b_0.conda#333e31fbfbb5057c92fa845ad6adef93
@@ -56,9 +56,10 @@ https://repo.anaconda.com/pkgs/main/linux-64/gst-plugins-base-1.14.1-h6a678d5_1.
 https://repo.anaconda.com/pkgs/main/linux-64/lcms2-2.12-h3be6417_0.conda#719db47afba9f6586eecb5eacac70bff
 https://repo.anaconda.com/pkgs/main/linux-64/libclang-14.0.6-default_hc6dbbc7_1.conda#8f12583c4027b2861cff470f6b8837c4
 https://repo.anaconda.com/pkgs/main/linux-64/libpq-12.15-hdbd6064_1.conda#218227d255f6056b6f49f52dd0d1731f
-https://repo.anaconda.com/pkgs/main/linux-64/libwebp-1.2.4-h11a3e52_1.conda#9f9153b30e58e9ce896f74634622cbf1
+https://repo.anaconda.com/pkgs/main/linux-64/libwebp-1.3.2-h11a3e52_0.conda#9e0d6c9abdd97b076c66d4cf488589ee
 https://repo.anaconda.com/pkgs/main/linux-64/nss-3.89.1-h6a678d5_0.conda#4d9d28fc3a0ca4916f281d2f5429ac50
-https://repo.anaconda.com/pkgs/main/linux-64/python-3.8.17-h955ad1f_0.conda#f901f4fd76d24a2d598788a24e4d7246
+https://repo.anaconda.com/pkgs/main/linux-64/python-3.8.18-h955ad1f_0.conda#fa35c1028f48db26df051ee75dd9422f
+https://repo.anaconda.com/pkgs/main/linux-64/certifi-2023.7.22-py38h06a4308_0.conda#59416ad8979a654bb8f5184b62d8a9e7
 https://repo.anaconda.com/pkgs/main/noarch/cycler-0.11.0-pyhd3eb1b0_0.conda#f5e365d2cdb66d547eb8c3ab93843aab
 https://repo.anaconda.com/pkgs/main/linux-64/cython-0.29.36-py38h5eee18b_0.conda#0465e461450c86b395da9ccc3853d7dc
 https://repo.anaconda.com/pkgs/main/linux-64/exceptiongroup-1.0.4-py38h06a4308_0.conda#db954e73dca6076c64a1004d71b45784
@@ -69,12 +70,12 @@ https://repo.anaconda.com/pkgs/main/linux-64/kiwisolver-1.4.4-py38h6a678d5_0.con
 https://repo.anaconda.com/pkgs/main/linux-64/mysql-5.7.24-h721c034_2.conda#dfc19ca2466d275c4c1f73b62c57f37b
 https://repo.anaconda.com/pkgs/main/linux-64/numpy-base-1.17.3-py38h2f8d375_0.conda#40edbb76ecacefb1e6ab639b514822b1
 https://repo.anaconda.com/pkgs/main/linux-64/packaging-23.1-py38h06a4308_0.conda#9ec9b6ee22dad7f49806c51218befd5b
-https://repo.anaconda.com/pkgs/main/linux-64/pillow-9.4.0-py38h6a678d5_0.conda#8afd1f4f8b23a1c44fca4975253b17f7
+https://repo.anaconda.com/pkgs/main/linux-64/pillow-9.4.0-py38h6a678d5_1.conda#3cc4f7f7c7ca5d7a5c5f26ad5425d8ef
 https://repo.anaconda.com/pkgs/main/linux-64/pluggy-1.0.0-py38h06a4308_1.conda#87bb1d3f6cf3e409a1dac38cee99918e
 https://repo.anaconda.com/pkgs/main/linux-64/ply-3.11-py38_0.conda#d6a69c576c6e4d19e3074eaae3d149f2
 https://repo.anaconda.com/pkgs/main/noarch/py-1.11.0-pyhd3eb1b0_0.conda#7205a898ed2abbf6e9b903dff6abe08e
 https://repo.anaconda.com/pkgs/main/linux-64/pyparsing-3.0.9-py38h06a4308_0.conda#becbbf51d2b05de228eed968e20f963d
-https://repo.anaconda.com/pkgs/main/linux-64/pytz-2022.7-py38h06a4308_0.conda#19c9f6a24d5c6f779c645d00f646666b
+https://repo.anaconda.com/pkgs/main/linux-64/pytz-2023.3.post1-py38h06a4308_0.conda#351d59ddfed216ab9b05481d3bb63106
 https://repo.anaconda.com/pkgs/main/linux-64/setuptools-68.0.0-py38h06a4308_0.conda#24f9c895455f3992d6b04957fd0e7546
 https://repo.anaconda.com/pkgs/main/noarch/six-1.16.0-pyhd3eb1b0_1.conda#34586824d411d36af2fa40e799c172d0
 https://repo.anaconda.com/pkgs/main/noarch/threadpoolctl-2.2.0-pyh0d69192_0.conda#bbfdbae4934150b902f97daaf287efe2
@@ -87,15 +88,15 @@ https://repo.anaconda.com/pkgs/main/linux-64/pytest-7.4.0-py38h06a4308_0.conda#b
 https://repo.anaconda.com/pkgs/main/noarch/python-dateutil-2.8.2-pyhd3eb1b0_0.conda#211ee00320b08a1ac9fea6677649f6c9
 https://repo.anaconda.com/pkgs/main/linux-64/qt-main-5.15.2-h7358343_9.conda#d3eac069d7e4e93b866a07c2274c9ee7
 https://repo.anaconda.com/pkgs/main/linux-64/sip-6.6.2-py38h6a678d5_0.conda#cb3f0d10f7f79870945f4dbbe0000f92
-https://repo.anaconda.com/pkgs/main/linux-64/matplotlib-base-3.1.3-py38hef1b27d_0.conda#a7ad7d097c25b7beeb76f370d51687a1
+https://repo.anaconda.com/pkgs/main/linux-64/matplotlib-base-3.3.4-py38h62a2d02_0.conda#7156fafe3362d0b6a2de43e0002febb3
 https://repo.anaconda.com/pkgs/main/linux-64/pandas-1.2.4-py38ha9443f7_0.conda#5bd3fd807a294f387feabc65821b75d0
 https://repo.anaconda.com/pkgs/main/linux-64/pyqt5-sip-12.11.0-py38h6a678d5_1.conda#7bc403c7d55f1465e922964d293d2186
-https://repo.anaconda.com/pkgs/main/linux-64/pytest-cov-4.0.0-py38h06a4308_0.conda#54035e39255f285f98ca1141b7f098e7
+https://repo.anaconda.com/pkgs/main/linux-64/pytest-cov-4.1.0-py38h06a4308_0.conda#ef981a8b88a9ecf7a84bf50516211e0c
 https://repo.anaconda.com/pkgs/main/noarch/pytest-forked-1.3.0-pyhd3eb1b0_0.tar.bz2#07970bffdc78f417d7f8f1c7e620f5c4
 https://repo.anaconda.com/pkgs/main/linux-64/qt-webengine-5.15.9-h9ab4d14_7.conda#907aa480f11eabd16bd6c72c81720ef2
 https://repo.anaconda.com/pkgs/main/linux-64/scipy-1.5.0-py38habc2bb6_0.conda#a27a97fc2377ab74cbd33ce22d3c3353
 https://repo.anaconda.com/pkgs/main/linux-64/pyamg-4.2.3-py38h79cecc1_0.conda#6e7f4f94000b244396de8bf4e6ae8dc4
 https://repo.anaconda.com/pkgs/main/noarch/pytest-xdist-2.5.0-pyhd3eb1b0_0.conda#d15cdc4207bcf8ca920822597f1d138d
 https://repo.anaconda.com/pkgs/main/linux-64/qtwebkit-5.212-h3fafdc1_5.conda#e811bbc0456e3d3a02cab199492153ee
 https://repo.anaconda.com/pkgs/main/linux-64/pyqt-5.15.7-py38h6a678d5_1.conda#62232dc285be8e7e85ae9596d89b3b95
-https://repo.anaconda.com/pkgs/main/linux-64/matplotlib-3.1.3-py38_0.conda#70d5f6df438d469dc78f082389ada23d
+https://repo.anaconda.com/pkgs/main/linux-64/matplotlib-3.3.4-py38h06a4308_0.conda#96033fd3465abc467ae394c6852930de

build_tools/azure/pytest-pyodide.js

@@ -0,0 +1,53 @@
+const { opendir } = require('node:fs/promises');
+const { loadPyodide } = require("pyodide");
+
+async function main() {
+    let exit_code = 0;
+    try {
+        global.pyodide = await loadPyodide();
+        let pyodide = global.pyodide;
+        const FS = pyodide.FS;
+        const NODEFS = FS.filesystems.NODEFS;
+
+        let mountDir = "/mnt";
+        pyodide.FS.mkdir(mountDir);
+        pyodide.FS.mount(pyodide.FS.filesystems.NODEFS, { root: "." }, mountDir);
+
+        await pyodide.loadPackage(["micropip"]);
+        await pyodide.runPythonAsync(`
+            import glob
+            import micropip
+
+            wheels = glob.glob('/mnt/dist/*.whl')
+            wheels = [f'emfs://{wheel}' for wheel in wheels]
+            print(f'installing wheels: {wheels}')
+            await micropip.install(wheels);
+
+            pkg_list = micropip.list()
+            print(pkg_list)
+        `);
+
+        // Pyodide is built without OpenMP, need to set environment variable to
+        // skip related test
+        await pyodide.runPythonAsync(`
+            import os
+            os.environ['SKLEARN_SKIP_OPENMP_TEST'] = 'true'
+        `);
+
+        await pyodide.runPythonAsync("import micropip; micropip.install('pytest')");
+        let pytest = pyodide.pyimport("pytest");
+        let args = process.argv.slice(2);
+        console.log('pytest args:', args);
+        exit_code = pytest.main(pyodide.toPy(args));
+    } catch (e) {
+        console.error(e);
+        // Arbitrary exit code here. I have seen this code reached instead of a
+        // Pyodide fatal error sometimes
+        exit_code = 66;
+
+    } finally {
+        process.exit(exit_code);
+    }
+}
+
+main();

build_tools/azure/test_script_pyodide.sh

@@ -2,14 +2,8 @@
 
 set -e
 
-source pyodide-venv/bin/activate
-
-pip list
-
-# Need to be outside of the git clone otherwise finds non build sklearn folder
-cd /tmp
-
-# TODO for now only testing sklearn import to make sure the wheel is not badly
-# broken. When Pyodide 0.24 is released we should run the full test suite and
-# xfail tests that fail due to Pyodide limitations
-python -c 'import sklearn'
+# We are using a pytest js wrapper script to run tests inside Pyodide. Maybe
+# one day we can use a Pyodide venv instead but at the time of writing
+# (2023-09-27) there is an issue with scipy.linalg in a Pyodide venv, see
+# https://github.com/pyodide/pyodide/issues/3865 for more details.
+node build_tools/azure/pytest-pyodide.js --pyargs sklearn --durations 20 --showlocals

build_tools/circle/doc_min_dependencies_environment.yml

@@ -11,7 +11,7 @@ dependencies:
   - cython=0.29.33  # min
   - joblib
   - threadpoolctl
-  - matplotlib=3.1.3  # min
+  - matplotlib=3.3.4  # min
   - pandas=1.0.5  # min
   - pyamg
   - pytest