CI Run test suite inside Pyodide (scikit-learn#27346)

lesteve · thomasjpfan · web-flow · commit 751ccc0a4d4c · 2023-09-28T17:57:20.000-04:00
Co-authored-by: Thomas J. Fan &lt;thomasjpfan@gmail.com&gt;
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -127,11 +127,11 @@ jobs:
     vmImage: ubuntu-22.04
   variables:
     # Need to match Python version and Emscripten version for the correct
-    # Pyodide version. For example, for Pyodide version 0.23.4, see
-    # https://github.com/pyodide/pyodide/blob/0.23.4/Makefile.envs
-    PYODIDE_VERSION: '0.23.4'
-    EMSCRIPTEN_VERSION: '3.1.32'
-    PYTHON_VERSION: '3.11.2'
+    # Pyodide version. For example, for Pyodide version 0.24.1, see
+    # https://github.com/pyodide/pyodide/blob/0.24.1/Makefile.envs
+    PYODIDE_VERSION: '0.24.1'
+    EMSCRIPTEN_VERSION: '3.1.45'
+    PYTHON_VERSION: '3.11.3'
 
   dependsOn: [git_commit, linting]
   condition: |
diff --git a/build_tools/azure/install_pyodide.sh b/build_tools/azure/install_pyodide.sh
@@ -15,8 +15,6 @@ pyodide build
 
 ls -ltrh dist
 
-pyodide venv pyodide-venv
-source pyodide-venv/bin/activate
-
-pip install dist/*.whl
-pip list
+# The Pyodide js library is needed by build_tools/azure/test_script_pyodide.sh
+# to run tests inside Pyodide
+npm install pyodide@$PYODIDE_VERSION
diff --git a/build_tools/azure/pytest-pyodide.js b/build_tools/azure/pytest-pyodide.js
@@ -0,0 +1,53 @@
+const { opendir } = require('node:fs/promises');
+const { loadPyodide } = require("pyodide");
+
+async function main() {
+    let exit_code = 0;
+    try {
+        global.pyodide = await loadPyodide();
+        let pyodide = global.pyodide;
+        const FS = pyodide.FS;
+        const NODEFS = FS.filesystems.NODEFS;
+
+        let mountDir = "/mnt";
+        pyodide.FS.mkdir(mountDir);
+        pyodide.FS.mount(pyodide.FS.filesystems.NODEFS, { root: "." }, mountDir);
+
+        await pyodide.loadPackage(["micropip"]);
+        await pyodide.runPythonAsync(`
+            import glob
+            import micropip
+
+            wheels = glob.glob('/mnt/dist/*.whl')
+            wheels = [f'emfs://{wheel}' for wheel in wheels]
+            print(f'installing wheels: {wheels}')
+            await micropip.install(wheels);
+
+            pkg_list = micropip.list()
+            print(pkg_list)
+        `);
+
+        // Pyodide is built without OpenMP, need to set environment variable to
+        // skip related test
+        await pyodide.runPythonAsync(`
+            import os
+            os.environ['SKLEARN_SKIP_OPENMP_TEST'] = 'true'
+        `);
+
+        await pyodide.runPythonAsync("import micropip; micropip.install('pytest')");
+        let pytest = pyodide.pyimport("pytest");
+        let args = process.argv.slice(2);
+        console.log('pytest args:', args);
+        exit_code = pytest.main(pyodide.toPy(args));
+    } catch (e) {
+        console.error(e);
+        // Arbitrary exit code here. I have seen this code reached instead of a
+        // Pyodide fatal error sometimes
+        exit_code = 66;
+
+    } finally {
+        process.exit(exit_code);
+    }
+}
+
+main();
diff --git a/build_tools/azure/test_script_pyodide.sh b/build_tools/azure/test_script_pyodide.sh
@@ -2,14 +2,8 @@
 
 set -e
 
-source pyodide-venv/bin/activate
-
-pip list
-
-# Need to be outside of the git clone otherwise finds non build sklearn folder
-cd /tmp
-
-# TODO for now only testing sklearn import to make sure the wheel is not badly
-# broken. When Pyodide 0.24 is released we should run the full test suite and
-# xfail tests that fail due to Pyodide limitations
-python -c 'import sklearn'
+# We are using a pytest js wrapper script to run tests inside Pyodide. Maybe
+# one day we can use a Pyodide venv instead but at the time of writing
+# (2023-09-27) there is an issue with scipy.linalg in a Pyodide venv, see
+# https://github.com/pyodide/pyodide/issues/3865 for more details.
+node build_tools/azure/pytest-pyodide.js --pyargs sklearn --durations 20 --showlocals
diff --git a/sklearn/_loss/tests/test_loss.py b/sklearn/_loss/tests/test_loss.py
@@ -27,7 +27,7 @@
     HuberLoss,
     PinballLoss,
 )
-from sklearn.utils import assert_all_finite
+from sklearn.utils import _IS_WASM, assert_all_finite
 from sklearn.utils._testing import create_memmap_backed_data, skip_if_32bit
 
 ALL_LOSSES = list(_LOSSES.values())
@@ -286,6 +286,9 @@ def test_loss_dtype(
 
     Also check that input arrays can be readonly, e.g. memory mapped.
     """
+    if _IS_WASM and readonly_memmap:  # pragma: nocover
+        pytest.xfail(reason="memmap not fully supported")
+
     loss = loss()
     # generate a y_true and raw_prediction in valid range
     n_samples = 5
diff --git a/sklearn/experimental/tests/test_enable_hist_gradient_boosting.py b/sklearn/experimental/tests/test_enable_hist_gradient_boosting.py
@@ -2,9 +2,13 @@
 
 import textwrap
 
+import pytest
+
+from sklearn.utils import _IS_WASM
 from sklearn.utils._testing import assert_run_python_script
 
 
+@pytest.mark.xfail(_IS_WASM, reason="cannot start subprocess")
 def test_import_raises_warning():
     code = """
     import pytest
diff --git a/sklearn/experimental/tests/test_enable_iterative_imputer.py b/sklearn/experimental/tests/test_enable_iterative_imputer.py
@@ -2,9 +2,13 @@
 
 import textwrap
 
+import pytest
+
+from sklearn.utils import _IS_WASM
 from sklearn.utils._testing import assert_run_python_script
 
 
+@pytest.mark.xfail(_IS_WASM, reason="cannot start subprocess")
 def test_imports_strategies():
     # Make sure different import strategies work or fail as expected.
 
diff --git a/sklearn/experimental/tests/test_enable_successive_halving.py b/sklearn/experimental/tests/test_enable_successive_halving.py
@@ -2,9 +2,13 @@
 
 import textwrap
 
+import pytest
+
+from sklearn.utils import _IS_WASM
 from sklearn.utils._testing import assert_run_python_script
 
 
+@pytest.mark.xfail(_IS_WASM, reason="cannot start subprocess")
 def test_imports_strategies():
     # Make sure different import strategies work or fail as expected.
 
diff --git a/sklearn/feature_extraction/tests/test_text.py b/sklearn/feature_extraction/tests/test_text.py
@@ -26,7 +26,7 @@
 from sklearn.model_selection import GridSearchCV, cross_val_score, train_test_split
 from sklearn.pipeline import Pipeline
 from sklearn.svm import LinearSVC
-from sklearn.utils import IS_PYPY
+from sklearn.utils import _IS_WASM, IS_PYPY
 from sklearn.utils._testing import (
     assert_allclose_dense_sparse,
     assert_almost_equal,
@@ -475,6 +475,13 @@ def test_tf_idf_smoothing():
     assert (tfidf >= 0).all()
 
 
+@pytest.mark.xfail(
+    _IS_WASM,
+    reason=(
+        "no floating point exceptions, see"
+        " https://github.com/numpy/numpy/pull/21895#issuecomment-1311525881"
+    ),
+)
 def test_tfidf_no_smoothing():
     X = [[1, 1, 1], [1, 1, 0], [1, 0, 0]]
     tr = TfidfTransformer(smooth_idf=False, norm="l2")
diff --git a/sklearn/tests/test_common.py b/sklearn/tests/test_common.py
@@ -60,7 +60,7 @@
     StandardScaler,
 )
 from sklearn.semi_supervised import LabelPropagation, LabelSpreading
-from sklearn.utils import IS_PYPY, all_estimators
+from sklearn.utils import _IS_WASM, IS_PYPY, all_estimators
 from sklearn.utils._tags import _DEFAULT_TAGS, _safe_tags
 from sklearn.utils._testing import (
     SkipTest,
@@ -206,6 +206,7 @@ def test_class_weight_balanced_linear_classifiers(name, Classifier):
     check_class_weight_balanced_linear_classifier(name, Classifier)
 
 
+@pytest.mark.xfail(_IS_WASM, reason="importlib not supported for Pyodide packages")
 @ignore_warnings
 def test_import_all_consistency():
     # Smoke test to check that any name in a __all__ list is actually defined
diff --git a/sklearn/tests/test_config.py b/sklearn/tests/test_config.py
@@ -6,6 +6,7 @@
 
 import sklearn
 from sklearn import config_context, get_config, set_config
+from sklearn.utils import _IS_WASM
 from sklearn.utils.parallel import Parallel, delayed
 
 
@@ -138,6 +139,7 @@ def test_config_threadsafe_joblib(backend):
     assert items == [False, True, False, True]
 
 
+@pytest.mark.xfail(_IS_WASM, reason="cannot start threads")
 def test_config_threadsafe():
     """Uses threads directly to test that the global config does not change
     between threads. Same test as `test_config_threadsafe_joblib` but with
diff --git a/sklearn/tests/test_discriminant_analysis.py b/sklearn/tests/test_discriminant_analysis.py
@@ -11,7 +11,7 @@
     _cov,
 )
 from sklearn.preprocessing import StandardScaler
-from sklearn.utils import check_random_state
+from sklearn.utils import _IS_WASM, check_random_state
 from sklearn.utils._testing import (
     _convert_container,
     assert_allclose,
@@ -591,6 +591,13 @@ def test_qda_store_covariance():
     )
 
 
+@pytest.mark.xfail(
+    _IS_WASM,
+    reason=(
+        "no floating point exceptions, see"
+        " https://github.com/numpy/numpy/pull/21895#issuecomment-1311525881"
+    ),
+)
 def test_qda_regularization():
     # The default is reg_param=0. and will cause issues when there is a
     # constant variable.
diff --git a/sklearn/utils/__init__.py b/sklearn/utils/__init__.py
@@ -77,6 +77,7 @@
 
 IS_PYPY = platform.python_implementation() == "PyPy"
 _IS_32BIT = 8 * struct.calcsize("P") == 32
+_IS_WASM = platform.machine() in ["wasm32", "wasm64"]
 
 
 def _in_unstable_openblas_configuration():
diff --git a/sklearn/utils/tests/test_sparsefuncs.py b/sklearn/utils/tests/test_sparsefuncs.py
@@ -6,6 +6,7 @@
 from scipy import linalg
 
 from sklearn.datasets import make_classification
+from sklearn.utils import _IS_WASM
 from sklearn.utils._testing import assert_allclose
 from sklearn.utils.fixes import CSC_CONTAINERS, CSR_CONTAINERS, LIL_CONTAINERS
 from sklearn.utils.sparsefuncs import (
@@ -794,6 +795,18 @@ def test_min_max(
         dtype=dtype,
     )
     X_sparse = sparse_format(X)
+
+    if (
+        _IS_WASM and large_indices and isinstance(X_sparse, sp.sparray)
+    ):  # pragma: nocover
+        pytest.xfail(
+            reason=(
+                "temporary xfailing test until it is fixed in main, see"
+                " https://github.com/scikit-learn/scikit-learn/issues/27470 for more"
+                " details."
+            )
+        )
+
     if large_indices:
         X_sparse.indices = X_sparse.indices.astype("int64")
         X_sparse.indptr = X_sparse.indptr.astype("int64")
diff --git a/sklearn/utils/tests/test_testing.py b/sklearn/utils/tests/test_testing.py
@@ -9,6 +9,7 @@
 
 from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
 from sklearn.tree import DecisionTreeClassifier
+from sklearn.utils import _IS_WASM
 from sklearn.utils._testing import (
     TempMemmap,
     _convert_container,
@@ -607,6 +608,7 @@ def test_tempmemmap(monkeypatch):
     assert registration_counter.nb_calls == 2
 
 
+@pytest.mark.xfail(_IS_WASM, reason="memmap not fully supported")
 @pytest.mark.parametrize("aligned", [False, True])
 def test_create_memmap_backed_data(monkeypatch, aligned):
     registration_counter = RegistrationCounter()
