Migrate tensor/dataframe/learn tests to oscar (#2106)

Co-authored-by: hekaisheng <kaisheng.hks@alibaba-inc.com>
Co-authored-by: wenjun.swj <wenjun.swj@alibaba-inc.com>
Co-authored-by: chaokunyang <shawn.ck.yang@gmail.com>
Co-authored-by: wjsi <swj0066@gmail.com>

Author: 4 people

.codecov.yml

@@ -24,7 +24,6 @@ ignore:
   - "mars/lib/futures"
   - "mars/lib/enum.py"
   - "mars/lib/six.py"
-  - "mars/lib/gipc.pyx"
   - "mars/lib/nvutils.py"
   - "mars/lib/tblib"
   - "mars/lib/uhashring"

.coveragerc

@@ -1,9 +1,13 @@
+# this configuration is for mars.tensor module as
+# coverage.py currently doesn't support measuring
+# gevent with threading.
+
 [run]
 branch = True
 relative_files = True
 cover_pylib = False
-concurrency = multiprocessing,gevent
 parallel = True
+concurrency = multiprocessing,thread
 plugins = Cython.Coverage
 include =
     mars/*
@@ -15,7 +19,6 @@ omit =
     mars/lib/futures/*
     mars/lib/enum.py
     mars/lib/six.py
-    mars/lib/gipc.pyx
     mars/lib/nvutils.py
     mars/lib/tblib/*
     mars/lib/uhashring/*

.coveragerc-threaded

@@ -89,7 +89,7 @@ jobs:
       run: |
         source ./ci/reload-env.sh
         # stop the build if there are Python syntax errors or undefined names
-        flake8 mars --count --select=E9,E111,E225,E302,E303,E901,E999,F7,F63,F82,F401,F821,F822,F823,F841,W291,W292,W391,W605 --show-source --statistics
+        flake8 mars --count --select=E9,E111,E225,E302,E303,E901,E999,F7,F63,F82,F401,F811,F821,F822,F823,F841,W291,W292,W391,W605 --show-source --statistics
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
         flake8 mars --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
 

.github/workflows/core-ci.yml

@@ -75,16 +75,12 @@ jobs:
 
         export NO_SERIALIZE_IN_TEST_EXECUTOR=1
         mkdir -p build
-        python -m pytest $PYTEST_CONFIG --cov-config .coveragerc-threaded --timeout=1500 \
-          mars/dataframe mars/learn mars/remote mars/serialize mars/storage mars/tensor
-        mv .coverage build/.coverage.tensor.file
         if [[ $UNAME == "darwin" ]]; then
-          python -m pytest $PYTEST_CONFIG --cov-config .coveragerc --forked --timeout=1500 \
-            --ignore mars/tests/test_kvstore.py mars/oscar mars/storage mars/deploy/local \
-            mars/web mars/lib mars/tests
+          python -m pytest $PYTEST_CONFIG --forked --timeout=1500 \
+            mars/oscar mars/storage mars/lib mars/tests
           mv .coverage build/.coverage.dist.file
         elif [[ $UNAME == "windows" ]]; then
-          python -m pytest $PYTEST_CONFIG --cov-config .coveragerc-threaded --timeout=1500 \
+          python -m pytest $PYTEST_CONFIG --timeout=1500 \
             mars/oscar
           mv .coverage build/.coverage.dist.file
         fi

.github/workflows/os-compat-ci.yml

@@ -9,14 +9,14 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        python-version: [3.8-kubernetes, 3.8-hadoop, 3.8-vineyard, 3.8-ray]
+        python-version: [3.8-kubernetes, 3.8-ray] # [3.8-kubernetes, 3.8-hadoop, 3.8-vineyard, 3.8-ray]
         include:
           - { os: ubuntu-latest, python-version: 3.8-kubernetes, no-common-tests: 1,
               no-deploy: 1, with-kubernetes: "with Kubernetes" }
-          - { os: ubuntu-latest, python-version: 3.8-hadoop, no-common-tests: 1,
-              no-deploy: 1, with-hadoop: "with hadoop" }
-          - { os: ubuntu-latest, python-version: 3.8-vineyard, no-common-tests: 1,
-              no-deploy: 1, with-vineyard: "with vineyard" }
+          #- { os: ubuntu-latest, python-version: 3.8-hadoop, no-common-tests: 1,
+          #    no-deploy: 1, with-hadoop: "with hadoop" }
+          #- { os: ubuntu-latest, python-version: 3.8-vineyard, no-common-tests: 1,
+          #    no-deploy: 1, with-vineyard: "with vineyard" }
           - { os: ubuntu-latest, python-version: 3.8-ray, no-common-tests: 1,
               no-deploy: 1, with-ray: "with ray" }
 
@@ -122,31 +122,28 @@ jobs:
 
           if [ -n "$WITH_HADOOP" ]; then
             source $CONDA/bin/activate test
-            pytest $PYTEST_CONFIG --cov-config .coveragerc-threaded -m hadoop
+            pytest $PYTEST_CONFIG -m hadoop
             coverage report
           fi
           if [ -n "$WITH_KUBERNETES" ]; then
-            pytest $PYTEST_CONFIG --cov-config .coveragerc --forked mars/deploy/kubernetes
+            pytest $PYTEST_CONFIG --forked mars/deploy/kubernetes
             coverage report
           fi
           if [ -n "$WITH_VINEYARD" ]; then
             export VINEYARD_IPC_SOCKET=/tmp/vineyard/vineyard.sock
             mkdir -p build
 
-            pytest $PYTEST_CONFIG --cov-config .coveragerc mars/storage/tests/test_libs.py
+            pytest $PYTEST_CONFIG mars/storage/tests/test_libs.py
             mv .coverage build/.coverage.test_lib.file
 
-            pytest $PYTEST_CONFIG --cov-config .coveragerc-threaded mars/dataframe/datastore/tests/test_datastore_execute.py \
+            pytest $PYTEST_CONFIG mars/dataframe/datastore/tests/test_datastore_execute.py \
                                                                     mars/tensor/datastore/tests/test_datastore_execute.py -k "vineyard"
             mv .coverage build/.coverage.test_tensor.file
 
-            pytest $PYTEST_CONFIG --cov-config .coveragerc-threaded mars/deploy/local/tests/test_cluster.py -k "VineyardEnabledTest"
-            mv .coverage build/.coverage.test_cluster.file
-
             coverage combine build/ && coverage report
           fi
           if [ -n "$WITH_RAY" ]; then
-            pytest $PYTEST_CONFIG --cov-config .coveragerc-threaded -m ray
+            pytest $PYTEST_CONFIG -m ray
             coverage report
           fi
           coverage xml

.github/workflows/platform-ci.yml

@@ -2,29 +2,23 @@
 set -e
 if [ -n "$WITH_CYTHON" ]; then
   mkdir -p build
-  pytest $PYTEST_CONFIG --cov-config .coveragerc --ignore-glob "*/integrated/*" \
-    --ignore mars/tests/test_mutable.py mars/serialize mars/optimizes mars/tests
-  mv .coverage build/.coverage.non-fork.file
-
   export POOL_START_METHOD=forkserver
 
-  retry -n 20 -g INTERNALERROR pytest $PYTEST_CONFIG --cov-config .coveragerc mars/oscar
+  retry -n 20 -g INTERNALERROR pytest $PYTEST_CONFIG --forked mars/tests mars/core/graph
+  mv .coverage build/.coverage.non-oscar.file
+
+  retry -n 20 -g INTERNALERROR pytest $PYTEST_CONFIG --forked mars/oscar
   mv .coverage build/.coverage.oscar_ctx.file
 
-  pytest $PYTEST_CONFIG --cov-config .coveragerc --forked mars/actors mars/deploy/local \
-    mars/scheduler mars/web
-  mv .coverage build/.coverage.fork.file
   coverage combine build/ && coverage report
 fi
 if [ -z "$NO_COMMON_TESTS" ]; then
   mkdir -p build
-  pytest $PYTEST_CONFIG --cov-config .coveragerc-threaded mars/tensor mars/dataframe mars/web \
-    mars/learn mars/remote mars/storage mars/lib
-  mv .coverage build/.coverage.tensor.file
-  pytest $PYTEST_CONFIG --cov-config .coveragerc-threaded --forked mars/oscar mars/services
-  mv .coverage build/.coverage.oscar.file
-  pytest $PYTEST_CONFIG --cov-config .coveragerc --forked --ignore mars/tensor --ignore mars/dataframe \
-    --ignore mars/learn --ignore mars/remote --ignore mars/services --ignore mars/oscar mars
+  pytest $PYTEST_CONFIG mars/remote mars/storage mars/lib
+  mv .coverage build/.coverage.tileable.file
+
+  pytest $PYTEST_CONFIG --forked --ignore mars/tensor --ignore mars/dataframe \
+    --ignore mars/learn --ignore mars/remote mars
   mv .coverage build/.coverage.main.file
   coverage combine build/ && coverage report
 

.github/workflows/run-tests.sh

@@ -68,25 +68,19 @@ jobs:
 
         if [ -z "$NO_COMMON_TESTS" ]; then
           if [[ ! "$PYTHON" =~ "3.6" ]] && [[ ! "$PYTHON" =~ "3.9" ]]; then
-            pip install h5py zarr matplotlib
+            pip install h5py zarr matplotlib fastparquet
             conda install -n test --quiet --yes -c conda-forge python=$PYTHON \
               "tiledb-py>=0.4.3,<0.6.0" "tiledb<2.0.0" || true
           fi
 
           conda install -n test --quiet --yes -c pkgs/main python=$PYTHON certifi
 
           if [[ "$(mars.test.module)" == "learn" ]]; then
-            if [[ ! "$PYTHON" =~ "3.9" ]]; then
-              conda install --quiet --yes -n test -c conda-forge --no-deps python=$PYTHON \
-                libxgboost py-xgboost xgboost lightgbm tensorflow
-            fi
-            if [[ "$PYTHON" =~ "3.6" ]]; then
-              pip install torch==1.4.0 torchvision==0.5.0 faiss-cpu fastparquet
-            fi
-            if [[ ! "$PYTHON" =~ "3.6" ]] && [[ ! "$PYTHON" =~ "3.9" ]]; then
-              pip install torch torchvision
-              pip install statsmodels tsfresh
-            fi
+            conda install --quiet --yes -n test -c conda-forge --no-deps python=$PYTHON \
+              libxgboost py-xgboost xgboost lightgbm tensorflow
+            pip install faiss-cpu
+            pip install torch torchvision
+            pip install statsmodels tsfresh
           fi
         fi
         conda list -n test
@@ -101,7 +95,7 @@ jobs:
     - bash: |
         set -e
         source ci/reload-env.sh
-        pytest $PYTEST_CONFIG --cov-config .coveragerc-threaded mars/$(mars.test.module)
+        pytest $PYTEST_CONFIG mars/$(mars.test.module)
         coverage report
         coverage xml
       displayName: 'Run tests'

azure-pipelines.yml

@@ -16,8 +16,4 @@
 
 from ._version import __version__
 from .config import options
-
-from .core.session import new_session, execute, fetch, stop_server
-# register
-from .deploy.oscar.session import Session
-del Session
+from .session import new_session, execute, fetch, fetch_log, stop_server

mars/__init__.py

@@ -16,7 +16,7 @@
 import os
 from typing import NamedTuple, Optional
 
-version_info = (0, 7, 0, 'b1')
+version_info = (0, 7, 0, 'b2')
 _num_index = max(idx if isinstance(v, int) else 0
                  for idx, v in enumerate(version_info))
 __version__ = '.'.join(map(str, version_info[:_num_index + 1])) + \