Add appveyor and fix upload config (#46)

* add appveyor

* refine install

Author: hekaisheng

.travis.yml

@@ -1,15 +1,32 @@
 language: python
-python:
-  - "2.7"
-  - "3.5"
-  - "3.6"
 
 matrix:
   include:
-    - os: linux
+    - sudo: required
+      os: linux
+      python: "2.7"
+      services:
+        - docker
+      env: DOCKER_IMAGE=quay.io/pypa/manylinux1_x86_64 PYVER=cp27-cp27mu
+    - sudo: required
+      os: linux
+      python: "3.5"
+      services:
+        - docker
+      env: DOCKER_IMAGE=quay.io/pypa/manylinux1_x86_64 PYVER=cp35-cp35m
+    - sudo: required
+      os: linux
+      python: "3.6"
+      services:
+        - docker
+      env: DOCKER_IMAGE=quay.io/pypa/manylinux1_x86_64 PYVER=cp36-cp36m
+    - sudo: required
+      os: linux
       python: "3.7"
       dist: xenial
-      sudo: true
+      services:
+        - docker
+      env: DOCKER_IMAGE=quay.io/pypa/manylinux1_x86_64 PYVER=cp37-cp37m
     - os: osx
       language: generic
       env: PYTHON=2.7.12

appveyor.yml

@@ -0,0 +1,73 @@
+environment:
+  SECUREPSD:
+    secure: W32PjuuRBQGf05cN6X5EXg8t3/2UsFwvefBlSbioclY=
+
+  matrix:
+
+    - PYTHON: "C:\\Python35-x64"
+      PYTHON_VERSION: "3.5.x"
+      PYTHON_ARCH: "64"
+
+    - PYTHON: "C:\\Python36-x64"
+      PYTHON_VERSION: "3.6.x"
+      PYTHON_ARCH: "64"
+
+    - PYTHON: "C:\\Python37-x64"
+      PYTHON_VERSION: "3.7.0"
+      PYTHON_ARCH: "64"
+
+install:
+  # Prepend newly installed Python to the PATH of this build (this cannot be
+  # done from inside the powershell script as it would require to restart
+  # the parent CMD process).
+  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
+
+  # Check that we have the expected version and architecture for Python
+  - "python --version"
+  - "python -c \"import struct; print(struct.calcsize('P') * 8)\""
+
+  # Upgrade to the latest version of pip to avoid it displaying warnings
+  # about it being out of date.
+  - "python -m pip install --disable-pip-version-check --user --upgrade pip"
+
+  # Install the build dependencies of the project.
+  - "pip install --upgrade wheel twine setuptools coveralls"
+  - "pip install -r requirements-dev.txt"
+  - "pip install -r requirements-extra.txt"
+
+
+build_script:
+  - "python setup.py build_ext -i"
+  # Build tar zip only once
+
+
+test_script:
+  - "python -m pytest -p no:warnings --ignore=mars/worker --timeout=1500"
+
+after_test:
+
+  # Specify account details for PyPI
+  - echo [distutils]                                  > %USERPROFILE%\\.pypirc
+  - echo index-servers =                             >> %USERPROFILE%\\.pypirc
+  - echo     pypi                                    >> %USERPROFILE%\\.pypirc
+  - echo [pypi]                                      >> %USERPROFILE%\\.pypirc
+  - echo repository=https://upload.pypi.org/legacy/  >> %USERPROFILE%\\.pypirc
+  - echo username=pyodps                             >> %USERPROFILE%\\.pypirc
+  - echo password=%SECUREPSD%                        >> %USERPROFILE%\\.pypirc
+
+on_success:
+  - echo %APPVEYOR_REPO_TAG%
+  - echo %APPVEYOR_REPO_TAG_NAME%
+  - echo %APPVEYOR_REPO_BRANCH%
+  - ps: >-
+      If ($env:APPVEYOR_REPO_TAG -eq "true"){
+          Invoke-Expression "python setup.py bdist_wheel"
+          if ($env:PYTHON_VERSION -eq "3.6.x" -and $env:PYTHON_ARCH -eq "64")
+          {
+              Invoke-Expression "python setup.py sdist --formats=gztar"
+          }
+
+          Invoke-Expression "twine upload --skip-existing dist/*"
+      }Else {
+          write-output "Not on a tag commit, won't deploy to pypi"
+      }

bin/travis-build-wheels.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -e -x
+
+# Install a system package required by our library
+yum install -y atlas-devel
+
+# Compile wheels
+PYBIN=/opt/python/${PYVER}/bin
+"${PYBIN}/pip" install -r /io/requirements-dev.txt
+"${PYBIN}/pip" install -r /io/requirements-extra.txt
+
+cd /io
+"${PYBIN}/python" setup.py bdist_wheel
+
+
+# Bundle external shared libraries into the wheels
+for whl in dist/*.whl; do
+    auditwheel repair "$whl" -w dist/
+done
+
+rm dist/*-linux*.whl

bin/travis-upload.sh

@@ -1,6 +1,20 @@
 #!/bin/bash
 
 if [ "$TRAVIS_TAG" ]; then
+  if [ "$TRAVIS_OS_NAME" = "linux" ]; then
+    sudo chmod 777 bin/*
+    docker pull $DOCKER_IMAGE
+    docker run --rm -e "PYVER=$PYVER" -v `pwd`:/io $DOCKER_IMAGE $PRE_CMD /io/bin/travis-build-wheels.sh
+  else
+    pip wheel --no-deps .
+    mkdir dist
+    cp *.whl dist/
+    pip install delocate
+    delocate-wheel dist/*.whl
+    delocate-addplat --rm-orig -x 10_9 -x 10_10 dist/*.whl
+  fi
+  ls dist/
+
   echo "[distutils]"                                  > ~/.pypirc
   echo "index-servers ="                             >> ~/.pypirc
   echo "    pypi"                                    >> ~/.pypirc
@@ -10,16 +24,7 @@ if [ "$TRAVIS_TAG" ]; then
   echo "password=$PASSWD"                            >> ~/.pypirc
 
   python -m pip install twine
-
-  python setup.py sdist --formats=gztar
-  python setup.py bdist_wheel
-
-  for whl in dist/*.whl; do
-	auditwheel repair $whl -w dist/
-  done
-  rm dist/*-linux*.whl
-
-  python -m twine upload -r pypi --skip-existing dist/*.tar.gz;
+  python -m twine upload -r pypi --skip-existing dist/*.whl;
 else
   echo "Not on a tag, won't deploy to pypi";
 fi

mars/actors/pool/tests/test_gevent_pool.py

@@ -20,6 +20,7 @@
 import uuid
 import time
 import unittest
+import sys
 
 import gevent
 
@@ -172,6 +173,7 @@ def distribute(self, uid):
         return int(hashlib.sha1(to_binary(uid)).hexdigest(), 16) % (self.n_process - 1) + 1
 
 
+@unittest.skipIf(sys.platform == 'win32', 'does not run in windows')
 class Test(unittest.TestCase):
     def setUp(self):
         self.exceptions = gevent.hub.Hub.NOT_ERROR

mars/deploy/local/tests/test_cluster.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 
 import unittest
+import sys
 
 import numpy as np
 
@@ -43,6 +44,7 @@ def __init__(self, f=None, **kw):
         super(SerializeMustFailOperand, self).__init__(_f=f, **kw)
 
 
+@unittest.skipIf(sys.platform == 'win32', 'does not run in windows')
 class Test(unittest.TestCase):
     def setUp(self):
         self._old_cache_memory_limit = options.worker.cache_memory_limit

mars/scheduler/tests/test_assigner.py

@@ -41,7 +41,7 @@ def get_reply(self):
 class Test(unittest.TestCase):
 
     def testAssignerActor(self):
-        with create_actor_pool(backend='gevent') as pool:
+        with create_actor_pool(n_process=1, backend='gevent') as pool:
             pool.create_actor(ClusterInfoActor, [pool.cluster_info.address],
                               uid=ClusterInfoActor.default_name())
             resource_ref = pool.create_actor(ResourceActor, uid=ResourceActor.default_name())

mars/scheduler/tests/test_main.py

@@ -34,6 +34,7 @@
 from mars.scheduler.graph import GraphState
 
 
+@unittest.skipIf(sys.platform == 'win32', "plasma don't support windows")
 class Test(unittest.TestCase):
 
     @classmethod

mars/tensor/execution/tests/test_datasource_execute.py

@@ -54,7 +54,7 @@ def testCreateSparseExecution(self):
 
         res = self.executor.execute_tensor(t3)
         self.assertIsInstance(res[0], SparseNDArray)
-        self.assertEqual(res[0].dtype, np.int64)
+        self.assertEqual(res[0].dtype, np.int_)
         np.testing.assert_array_equal(res[0].toarray(), mat[..., :2].toarray())
         np.testing.assert_array_equal(res[1].toarray(), mat[..., 2:].toarray())
 
@@ -80,7 +80,7 @@ def testEmptyExecution(self):
 
         res = self.executor.execute_tensor(t, concat=True)
         self.assertEqual(res[0].shape, (20, 30))
-        self.assertTrue(res[0].dtype, np.int64)
+        self.assertEqual(res[0].dtype, np.int64)
         self.assertFalse(np.array_equal(res, np.zeros((20, 30))))
 
         t = empty((20, 30), chunks=5)
@@ -91,7 +91,7 @@ def testEmptyExecution(self):
         t2 = empty_like(t)
         res = self.executor.execute_tensor(t2, concat=True)
         self.assertEqual(res[0].shape, (20, 30))
-        self.assertTrue(res[0].dtype, np.int64)
+        self.assertEqual(res[0].dtype, np.float64)
 
     def testFullExecution(self):
         t = full((2, 2), 1, dtype='f4', chunks=1)
@@ -328,7 +328,7 @@ def testEyeExecution(self):
         t = eye(2, dtype=int)
 
         res = self.executor.execute_tensor(t, concat=True)[0]
-        self.assertEqual(res.dtype, np.int64)
+        self.assertEqual(res.dtype, np.int_)
 
         # test sparse
         t = eye(5, sparse=True, chunks=2)
@@ -410,7 +410,7 @@ def testLinspaceExecution(self):
         a = linspace(2.0, 9.0, num=11, chunks=3, dtype=int)
 
         res = self.executor.execute_tensor(a, concat=True)[0]
-        self.assertEqual(res.dtype, np.int64)
+        self.assertEqual(res.dtype, np.int_)
 
     def testMeshgridExecution(self):
         a = arange(5, chunks=2)

mars/tests/test_promise.py

@@ -18,6 +18,7 @@
 import time
 import unittest
 import weakref
+import sys
 
 import gevent
 
@@ -87,6 +88,7 @@ def _raise_exception(exc):
     raise exc
 
 
+@unittest.skipIf(sys.platform == 'win32', 'does not run in windows')
 class Test(unittest.TestCase):
     def testPromise(self):
         promises = weakref.WeakValueDictionary()

mars/web/tests/test_api.py

@@ -14,25 +14,34 @@
 # limitations under the License.
 
 import time
+import requests
+import json
+import unittest
+import mock
 import os
 import sys
 import signal
 import subprocess
-import unittest
 
+import gevent
 import numpy as np
 from numpy.testing import assert_array_equal
-import gevent
-import requests
 
-from mars.config import options
+
 from mars import tensor as mt
+from mars.tensor.execution.core import Executor
+from mars.actors import create_actor_pool, new_client
 from mars.utils import get_next_port
-from mars.actors.core import new_client
-from mars.scheduler import KVStoreActor
+from mars.cluster_info import ClusterInfoActor
+from mars.scheduler import SessionManagerActor, KVStoreActor, ResourceActor
+from mars.scheduler.graph import GraphActor
+from mars.web import MarsWeb
 from mars.session import new_session
+from mars.serialize.dataserializer import dumps, loads
+from mars.config import options
 
 
+@unittest.skipIf(sys.platform == 'win32', 'does not run in windows')
 class Test(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
@@ -179,3 +188,67 @@ def testApi(self):
             res = requests.get(service_ep + '/worker')
             self.assertEqual(res.status_code, 200)
 
+
+class TestWithMockServer(unittest.TestCase):
+    def setUp(self):
+        self._executor = Executor('numpy')
+
+        # create scheduler pool with needed actor
+        scheduler_address = '127.0.0.1:' + str(get_next_port())
+        pool = create_actor_pool(address=scheduler_address, n_process=1, backend='gevent')
+        pool.create_actor(ClusterInfoActor, [scheduler_address], uid=ClusterInfoActor.default_name())
+        pool.create_actor(ResourceActor, uid=ResourceActor.default_name())
+        pool.create_actor(SessionManagerActor, uid=SessionManagerActor.default_name())
+        self._kv_store_ref = pool.create_actor(KVStoreActor, uid=KVStoreActor.default_name())
+        self._pool = pool
+
+        self.start_web(scheduler_address)
+
+        check_time = time.time()
+        while True:
+            if time.time() - check_time > 30:
+                raise SystemError('Wait for service start timeout')
+            try:
+                resp = requests.get(self._service_ep + '/api', timeout=1)
+            except (requests.ConnectionError, requests.Timeout):
+                time.sleep(1)
+                continue
+            if resp.status_code >= 400:
+                time.sleep(1)
+                continue
+            break
+
+    def tearDown(self):
+        self._web.stop()
+        self._pool.stop()
+
+    def start_web(self, scheduler_address):
+        import gevent.monkey
+        gevent.monkey.patch_all()
+
+        web_port = str(get_next_port())
+        mars_web = MarsWeb(port=int(web_port), scheduler_ip=scheduler_address)
+        mars_web.start()
+        service_ep = 'http://127.0.0.1:' + web_port
+        self._web = mars_web
+        self._service_ep = service_ep
+
+    @mock.patch(GraphActor.__module__ + '.GraphActor.execute_graph')
+    @mock.patch(GraphActor.__module__ + '.ResultReceiverActor.fetch_tensor')
+    def testApi(self, mock_fetch_tensor, _):
+        with new_session(self._service_ep) as sess:
+            self._kv_store_ref.write('/workers/meta/%s' % 'mock_endpoint', 'mock_meta')
+            self.assertEqual(sess.count_workers(), 1)
+
+            a = mt.ones((100, 100), chunks=30)
+            b = mt.ones((100, 100), chunks=30)
+            c = a.dot(b)
+            graph_key = sess.run(c, timeout=120, wait=False)
+            self._kv_store_ref.write('/sessions/%s/graph/%s/state' % (sess.session_id, graph_key), 'SUCCEEDED')
+            graph_url = '%s/api/session/%s/graph/%s' % (self._service_ep, sess.session_id, graph_key)
+            graph_state = json.loads(requests.get(graph_url).text)
+            self.assertEqual(graph_state['state'], 'success')
+            mock_fetch_tensor.return_value = dumps(self._executor.execute_tensor(c, concat=True)[0])
+            data_url = graph_url + '/data/' + c.key
+            data = loads(requests.get(data_url).content)
+            assert_array_equal(data, np.ones((100, 100)) * 100)