Update code for 2.0.5 release

Mostly done by ecederstrand
(#1529)

Author: hainm

.gitignore

@@ -9,6 +9,9 @@ tests/TestListTravis.sh
 *nogit*
 Makefile
 set_env.sh
+venv
+__pycache__
 
 # editors
+.idea
 .vscode

devtools/builder/build_all.sh

@@ -1,39 +1,61 @@
-#!/bin/sh
-
-# Must be in pytraj root folder
-
-# add cpptraj folder
-python -c "import auditwheel" || exit 1
+#!/usr/bin/env bash
+set -e
 
 function main(){
-    # this function will be run in the end of this script
+    # Must be in pytraj root folder
+
     devtools/mkrelease
     clone_or_update_cpptraj
-    pip_linux
+
+    # osx
+    build_libcpptraj_osx
     pip_osx
-    conda_linux
     conda_osx
+
+    # linux
+    build_libcpptraj_linux
+    pip_linux
+    conda_linux
 }
 
 
+function create_venv(){
+    if [ ! -d /tmp/pytraj_venv ]; then
+        # We need to place the venv outside the checkout so it does not get wiped out by 'git clean'
+        python3 -m venv /tmp/pytraj_venv
+        /tmp/pytraj_venv/bin/python -m pip install -U pip setuptools wheel
+        /tmp/pytraj_venv/bin/python -m pip install auditwheel numpy Cython
+    fi
+}
+
 function clone_or_update_cpptraj(){
     if [ ! -d cpptraj ]; then
         git clone https://github.com/amber-md/cpptraj
     else
-        (cd cpptraj && git pull && git clean -fdx .)
+        # (cd cpptraj && git pull && git clean -fdx .)
+        echo "bla bla"
     fi
 }
 
 
 function pip_linux(){
-    sh devtools/builder/run_docker_build_wheels_linux.sh
+    devtools/builder/run_docker_build_wheels_linux.sh
 }
 
 
-function pip_osx(){
-    (cd cpptraj && git clean -fdx .)
+function build_libcpptraj_osx(){
+    # (cd cpptraj && git clean -fdx .)
     export CPPTRAJHOME=`pwd`/cpptraj
     python scripts/install_libcpptraj.py
+}
+
+
+function build_libcpptraj_linux(){
+    bash devtools/builder/run_docker_build_libcpptraj_linux.sh
+}
+
+
+function pip_osx(){
     (cd dist && python ../scripts/build_wheel.py pytraj*.tar.gz)
 }
 
@@ -44,10 +66,11 @@ function conda_linux(){
 
 
 function conda_osx(){
-    for pyver in 2.7 3.7 3.5 3.6; do
+    # for pyver in 3.5 3.6 3.7 3.8; do
+    for pyver in 3.8; do
         conda build devtools/conda-recipe/pytraj --py $pyver
         tarfile=`conda build devtools/conda-recipe/pytraj --py $pyver --output`
-    
+
         build_dir=dist/conda/osx-64
         if [ ! -d $build_dir ]; then
             mkdir -p $build_dir

devtools/builder/run_docker_build_conda_linux.sh

@@ -1,8 +1,9 @@
 #!/usr/bin/env bash
+set -e
 
 FEEDSTOCK_ROOT=$(cd "$(dirname "$0")/../../"; pwd;)
-# DOCKER_IMAGE=hainm/pytraj-build-box:2019-03 # centos-5; does not have python 3.7
-DOCKER_IMAGE=condaforge/linux-anvil # Miniconda only has python 3.7 on centos >= 6
+# DOCKER_IMAGE=hainm/pytraj-build-box:2020-04-24  # centos-7
+DOCKER_IMAGE=hainm/pytraj-build-box:2019-03
 
 docker info
 cat << EOF | docker run -i \
@@ -12,11 +13,12 @@ cat << EOF | docker run -i \
                         bash || exit $?
 
 set -x
+set -e
 cd /feedstock_root/
 
 export PATH=\$HOME/miniconda3/bin:\$PATH
 
-for pyver in 3.7 2.7 3.5 3.6; do
+for pyver in  3.5 3.6 3.7 3.8; do
     conda build devtools/conda-recipe/pytraj --py \$pyver
     tarfile=\`conda build devtools/conda-recipe/pytraj --py \$pyver --output\`
     echo "\$tarfile"

devtools/builder/run_docker_build_libcpptraj_linux.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -e
+
+FEEDSTOCK_ROOT=$(cd "$(dirname "$0")/../../"; pwd;)
+DOCKER_IMAGE=hainm/pytraj-build-box:2019-03
+
+docker info
+cat << EOF | docker run -i \
+                        -v ${FEEDSTOCK_ROOT}:/feedstock_root \
+                        -a stdin -a stdout -a stderr \
+                        ${DOCKER_IMAGE}\
+                        bash || exit $?
+
+set -x
+set -e
+cd /feedstock_root/
+
+if [ ! -d cpptraj ]; then
+    python scripts/install_libcpptraj.py github -openmp
+else
+    # (cd cpptraj && git pull && git clean -fdx .)
+    python scripts/install_libcpptraj.py -openmp
+fi
+EOF

devtools/builder/run_docker_build_wheels_linux.sh

@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# Assume libcpptraj.so is already built
+set -e
 
 FEEDSTOCK_ROOT=$(cd "$(dirname "$0")/../../"; pwd;)
 DOCKER_IMAGE=hainm/pytraj-build-box:2019-03
@@ -11,23 +13,18 @@ cat << EOF | docker run -i \
                         bash || exit $?
 
 set -x
+set -e
 cd /feedstock_root/
 
-export python=/opt/python/cp36-cp36m/bin/python
+export PATH="/opt/python/cp38-cp38/bin:\$PATH"
 
 if [ ! -d dist ]; then
-    \$python ./devtools/mkrelease
-fi
-
-if [ ! -d cpptraj ]; then
-    \$python scripts/install_libcpptraj.py github -openmp
-else
-    # (cd cpptraj && git pull && git clean -fdx .)
-    \$python scripts/install_libcpptraj.py -openmp
+    devtools/mkrelease
 fi
 
+ls -la dist
 cd dist
-\$python ../scripts/build_wheel.py pytraj*gz --manylinux-docker
+python ../scripts/build_wheel.py ./pytraj*gz --manylinux-docker --py 3.5 3.6 3.7 3.8
 
-cd ..
+rm -rf scripts/__pycache__
 EOF

devtools/builder/upload.sh

@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+set -e
 
 function main(){
     upload_pip

devtools/ci/install_ambertools.sh

@@ -3,6 +3,5 @@ conda install ambertools=17 -c http://ambermd.org/downloads/ambertools/conda/ -y
 python_ver=$(python -c 'import sys; print(".".join(map(str, sys.version_info[:2])))')
 echo "python_ver" $python_ver
 
-rm -rf $HOME/miniconda3/envs/$myenv/lib/python$python_ver/site-packages/pytraj
-rm -rf $HOME/miniconda3/envs/$myenv/lib/python$python_ver/site-packages/pytraj-2.0.0-py3.6.egg-info
+rm -rf $HOME/miniconda3/envs/$myenv/lib/python$python_ver/site-packages/pytraj*
 rm $HOME/miniconda3/envs/$myenv/lib/libcpptraj*

devtools/ci/test_setup_command.py

@@ -98,7 +98,7 @@ def test_install_libcpptraj_if_having_cpptraj_folder_here():
 
 def test_install_to_amberhome():
     fn = './fake_amberhome'
-    python_path = '{}/lib/python3.6/site-packages/'.format(fn)
+    python_path = '{}/lib/python3.8/site-packages/'.format(fn)
     mkdir_cmd = 'mkdir -p {}'.format(python_path)
     subprocess.check_call(mkdir_cmd, shell=True)
     full_name = os.path.abspath(fn)

devtools/conda-recipe/pytraj/build.sh

@@ -5,16 +5,12 @@ pyver=`python -c "import sys; print('.'.join(str(x) for x in sys.version_info[:2
 pyver2=`python -c "import sys; print(''.join(str(x) for x in sys.version_info[:2]))"`
 
 # FIXME: why hard code the version here?
-version='2.0.4'
+version='2.0.5'
 
 if [ "$isosx" = "True" ]; then
-   whlfile=pytraj-$version-cp$pyver2-cp${pyver2}m-macosx_*_x86_64.whl
+   whlfile=pytraj-$version-cp$pyver2-cp${pyver2}*-macosx_*_x86_64.whl
 else
-   if [ "$pyver2" = "27" ]; then
-       whlfile=pytraj-$version-cp$pyver2-cp${pyver2}mu-manylinux1_x86_64.whl
-   else
-       whlfile=pytraj-$version-cp$pyver2-cp${pyver2}m-manylinux1_x86_64.whl
-   fi
+   whlfile=pytraj-$version-cp$pyver2-cp${pyver2}*-manylinux1_x86_64.whl
 fi
 
 wheel unpack `ls $RECIPE_DIR/../../../dist/wheelhouse/$whlfile`

devtools/conda-recipe/pytraj/meta.yaml

@@ -1,5 +1,5 @@
 {% set name = "pytraj" %}
-{% set version = "2.0.4" %} # 
+{% set version = "2.0.5" %}
 
 package:
   name: {{ name }}

scripts/build_wheel.py

@@ -32,30 +32,33 @@ class PipBuilder(object):
     Build wheel package from conda package?
     '''
     REQUIRED_PACKAGES = ['auditwheel']
-    SUPPORTED_VERSIONS = ['2.7mu', '2.7', '3.7', '3.5', '3.6']
+    SUPPORTED_VERSIONS = {
+        '3.5': 'cp35-cp35m',
+        '3.6': 'cp36-cp36m',
+        '3.7': 'cp37-cp37m',
+        '3.8': 'cp38-cp38',
+    }
 
     if sys.platform.startswith('darwin'):
         REQUIRED_PACKAGES.append('conda_build')
-        SUPPORTED_VERSIONS.remove('2.7mu')
 
-    MANY_LINUX_PYTHONS = dict((py_version,
-                               '/opt/python/cp{py}-cp{py}m/bin/python'.format(
-                                   py=py_version.replace('.', '')))
-                              for py_version in SUPPORTED_VERSIONS)
-    # wide-unicode (to be compatible with miniconda/anaconda python)
-    if '2.7mu' in MANY_LINUX_PYTHONS:
-        MANY_LINUX_PYTHONS['2.7mu'] = '/opt/python/cp27-cp27mu/bin/python'
+    MANY_LINUX_PYTHONS = {
+        py_version: '/opt/python/{tag}/bin/python'.format(tag=tag)
+        for py_version, tag in SUPPORTED_VERSIONS.items()
+    }
 
     def __init__(self,
                  tarfile,
                  pytraj_home,
                  python_versions,
                  use_manylinux=False,
-                 cpptraj_dir=''):
+                 cpptraj_dir='',
+                 validate_only=False):
         self.libcpptraj = ''  # will be updated later
         self.is_osx = sys.platform.startswith('darwin')
         self.python_versions = python_versions
         self.use_manylinux = use_manylinux
+        self.validate_only = validate_only
         if cpptraj_dir:
             self.cpptraj_dir = os.path.abspath(cpptraj_dir)
         else:
@@ -78,10 +81,13 @@ def __init__(self,
     def run(self):
         self.check_cpptraj_and_required_libs()
         for python_version in self.python_versions:
-            self.initialize_env(python_version)
-            self.build_original_wheel(python_version)
-            self.repair_wheel(python_version)
-            self.validate_install(python_version)
+            if self.validate_only:
+                self.validate_install(python_version)
+            else:
+                self.initialize_env(python_version)
+                self.build_original_wheel(python_version)
+                self.repair_wheel(python_version)
+                self.validate_install(python_version)
 
     def initialize_env(self, python_version):
         if not self.use_manylinux:
@@ -157,7 +163,7 @@ def validate_install(self, py_version):
         env = 'pytraj' + py_version
         # e.g: change 2.7 to 27
         print('Testing pytraj build')
-        whl_file = self._get_wheel_file(py_version, folder='wheelhouse')
+        whl_file = os.path.abspath(self._get_wheel_file(py_version, folder='wheelhouse'))
         print('Testing wheel file {}'.format(whl_file))
         try:
             subprocess.check_call(
@@ -228,27 +234,32 @@ def check_cpptraj_and_required_libs(self):
     parser.add_argument(
         '--py',
         default=None,
-        help='Python version. Default: build all versions (2.7, 3.4, 3.5)')
+        nargs='+',
+        help='Python version (e.g. 2.7 or 3.8). Default: build all supported versions')
     parser.add_argument(
         '--cpptraj-dir', default='', help='cpptraj dir, optional')
     parser.add_argument(
         '--manylinux-docker',
         action='store_true',
         help='If specified, use Python versions from manylinux')
+    parser.add_argument(
+        '--validate-only',
+        action='store_true',
+        help='Only validate the exisint repaired wheel')
     args = parser.parse_args()
 
     tarfile = os.path.abspath(args.tarfile)
-    python_versions = PipBuilder.SUPPORTED_VERSIONS if args.py is None else [
-        args.py,
-    ]
+    python_versions = args.py or list(PipBuilder.SUPPORTED_VERSIONS.keys())
+    print(f"Building package for python: {python_versions}")
     # pytraj tar file
     pytraj_home = os.path.dirname(__file__).strip('scripts')
     builder = PipBuilder(
         tarfile=tarfile,
         pytraj_home=pytraj_home,
         python_versions=python_versions,
         use_manylinux=args.manylinux_docker,
-        cpptraj_dir=args.cpptraj_dir)
+        cpptraj_dir=args.cpptraj_dir,
+        validate_only=args.validate_only)
     builder.run()
     # builder.libcpptraj = '../cpptraj/lib/libcpptraj.dylib'
     # builder.validate_install('3.5')

scripts/cythonize.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python
 import os
 import sys
 import Cython

scripts/install_libcpptraj.py

@@ -165,32 +165,18 @@ def install_libcpptraj(compiler='', build_flag='', n_cpus=4):
     except OSError:
         pass
 
-    if sys.platform.startswith('win'):
-        _install_libcpptraj_win_msys2()
-    else:
-        cxx_overwrite = ''
-
-        # We don't need this anymore?
-        # if IS_OSX:
-        #     if ((compiler == 'clang' or 'clang' in os.getenv('CXX', '')) or
-        #         (os.getenv('CXX') and is_clang(os.getenv('CXX')))):
-        #         # cxx_overwrite = 'CXX="clang++ -stdlib=libstdc++"'
-        print('cxx_overwrite flag', cxx_overwrite)
+    cm = 'bash configure {build_flag} {compiler}'.format(
+        build_flag=build_flag,
+        compiler=compiler)
 
-        cm = 'bash configure {build_flag} {compiler} {cxx_overwrite}'.format(
-            build_flag=build_flag,
-            compiler=compiler,
-            cxx_overwrite=cxx_overwrite)
+    print('configure command: ', cm)
+    subprocess.check_call(cm, shell=True)
 
-        print('configure command: ', cm)
-        # do not use subprocess to avoid split cxx_overwrite command
-        os.system(cm)
-
-        if IS_OSX:
-            add_cpptraj_cxx_to_config('config.h', CPPTRAJ_CXX)
+    if IS_OSX:
+        add_cpptraj_cxx_to_config('config.h', CPPTRAJ_CXX)
 
-        subprocess.check_call('make libcpptraj -j{}'.format(n_cpus).split())
-        os.chdir(cwd)
+    subprocess.check_call('make libcpptraj -j{}'.format(n_cpus).split())
+    os.chdir(cwd)
 
 
 def parse_args():