Merge branch 'release-4.1.1'

Author: mpenkov

.circleci/config.yml

@@ -2,7 +2,7 @@ version: 2
 jobs:
   build:
     docker:
-      - image: circleci/python:3.7.4
+      - image: cimg/python:3.8.11
 
     working_directory: ~/gensim
 
@@ -18,12 +18,12 @@ jobs:
             sudo apt-get -yq update
             sudo apt-get -yq remove texlive-binaries --purge
             sudo apt-get -yq --no-install-suggests --no-install-recommends --force-yes install dvipng texlive-latex-base texlive-latex-extra texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended latexmk
-            sudo apt-get -yq install build-essential python3.7-dev
+            sudo apt-get -yq install build-essential python3.8-dev
 
       - run:
           name: Basic installation (tox)
           command: |
-            python3.7 -m virtualenv venv
+            python3.8 -m virtualenv venv
             source venv/bin/activate
             pip install tox --progress-bar off
 

.github/workflows/build-wheels.yml

@@ -17,19 +17,96 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.6, 3.7, 3.8]
-        os: [ubuntu-latest, macos-latest]
+        python-version: [3.6, 3.7, 3.8, 3.9]
+        os: [ubuntu-latest, macos-latest, windows-latest]
         platform: [x64]
         include:
+        #
+        # We want the _oldest_ possible manylinux version to ensure our
+        # wheels work on the widest possible range of distros.  Version 1
+        # seems to break for certain Python versions under Linux and Windows,
+        # so we use 2010, which is the next oldest.
+        #
+        # When selecting the numpy version to build against, we need to satisfy
+        # two conditions.  First, we want the wheel to be available for the
+        # version of Python we're building against, because building numpy
+        # wheels on our own is too much work.
+        #
+        # Second, in order to guarantee compatibility with the greatest range
+        # of numpy versions, we want to build against the oldest possible numpy
+        # version, as long as it's 1.17.0 or newer.  Building versions earlier
+        # than 1.17.0 yields wheels that are incompatible with some newer
+        # versions of numpy.  See https://github.com/RaRe-Technologies/gensim/issues/3226
+        # for details.
+        #
+        # The logic for numpy version selection is based on
+        # https://github.com/scipy/oldest-supported-numpy/blob/master/setup.cfg
+        # with the exception that we enforce the minimum version to be 1.17.0.
+        #
         - os: ubuntu-latest
+          manylinux-version: 2010
+          python-version: 3.6
+          build-depends: numpy==1.17.0
+
+        - os: ubuntu-latest
+          manylinux-version: 2010
           python-version: 3.7
-          skip-network-tests: 1
+          build-depends: numpy==1.17.0
+
+        - os: ubuntu-latest
+          manylinux-version: 2010
+          python-version: 3.8
+          build-depends: numpy==1.17.3
+
         - os: ubuntu-latest
+          manylinux-version: 2010
+          python-version: 3.9
+          build-depends: numpy==1.19.3
+
+        - os: macos-latest
+          travis-os-name: osx
+          manylinux-version: 1
+          python-version: 3.6
+          build-depends: numpy==1.17.0
+
+        - os: macos-latest
+          travis-os-name: osx
+          manylinux-version: 1
+          python-version: 3.7
+          build-depends: numpy==1.17.0
+
+        - os: macos-latest
+          travis-os-name: osx
+          manylinux-version: 1
           python-version: 3.8
-          skip-network-tests: 1
+          build-depends: numpy==1.21.0
+
         - os: macos-latest
-          travis-os-name: osx  # For multibuild
-          skip-network-tests: 1
+          travis-os-name: osx
+          manylinux-version: 1
+          python-version: 3.9
+          build-depends: numpy==1.21.0
+
+        - os: windows-latest
+          manylinux-version: 2010
+          python-version: 3.6
+          build-depends: numpy==1.17.0
+
+        - os: windows-latest
+          manylinux-version: 2010
+          python-version: 3.7
+          build-depends: numpy==1.17.0
+
+        - os: windows-latest
+          manylinux-version: 2010
+          python-version: 3.8
+          build-depends: numpy==1.17.3
+
+        - os: windows-latest
+          manylinux-version: 2010
+          python-version: 3.9
+          build-depends: numpy==1.19.3
+
     env:
       PKG_NAME: gensim
       REPO_DIR: gensim
@@ -40,7 +117,11 @@ jobs:
       TEST_DEPENDS: Morfessor==2.0.2a4 python-levenshtein==0.12.0 visdom==0.1.8.9 pytest mock cython nmslib pyemd testfixtures scikit-learn pyemd
       DOCKER_TEST_IMAGE: multibuild/xenial_x86_64
       TRAVIS_OS_NAME: ${{ matrix.travis-os-name }}
-      SKIP_NETWORK_TESTS: ${{ matrix.skip-network-tests }}
+      SKIP_NETWORK_TESTS: 1
+      MB_ML_VER: ${{ matrix.manylinux-version }}
+      WHEELHOUSE_UPLOADER_USERNAME: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      WHEELHOUSE_UPLOADER_SECRET: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      BUILD_DEPENDS: ${{ matrix.build-depends }}
 
     steps:
     - uses: actions/checkout@v2
@@ -50,6 +131,7 @@ jobs:
     - name: Print environment variables
       run: |
         echo "PLAT: ${PLAT}"
+        echo "MB_ML_VER: ${MB_ML_VER}"
         echo "DOCKER_TEST_IMAGE: ${DOCKER_TEST_IMAGE}"
         echo "TEST_DEPENDS: ${TEST_DEPENDS}"
         echo "TRAVIS_OS_NAME: ${TRAVIS_OS_NAME}"
@@ -62,7 +144,8 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install virtualenv
-    - name: Build and Install Wheels
+    - name: Build and Install Wheels (Multibuild)
+      if: matrix.os != 'windows-latest'
       run: |
         echo ::group::Set up Multibuild
         source multibuild/common_utils.sh
@@ -78,9 +161,47 @@ jobs:
         echo ::group::Install run
         install_run ${{ matrix.PLAT }}
         echo ::endgroup::
+    #
+    # We can't use multibuild on Windows, so we have to roll our own build script.
+    # Adapted from
+    # https://github.com/RaRe-Technologies/gensim-wheels/commit/084b863390edee05bbe15d4ec05d1ab726e52202
+    #
+    - name: Build and Install Wheels (Windows)
+      if: matrix.os == 'windows-latest'
+      run: |
+        echo ::group::Set up dependencies
+        python --version
+        python -c "import struct; print(struct.calcsize('P') * 8)"
+
+        python -m pip install -U pip setuptools wheel wheelhouse_uploader ${{ env.BUILD_DEPENDS }}
+        echo ::endgroup::
+
+        echo ::group::Build wheel
+        python setup.py bdist_wheel
+        echo ::endgroup
+
+        echo ::group::Install run
+        ls dist
+        python continuous_integration/install_wheel.py
+        echo ::endgroup::
+
+        #
+        # For consistency with the multibuild step.
+        #
+        mv dist wheelhouse
+
     - name: Upload wheels to s3://gensim-wheels
-      if: always()
+      #
+      # Only do this if the credentials are set.
+      # This means that PRs will still build wheels, but not upload them.
+      # (PRs do not have access to secrets).
+      #
+      # The always() ensures this step runs even if a previous step fails.
+      # We want to upload wheels whenever possible (even if e.g. tests failed)
+      # because we don't want an innocuous test failure from blocking a release.
+      #
+      if: ${{ always() && env.WHEELHOUSE_UPLOADER_USERNAME && env.WHEELHOUSE_UPLOADER_SECRET }}
       run: |
-            pip install wheelhouse-uploader
-            ls wheelhouse/*.whl
-            python -m wheelhouse_uploader upload --local-folder wheelhouse/ --no-ssl-check gensim-wheels --provider S3 --no-enable-cdn
+        pip install wheelhouse-uploader
+        ls wheelhouse/*.whl
+        python -m wheelhouse_uploader upload --local-folder wheelhouse/ --no-ssl-check gensim-wheels --provider S3 --no-enable-cdn

.travis.yml

@@ -4,6 +4,9 @@ branches:
     - travis
 language: python
 arch: arm64-graviton2
+#
+# https://docs.travis-ci.com/user/reference/focal/
+#
 dist: focal
 virt: vm
 group: edge
@@ -17,13 +20,30 @@ env:
       - MB_ML_VER=2014
       - SKIP_NETWORK_TESTS=1
       - DOCKER_TEST_IMAGE=multibuild/xenial_arm64v8
-      - BUILD_DEPENDS="numpy==1.19.2 scipy==1.5.3"
+        #
+        # Build wheels with the oldest possible numpy version to avoid
+        # the problem encountered by:
+        #
+        # https://github.com/RaRe-Technologies/gensim/issues/3085
+        #
+      - BUILD_DEPENDS="oldest-supported-numpy scipy==1.7.0"
+        #
+        # The contents of this file mirror the linux_testenv list
+        # in gensim's setup.py.  I can't think of a better way to include
+        # them here for now.  They'll get picked up by the multibuild stuff
+        # running in multibuild/common_utils.sh.
+        #
       - TEST_DEPENDS="pytest mock cython nmslib pyemd testfixtures Morfessor==2.0.2a4 python-levenshtein==0.12.0 visdom==0.1.8.9 scikit-learn"
 
 matrix:
     - os: linux
       env:
         - MB_PYTHON_VERSION=3.6
+        #
+        # scipy 1.7.0 wheels not available for Py3.6, so we have to build using
+        # an older version.
+        #
+        - BUILD_DEPENDS="oldest-supported-numpy scipy==1.5.3"
     - os: linux
       env:
         - MB_PYTHON_VERSION=3.7
@@ -33,6 +53,7 @@ matrix:
     - os: linux
       env:
         - MB_PYTHON_VERSION=3.9
+
 before_install:
     - source multibuild/common_utils.sh
     - source multibuild/travis_steps.sh
@@ -45,9 +66,3 @@ after_script:
     - ls -laht ${TRAVIS_BUILD_DIR}/wheelhouse/
     - pip install wheelhouse-uploader
     - python -m wheelhouse_uploader upload --local-folder ${TRAVIS_BUILD_DIR}/wheelhouse/ --no-ssl-check gensim-wheels --provider S3 --no-enable-cdn
-
-notifications:
-  email:
-    - penkov+gensimwheels@pm.me
-  on_success: always
-  on_failure: always

CHANGELOG.md

@@ -3,6 +3,10 @@ Changes
 
 ## Unreleased
 
+## 4.1.1, 2021-09-14
+
+This is a bugfix release that addresses compatibility issues with older versions of numpy.
+
 ## 4.1.0, 2021-08-15
 
 Gensim 4.1 brings two major new functionalities:

README.md

@@ -176,3 +176,4 @@ BibTeX entry:
   [OpenBLAS]: http://xianyi.github.io/OpenBLAS/
   [source tar.gz]: http://pypi.python.org/pypi/gensim
   [documentation]: http://radimrehurek.com/gensim/install.html
+

config.sh

@@ -6,7 +6,13 @@ function pre_build {
     :
 }
 
+function pip_opts {
+    [ -n "$MANYLINUX_URL" ] && echo "--find-links $MANYLINUX_URL"
+    echo "-v"
+}
+
 function build_wheel_cmd {
+    set -x
     local cmd=${1:-pip_wheel_cmd}
     local wheelhouse=$(abspath ${WHEEL_SDIR:-wheelhouse})
     start_spinner
@@ -15,13 +21,18 @@ function build_wheel_cmd {
     if [ -n "$BUILD_DEPENDS" ]; then
 	pip install $(pip_opts) $BUILD_DEPENDS
     fi
+    pip --version
+    pip freeze
     $cmd $wheelhouse
     repair_wheelhouse $wheelhouse
+    set +x
 }
 
 function run_tests {
     # Runs tests on installed distribution from an empty directory
+    set -x
     python --version
     pip freeze
     pytest -rfxEXs --durations=20 --disable-warnings --showlocals --pyargs gensim
+    set +x
 }

continuous_integration/install_wheel.py

@@ -0,0 +1,17 @@
+"""Install the built wheel for testing under AppVeyor.
+
+Assumes that gensim/dist contains a single wheel to install.
+"""
+import os
+import subprocess
+
+curr_dir = os.path.dirname(__file__)
+dist_path = os.path.join(curr_dir, '..', 'dist')
+wheels = [
+    os.path.join(dist_path, f)
+    for f in os.listdir(dist_path) if f.endswith('.whl')
+]
+assert len(wheels) == 1, "wheels = %r" % wheels
+
+command = 'pip install --pre --force-reinstall'.split() + [wheels[0]]
+subprocess.check_call(command)

docs/src/conf.py

@@ -63,7 +63,7 @@
 # The short X.Y version.
 version = '4.1'
 # The full version, including alpha/beta/rc tags.
-release = '4.1.0'
+release = '4.1.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

gensim/__init__.py

@@ -4,7 +4,7 @@
 
 """
 
-__version__ = '4.1.0'
+__version__ = '4.1.1'
 
 import logging