Merge pull request numpy#25108 from charris/add-qemu-tests

charris · web-flow · commit 29c20300d782 · 2023-11-11T04:10:49.000-07:00
CI: Add linux_qemu action and remove travis testing.
diff --git a/.github/workflows/linux_qemu.yml b/.github/workflows/linux_qemu.yml
@@ -0,0 +1,157 @@
+# Meson's Python module doesn't support crosscompiling,
+# and python dependencies may be another potential hurdle.
+# There might also be a need to run runtime tests during configure time.
+#
+# The recommended practice is to rely on Docker to provide the x86_64 crosscompile toolchain,
+# enabling native execution via binfmt.
+#
+# In simpler terms, everything except the crosscompile toolchain will be emulated.
+
+name: Linux Qemu tests
+
+on:
+  pull_request:
+    branches:
+      - main
+      - maintenance/**
+
+defaults:
+  run:
+    shell: bash
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  linux_qemu:
+    if: "github.repository == 'numpy/numpy'"
+    runs-on: ubuntu-22.04
+    continue-on-error: true
+    strategy:
+      matrix:
+        BUILD_PROP:
+          - [
+              "armhf",
+              "arm-linux-gnueabihf",
+              "arm32v7/ubuntu:22.04",
+              "-Dallow-noblas=true",
+              # test_unary_spurious_fpexception is currently skipped
+              # FIXME(@seiko2plus): Requires confirmation for the following issue:
+              # The presence of an FP invalid exception caused by sqrt. Unsure if this is a qemu bug or not.
+              "(test_kind or test_multiarray or test_simd or test_umath or test_ufunc) and not test_unary_spurious_fpexception"
+          ]
+          - [
+              "ppc64le",
+              "powerpc64le-linux-gnu",
+              "ppc64le/ubuntu:22.04",
+              "-Dallow-noblas=true",
+              "test_kind or test_multiarray or test_simd or test_umath or test_ufunc",
+          ]
+          - [
+              "ppc64le - baseline(Power9)",
+              "powerpc64le-linux-gnu",
+              "ppc64le/ubuntu:22.04",
+              "-Dallow-noblas=true -Dcpu-baseline=vsx3",
+              "test_kind or test_multiarray or test_simd or test_umath or test_ufunc",
+          ]
+          - [
+              "s390x",
+              "s390x-linux-gnu",
+              "s390x/ubuntu:22.04",
+              "-Dallow-noblas=true",
+              # Skipping TestRationalFunctions.test_gcd_overflow test
+              # because of a possible qemu bug that appears to be related to int64 overflow in absolute operation.
+              # TODO(@seiko2plus): Confirm the bug and provide a minimal reproducer, then report it to upstream.
+              "(test_kind or test_multiarray or test_simd or test_umath or test_ufunc) and not test_gcd_overflow"
+          ]
+          - [
+              "s390x - baseline(Z13)",
+              "s390x-linux-gnu",
+              "s390x/ubuntu:22.04",
+              "-Dallow-noblas=true -Dcpu-baseline=vx",
+              "(test_kind or test_multiarray or test_simd or test_umath or test_ufunc) and not test_gcd_overflow"
+          ]
+    env:
+      TOOLCHAIN_NAME: ${{ matrix.BUILD_PROP[1] }}
+      DOCKER_CONTAINER: ${{ matrix.BUILD_PROP[2] }}
+      MESON_OPTIONS: ${{ matrix.BUILD_PROP[3] }}
+      RUNTIME_TEST_FILTER: ${{ matrix.BUILD_PROP[4] }}
+      TERM: xterm-256color
+
+    name: "${{ matrix.BUILD_PROP[0] }}"
+    steps:
+    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      with:
+        submodules: recursive
+        fetch-depth: 0
+
+    - name: Initialize binfmt_misc for qemu-user-static
+      run: |
+        docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+
+    - name: Install GCC cross-compilers
+      run: |
+        sudo apt update
+        sudo apt install -y ninja-build gcc-${TOOLCHAIN_NAME} g++-${TOOLCHAIN_NAME} gfortran-${TOOLCHAIN_NAME}
+
+    - name: Cache docker container
+      uses: actions/cache@v3
+      id: container-cache
+      with:
+        path: ~/docker_${{ matrix.BUILD_PROP[1] }}
+        key: container-${{ runner.os }}-${{ matrix.BUILD_PROP[1] }}-${{ matrix.BUILD_PROP[2] }}-${{ hashFiles('build_requirements.txt') }}
+
+    - name: Creates new container
+      if: steps.container-cache.outputs.cache-hit != 'true'
+      run: |
+        docker run --name the_container --interactive -v /:/host -v $(pwd):/numpy ${DOCKER_CONTAINER} /bin/bash -c "
+          apt update &&
+          apt install -y cmake git python3 python-is-python3 python3-dev python3-pip &&
+          mkdir -p /lib64 && ln -s /host/lib64/ld-* /lib64/ &&
+          ln -s /host/lib/x86_64-linux-gnu /lib/x86_64-linux-gnu &&
+          rm -rf /usr/${TOOLCHAIN_NAME} && ln -s /host/usr/${TOOLCHAIN_NAME} /usr/${TOOLCHAIN_NAME} &&
+          rm -rf /usr/lib/gcc/${TOOLCHAIN_NAME} && ln -s /host/usr/lib/gcc-cross/${TOOLCHAIN_NAME} /usr/lib/gcc/${TOOLCHAIN_NAME} &&
+          rm -f /usr/bin/gcc && ln -s /host/usr/bin/${TOOLCHAIN_NAME}-gcc /usr/bin/gcc &&
+          rm -f /usr/bin/g++ && ln -s /host/usr/bin/${TOOLCHAIN_NAME}-g++ /usr/bin/g++ &&
+          rm -f /usr/bin/gfortran && ln -s /host/usr/bin/${TOOLCHAIN_NAME}-gfortran /usr/bin/gfortran &&
+          rm -f /usr/bin/ar && ln -s /host/usr/bin/${TOOLCHAIN_NAME}-ar /usr/bin/ar &&
+          rm -f /usr/bin/as && ln -s /host/usr/bin/${TOOLCHAIN_NAME}-as /usr/bin/as &&
+          rm -f /usr/bin/ld && ln -s /host/usr/bin/${TOOLCHAIN_NAME}-ld /usr/bin/ld &&
+          rm -f /usr/bin/ld.bfd && ln -s /host/usr/bin/${TOOLCHAIN_NAME}-ld.bfd /usr/bin/ld.bfd &&
+          rm -f /usr/bin/ninja && ln -s /host/usr/bin/ninja /usr/bin/ninja &&
+          git config --global --add safe.directory /numpy &&
+          python -m pip install -r /numpy/build_requirements.txt &&
+          python -m pip install pytest pytest-xdist hypothesis typing_extensions &&
+          rm -f /usr/local/bin/ninja && mkdir -p /usr/local/bin && ln -s /host/usr/bin/ninja /usr/local/bin/ninja
+        "
+        docker commit the_container the_container
+        mkdir -p "~/docker_${TOOLCHAIN_NAME}"
+        docker save -o "~/docker_${TOOLCHAIN_NAME}/the_container.tar" the_container
+
+    - name: Load container from cache
+      if: steps.container-cache.outputs.cache-hit == 'true'
+      run: docker load -i "~/docker_${TOOLCHAIN_NAME}/the_container.tar"
+
+    - name: Meson Build
+      run: |
+        docker run --rm -e "TERM=xterm-256color" -v $(pwd):/numpy -v /:/host the_container \
+        /bin/script -e -q -c "/bin/bash --noprofile --norc -eo pipefail -c '
+          cd /numpy && spin build --clean -- ${MESON_OPTIONS}
+        '"
+
+    - name: Meson Log
+      if: always()
+      run: 'cat build/meson-logs/meson-log.txt'
+
+    - name: Run Tests
+      run: |
+        docker run --rm -e "TERM=xterm-256color" -v $(pwd):/numpy -v /:/host the_container \
+        /bin/script -e -q -c "/bin/bash --noprofile --norc -eo pipefail -c '
+          export F90=/usr/bin/gfortran
+          cd /numpy && spin test -- -k \"${RUNTIME_TEST_FILTER}\"
+        '"
+
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
@@ -23,30 +23,23 @@ jobs:
         compiler: ["MSVC", "Clang-cl"]
     steps:
     - name: Checkout
-      uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       with:
         submodules: recursive
         fetch-depth: 0
 
     - name: Setup Python
-      uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
+      uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1
       with:
         python-version: '3.11'
 
     - name: Install build dependencies from PyPI
       run: |
         python -m pip install spin Cython
 
-    - name: Install OpenBLAS (MacPython build)
+    - name: Install pkg-config
       run: |
-        # Download and install pre-built OpenBLAS library with 32-bit
-        # interfaces. Unpack it in the pkg-config hardcoded path
-        choco install unzip -y
-        choco install wget -y
         choco install -y --checksum 6004DF17818F5A6DBF19CB335CC92702 pkgconfiglite
-        wget https://anaconda.org/multibuild-wheels-staging/openblas-libs/v0.3.21/download/openblas-v0.3.21-win_amd64-gcc_10_3_0.zip
-        unzip -d c:\opt openblas-v0.3.21-win_amd64-gcc_10_3_0.zip
-        echo "PKG_CONFIG_PATH=c:\opt\64\lib\pkgconfig;" >> $env:GITHUB_ENV
 
     - name: Install Clang-cl
       if: matrix.compiler == 'Clang-cl'
@@ -55,26 +48,26 @@ jobs:
 
     - name: Install NumPy (MSVC)
       if: matrix.compiler == 'MSVC'
+      env:
+        PKG_CONFIG_PATH: ${{ github.workspace }}/.openblas
       run: |
-        spin build -j2 -- --vsenv
+        python -m pip install scipy-openblas32
+        spin build --with-scipy-openblas=32 -j2 -- --vsenv
 
     - name: Install NumPy (Clang-cl)
       if: matrix.compiler == 'Clang-cl'
+      env:
+        PKG_CONFIG_PATH: ${{ github.workspace }}/.openblas
       run: |
         "[binaries]","c = 'clang-cl'","cpp = 'clang-cl'","ar = 'llvm-lib'","c_ld = 'lld-link'","cpp_ld = 'lld-link'" | Out-File $PWD/clang-cl-build.ini -Encoding ascii
-        spin build -j2 -- --vsenv --native-file=$PWD/clang-cl-build.ini
+        python -m pip install scipy-openblas32
+        spin build --with-scipy-openblas=32 -j2 -- --vsenv --native-file=$PWD/clang-cl-build.ini
 
-    - name: Copy OpenBLAS DLL, write _distributor_init.py
+    - name: Meson Log
+      shell: bash
+      if: ${{ failure()  }}
       run: |
-        # Getting the OpenBLAS DLL to the right place so it loads
-        $installed_path = "$PWD\build-install\usr\Lib\site-packages"
-        $numpy_path = "${installed_path}\numpy"
-        $libs_path = "${installed_path}\numpy.libs"
-        mkdir ${libs_path}
-        $ob_path = "C:/opt/64/bin/"
-        cp $ob_path/*.dll $libs_path
-        # Write _distributor_init.py to load .libs DLLs.
-        python -c "from tools import openblas_support; openblas_support.make_init(r'${numpy_path}')"
+        cat build/meson-logs/meson-log.txt
 
     - name: Install test dependencies
       run: |
@@ -85,19 +78,19 @@ jobs:
       run: |
         spin test
 
-  msvc_32bit_python_openblas:
+  msvc_32bit_python_no_openblas:
     name: MSVC, 32-bit Python, no BLAS
     runs-on: windows-2019
     if: "github.repository == 'numpy/numpy'"
     steps:
       - name: Checkout
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
           submodules: recursive
           fetch-depth: 0
 
       - name: Setup Python (32-bit)
-        uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
+        uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1
         with:
           python-version: '3.10'
           architecture: 'x86'
diff --git a/.travis.yml b/.travis.yml
diff --git a/meson_cpu/ppc64/meson.build b/meson_cpu/ppc64/meson.build
@@ -33,7 +33,7 @@ VSX3 = mod_features.new(
 VSX4 = mod_features.new(
   'VSX4', 4, implies: VSX3, args: {'val': '-mcpu=power10', 'match': '.*[mcpu=|vsx].*'},
   detect: {'val': 'VSX4', 'match': 'VSX.*'},
-  test_code: files(source_root + '/numpy/distutils/checks/cpu_vsx3.c')[0],
+  test_code: files(source_root + '/numpy/distutils/checks/cpu_vsx4.c')[0],
   extra_tests: {
     'VSX4_MMA': files(source_root + '/numpy/distutils/checks/extra_vsx4_mma.c')[0]
   }
diff --git a/numpy/core/src/common/half.hpp b/numpy/core/src/common/half.hpp
@@ -47,7 +47,7 @@ class Half final {
     /// Default constructor. initialize nothing.
     Half() = default;
 
-    /// Constract from float
+    /// Construct from float
     /// If there are no hardware optimization available, rounding will always
     /// be set to ties to even.
     explicit Half(float f)
@@ -118,7 +118,7 @@ class Half final {
     #endif
     }
 
-    /// Returns a new Half constracted from the IEEE 754 binary16.
+    /// Returns a new Half constructed from the IEEE 754 binary16.
     static constexpr Half FromBits(uint16_t bits)
     {
         Half h{};
@@ -131,7 +131,7 @@ class Half final {
         return bits_;
     }
 
-    /// @name Comparison operators (orderd)
+    /// @name Comparison operators (ordered)
     /// @{
     constexpr bool operator==(Half r) const
     {
@@ -155,7 +155,7 @@ class Half final {
     }
     /// @}
 
-    /// @name Comparison operators (unorderd)
+    /// @name Comparison operators (unordered)
     /// @{
     constexpr bool operator!=(Half r) const
     {
diff --git a/numpy/core/tests/test_half.py b/numpy/core/tests/test_half.py
@@ -274,17 +274,17 @@ def test_half_correctness(self):
         if len(a32_fail) != 0:
             bad_index = a32_fail[0]
             assert_equal(self.finite_f32, a_manual,
-                 "First non-equal is half value %x -> %g != %g" %
-                            (self.finite_f16[bad_index],
+                 "First non-equal is half value 0x%x -> %g != %g" %
+                            (a_bits[bad_index],
                              self.finite_f32[bad_index],
                              a_manual[bad_index]))
 
         a64_fail = np.nonzero(self.finite_f64 != a_manual)[0]
         if len(a64_fail) != 0:
             bad_index = a64_fail[0]
             assert_equal(self.finite_f64, a_manual,
-                 "First non-equal is half value %x -> %g != %g" %
-                            (self.finite_f16[bad_index],
+                 "First non-equal is half value 0x%x -> %g != %g" %
+                            (a_bits[bad_index],
                              self.finite_f64[bad_index],
                              a_manual[bad_index]))
 
@@ -327,7 +327,8 @@ def test_half_funcs(self):
         a = np.array([0, 0, -1, -1/1e20, 0, 2.0**-24, 7.629e-6], dtype=float16)
         assert_equal(a.nonzero()[0],
                      [2, 5, 6])
-        a = a.byteswap().newbyteorder()
+        a = a.byteswap()
+        a = a.view(a.dtype.newbyteorder())
         assert_equal(a.nonzero()[0],
                      [2, 5, 6])
 
diff --git a/numpy/core/tests/test_ufunc.py b/numpy/core/tests/test_ufunc.py
@@ -1617,6 +1617,8 @@ def check_identityless_reduction(self, a):
         assert_equal(np.minimum.reduce(a, axis=()), a)
 
     @requires_memory(6 * 1024**3)
+    @pytest.mark.skipif(sys.maxsize < 2**32,
+            reason="test array too large for 32bit platform")
     def test_identityless_reduction_huge_array(self):
         # Regression test for gh-20921 (copying identity incorrectly failed)
         arr = np.zeros((2, 2**31), 'uint8')
diff --git a/numpy/distutils/ccompiler_opt.py b/numpy/distutils/ccompiler_opt.py

Original file line number	Diff line number	Diff line change
`@@ -33,7 +33,7 @@ VSX3 = mod_features.new(`
`33`	`33`	`VSX4 = mod_features.new(`
`34`	`34`	`'VSX4', 4, implies: VSX3, args: {'val': '-mcpu=power10', 'match': '.[mcpu=\|vsx].'},`
`35`	`35`	`detect: {'val': 'VSX4', 'match': 'VSX.*'},`
`36`		`- test_code: files(source_root + '/numpy/distutils/checks/cpu_vsx3.c')[0],`
	`36`	`+ test_code: files(source_root + '/numpy/distutils/checks/cpu_vsx4.c')[0],`
`37`	`37`	`extra_tests: {`
`38`	`38`	`'VSX4_MMA': files(source_root + '/numpy/distutils/checks/extra_vsx4_mma.c')[0]`
`39`	`39`	`}`
Original file line number	Diff line number	Diff line change
`@@ -47,7 +47,7 @@ class Half final {`
`47`	`47`	`/// Default constructor. initialize nothing.`
`48`	`48`	`Half() = default;`
`49`	`49`
`50`		`- /// Constract from float`
	`50`	`+ /// Construct from float`
`51`	`51`	`/// If there are no hardware optimization available, rounding will always`
`52`	`52`	`/// be set to ties to even.`
`53`	`53`	`explicit Half(float f)`
`@@ -118,7 +118,7 @@ class Half final {`
`118`	`118`	`#endif`
`119`	`119`	`}`
`120`	`120`
`121`		`- /// Returns a new Half constracted from the IEEE 754 binary16.`
	`121`	`+ /// Returns a new Half constructed from the IEEE 754 binary16.`
`122`	`122`	`static constexpr Half FromBits(uint16_t bits)`
`123`	`123`	`{`
`124`	`124`	`Half h{};`
`@@ -131,7 +131,7 @@ class Half final {`
`131`	`131`	`return bits_;`
`132`	`132`	`}`
`133`	`133`
`134`		`- /// @name Comparison operators (orderd)`
	`134`	`+ /// @name Comparison operators (ordered)`
`135`	`135`	`/// @{`
`136`	`136`	`constexpr bool operator==(Half r) const`
`137`	`137`	`{`
`@@ -155,7 +155,7 @@ class Half final {`
`155`	`155`	`}`
`156`	`156`	`/// @}`
`157`	`157`
`158`		`- /// @name Comparison operators (unorderd)`
	`158`	`+ /// @name Comparison operators (unordered)`
`159`	`159`	`/// @{`
`160`	`160`	`constexpr bool operator!=(Half r) const`
`161`	`161`	`{`