Merge pull request data-apis#244 from honno/ci-updates

CI updates

Author: honno

.github/workflows/numpy.yml renamed to .github/workflows/test.yml

@@ -1,4 +1,4 @@
-name: NumPy Array API
+name: Test Array API Strict
 
 on: [push, pull_request]
 
@@ -22,10 +22,12 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install numpy==1.26.2
+        python -m pip install array-api-strict
         python -m pip install -r requirements.txt
     - name: Run the test suite
       env:
-        ARRAY_API_TESTS_MODULE: numpy.array_api
+        ARRAY_API_TESTS_MODULE: array_api_strict
       run: |
-        pytest -v -rxXfE --ci --skips-file numpy-skips.txt
+        pytest -v -rxXfE --skips-file array-api-strict-skips.txt array_api_tests/
+        # We also have internal tests that isn't really necessary for adopters
+        pytest -v -rxXfE meta_tests/

README.md

@@ -138,9 +138,9 @@ issues](https://github.com/data-apis/array-api-tests/issues/) to us.
 
 ## Running on CI
 
-See our existing [GitHub Actions workflow for
-Numpy](https://github.com/data-apis/array-api-tests/blob/master/.github/workflows/numpy.yml)
-for an example of using the test suite on CI.
+See our existing [GitHub Actions workflow for `array-api-strict`](https://github.com/data-apis/array-api-tests/blob/master/.github/workflows/test.yml)
+for an example of using the test suite on CI. Note [`array-api-strict`](https://github.com/data-apis/array-api-strict)
+is an implementation of the array API that uses NumPy under the hood.
 
 ### Releases
 
@@ -161,12 +161,6 @@ You can specify the API version to use when testing via the
 array module's `__array_api_version__` value, and if that attribute doesn't
 exist then we fallback to `"2021.12"`.
 
-#### CI flag
-
-Use the `--ci` flag to run only the primary and special cases tests. You can
-ignore the other test cases as they are redundant for the purposes of checking
-compliance.
-
 #### Data-dependent shapes
 
 Use the `--disable-data-dependent-shapes` flag to skip testing functions which have

array-api-strict-skips.txt

@@ -0,0 +1,8 @@
+# Known special case issue in NumPy. Not worth working around here
+# https://github.com/numpy/numpy/issues/21213
+array_api_tests/test_special_cases.py::test_iop[__ipow__(x1_i is -infinity and x2_i > 0 and not (x2_i.is_integer() and x2_i % 2 == 1)) -> +infinity]
+array_api_tests/test_special_cases.py::test_iop[__ipow__(x1_i is -0 and x2_i > 0 and not (x2_i.is_integer() and x2_i % 2 == 1)) -> +0]
+
+# The test suite is incorrectly checking sums that have loss of significance
+# (https://github.com/data-apis/array-api-tests/issues/168)
+array_api_tests/test_statistical_functions.py::test_sum

array_api_tests/pytest_helpers.py

@@ -485,7 +485,7 @@ def assert_array_elements(
         >>> assert xp.all(out == x)
 
     """
-    # __tracebackhide__ = True
+    __tracebackhide__ = True
     dh.result_type(out.dtype, expected.dtype)  # sanity check
     assert_shape(func_name, out_shape=out.shape, expected=expected.shape, kw=kw)  # sanity check
     f_func = f"[{func_name}({fmt_kw(kw)})]"

array_api_tests/test_array_object.py

@@ -16,8 +16,6 @@
 from . import xp as _xp
 from .typing import DataType, Index, Param, Scalar, ScalarType, Shape
 
-pytestmark = pytest.mark.ci
-
 
 def scalar_objects(
     dtype: DataType, shape: Shape
@@ -107,6 +105,7 @@ def test_getitem(shape, dtype, data):
         ph.assert_array_elements("__getitem__", out=out, expected=expected)
 
 
+@pytest.mark.unvectorized
 @given(
     shape=hh.shapes(),
     dtypes=hh.oneway_promotable_dtypes(dh.all_dtypes),
@@ -154,6 +153,7 @@ def test_setitem(shape, dtypes, data):
         )
 
 
+@pytest.mark.unvectorized
 @pytest.mark.data_dependent_shapes
 @given(hh.shapes(), st.data())
 def test_getitem_masking(shape, data):
@@ -199,6 +199,7 @@ def test_getitem_masking(shape, data):
                 )
 
 
+@pytest.mark.unvectorized
 @given(hh.shapes(), st.data())
 def test_setitem_masking(shape, data):
     x = data.draw(hh.arrays(xps.scalar_dtypes(), shape=shape), label="x")

array_api_tests/test_constants.py

@@ -7,8 +7,6 @@
 from . import xp
 from .typing import Array
 
-pytestmark = pytest.mark.ci
-
 
 def assert_scalar_float(name: str, c: Any):
     assert isinstance(c, SupportsFloat), f"{name}={c!r} does not look like a float"

array_api_tests/test_creation_functions.py

@@ -3,7 +3,6 @@
 from itertools import count
 from typing import Iterator, NamedTuple, Union
 
-import pytest
 from hypothesis import assume, given, note
 from hypothesis import strategies as st
 
@@ -15,8 +14,6 @@
 from . import xps
 from .typing import DataType, Scalar
 
-pytestmark = pytest.mark.ci
-
 
 class frange(NamedTuple):
     start: float

array_api_tests/test_data_type_functions.py

@@ -14,8 +14,6 @@
 from . import xp as _xp
 from .typing import DataType
 
-pytestmark = pytest.mark.ci
-
 
 # TODO: test with complex dtypes
 def non_complex_dtypes():

array_api_tests/test_fft.py

@@ -17,7 +17,6 @@
 from . import xp
 
 pytestmark = [
-    pytest.mark.ci,
     pytest.mark.xp_extension("fft"),
     pytest.mark.min_version("2022.12"),
 ]

array_api_tests/test_has_names.py

@@ -9,8 +9,6 @@
 from .stubs import (array_attributes, array_methods, category_to_funcs,
                     extension_to_funcs, EXTENSIONS)
 
-pytestmark = pytest.mark.ci
-
 has_name_params = []
 for ext, stubs in extension_to_funcs.items():
     for stub in stubs: