Merge branch 'master' into more-linalg2

Author: asmeurer

.github/workflows/numpy.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.8, 3.9]
+        python-version: ["3.10", "3.11"]
 
     steps:
     - name: Checkout array-api-tests
@@ -22,60 +22,10 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install numpy==1.22.1
+        python -m pip install numpy==1.26.2
         python -m pip install -r requirements.txt
     - name: Run the test suite
       env:
         ARRAY_API_TESTS_MODULE: numpy.array_api
       run: |
-        # Skip testing functions with known issues
-        cat << EOF >> skips.txt
-
-        # copy not implemented
-        array_api_tests/test_creation_functions.py::test_asarray_arrays
-        # https://github.com/numpy/numpy/issues/20870
-        array_api_tests/test_data_type_functions.py::test_can_cast
-        # The return dtype for trace is not consistent in the spec
-        # https://github.com/data-apis/array-api/issues/202#issuecomment-952529197
-        array_api_tests/test_linalg.py::test_trace
-        # Various fixes to vector_norm are in
-        # https://github.com/numpy/numpy/pull/21084.
-        array_api_tests/test_linalg.py::test_vector_norm
-        # waiting on NumPy to allow/revert distinct NaNs for np.unique
-        # https://github.com/numpy/numpy/issues/20326#issuecomment-1012380448
-        array_api_tests/test_set_functions.py
-
-        # https://github.com/numpy/numpy/issues/21373
-        array_api_tests/test_array_object.py::test_getitem
-
-        # missing copy arg
-        array_api_tests/test_signatures.py::test_func_signature[reshape]
-
-        # https://github.com/numpy/numpy/issues/21211
-        array_api_tests/test_special_cases.py::test_iop[__iadd__(x1_i is -0 and x2_i is -0) -> -0]
-        # https://github.com/numpy/numpy/issues/21213
-        array_api_tests/test_special_cases.py::test_iop[__ipow__(x1_i is -infinity and x2_i > 0 and not (x2_i.is_integer() and x2_i % 2 == 1)) -> +infinity]
-        array_api_tests/test_special_cases.py::test_iop[__ipow__(x1_i is -0 and x2_i > 0 and not (x2_i.is_integer() and x2_i % 2 == 1)) -> +0]
-        # noted diversions from spec
-        array_api_tests/test_special_cases.py::test_binary[floor_divide(x1_i is +infinity and isfinite(x2_i) and x2_i > 0) -> +infinity]
-        array_api_tests/test_special_cases.py::test_binary[floor_divide(x1_i is +infinity and isfinite(x2_i) and x2_i < 0) -> -infinity]
-        array_api_tests/test_special_cases.py::test_binary[floor_divide(x1_i is -infinity and isfinite(x2_i) and x2_i > 0) -> -infinity]
-        array_api_tests/test_special_cases.py::test_binary[floor_divide(x1_i is -infinity and isfinite(x2_i) and x2_i < 0) -> +infinity]
-        array_api_tests/test_special_cases.py::test_binary[floor_divide(isfinite(x1_i) and x1_i > 0 and x2_i is -infinity) -> -0]
-        array_api_tests/test_special_cases.py::test_binary[floor_divide(isfinite(x1_i) and x1_i < 0 and x2_i is +infinity) -> -0]
-        array_api_tests/test_special_cases.py::test_binary[__floordiv__(x1_i is +infinity and isfinite(x2_i) and x2_i > 0) -> +infinity]
-        array_api_tests/test_special_cases.py::test_binary[__floordiv__(x1_i is +infinity and isfinite(x2_i) and x2_i < 0) -> -infinity]
-        array_api_tests/test_special_cases.py::test_binary[__floordiv__(x1_i is -infinity and isfinite(x2_i) and x2_i > 0) -> -infinity]
-        array_api_tests/test_special_cases.py::test_binary[__floordiv__(x1_i is -infinity and isfinite(x2_i) and x2_i < 0) -> +infinity]
-        array_api_tests/test_special_cases.py::test_binary[__floordiv__(isfinite(x1_i) and x1_i > 0 and x2_i is -infinity) -> -0]
-        array_api_tests/test_special_cases.py::test_binary[__floordiv__(isfinite(x1_i) and x1_i < 0 and x2_i is +infinity) -> -0]
-        array_api_tests/test_special_cases.py::test_iop[__ifloordiv__(x1_i is +infinity and isfinite(x2_i) and x2_i > 0) -> +infinity]
-        array_api_tests/test_special_cases.py::test_iop[__ifloordiv__(x1_i is +infinity and isfinite(x2_i) and x2_i < 0) -> -infinity]
-        array_api_tests/test_special_cases.py::test_iop[__ifloordiv__(x1_i is -infinity and isfinite(x2_i) and x2_i > 0) -> -infinity]
-        array_api_tests/test_special_cases.py::test_iop[__ifloordiv__(x1_i is -infinity and isfinite(x2_i) and x2_i < 0) -> +infinity]
-        array_api_tests/test_special_cases.py::test_iop[__ifloordiv__(isfinite(x1_i) and x1_i > 0 and x2_i is -infinity) -> -0]
-        array_api_tests/test_special_cases.py::test_iop[__ifloordiv__(isfinite(x1_i) and x1_i < 0 and x2_i is +infinity) -> -0]
-
-        EOF
-
-        pytest -v -rxXfE --ci
+        pytest -v -rxXfE --ci --skips-file numpy-skips.txt

.gitignore

@@ -117,6 +117,10 @@ venv.bak/
 # Rope project settings
 .ropeproject
 
+# IDE
+.idea/
+.vscode/
+
 # mkdocs documentation
 /site
 

README.md

@@ -36,13 +36,7 @@ You need to specify the array library to test. It can be specified via the
 $ export ARRAY_API_TESTS_MODULE=numpy.array_api
 ```
 
-Alternately, change the `array_module` variable in `array_api_tests/_array_module.py`
-line, e.g.
-
-```diff
-- array_module = None
-+ import numpy.array_api as array_module
-```
+Alternately, import/define the `xp` variable in `array_api_tests/__init__.py`.
 
 ### Run the suite
 
@@ -160,6 +154,13 @@ library to fail.
 
 ### Configuration
 
+#### API version
+
+You can specify the API version to use when testing via the
+`ARRAY_API_TESTS_VERSION` environment variable. Currently this defaults to the
+array module's `__array_api_version__` value, and if that attribute doesn't
+exist then we fallback to `"2021.12"`.
+
 #### CI flag
 
 Use the `--ci` flag to run only the primary and special cases tests. You can
@@ -178,13 +179,22 @@ By default, tests for the optional Array API extensions such as
 will be skipped if not present in the specified array module. You can purposely
 skip testing extension(s) via the `--disable-extension` option.
 
-#### Skip test cases
+#### Skip or XFAIL test cases
+
+Test cases you want to skip can be specified in a skips or XFAILS file. The
+difference between skip and XFAIL is that XFAIL tests are still run and
+reported as XPASS if they pass.
+
+By default, the skips and xfails files are `skips.txt` and `fails.txt` in the root
+of this repository, but any file can be specified with the `--skips-file` and
+`--xfails-file` command line flags.
 
-Test cases you want to skip can be specified in a `skips.txt` file in the root
-of this repository, e.g.:
+The files should list the test ids to be skipped/xfailed. Empty lines and
+lines starting with `#` are ignored. The test id can be any substring of the
+test ids to skip/xfail.
 
 ```
-# ./skips.txt
+# skips.txt or xfails.txt
 # Line comments can be denoted with the hash symbol (#)
 
 # Skip specific test case, e.g. when argsort() does not respect relative order
@@ -200,39 +210,81 @@ array_api_tests/test_add[__iadd__(x, s)]
 array_api_tests/test_set_functions.py
 ```
 
-For GitHub Actions, you might like to keep everything in the workflow config
-instead of having a seperate `skips.txt` file, e.g.:
+Here is an example GitHub Actions workflow file, where the xfails are stored
+in `array-api-tests.xfails.txt` in the base of the `your-array-library` repo.
+
+If you want, you can use `-o xfail_strict=True`, which causes XPASS tests (XFAIL
+tests that actually pass) to fail the test suite. However, be aware that
+XFAILures can be flaky (see below, so this may not be a good idea unless you
+use some other mitigation of such flakyness).
+
+If you don't want this behavior, you can remove it, or use `--skips-file`
+instead of `--xfails-file`.
 
 ```yaml
 # ./.github/workflows/array_api.yml
-...
-    ...
-    - name: Run the test suite
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.8', '3.9', '3.10', '3.11']
+
+    steps:
+    - name: Checkout <your array library>
+      uses: actions/checkout@v3
+      with:
+        path: your-array-library
+
+    - name: Checkout array-api-tests
+      uses: actions/checkout@v3
+      with:
+        repository: data-apis/array-api-tests
+        submodules: 'true'
+        path: array-api-tests
+
+    - name: Run the array API test suite
       env:
         ARRAY_API_TESTS_MODULE: your.array.api.namespace
       run: |
-        # Skip test cases with known issues
-        cat << EOF >> skips.txt
-
-        # Comments can still work here
-        array_api_tests/test_sorting_functions.py::test_argsort
-        array_api_tests/test_add[__iadd__(x1, x2)]
-        array_api_tests/test_add[__iadd__(x, s)]
-        array_api_tests/test_set_functions.py
-
-        EOF
-
-        pytest -v -rxXfE --ci
+        export PYTHONPATH="${GITHUB_WORKSPACE}/your-array-library"
+        cd ${GITHUB_WORKSPACE}/array-api-tests
+        pytest -v -rxXfE --ci --xfails-file ${GITHUB_WORKSPACE}/your-array-library/array-api-tests-xfails.txt array_api_tests/
 ```
 
+> **Warning**
+>
+> XFAIL tests that use Hypothesis (basically every test in the test suite except
+> those in test_has_names.py) can be flaky, due to the fact that Hypothesis
+> might not always run the test with an input that causes the test to fail.
+> There are several ways to avoid this problem:
+>
+> - Increase the maximum number of examples, e.g., by adding `--max-examples
+>   200` to the test command (the default is `100`, see below). This will
+>   make it more likely that the failing case will be found, but it will also
+>   make the tests take longer to run.
+> - Don't use `-o xfail_strict=True`. This will make it so that if an XFAIL
+>   test passes, it will alert you in the test summary but will not cause the
+>   test run to register as failed.
+> - Use skips instead of XFAILS. The difference between XFAIL and skip is that
+>   a skipped test is never run at all, whereas an XFAIL test is always run
+>   but ignored if it fails.
+> - Save the [Hypothesis examples
+>   database](https://hypothesis.readthedocs.io/en/latest/database.html)
+>   persistently on CI. That way as soon as a run finds one failing example,
+>   it will always re-run future runs with that example. But note that the
+>   Hypothesis examples database may be cleared when a new version of
+>   Hypothesis or the test suite is released.
+
 #### Max examples
 
 The tests make heavy use
 [Hypothesis](https://hypothesis.readthedocs.io/en/latest/). You can configure
-how many examples are generated using the `--max-examples` flag, which defaults
-to 100. Lower values can be useful for quick checks, and larger values should
-result in more rigorous runs. For example, `--max-examples 10_000` may find bugs
-where default runs don't but will take much longer to run.
+how many examples are generated using the `--max-examples` flag, which
+defaults to `100`. Lower values can be useful for quick checks, and larger
+values should result in more rigorous runs. For example, `--max-examples
+10_000` may find bugs where default runs don't but will take much longer to
+run.
 
 
 ## Contributing

array-api

@@ -1,11 +1,36 @@
+import os
 from functools import wraps
+from importlib import import_module
 
 from hypothesis import strategies as st
 from hypothesis.extra import array_api
 
-from ._array_module import mod as _xp
+from . import _version
 
-__all__ = ["xps"]
+__all__ = ["xp", "api_version", "xps"]
+
+
+# You can comment the following out and instead import the specific array module
+# you want to test, e.g. `import numpy.array_api as xp`.
+if "ARRAY_API_TESTS_MODULE" in os.environ:
+    xp_name = os.environ["ARRAY_API_TESTS_MODULE"]
+    _module, _sub = xp_name, None
+    if "." in xp_name:
+        _module, _sub = xp_name.split(".", 1)
+    xp = import_module(_module)
+    if _sub:
+        try:
+            xp = getattr(xp, _sub)
+        except AttributeError:
+            # _sub may be a submodule that needs to be imported. WE can't
+            # do this in every case because some array modules are not
+            # submodules that can be imported (like mxnet.nd).
+            xp = import_module(xp_name)
+else:
+    raise RuntimeError(
+        "No array module specified - either edit __init__.py or set the "
+        "ARRAY_API_TESTS_MODULE environment variable."
+    )
 
 
 # We monkey patch floats() to always disable subnormals as they are out-of-scope
@@ -41,9 +66,9 @@ def _from_dtype(*a, **kw):
     pass
 
 
-xps = array_api.make_strategies_namespace(_xp, api_version="2021.12")
-
-
-from . import _version
+api_version = os.getenv(
+    "ARRAY_API_TESTS_VERSION", getattr(xp, "__array_api_version__", "2021.12")
+)
+xps = array_api.make_strategies_namespace(xp, api_version=api_version)
 
 __version__ = _version.get_versions()["version"]

array_api_tests/__init__.py

@@ -1,35 +1,5 @@
-import os
-from importlib import import_module
+from . import stubs, xp
 
-from . import stubs
-
-# Replace this with a specific array module to test it, for example,
-#
-# import numpy as array_module
-array_module = None
-
-if array_module is None:
-    if 'ARRAY_API_TESTS_MODULE' in os.environ:
-        mod_name = os.environ['ARRAY_API_TESTS_MODULE']
-        _module, _sub = mod_name, None
-        if '.' in mod_name:
-            _module, _sub = mod_name.split('.', 1)
-        mod = import_module(_module)
-        if _sub:
-            try:
-                mod = getattr(mod, _sub)
-            except AttributeError:
-                # _sub may be a submodule that needs to be imported. WE can't
-                # do this in every case because some array modules are not
-                # submodules that can be imported (like mxnet.nd).
-                mod = import_module(mod_name)
-    else:
-        raise RuntimeError("No array module specified. Either edit _array_module.py or set the ARRAY_API_TESTS_MODULE environment variable")
-else:
-    mod = array_module
-    mod_name = mod.__name__
-# Names from the spec. This is what should actually be imported from this
-# file.
 
 class _UndefinedStub:
     """
@@ -45,7 +15,7 @@ def __init__(self, name):
         self.name = name
 
     def _raise(self, *args, **kwargs):
-        raise AssertionError(f"{self.name} is not defined in {mod_name}")
+        raise AssertionError(f"{self.name} is not defined in {xp.__name__}")
 
     def __repr__(self):
         return f"<undefined stub for {self.name!r}>"
@@ -58,13 +28,15 @@ def __repr__(self):
     "uint8", "uint16", "uint32", "uint64",
     "int8", "int16", "int32", "int64",
     "float32", "float64",
+    "complex64", "complex128",
 ]
 _constants = ["e", "inf", "nan", "pi"]
 _funcs = [f.__name__ for funcs in stubs.category_to_funcs.values() for f in funcs]
-_top_level_attrs = _dtypes + _constants + _funcs + stubs.EXTENSIONS
+_funcs += ["take", "isdtype", "conj", "imag", "real"]  # TODO: bump spec and update array-api-tests to new spec layout
+_top_level_attrs = _dtypes + _constants + _funcs + stubs.EXTENSIONS + ["fft"]
 
 for attr in _top_level_attrs:
     try:
-        globals()[attr] = getattr(mod, attr)
+        globals()[attr] = getattr(xp, attr)
     except AttributeError:
         globals()[attr] = _UndefinedStub(attr)