From 288c2a0c6a69b2c0df008a22b531f98f6fea9525 Mon Sep 17 00:00:00 2001 From: crusaderky Date: Fri, 28 Feb 2025 14:39:12 +0000 Subject: [PATCH 1/3] TST: dask tests are extremely flaky --- .github/workflows/array-api-tests-dask.yml | 5 ++++- .github/workflows/array-api-tests.yml | 2 +- dask-skips.txt | 6 +++--- dask-xfails.txt | 18 ++++++++---------- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/.github/workflows/array-api-tests-dask.yml b/.github/workflows/array-api-tests-dask.yml index 78010233..9ce06129 100644 --- a/.github/workflows/array-api-tests-dask.yml +++ b/.github/workflows/array-api-tests-dask.yml @@ -10,4 +10,7 @@ jobs: package-version: '>= 2024.9.0' module-name: dask.array extra-requires: numpy - pytest-extra-args: --disable-deadline --max-examples=5 + # Dask is quite slower then other libraries on unit tests + # Reduce the number of examples to speed up CI, even though + # this can cause flakiness. + pytest-extra-args: --max-examples=50 diff --git a/.github/workflows/array-api-tests.yml b/.github/workflows/array-api-tests.yml index d60176ce..6ace193a 100644 --- a/.github/workflows/array-api-tests.yml +++ b/.github/workflows/array-api-tests.yml @@ -33,7 +33,7 @@ on: description: "Multiline string of environment variables to set for the test run." env: - PYTEST_ARGS: "--max-examples 200 -v -rxXfE --ci ${{ inputs.pytest-extra-args }} --hypothesis-disable-deadline" + PYTEST_ARGS: "--max-examples 200 -v -rxXfE --ci ${{ inputs.pytest-extra-args }} --hypothesis-disable-deadline --durations 10" jobs: tests: diff --git a/dask-skips.txt b/dask-skips.txt index f4c0c282..17cd8deb 100644 --- a/dask-skips.txt +++ b/dask-skips.txt @@ -1,5 +1,5 @@ -# slow and not implemented in dask -array_api_tests/test_linalg.py::test_matrix_power +# Passes, but extremely slow +array_api_tests/test_linalg.py::test_outer -# hangs on 2024.12 +# Hangs array_api_tests/test_creation_functions.py::test_eye diff --git a/dask-xfails.txt b/dask-xfails.txt index 4d0e2ee8..e98ab3cd 100644 --- a/dask-xfails.txt +++ b/dask-xfails.txt @@ -1,3 +1,7 @@ +# Shape mismatch; needs high priority investigation +array_api_tests/test_array_object.py::test_setitem +array_api_tests/test_indexing_functions.py::test_take + # Various indexing errors array_api_tests/test_array_object.py::test_getitem_masking @@ -89,28 +93,24 @@ array_api_tests/meta/test_hypothesis_helpers.py::test_symmetric_matrices # https://github.com/dask/dask/issues/11706 array_api_tests/test_creation_functions.py::test_arange +# da.searchsorted with a sorter argument is not supported +array_api_tests/test_searching_functions.py::test_searchsorted + # 2023.12 support array_api_tests/test_manipulation_functions.py::test_repeat # 2024.12 support -array_api_tests/test_array_object.py::test_setitem array_api_tests/test_array_object.py::test_getitem_arrays_and_ints_1[1] array_api_tests/test_array_object.py::test_getitem_arrays_and_ints_1[None] array_api_tests/test_array_object.py::test_getitem_arrays_and_ints_2[1] array_api_tests/test_array_object.py::test_getitem_arrays_and_ints_2[None] array_api_tests/test_has_names.py::test_has_names[indexing-take_along_axis] array_api_tests/test_signatures.py::test_func_signature[take_along_axis] - array_api_tests/test_linalg.py::test_cholesky array_api_tests/test_linalg.py::test_linalg_matmul +array_api_tests/test_linalg.py::test_matmul array_api_tests/test_linalg.py::test_matrix_norm array_api_tests/test_linalg.py::test_qr -array_api_tests/test_manipulation_functions.py::test_concat -array_api_tests/test_manipulation_functions.py::test_roll -array_api_tests/test_operators_and_elementwise_functions.py::test_add[add(x1, x2)] -array_api_tests/test_operators_and_elementwise_functions.py::test_bitwise_left_shift[bitwise_left_shift(x1, x2)] -array_api_tests/test_operators_and_elementwise_functions.py::test_bitwise_right_shift[bitwise_right_shift(x1, x2)] -array_api_tests/test_operators_and_elementwise_functions.py::test_greater[__gt__(x1, x2)] array_api_tests/test_signatures.py::test_func_signature[count_nonzero] array_api_tests/test_special_cases.py::test_binary[floor_divide(x1_i is +infinity and isfinite(x2_i) and x2_i > 0) -> +infinity] array_api_tests/test_special_cases.py::test_binary[floor_divide(x1_i is +infinity and isfinite(x2_i) and x2_i < 0) -> -infinity] @@ -131,5 +131,3 @@ array_api_tests/test_special_cases.py::test_iop[__ifloordiv__(x1_i is -infinity array_api_tests/test_special_cases.py::test_iop[__ifloordiv__(isfinite(x1_i) and x1_i > 0 and x2_i is -infinity) -> -0] array_api_tests/test_special_cases.py::test_iop[__ifloordiv__(isfinite(x1_i) and x1_i < 0 and x2_i is +infinity) -> -0] array_api_tests/test_special_cases.py::test_nan_propagation[cumulative_prod] - - From 1a0ac93bba0bed5100e9cf0c6586426a7b8ea394 Mon Sep 17 00:00:00 2001 From: crusaderky Date: Fri, 28 Feb 2025 16:42:10 +0000 Subject: [PATCH 2/3] fixes and reverts --- .github/workflows/array-api-tests-dask.yml | 10 ++++++---- dask-skips.txt | 4 ++++ dask-xfails.txt | 11 +++++++++-- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/.github/workflows/array-api-tests-dask.yml b/.github/workflows/array-api-tests-dask.yml index 9ce06129..2ad98586 100644 --- a/.github/workflows/array-api-tests-dask.yml +++ b/.github/workflows/array-api-tests-dask.yml @@ -10,7 +10,9 @@ jobs: package-version: '>= 2024.9.0' module-name: dask.array extra-requires: numpy - # Dask is quite slower then other libraries on unit tests - # Reduce the number of examples to speed up CI, even though - # this can cause flakiness. - pytest-extra-args: --max-examples=50 + # Dask is substantially slower then other libraries on unit tests. + # Reduce the number of examples to speed up CI, even though this means that this + # workflow is barely more than a smoke test, and one should expect extreme + # flakiness. Before changes to dask-xfails.txt or dask-skips.txt, please run + # the full test suite with at least 200 examples. + pytest-extra-args: --max-examples=5 diff --git a/dask-skips.txt b/dask-skips.txt index 17cd8deb..a16a8588 100644 --- a/dask-skips.txt +++ b/dask-skips.txt @@ -1,3 +1,7 @@ +# NOTE: dask tests run on a very small number of examples in CI due to +# slowness. This causes very high flakiness in the tests. +# Before changing this file, please run with at least 200 examples. + # Passes, but extremely slow array_api_tests/test_linalg.py::test_outer diff --git a/dask-xfails.txt b/dask-xfails.txt index e98ab3cd..667d7ed0 100644 --- a/dask-xfails.txt +++ b/dask-xfails.txt @@ -1,3 +1,7 @@ +# NOTE: dask tests run on a very small number of examples in CI due to +# slowness. This causes very high flakiness in the tests. +# Before changing this file, please run with at least 200 examples. + # Shape mismatch; needs high priority investigation array_api_tests/test_array_object.py::test_setitem array_api_tests/test_indexing_functions.py::test_take @@ -35,7 +39,7 @@ array_api_tests/test_set_functions.py::test_unique_values # fails for ndim > 2 array_api_tests/test_linalg.py::test_svdvals -# dtype mismatch got uint64, but should be uint8, NPY_PROMOTION_STATE=weak doesn't help :( +# dtype mismatch got uint64, but should be uint8; NPY_PROMOTION_STATE=weak doesn't help array_api_tests/test_linalg.py::test_tensordot # AssertionError: out.dtype=uint64, but should be uint8 [tensordot(uint8, uint8)] @@ -105,13 +109,16 @@ array_api_tests/test_array_object.py::test_getitem_arrays_and_ints_1[None] array_api_tests/test_array_object.py::test_getitem_arrays_and_ints_2[1] array_api_tests/test_array_object.py::test_getitem_arrays_and_ints_2[None] array_api_tests/test_has_names.py::test_has_names[indexing-take_along_axis] +array_api_tests/test_signatures.py::test_func_signature[count_nonzero] array_api_tests/test_signatures.py::test_func_signature[take_along_axis] + array_api_tests/test_linalg.py::test_cholesky array_api_tests/test_linalg.py::test_linalg_matmul array_api_tests/test_linalg.py::test_matmul array_api_tests/test_linalg.py::test_matrix_norm array_api_tests/test_linalg.py::test_qr -array_api_tests/test_signatures.py::test_func_signature[count_nonzero] +array_api_tests/test_manipulation_functions.py::test_roll + array_api_tests/test_special_cases.py::test_binary[floor_divide(x1_i is +infinity and isfinite(x2_i) and x2_i > 0) -> +infinity] array_api_tests/test_special_cases.py::test_binary[floor_divide(x1_i is +infinity and isfinite(x2_i) and x2_i < 0) -> -infinity] array_api_tests/test_special_cases.py::test_binary[floor_divide(x1_i is -infinity and isfinite(x2_i) and x2_i > 0) -> -infinity] From 2a19a61dcdfdadad2e79705a9a683452c5e9ea24 Mon Sep 17 00:00:00 2001 From: crusaderky Date: Fri, 28 Feb 2025 16:55:37 +0000 Subject: [PATCH 3/3] nit --- dask-xfails.txt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dask-xfails.txt b/dask-xfails.txt index 667d7ed0..e2314cf6 100644 --- a/dask-xfails.txt +++ b/dask-xfails.txt @@ -2,9 +2,9 @@ # slowness. This causes very high flakiness in the tests. # Before changing this file, please run with at least 200 examples. -# Shape mismatch; needs high priority investigation +# Broken edge case with shape 0 +# https://github.com/dask/dask/issues/11800 array_api_tests/test_array_object.py::test_setitem -array_api_tests/test_indexing_functions.py::test_take # Various indexing errors array_api_tests/test_array_object.py::test_getitem_masking @@ -19,6 +19,9 @@ array_api_tests/test_data_type_functions.py::test_finfo[float32] # (I think the test is not forcing the op to be computed?) array_api_tests/test_creation_functions.py::test_linspace +# Shape mismatch +array_api_tests/test_indexing_functions.py::test_take + # Array methods and attributes not already on da.Array cannot be wrapped array_api_tests/test_has_names.py::test_has_names[array_method-__array_namespace__] array_api_tests/test_has_names.py::test_has_names[array_method-to_device]