Skip to content

Use peak memory usage as a better proxy for ctest parallelism #18603

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 8 commits into
base: branch-25.06
Choose a base branch
from
Open

Use peak memory usage as a better proxy for ctest parallelism #18603

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
94a8056
Use peak memory usage as a better proxy for ctest parallelism
bdice Apr 30, 2025
903d818
Use 100% for LARGE_STRINGS_TEST
bdice Apr 30, 2025
66dd8f0
Try allowing 14 tests to run simultaneously.
bdice Apr 30, 2025
c2ae135
Try 10%
bdice Apr 30, 2025
a4403b9
Try 15% default, 70% for most of the large tests.
bdice May 1, 2025
32c77b7
Use async and 14-way parallelism for small tests.
bdice May 1, 2025
b91f12f
Merge branch 'branch-25.06' into ctest-parallelism
bdice May 1, 2025
8cec680
Disable streams testing for now.
bdice May 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion conda/recipes/libcudf/recipe.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ cache:
./build.sh -n -v \
libcudf libcudf_kafka benchmarks tests \
--build_metrics --incl_cache_stats --allgpuarch \
--cmake-args=\"-DCUDF_ENABLE_ARROW_S3=ON\"
--cmake-args=\"-DCUDF_ENABLE_ARROW_S3=ON -DCUDF_BUILD_STREAMS_TEST_UTIL=OFF\"
secrets:
- AWS_ACCESS_KEY_ID
- AWS_SECRET_ACCESS_KEY
Expand Down
2 changes: 1 addition & 1 deletion cpp/include/cudf_test/testing_main.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ inline auto parse_cudf_test_opts(int argc, char** argv)
std::getenv("GTEST_CUDF_STREAM_MODE"); // Overridden by CLI options
char const* env_stream_error_mode =
std::getenv("GTEST_CUDF_STREAM_ERROR_MODE"); // Overridden by CLI options
auto default_rmm_mode = env_rmm_mode ? env_rmm_mode : "pool";
auto default_rmm_mode = env_rmm_mode ? env_rmm_mode : "async";
auto default_stream_mode = env_stream_mode ? env_stream_mode : "default";
auto default_stream_error_mode = env_stream_error_mode ? env_stream_error_mode : "error";
options.allow_unrecognised_options().add_options()(
Expand Down
46 changes: 10 additions & 36 deletions cpp/tests/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ function(ConfigureTest CMAKE_TEST_NAME)

if(NOT DEFINED _CUDF_TEST_GPUS AND NOT DEFINED _CUDF_TEST_PERCENT)
set(_CUDF_TEST_GPUS 1)
set(_CUDF_TEST_PERCENT 15)
set(_CUDF_TEST_PERCENT 7)
endif()

if(NOT DEFINED _CUDF_TEST_GPUS)
Expand Down Expand Up @@ -163,8 +163,6 @@ ConfigureTest(
groupby/sum_tests.cpp
groupby/tdigest_tests.cu
groupby/var_tests.cpp
GPUS 1
PERCENT 100
)

# ##################################################################################################
Expand Down Expand Up @@ -203,17 +201,13 @@ ConfigureTest(
ConfigureTest(
PARTITIONING_TEST partitioning/hash_partition_test.cpp partitioning/round_robin_test.cpp
partitioning/partition_test.cpp
GPUS 1
PERCENT 70
)

# ##################################################################################################
# * quantiles tests -------------------------------------------------------------------------------
ConfigureTest(
QUANTILES_TEST quantiles/percentile_approx_test.cpp quantiles/quantile_test.cpp
quantiles/quantiles_test.cpp
GPUS 1
PERCENT 70 EXTRA_LIBS ${ARROW_LIBRARIES}
quantiles/quantiles_test.cpp EXTRA_LIBS ${ARROW_LIBRARIES}
)

# ##################################################################################################
Expand All @@ -229,8 +223,6 @@ ConfigureTest(
reductions/scan_tests.cpp
reductions/segmented_reduction_tests.cpp
reductions/tdigest_tests.cu
GPUS 1
PERCENT 70
)

# ##################################################################################################
Expand Down Expand Up @@ -266,6 +258,8 @@ ConfigureTest(
binaryop/binop-compiled-test.cpp
binaryop/binop-compiled-fixed_point-test.cpp
binaryop/binop-generic-ptx-test.cpp
GPUS 1
PERCENT 70
)

# ##################################################################################################
Expand Down Expand Up @@ -304,15 +298,11 @@ ConfigureTest(
ConfigureTest(COMPRESSION_TEST io/comp/comp_test.cpp)
ConfigureTest(ROW_SELECTION_TEST io/row_selection_test.cpp)

ConfigureTest(
CSV_TEST io/csv_test.cpp
GPUS 1
PERCENT 30 EXTRA_LIBS ${ARROW_LIBRARIES}
)
ConfigureTest(CSV_TEST io/csv_test.cpp EXTRA_LIBS ${ARROW_LIBRARIES})
ConfigureTest(
ORC_TEST io/orc_chunked_reader_test.cu io/orc_test.cpp
GPUS 1
PERCENT 100
PERCENT 70
)
ConfigureTest(
PARQUET_TEST
Expand All @@ -327,12 +317,10 @@ ConfigureTest(
io/parquet_v2_test.cpp
io/parquet_writer_test.cpp
GPUS 1
PERCENT 30
PERCENT 70
)
ConfigureTest(
JSON_TEST io/json/json_test.cpp io/json/json_chunked_reader.cu
GPUS 1
PERCENT 30 EXTRA_LIBS ${ARROW_LIBRARIES}
JSON_TEST io/json/json_test.cpp io/json/json_chunked_reader.cu EXTRA_LIBS ${ARROW_LIBRARIES}
)
ConfigureTest(JSON_WRITER_TEST io/json/json_writer.cpp)
ConfigureTest(JSON_TYPE_CAST_TEST io/json/json_type_cast_test.cu)
Expand All @@ -341,11 +329,7 @@ ConfigureTest(MULTIBYTE_SPLIT_TEST io/text/multibyte_split_test.cpp)
ConfigureTest(JSON_QUOTE_NORMALIZATION io/json/json_quote_normalization_test.cpp)
ConfigureTest(JSON_WHITESPACE_NORMALIZATION io/json/json_whitespace_normalization_test.cu)
ConfigureTest(JSON_TREE_CSR io/json/json_tree_csr.cu)
ConfigureTest(
DATA_CHUNK_SOURCE_TEST io/text/data_chunk_source_test.cpp
GPUS 1
PERCENT 100
)
ConfigureTest(DATA_CHUNK_SOURCE_TEST io/text/data_chunk_source_test.cpp)
target_link_libraries(DATA_CHUNK_SOURCE_TEST PRIVATE ZLIB::ZLIB)
ConfigureTest(LOGICAL_STACK_TEST io/fst/logical_stack_test.cu)
ConfigureTest(FST_TEST io/fst/fst_test.cu)
Expand All @@ -356,8 +340,6 @@ ConfigureTest(TYPE_INFERENCE_TEST io/type_inference_test.cu)
ConfigureTest(
SORT_TEST sort/segmented_sort_tests.cpp sort/sort_nested_types_tests.cpp sort/sort_test.cpp
sort/stable_sort_tests.cpp sort/rank_test.cpp
GPUS 1
PERCENT 70
)

# ##################################################################################################
Expand Down Expand Up @@ -452,11 +434,7 @@ ConfigureTest(DEVICE_ATOMICS_TEST device_atomics/device_atomics_test.cu)

# ##################################################################################################
# * transpose tests -------------------------------------------------------------------------------
ConfigureTest(
TRANSPOSE_TEST transpose/transpose_test.cpp
GPUS 1
PERCENT 70
)
ConfigureTest(TRANSPOSE_TEST transpose/transpose_test.cpp)

# ##################################################################################################
# * table tests -----------------------------------------------------------------------------------
Expand Down Expand Up @@ -506,8 +484,6 @@ ConfigureTest(
rolling/range_window_bounds_test.cpp
rolling/range_window_type_test.cpp
rolling/rolling_test.cpp
GPUS 1
PERCENT 70
)

# ##################################################################################################
Expand Down Expand Up @@ -675,8 +651,6 @@ ConfigureTest(
lists/sort_lists_tests.cpp
lists/stream_compaction/apply_boolean_mask_tests.cpp
lists/stream_compaction/distinct_tests.cpp
GPUS 1
PERCENT 70
)

# ##################################################################################################
Expand Down
Loading