From 6fc58d05c5b99f8aaeb97b6a84c3f16dfd192144 Mon Sep 17 00:00:00 2001 From: Krzysztof Swiecicki Date: Thu, 4 Apr 2024 12:13:18 +0200 Subject: [PATCH 1/5] Add memspace "highest bandwidth" This memspace contains an aggregated list of NUMA nodes identified as best targets after selecting each NUMA node as the initiator. Querying the bandwidth value requires HMAT support on the platform, calling umfMemspaceHighestBandwidthGet() will return NULL if it's not supported. --- include/umf/memspace.h | 5 + src/CMakeLists.txt | 3 +- src/libumf.map | 1 + src/libumf_linux.c | 1 + src/memory_target.c | 11 ++ src/memory_target.h | 4 + src/memory_target_ops.h | 2 + src/memory_targets/memory_target_numa.c | 46 ++++++++ src/memspace.c | 82 ++++++++++++++ src/memspace_internal.h | 13 +++ src/memspaces/memspace_highest_bandwidth.c | 103 ++++++++++++++++++ test/CMakeLists.txt | 4 + test/memspaces/memspace_highest_bandwidth.cpp | 19 ++++ 13 files changed, 293 insertions(+), 1 deletion(-) create mode 100644 src/memspaces/memspace_highest_bandwidth.c create mode 100644 test/memspaces/memspace_highest_bandwidth.cpp diff --git a/include/umf/memspace.h b/include/umf/memspace.h index 6c7cc7660e..f262d69f53 100644 --- a/include/umf/memspace.h +++ b/include/umf/memspace.h @@ -56,6 +56,11 @@ umf_memspace_handle_t umfMemspaceHostAllGet(void); /// umf_memspace_handle_t umfMemspaceHighestCapacityGet(void); +/// \brief Retrieves predefined highest bandwidth memspace. +/// \return highest bandwidth memspace handle on success or NULL on failure. +/// +umf_memspace_handle_t umfMemspaceHighestBandwidthGet(void); + #ifdef __cplusplus } #endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6f2041075e..5e4dd0b7d7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -81,7 +81,8 @@ set(UMF_SOURCES_COMMON_LINUX_MACOSX memory_targets/memory_target_numa.c memspaces/memspace_numa.c memspaces/memspace_host_all.c - memspaces/memspace_highest_capacity.c) + memspaces/memspace_highest_capacity.c + memspaces/memspace_highest_bandwidth.c) set(UMF_SOURCES_LINUX ${UMF_SOURCES_LINUX} ${UMF_SOURCES_COMMON_LINUX_MACOSX} provider/provider_os_memory_linux.c) diff --git a/src/libumf.map b/src/libumf.map index 69a472e414..82b5925b5b 100644 --- a/src/libumf.map +++ b/src/libumf.map @@ -30,6 +30,7 @@ UMF_1.0 { umfMemoryProviderPutIPCHandle; umfMemspaceCreateFromNumaArray; umfMemspaceDestroy; + umfMemspaceHighestBandwidthGet; umfMemspaceHighestCapacityGet; umfMemspaceHostAllGet; umfOpenIPCHandle; diff --git a/src/libumf_linux.c b/src/libumf_linux.c index 64c2cef021..00470dcb16 100644 --- a/src/libumf_linux.c +++ b/src/libumf_linux.c @@ -29,6 +29,7 @@ void __attribute__((destructor)) umfDestroy(void) { umfMemoryTrackerDestroy(t); umfMemspaceHostAllDestroy(); umfMemspaceHighestCapacityDestroy(); + umfMemspaceHighestBandwidthDestroy(); umfDestroyTopology(); } diff --git a/src/memory_target.c b/src/memory_target.c index 54b70f8c5c..a397f8876e 100644 --- a/src/memory_target.c +++ b/src/memory_target.c @@ -83,3 +83,14 @@ umf_result_t umfMemoryTargetGetCapacity(umf_memory_target_handle_t memoryTarget, assert(capacity); return memoryTarget->ops->get_capacity(memoryTarget->priv, capacity); } + +umf_result_t +umfMemoryTargetGetBandwidth(umf_memory_target_handle_t srcMemoryTarget, + umf_memory_target_handle_t dstMemoryTarget, + size_t *bandwidth) { + assert(srcMemoryTarget); + assert(dstMemoryTarget); + assert(bandwidth); + return srcMemoryTarget->ops->get_bandwidth( + srcMemoryTarget->priv, dstMemoryTarget->priv, bandwidth); +} diff --git a/src/memory_target.h b/src/memory_target.h index 4811cd5278..edcb8cb23b 100644 --- a/src/memory_target.h +++ b/src/memory_target.h @@ -37,6 +37,10 @@ umf_result_t umfMemoryTargetClone(umf_memory_target_handle_t memoryTarget, umf_memory_target_handle_t *outHandle); umf_result_t umfMemoryTargetGetCapacity(umf_memory_target_handle_t memoryTarget, size_t *capacity); +umf_result_t +umfMemoryTargetGetBandwidth(umf_memory_target_handle_t srcMemoryTarget, + umf_memory_target_handle_t dstMemoryTarget, + size_t *bandwidth); #ifdef __cplusplus } diff --git a/src/memory_target_ops.h b/src/memory_target_ops.h index c36a063260..5777cc58dc 100644 --- a/src/memory_target_ops.h +++ b/src/memory_target_ops.h @@ -41,6 +41,8 @@ typedef struct umf_memory_target_ops_t { umf_memory_provider_handle_t *provider); umf_result_t (*get_capacity)(void *memoryTarget, size_t *capacity); + umf_result_t (*get_bandwidth)(void *srcMemoryTarget, void *dstMemoryTarget, + size_t *bandwidth); } umf_memory_target_ops_t; #ifdef __cplusplus diff --git a/src/memory_targets/memory_target_numa.c b/src/memory_targets/memory_target_numa.c index ed1ee95ca5..0789db047e 100644 --- a/src/memory_targets/memory_target_numa.c +++ b/src/memory_targets/memory_target_numa.c @@ -19,6 +19,7 @@ #include "base_alloc_global.h" #include "memory_target_numa.h" #include "topology.h" +#include "utils_log.h" struct numa_memory_target_t { unsigned physical_id; @@ -143,6 +144,50 @@ static umf_result_t numa_get_capacity(void *memTarget, size_t *capacity) { return UMF_RESULT_SUCCESS; } +static umf_result_t numa_get_bandwidth(void *srcMemoryTarget, + void *dstMemoryTarget, + size_t *bandwidth) { + hwloc_topology_t topology = umfGetTopology(); + if (!topology) { + return UMF_RESULT_ERROR_NOT_SUPPORTED; + } + + hwloc_obj_t srcNumaNode = hwloc_get_obj_by_type( + topology, HWLOC_OBJ_NUMANODE, + ((struct numa_memory_target_t *)srcMemoryTarget)->physical_id); + if (!srcNumaNode) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + hwloc_obj_t dstNumaNode = hwloc_get_obj_by_type( + topology, HWLOC_OBJ_NUMANODE, + ((struct numa_memory_target_t *)dstMemoryTarget)->physical_id); + if (!dstNumaNode) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + // Given NUMA nodes aren't local, HWLOC returns an error in such case. + if (!hwloc_bitmap_intersects(srcNumaNode->cpuset, dstNumaNode->cpuset)) { + *bandwidth = 0; + return UMF_RESULT_SUCCESS; + } + + struct hwloc_location initiator = {.location.cpuset = srcNumaNode->cpuset, + .type = HWLOC_LOCATION_TYPE_CPUSET}; + hwloc_uint64_t value = 0; + int ret = hwloc_memattr_get_value(topology, HWLOC_MEMATTR_ID_BANDWIDTH, + dstNumaNode, &initiator, 0, &value); + if (ret) { + LOG_ERR("Retrieving bandwidth for initiator node %u to node %u failed.", + srcNumaNode->os_index, dstNumaNode->os_index); + return (errno == EINVAL) ? UMF_RESULT_ERROR_NOT_SUPPORTED + : UMF_RESULT_ERROR_UNKNOWN; + } + + *bandwidth = value; + return UMF_RESULT_SUCCESS; +} + struct umf_memory_target_ops_t UMF_MEMORY_TARGET_NUMA_OPS = { .version = UMF_VERSION_CURRENT, .initialize = numa_initialize, @@ -150,5 +195,6 @@ struct umf_memory_target_ops_t UMF_MEMORY_TARGET_NUMA_OPS = { .pool_create_from_memspace = numa_pool_create_from_memspace, .clone = numa_clone, .get_capacity = numa_get_capacity, + .get_bandwidth = numa_get_bandwidth, .memory_provider_create_from_memspace = numa_memory_provider_create_from_memspace}; diff --git a/src/memspace.c b/src/memspace.c index 5f461931de..6b84e92cb9 100644 --- a/src/memspace.c +++ b/src/memspace.c @@ -208,3 +208,85 @@ umfMemspaceSortDesc(umf_memspace_handle_t hMemspace, return UMF_RESULT_SUCCESS; } + +umf_result_t umfMemspaceFilter(umf_memspace_handle_t hMemspace, + umfGetTargetFn getTarget, + umf_memspace_handle_t *filteredMemspace) { + if (!hMemspace || !getTarget) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + umf_memory_target_handle_t *uniqueBestNodes = + umf_ba_global_alloc(hMemspace->size * sizeof(*uniqueBestNodes)); + if (!uniqueBestNodes) { + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + + umf_result_t ret = UMF_RESULT_SUCCESS; + + size_t numUniqueBestNodes = 0; + for (size_t nodeIdx = 0; nodeIdx < hMemspace->size; nodeIdx++) { + umf_memory_target_handle_t target = NULL; + ret = getTarget(hMemspace->nodes[nodeIdx], hMemspace->nodes, + hMemspace->size, &target); + if (ret != UMF_RESULT_SUCCESS) { + goto err_free_best_targets; + } + + // check if the target is already present in the best nodes + size_t bestTargetIdx; + for (bestTargetIdx = 0; bestTargetIdx < numUniqueBestNodes; + bestTargetIdx++) { + if (uniqueBestNodes[bestTargetIdx] == target) { + break; + } + } + + // if the target is not present, add it to the best nodes + if (bestTargetIdx == numUniqueBestNodes) { + uniqueBestNodes[numUniqueBestNodes++] = target; + } + } + + // copy the unique best nodes into a new memspace + umf_memspace_handle_t newMemspace = + umf_ba_global_alloc(sizeof(*newMemspace)); + if (!newMemspace) { + ret = UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + goto err_free_best_targets; + } + + newMemspace->size = numUniqueBestNodes; + newMemspace->nodes = + umf_ba_global_alloc(sizeof(*newMemspace->nodes) * newMemspace->size); + if (!newMemspace->nodes) { + ret = UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + goto err_free_new_memspace; + } + + size_t cloneIdx = 0; + for (size_t cloneIdx = 0; cloneIdx < newMemspace->size; cloneIdx++) { + ret = umfMemoryTargetClone(uniqueBestNodes[cloneIdx], + &newMemspace->nodes[cloneIdx]); + if (ret != UMF_RESULT_SUCCESS) { + goto err_free_cloned_nodes; + } + } + + *filteredMemspace = newMemspace; + umf_ba_global_free(uniqueBestNodes); + + return UMF_RESULT_SUCCESS; + +err_free_cloned_nodes: + while (cloneIdx != 0) { + cloneIdx--; + umfMemoryTargetDestroy(newMemspace->nodes[cloneIdx]); + } + umf_ba_global_free(newMemspace->nodes); +err_free_new_memspace: + umf_ba_global_free(newMemspace); +err_free_best_targets: + umf_ba_global_free(uniqueBestNodes); + return ret; +} diff --git a/src/memspace_internal.h b/src/memspace_internal.h index 3e694abe23..6ced673033 100644 --- a/src/memspace_internal.h +++ b/src/memspace_internal.h @@ -39,6 +39,18 @@ typedef umf_result_t (*umfGetPropertyFn)(umf_memory_target_handle_t, umf_result_t umfMemspaceSortDesc(umf_memspace_handle_t hMemspace, umfGetPropertyFn getProperty); +typedef umf_result_t (*umfGetTargetFn)(umf_memory_target_handle_t initiator, + umf_memory_target_handle_t *nodes, + size_t numNodes, + umf_memory_target_handle_t *target); + +/// +/// \brief Filters the targets using getTarget() to create a new memspace +/// +umf_result_t umfMemspaceFilter(umf_memspace_handle_t hMemspace, + umfGetTargetFn getTarget, + umf_memspace_handle_t *filteredMemspace); + /// /// \brief Destroys memspace /// \param hMemspace handle to memspace @@ -47,6 +59,7 @@ void umfMemspaceDestroy(umf_memspace_handle_t hMemspace); void umfMemspaceHostAllDestroy(void); void umfMemspaceHighestCapacityDestroy(void); +void umfMemspaceHighestBandwidthDestroy(void); #ifdef __cplusplus } diff --git a/src/memspaces/memspace_highest_bandwidth.c b/src/memspaces/memspace_highest_bandwidth.c new file mode 100644 index 0000000000..35475a5273 --- /dev/null +++ b/src/memspaces/memspace_highest_bandwidth.c @@ -0,0 +1,103 @@ +/* + * + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +#include +#include +#include +#include + +#include "base_alloc_global.h" +#include "memory_target_numa.h" +#include "memspace_internal.h" +#include "memspace_numa.h" +#include "topology.h" +#include "utils_common.h" +#include "utils_concurrency.h" +#include "utils_log.h" + +static umf_result_t getBestBandwidthTarget(umf_memory_target_handle_t initiator, + umf_memory_target_handle_t *nodes, + size_t numNodes, + umf_memory_target_handle_t *target) { + size_t bestNodeIdx = 0; + size_t bestBandwidth = 0; + for (size_t nodeIdx = 0; nodeIdx < numNodes; nodeIdx++) { + size_t bandwidth = 0; + umf_result_t ret = + umfMemoryTargetGetBandwidth(initiator, nodes[nodeIdx], &bandwidth); + if (ret) { + return ret; + } + + if (bandwidth > bestBandwidth) { + bestNodeIdx = nodeIdx; + bestBandwidth = bandwidth; + } + } + + *target = nodes[bestNodeIdx]; + + return UMF_RESULT_SUCCESS; +} + +static umf_result_t +umfMemspaceHighestBandwidthCreate(umf_memspace_handle_t *hMemspace) { + if (!hMemspace) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + umf_memspace_handle_t hostAllMemspace = umfMemspaceHostAllGet(); + if (!hostAllMemspace) { + return UMF_RESULT_ERROR_UNKNOWN; + } + + umf_memspace_handle_t highBandwidthMemspace = NULL; + umf_result_t ret = umfMemspaceFilter( + hostAllMemspace, getBestBandwidthTarget, &highBandwidthMemspace); + if (ret != UMF_RESULT_SUCCESS) { + // HWLOC could possibly return an 'EINVAL' error, which in this context + // means that the HMAT is unavailable and we can't obtain the + // 'bandwidth' value of any NUMA node. + return ret; + } + + *hMemspace = highBandwidthMemspace; + return UMF_RESULT_SUCCESS; +} + +static umf_memspace_handle_t UMF_MEMSPACE_HIGHEST_BANDWIDTH = NULL; +static UTIL_ONCE_FLAG UMF_MEMSPACE_HBW_INITIALIZED = UTIL_ONCE_FLAG_INIT; + +void umfMemspaceHighestBandwidthDestroy(void) { + if (UMF_MEMSPACE_HIGHEST_BANDWIDTH) { + umfMemspaceDestroy(UMF_MEMSPACE_HIGHEST_BANDWIDTH); + UMF_MEMSPACE_HIGHEST_BANDWIDTH = NULL; + } +} + +static void umfMemspaceHighestBandwidthInit(void) { + umf_result_t ret = + umfMemspaceHighestBandwidthCreate(&UMF_MEMSPACE_HIGHEST_BANDWIDTH); + if (ret != UMF_RESULT_SUCCESS) { + LOG_ERR( + "Creating the highest bandwidth memspace failed with a %u error\n", + ret); + assert(ret == UMF_RESULT_ERROR_NOT_SUPPORTED); + } + +#if defined(_WIN32) && !defined(UMF_SHARED_LIBRARY) + atexit(umfMemspaceHighestBandwidthDestroy); +#endif +} + +umf_memspace_handle_t umfMemspaceHighestBandwidthGet(void) { + util_init_once(&UMF_MEMSPACE_HBW_INITIALIZED, + umfMemspaceHighestBandwidthInit); + return UMF_MEMSPACE_HIGHEST_BANDWIDTH; +} diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 4136481453..05ddc72a05 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -173,6 +173,10 @@ if(LINUX) # OS-specific functions are implemented only for Linux now NAME memspace_highest_capacity SRCS memspaces/memspace_highest_capacity.cpp LIBS ${UMF_UTILS_FOR_TEST} ${LIBNUMA_LIBRARIES}) + add_umf_test( + NAME memspace_highest_bandwidth + SRCS memspaces/memspace_highest_bandwidth.cpp + LIBS ${UMF_UTILS_FOR_TEST} ${LIBNUMA_LIBRARIES}) endif() if(UMF_BUILD_GPU_TESTS AND UMF_BUILD_LEVEL_ZERO_PROVIDER) diff --git a/test/memspaces/memspace_highest_bandwidth.cpp b/test/memspaces/memspace_highest_bandwidth.cpp new file mode 100644 index 0000000000..c386086324 --- /dev/null +++ b/test/memspaces/memspace_highest_bandwidth.cpp @@ -0,0 +1,19 @@ +// Copyright (C) 2024 Intel Corporation +// Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "memory_target_numa.h" +#include "memspace_helpers.hpp" +#include "memspace_internal.h" +#include "test_helpers.h" + +#include +#include +#include + +using umf_test::test; + +TEST_F(numaNodesTest, memspaceGet) { + umf_memspace_handle_t hMemspace = umfMemspaceHighestBandwidthGet(); + UT_ASSERTne(hMemspace, nullptr); +} From a49c26e8ede6ac30e818856a1bb468288d1d16de Mon Sep 17 00:00:00 2001 From: Krzysztof Swiecicki Date: Thu, 4 Apr 2024 11:55:21 +0200 Subject: [PATCH 2/5] Add tests for memspace "highest bandwidth" Those tests are skipped with GTEST_SKIP() when bandwidth property can't be queried (HMAT is not supported on the platform). --- test/CMakeLists.txt | 2 +- test/memspaces/memspace_helpers.hpp | 105 ++++------ test/memspaces/memspace_highest_bandwidth.cpp | 194 +++++++++++++++++- test/memspaces/memspace_host_all.cpp | 68 +++--- test/memspaces/memspace_numa.cpp | 41 ++++ test/test_valgrind.sh | 3 + 6 files changed, 302 insertions(+), 111 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 05ddc72a05..691da7430b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -176,7 +176,7 @@ if(LINUX) # OS-specific functions are implemented only for Linux now add_umf_test( NAME memspace_highest_bandwidth SRCS memspaces/memspace_highest_bandwidth.cpp - LIBS ${UMF_UTILS_FOR_TEST} ${LIBNUMA_LIBRARIES}) + LIBS ${UMF_UTILS_FOR_TEST} ${LIBNUMA_LIBRARIES} ${LIBHWLOC_LIBRARIES}) endif() if(UMF_BUILD_GPU_TESTS AND UMF_BUILD_LEVEL_ZERO_PROVIDER) diff --git a/test/memspaces/memspace_helpers.hpp b/test/memspaces/memspace_helpers.hpp index 32617deb7a..b92beba63d 100644 --- a/test/memspaces/memspace_helpers.hpp +++ b/test/memspaces/memspace_helpers.hpp @@ -8,8 +8,10 @@ #include "base.hpp" #include "memspace_internal.h" #include "memspaces/memspace_numa.h" +#include "test_helpers.h" #include +#include #include #define SIZE_4K (4096UL) @@ -40,75 +42,44 @@ struct numaNodesTest : ::umf_test::test { unsigned long maxNodeId = 0; }; -struct memspaceNumaTest : ::numaNodesTest { - void SetUp() override { - ::numaNodesTest::SetUp(); - - umf_result_t ret = umfMemspaceCreateFromNumaArray( - nodeIds.data(), nodeIds.size(), &hMemspace); - ASSERT_EQ(ret, UMF_RESULT_SUCCESS); - ASSERT_NE(hMemspace, nullptr); - } - - void TearDown() override { - ::numaNodesTest::TearDown(); - if (hMemspace) { - umfMemspaceDestroy(hMemspace); +/// +/// @brief Retrieves the memory policy information for \p ptr. +/// @param ptr allocation pointer. +/// @param maxNodeId maximum node id. +/// @param mode [out] memory policy. +/// @param boundNodeIds [out] node ids associated with the policy. +/// @param allocNodeId [out] id of the node that allocated the memory. +/// +void getAllocationPolicy(void *ptr, unsigned long maxNodeId, int &mode, + std::vector &boundNodeIds, + size_t &allocNodeId) { + const static unsigned bitsPerUlong = sizeof(unsigned long) * 8; + + const unsigned nrUlongs = (maxNodeId + bitsPerUlong) / bitsPerUlong; + std::vector memNodeMasks(nrUlongs, 0); + + int memMode = -1; + // Get policy and the nodes associated with this policy. + int ret = get_mempolicy(&memMode, memNodeMasks.data(), + nrUlongs * bitsPerUlong, ptr, MPOL_F_ADDR); + UT_ASSERTeq(ret, 0); + mode = memMode; + + UT_ASSERTeq(boundNodeIds.size(), 0); + for (size_t i = 0; i <= maxNodeId; i++) { + const size_t memNodeMaskIdx = ((i + bitsPerUlong) / bitsPerUlong) - 1; + const auto &memNodeMask = memNodeMasks.at(memNodeMaskIdx); + + if (memNodeMask && (1UL << (i % bitsPerUlong))) { + boundNodeIds.emplace_back(i); } } - umf_memspace_handle_t hMemspace = nullptr; -}; - -struct memspaceNumaProviderTest : ::memspaceNumaTest { - void SetUp() override { - ::memspaceNumaTest::SetUp(); - - umf_result_t ret = - umfMemoryProviderCreateFromMemspace(hMemspace, nullptr, &hProvider); - ASSERT_EQ(ret, UMF_RESULT_SUCCESS); - ASSERT_NE(hProvider, nullptr); - } - - void TearDown() override { - ::memspaceNumaTest::TearDown(); - - if (hProvider != nullptr) { - umfMemoryProviderDestroy(hProvider); - } - } - - umf_memory_provider_handle_t hProvider = nullptr; -}; - -struct memspaceHostAllTest : ::numaNodesTest { - void SetUp() override { - ::numaNodesTest::SetUp(); - - hMemspace = umfMemspaceHostAllGet(); - ASSERT_NE(hMemspace, nullptr); - } - - umf_memspace_handle_t hMemspace = nullptr; -}; - -struct memspaceHostAllProviderTest : ::memspaceHostAllTest { - void SetUp() override { - ::memspaceHostAllTest::SetUp(); - - umf_result_t ret = - umfMemoryProviderCreateFromMemspace(hMemspace, nullptr, &hProvider); - ASSERT_EQ(ret, UMF_RESULT_SUCCESS); - ASSERT_NE(hProvider, nullptr); - } - - void TearDown() override { - ::memspaceHostAllTest::TearDown(); - - umfMemoryProviderDestroy(hProvider); - } - - umf_memory_provider_handle_t hProvider = nullptr; -}; + // Get the node that allocated the memory at 'ptr'. + int nodeId = -1; + ret = get_mempolicy(&nodeId, nullptr, 0, ptr, MPOL_F_ADDR | MPOL_F_NODE); + UT_ASSERTeq(ret, 0); + allocNodeId = static_cast(nodeId); +} #endif /* UMF_MEMSPACE_HELPERS_HPP */ diff --git a/test/memspaces/memspace_highest_bandwidth.cpp b/test/memspaces/memspace_highest_bandwidth.cpp index c386086324..7a56eeb261 100644 --- a/test/memspaces/memspace_highest_bandwidth.cpp +++ b/test/memspaces/memspace_highest_bandwidth.cpp @@ -7,13 +7,197 @@ #include "memspace_internal.h" #include "test_helpers.h" -#include -#include +#include +#include #include using umf_test::test; -TEST_F(numaNodesTest, memspaceGet) { - umf_memspace_handle_t hMemspace = umfMemspaceHighestBandwidthGet(); - UT_ASSERTne(hMemspace, nullptr); +// In HWLOC v2.3.0, the 'hwloc_location_type_e' enum is defined inside an +// 'hwloc_location' struct. In newer versions, this enum is defined globally. +// To prevent compile errors in C++ tests related this scope change +// 'hwloc_location_type_e' has been aliased. +using hwloc_location_type_alias = decltype(hwloc_location::type); + +static bool canQueryBandwidth(size_t nodeId) { + hwloc_topology_t topology = nullptr; + int ret = hwloc_topology_init(&topology); + UT_ASSERTeq(ret, 0); + ret = hwloc_topology_load(topology); + UT_ASSERTeq(ret, 0); + + hwloc_obj_t numaNode = + hwloc_get_obj_by_type(topology, HWLOC_OBJ_NUMANODE, nodeId); + UT_ASSERTne(numaNode, nullptr); + + // Setup initiator structure. + struct hwloc_location initiator; + initiator.location.cpuset = numaNode->cpuset; + initiator.type = hwloc_location_type_alias::HWLOC_LOCATION_TYPE_CPUSET; + + hwloc_uint64_t value = 0; + ret = hwloc_memattr_get_value(topology, HWLOC_MEMATTR_ID_BANDWIDTH, + numaNode, &initiator, 0, &value); + + hwloc_topology_destroy(topology); + return (ret == 0); +} + +struct memspaceHighestBandwidthTest : ::numaNodesTest { + void SetUp() override { + ::numaNodesTest::SetUp(); + + if (!canQueryBandwidth(nodeIds.front())) { + GTEST_SKIP(); + } + + hMemspace = umfMemspaceHighestBandwidthGet(); + ASSERT_NE(hMemspace, nullptr); + } + + umf_memspace_handle_t hMemspace = nullptr; +}; + +struct memspaceHighestBandwidthProviderTest : ::memspaceHighestBandwidthTest { + void SetUp() override { + ::memspaceHighestBandwidthTest::SetUp(); + + if (!canQueryBandwidth(nodeIds.front())) { + GTEST_SKIP(); + } + + umf_result_t ret = + umfMemoryProviderCreateFromMemspace(hMemspace, nullptr, &hProvider); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ASSERT_NE(hProvider, nullptr); + } + + void TearDown() override { + ::memspaceHighestBandwidthTest::TearDown(); + + if (hProvider) { + umfMemoryProviderDestroy(hProvider); + } + } + + umf_memory_provider_handle_t hProvider = nullptr; +}; + +TEST_F(memspaceHighestBandwidthTest, providerFromMemspace) { + umf_memory_provider_handle_t hProvider = nullptr; + umf_result_t ret = + umfMemoryProviderCreateFromMemspace(hMemspace, nullptr, &hProvider); + UT_ASSERTeq(ret, UMF_RESULT_SUCCESS); + UT_ASSERTne(hProvider, nullptr); + + umfMemoryProviderDestroy(hProvider); +} + +TEST_F(memspaceHighestBandwidthProviderTest, allocFree) { + void *ptr = nullptr; + size_t size = SIZE_4K; + size_t alignment = 0; + + umf_result_t ret = umfMemoryProviderAlloc(hProvider, size, alignment, &ptr); + UT_ASSERTeq(ret, UMF_RESULT_SUCCESS); + UT_ASSERTne(ptr, nullptr); + + // Access the allocation, so that all the pages associated with it are + // allocated on some NUMA node. + memset(ptr, 0xFF, size); + + ret = umfMemoryProviderFree(hProvider, ptr, size); + UT_ASSERTeq(ret, UMF_RESULT_SUCCESS); +} + +static std::vector getAllCpus() { + std::vector allCpus; + for (int i = 0; i < numa_num_possible_cpus(); ++i) { + if (numa_bitmask_isbitset(numa_all_cpus_ptr, i)) { + allCpus.push_back(i); + } + } + + return allCpus; +} + +#define MAX_NODES 512 + +TEST_F(memspaceHighestBandwidthProviderTest, allocLocalMt) { + auto pinAllocValidate = [&](umf_memory_provider_handle_t hProvider, + int cpu) { + hwloc_topology_t topology = NULL; + UT_ASSERTeq(hwloc_topology_init(&topology), 0); + UT_ASSERTeq(hwloc_topology_load(topology), 0); + + // Pin current thread to the provided CPU. + hwloc_cpuset_t pinCpuset = hwloc_bitmap_alloc(); + UT_ASSERTeq(hwloc_bitmap_set(pinCpuset, cpu), 0); + UT_ASSERTeq( + hwloc_set_cpubind(topology, pinCpuset, HWLOC_CPUBIND_THREAD), 0); + + // Confirm that the thread is pinned to the provided CPU. + hwloc_cpuset_t curCpuset = hwloc_bitmap_alloc(); + UT_ASSERTeq( + hwloc_get_cpubind(topology, curCpuset, HWLOC_CPUBIND_THREAD), 0); + UT_ASSERT(hwloc_bitmap_isequal(curCpuset, pinCpuset)); + hwloc_bitmap_free(curCpuset); + hwloc_bitmap_free(pinCpuset); + + // Allocate some memory. + const size_t size = SIZE_4K; + const size_t alignment = 0; + void *ptr = nullptr; + + umf_result_t ret = + umfMemoryProviderAlloc(hProvider, size, alignment, &ptr); + UT_ASSERTeq(ret, UMF_RESULT_SUCCESS); + UT_ASSERTne(ptr, nullptr); + + // Access the allocation, so that all the pages associated with it are + // allocated on some NUMA node. + memset(ptr, 0xFF, size); + + // Get the NUMA node responsible for this allocation. + int mode = -1; + std::vector boundNodeIds; + size_t allocNodeId = SIZE_MAX; + getAllocationPolicy(ptr, maxNodeId, mode, boundNodeIds, allocNodeId); + + // Get the CPUs associated with the specified NUMA node. + hwloc_obj_t allocNodeObj = + hwloc_get_obj_by_type(topology, HWLOC_OBJ_NUMANODE, allocNodeId); + + unsigned nNodes = MAX_NODES; + std::vector localNodes(MAX_NODES); + hwloc_location loc; + loc.location.object = allocNodeObj, + loc.type = hwloc_location_type_alias::HWLOC_LOCATION_TYPE_OBJECT; + UT_ASSERTeq(hwloc_get_local_numanode_objs(topology, &loc, &nNodes, + localNodes.data(), 0), + 0); + UT_ASSERT(nNodes <= MAX_NODES); + + // Confirm that the allocation from this thread was made to a local + // NUMA node. + UT_ASSERT(std::any_of(localNodes.begin(), localNodes.end(), + [&allocNodeObj](hwloc_obj_t node) { + return node == allocNodeObj; + })); + + ret = umfMemoryProviderFree(hProvider, ptr, size); + UT_ASSERTeq(ret, UMF_RESULT_SUCCESS); + + hwloc_topology_destroy(topology); + }; + + const auto cpus = getAllCpus(); + std::vector threads; + for (auto cpu : cpus) { + threads.emplace_back(pinAllocValidate, hProvider, cpu); + } + + for (auto &thread : threads) { + thread.join(); + } } diff --git a/test/memspaces/memspace_host_all.cpp b/test/memspaces/memspace_host_all.cpp index a87df9c1ee..e0326709b8 100644 --- a/test/memspaces/memspace_host_all.cpp +++ b/test/memspaces/memspace_host_all.cpp @@ -16,6 +16,36 @@ using umf_test::test; +struct memspaceHostAllTest : ::numaNodesTest { + void SetUp() override { + ::numaNodesTest::SetUp(); + + hMemspace = umfMemspaceHostAllGet(); + ASSERT_NE(hMemspace, nullptr); + } + + umf_memspace_handle_t hMemspace = nullptr; +}; + +struct memspaceHostAllProviderTest : ::memspaceHostAllTest { + void SetUp() override { + ::memspaceHostAllTest::SetUp(); + + umf_result_t ret = + umfMemoryProviderCreateFromMemspace(hMemspace, nullptr, &hProvider); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ASSERT_NE(hProvider, nullptr); + } + + void TearDown() override { + ::memspaceHostAllTest::TearDown(); + + umfMemoryProviderDestroy(hProvider); + } + + umf_memory_provider_handle_t hProvider = nullptr; +}; + TEST_F(numaNodesTest, memspaceGet) { umf_memspace_handle_t hMemspace = umfMemspaceHostAllGet(); UT_ASSERTne(hMemspace, nullptr); @@ -57,44 +87,6 @@ TEST_F(memspaceHostAllProviderTest, allocFree) { UT_ASSERTeq(ret, UMF_RESULT_SUCCESS); } -/// -/// @brief Retrieves the memory policy information for \p ptr. -/// @param ptr allocation pointer. -/// @param maxNodeId maximum node id. -/// @param mode [out] memory policy. -/// @param boundNodeIds [out] node ids associated with the policy. -/// @param allocNodeId [out] id of the node that allocated the memory. -/// -static void getAllocationPolicy(void *ptr, unsigned long maxNodeId, int &mode, - std::vector &boundNodeIds, - size_t &allocNodeId) { - const static unsigned bitsPerUlong = sizeof(unsigned long) * 8; - - const unsigned nrUlongs = (maxNodeId + bitsPerUlong) / bitsPerUlong; - std::vector memNodeMasks(nrUlongs, 0); - - int memMode = -1; - // Get policy and the nodes associated with this policy. - int ret = get_mempolicy(&memMode, memNodeMasks.data(), - nrUlongs * bitsPerUlong, ptr, MPOL_F_ADDR); - UT_ASSERTeq(ret, 0); - mode = memMode; - - UT_ASSERTeq(boundNodeIds.size(), 0); - for (size_t i = 0; i <= maxNodeId; i++) { - const size_t memNodeMaskIdx = ((i + bitsPerUlong) / bitsPerUlong) - 1; - const auto &memNodeMask = memNodeMasks.at(memNodeMaskIdx); - - if (memNodeMask && (1UL << (i % bitsPerUlong))) { - boundNodeIds.emplace_back(i); - } - } - - // Get the node that allocated the memory at 'ptr'. - int nodeId = getNumaNodeByPtr(ptr); - allocNodeId = static_cast(nodeId); -} - TEST_F(memspaceHostAllProviderTest, allocsSpreadAcrossAllNumaNodes) { // This testcase is unsuitable for TSan. #ifdef __SANITIZE_THREAD__ diff --git a/test/memspaces/memspace_numa.cpp b/test/memspaces/memspace_numa.cpp index cfc7cf2eb3..c214ef189b 100644 --- a/test/memspaces/memspace_numa.cpp +++ b/test/memspaces/memspace_numa.cpp @@ -9,6 +9,47 @@ #include +struct memspaceNumaTest : ::numaNodesTest { + void SetUp() override { + ::numaNodesTest::SetUp(); + + umf_result_t ret = umfMemspaceCreateFromNumaArray( + nodeIds.data(), nodeIds.size(), &hMemspace); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ASSERT_NE(hMemspace, nullptr); + } + + void TearDown() override { + ::numaNodesTest::TearDown(); + if (hMemspace) { + umfMemspaceDestroy(hMemspace); + } + } + + umf_memspace_handle_t hMemspace = nullptr; +}; + +struct memspaceNumaProviderTest : ::memspaceNumaTest { + void SetUp() override { + ::memspaceNumaTest::SetUp(); + + umf_result_t ret = + umfMemoryProviderCreateFromMemspace(hMemspace, nullptr, &hProvider); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ASSERT_NE(hProvider, nullptr); + } + + void TearDown() override { + ::memspaceNumaTest::TearDown(); + + if (hProvider != nullptr) { + umfMemoryProviderDestroy(hProvider); + } + } + + umf_memory_provider_handle_t hProvider = nullptr; +}; + TEST_F(numaNodesTest, createDestroy) { umf_memspace_handle_t hMemspace = nullptr; umf_result_t ret = umfMemspaceCreateFromNumaArray( diff --git a/test/test_valgrind.sh b/test/test_valgrind.sh index 8cbd241685..5680fefef8 100755 --- a/test/test_valgrind.sh +++ b/test/test_valgrind.sh @@ -100,6 +100,9 @@ for test in $(ls -1 umf_test-*); do umf_test-provider_os_memory_multiple_numa_nodes) FILTER='--gtest_filter="-testNuma.checkModeInterleave*:testNumaNodesAllocations/testNumaOnEachNode.checkNumaNodesAllocations*:testNumaNodesAllocations/testNumaOnEachNode.checkModePreferred*:testNumaNodesAllocations/testNumaOnEachNode.checkModeInterleaveSingleNode*:testNumaNodesAllocationsAllCpus/testNumaOnEachCpu.checkModePreferredEmptyNodeset*:testNumaNodesAllocationsAllCpus/testNumaOnEachCpu.checkModeLocal*"' ;; + umf_test-memspace_highest_bandwidth) + FILTER='--gtest_filter="-*allocLocalMt*"' + ;; esac [ "$FILTER" != "" ] && echo -n "($FILTER) " From b102b1e2114589cc679c60ce5d5a4d443596e8e1 Mon Sep 17 00:00:00 2001 From: Krzysztof Swiecicki Date: Mon, 8 Apr 2024 13:00:23 +0200 Subject: [PATCH 3/5] Run tests on QEMU with verbose option It makes it easier to see which tests were actually completed without skipping. --- scripts/qemu/run-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/qemu/run-build.sh b/scripts/qemu/run-build.sh index 2aa634d94c..f486d43c95 100755 --- a/scripts/qemu/run-build.sh +++ b/scripts/qemu/run-build.sh @@ -40,7 +40,7 @@ make -j $(nproc) echo password | sudo sync; echo password | sudo sh -c "/usr/bin/echo 3 > /proc/sys/vm/drop_caches" -ctest --output-on-failure +ctest --verbose # run tests bound to a numa node numactl -N 0 ctest --output-on-failure From c202d66875f0def1a0edd5605c01f87cb676845e Mon Sep 17 00:00:00 2001 From: Krzysztof Swiecicki Date: Fri, 10 May 2024 09:23:03 +0000 Subject: [PATCH 4/5] Add brief introduction of HBW memspace --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 5c3284eb57..7e76c708fd 100644 --- a/README.md +++ b/README.md @@ -220,6 +220,11 @@ using umfMemspaceHostAllGet. Memspace backed by all available NUMA nodes discovered on the platform sorted by capacity. Can be retrieved using umfMemspaceHighestCapacityGet. +#### Highest bandwidth memspace + +Memspace backed by an aggregated list of NUMA nodes identified as highest bandwidth after selecting each available NUMA node as the initiator. +Querying the bandwidth value requires HMAT support on the platform. Calling `umfMemspaceHighestBandwidthGet()` will return NULL if it's not supported. + ### Proxy library UMF provides the UMF proxy library (`umf_proxy`) that makes it possible From c69bed76624eb89e88dfd4d26e1982b04f39a860 Mon Sep 17 00:00:00 2001 From: Krzysztof Swiecicki Date: Thu, 16 May 2024 14:07:22 +0000 Subject: [PATCH 5/5] Add missing param checks in mem target functions --- src/memory_target.c | 13 ++++++++----- src/memory_targets/memory_target_numa.c | 8 ++++++++ 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/memory_target.c b/src/memory_target.c index a397f8876e..6ec08eab83 100644 --- a/src/memory_target.c +++ b/src/memory_target.c @@ -79,8 +79,10 @@ umf_result_t umfMemoryTargetClone(umf_memory_target_handle_t memoryTarget, umf_result_t umfMemoryTargetGetCapacity(umf_memory_target_handle_t memoryTarget, size_t *capacity) { - assert(memoryTarget); - assert(capacity); + if (!memoryTarget || !capacity) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + return memoryTarget->ops->get_capacity(memoryTarget->priv, capacity); } @@ -88,9 +90,10 @@ umf_result_t umfMemoryTargetGetBandwidth(umf_memory_target_handle_t srcMemoryTarget, umf_memory_target_handle_t dstMemoryTarget, size_t *bandwidth) { - assert(srcMemoryTarget); - assert(dstMemoryTarget); - assert(bandwidth); + if (!srcMemoryTarget || !dstMemoryTarget || !bandwidth) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + return srcMemoryTarget->ops->get_bandwidth( srcMemoryTarget->priv, dstMemoryTarget->priv, bandwidth); } diff --git a/src/memory_targets/memory_target_numa.c b/src/memory_targets/memory_target_numa.c index 0789db047e..2efc2271e9 100644 --- a/src/memory_targets/memory_target_numa.c +++ b/src/memory_targets/memory_target_numa.c @@ -125,6 +125,10 @@ static umf_result_t numa_clone(void *memTarget, void **outMemTarget) { } static umf_result_t numa_get_capacity(void *memTarget, size_t *capacity) { + if (!memTarget || !capacity) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + hwloc_topology_t topology = umfGetTopology(); if (!topology) { return UMF_RESULT_ERROR_NOT_SUPPORTED; @@ -147,6 +151,10 @@ static umf_result_t numa_get_capacity(void *memTarget, size_t *capacity) { static umf_result_t numa_get_bandwidth(void *srcMemoryTarget, void *dstMemoryTarget, size_t *bandwidth) { + if (!srcMemoryTarget || !dstMemoryTarget || !bandwidth) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + hwloc_topology_t topology = umfGetTopology(); if (!topology) { return UMF_RESULT_ERROR_NOT_SUPPORTED;