Add memspace "lowest latency"

This memspace is analogous to the 'highest bandwidth' memspace in its
composition, but it focuses on the NUMA nodes with best latency.

Author: kswiecicki

include/umf/memspace.h

@@ -61,6 +61,11 @@ umf_memspace_handle_t umfMemspaceHighestCapacityGet(void);
 ///
 umf_memspace_handle_t umfMemspaceHighestBandwidthGet(void);
 
+/// \brief Retrieves predefined lowest latency memspace.
+/// \return lowest latency memspace handle on success or NULL on failure.
+///
+umf_memspace_handle_t umfMemspaceLowestLatencyGet(void);
+
 #ifdef __cplusplus
 }
 #endif

src/CMakeLists.txt

@@ -84,7 +84,8 @@ set(UMF_SOURCES_COMMON_LINUX_MACOSX
     memspaces/memspace_numa.c
     memspaces/memspace_host_all.c
     memspaces/memspace_highest_capacity.c
-    memspaces/memspace_highest_bandwidth.c)
+    memspaces/memspace_highest_bandwidth.c
+    memspaces/memspace_lowest_latency.c)
 
 set(UMF_SOURCES_LINUX ${UMF_SOURCES_LINUX} ${UMF_SOURCES_COMMON_LINUX_MACOSX}
                       provider/provider_os_memory_linux.c)

src/libumf.map

@@ -36,6 +36,7 @@ UMF_1.0 {
         umfMemspaceHighestBandwidthGet;
         umfMemspaceHighestCapacityGet;
         umfMemspaceHostAllGet;
+        umfMemspaceLowestLatencyGet;
         umfOpenIPCHandle;
         umfOsMemoryProviderOps;
         umfPoolAlignedMalloc;

src/libumf_linux.c

@@ -30,6 +30,7 @@ void __attribute__((destructor)) umfDestroy(void) {
     umfMemspaceHostAllDestroy();
     umfMemspaceHighestCapacityDestroy();
     umfMemspaceHighestBandwidthDestroy();
+    umfMemspaceLowestLatencyDestroy();
     umfDestroyTopology();
 }
 

src/memory_target.c

@@ -97,3 +97,15 @@ umfMemoryTargetGetBandwidth(umf_memory_target_handle_t srcMemoryTarget,
     return srcMemoryTarget->ops->get_bandwidth(
         srcMemoryTarget->priv, dstMemoryTarget->priv, bandwidth);
 }
+
+umf_result_t
+umfMemoryTargetGetLatency(umf_memory_target_handle_t srcMemoryTarget,
+                          umf_memory_target_handle_t dstMemoryTarget,
+                          size_t *latency) {
+    if (!srcMemoryTarget || !dstMemoryTarget || !latency) {
+        return UMF_RESULT_ERROR_INVALID_ARGUMENT;
+    }
+
+    return srcMemoryTarget->ops->get_latency(srcMemoryTarget->priv,
+                                             dstMemoryTarget->priv, latency);
+}

src/memory_target.h

@@ -39,6 +39,10 @@ umf_result_t
 umfMemoryTargetGetBandwidth(umf_memory_target_handle_t srcMemoryTarget,
                             umf_memory_target_handle_t dstMemoryTarget,
                             size_t *bandwidth);
+umf_result_t
+umfMemoryTargetGetLatency(umf_memory_target_handle_t srcMemoryTarget,
+                          umf_memory_target_handle_t dstMemoryTarget,
+                          size_t *latency);
 
 #ifdef __cplusplus
 }

src/memory_target_ops.h

@@ -41,6 +41,8 @@ typedef struct umf_memory_target_ops_t {
     umf_result_t (*get_capacity)(void *memoryTarget, size_t *capacity);
     umf_result_t (*get_bandwidth)(void *srcMemoryTarget, void *dstMemoryTarget,
                                   size_t *bandwidth);
+    umf_result_t (*get_latency)(void *srcMemoryTarget, void *dstMemoryTarget,
+                                size_t *latency);
 } umf_memory_target_ops_t;
 
 #ifdef __cplusplus

src/memory_targets/memory_target_numa.c

@@ -167,13 +167,26 @@ static umf_result_t numa_get_capacity(void *memTarget, size_t *capacity) {
     return UMF_RESULT_SUCCESS;
 }
 
-static umf_result_t numa_get_bandwidth(void *srcMemoryTarget,
-                                       void *dstMemoryTarget,
-                                       size_t *bandwidth) {
-    if (!srcMemoryTarget || !dstMemoryTarget || !bandwidth) {
-        return UMF_RESULT_ERROR_INVALID_ARGUMENT;
+typedef enum memattr_type {
+    MEMATTR_TYPE_BANDWIDTH,
+    MEMATTR_TYPE_LATENCY
+} memattr_type;
+
+static size_t memattr_get_worst_value(memattr_type type) {
+    switch (type) {
+    case MEMATTR_TYPE_BANDWIDTH:
+        return 0;
+    case MEMATTR_TYPE_LATENCY:
+        return SIZE_MAX;
+    default:
+        assert(0); // Should not be reachable
+        return 0;
     }
+}
 
+static umf_result_t query_attribute_value(void *srcMemoryTarget,
+                                          void *dstMemoryTarget, size_t *value,
+                                          memattr_type type) {
     hwloc_topology_t topology = umfGetTopology();
     if (!topology) {
         return UMF_RESULT_ERROR_NOT_SUPPORTED;
@@ -195,23 +208,75 @@ static umf_result_t numa_get_bandwidth(void *srcMemoryTarget,
 
     // Given NUMA nodes aren't local, HWLOC returns an error in such case.
     if (!hwloc_bitmap_intersects(srcNumaNode->cpuset, dstNumaNode->cpuset)) {
-        *bandwidth = 0;
+        // Since we want to skip such query, we return the worst possible
+        // value for given memory attribute.
+        *value = memattr_get_worst_value(type);
         return UMF_RESULT_SUCCESS;
     }
 
+    enum hwloc_memattr_id_e hwlocMemAttrType = INT_MAX;
+    switch (type) {
+    case MEMATTR_TYPE_BANDWIDTH:
+        hwlocMemAttrType = HWLOC_MEMATTR_ID_BANDWIDTH;
+        break;
+    case MEMATTR_TYPE_LATENCY:
+        hwlocMemAttrType = HWLOC_MEMATTR_ID_LATENCY;
+        break;
+    default:
+        assert(0); // Shouldn't be reachable.
+        return UMF_RESULT_ERROR_INVALID_ARGUMENT;
+    }
+
     struct hwloc_location initiator = {.location.cpuset = srcNumaNode->cpuset,
                                        .type = HWLOC_LOCATION_TYPE_CPUSET};
-    hwloc_uint64_t value = 0;
-    int ret = hwloc_memattr_get_value(topology, HWLOC_MEMATTR_ID_BANDWIDTH,
-                                      dstNumaNode, &initiator, 0, &value);
+
+    hwloc_uint64_t memAttrValue = 0;
+    int ret = hwloc_memattr_get_value(topology, hwlocMemAttrType, dstNumaNode,
+                                      &initiator, 0, &memAttrValue);
     if (ret) {
-        LOG_ERR("Retrieving bandwidth for initiator node %u to node %u failed.",
-                srcNumaNode->os_index, dstNumaNode->os_index);
         return (errno == EINVAL) ? UMF_RESULT_ERROR_NOT_SUPPORTED
                                  : UMF_RESULT_ERROR_UNKNOWN;
     }
 
-    *bandwidth = value;
+    *value = memAttrValue;
+
+    return UMF_RESULT_SUCCESS;
+}
+
+static umf_result_t numa_get_bandwidth(void *srcMemoryTarget,
+                                       void *dstMemoryTarget,
+                                       size_t *bandwidth) {
+    if (!srcMemoryTarget || !dstMemoryTarget || !bandwidth) {
+        return UMF_RESULT_ERROR_INVALID_ARGUMENT;
+    }
+
+    umf_result_t ret = query_attribute_value(srcMemoryTarget, dstMemoryTarget,
+                                             bandwidth, MEMATTR_TYPE_BANDWIDTH);
+    if (ret) {
+        LOG_ERR("Retrieving bandwidth for initiator node %u to node %u failed.",
+                ((struct numa_memory_target_t *)srcMemoryTarget)->physical_id,
+                ((struct numa_memory_target_t *)dstMemoryTarget)->physical_id);
+        return ret;
+    }
+
+    return UMF_RESULT_SUCCESS;
+}
+
+static umf_result_t numa_get_latency(void *srcMemoryTarget,
+                                     void *dstMemoryTarget, size_t *latency) {
+    if (!srcMemoryTarget || !dstMemoryTarget || !latency) {
+        return UMF_RESULT_ERROR_INVALID_ARGUMENT;
+    }
+
+    umf_result_t ret = query_attribute_value(srcMemoryTarget, dstMemoryTarget,
+                                             latency, MEMATTR_TYPE_LATENCY);
+    if (ret) {
+        LOG_ERR("Retrieving latency for initiator node %u to node %u failed.",
+                ((struct numa_memory_target_t *)srcMemoryTarget)->physical_id,
+                ((struct numa_memory_target_t *)dstMemoryTarget)->physical_id);
+        return ret;
+    }
+
     return UMF_RESULT_SUCCESS;
 }
 
@@ -223,5 +288,6 @@ struct umf_memory_target_ops_t UMF_MEMORY_TARGET_NUMA_OPS = {
     .clone = numa_clone,
     .get_capacity = numa_get_capacity,
     .get_bandwidth = numa_get_bandwidth,
+    .get_latency = numa_get_latency,
     .memory_provider_create_from_memspace =
         numa_memory_provider_create_from_memspace};

src/memspace_internal.h

@@ -60,6 +60,7 @@ void umfMemspaceDestroy(umf_memspace_handle_t hMemspace);
 void umfMemspaceHostAllDestroy(void);
 void umfMemspaceHighestCapacityDestroy(void);
 void umfMemspaceHighestBandwidthDestroy(void);
+void umfMemspaceLowestLatencyDestroy(void);
 
 #ifdef __cplusplus
 }

src/memspaces/memspace_lowest_latency.c

@@ -0,0 +1,103 @@
+/*
+ *
+ * Copyright (C) 2024 Intel Corporation
+ *
+ * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ */
+
+#include <assert.h>
+#include <ctype.h>
+#include <hwloc.h>
+#include <stdlib.h>
+
+#include "base_alloc_global.h"
+#include "memory_target_numa.h"
+#include "memspace_internal.h"
+#include "memspace_numa.h"
+#include "topology.h"
+#include "utils_common.h"
+#include "utils_concurrency.h"
+#include "utils_log.h"
+
+static umf_result_t getBestLatencyTarget(umf_memory_target_handle_t initiator,
+                                         umf_memory_target_handle_t *nodes,
+                                         size_t numNodes,
+                                         umf_memory_target_handle_t *target) {
+    size_t bestNodeIdx = 0;
+    size_t bestLatency = SIZE_MAX;
+    for (size_t nodeIdx = 0; nodeIdx < numNodes; nodeIdx++) {
+        size_t latency = SIZE_MAX;
+        umf_result_t ret =
+            umfMemoryTargetGetLatency(initiator, nodes[nodeIdx], &latency);
+        if (ret) {
+            return ret;
+        }
+
+        if (latency < bestLatency) {
+            bestNodeIdx = nodeIdx;
+            bestLatency = latency;
+        }
+    }
+
+    *target = nodes[bestNodeIdx];
+
+    return UMF_RESULT_SUCCESS;
+}
+
+static umf_result_t
+umfMemspaceLowestLatencyCreate(umf_memspace_handle_t *hMemspace) {
+    if (!hMemspace) {
+        return UMF_RESULT_ERROR_INVALID_ARGUMENT;
+    }
+
+    umf_memspace_handle_t hostAllMemspace = umfMemspaceHostAllGet();
+    if (!hostAllMemspace) {
+        return UMF_RESULT_ERROR_UNKNOWN;
+    }
+
+    umf_memspace_handle_t lowLatencyMemspace = NULL;
+    umf_result_t ret = umfMemspaceFilter(hostAllMemspace, getBestLatencyTarget,
+                                         &lowLatencyMemspace);
+    if (ret != UMF_RESULT_SUCCESS) {
+        // HWLOC could possibly return an 'EINVAL' error, which in this context
+        // means that the HMAT is unavailable and we can't obtain the
+        // 'latency' value of any NUMA node.
+        return ret;
+    }
+
+    *hMemspace = lowLatencyMemspace;
+    return UMF_RESULT_SUCCESS;
+}
+
+static umf_memspace_handle_t UMF_MEMSPACE_LOWEST_LATENCY = NULL;
+static UTIL_ONCE_FLAG UMF_MEMSPACE_LOWEST_LATENCY_INITIALIZED =
+    UTIL_ONCE_FLAG_INIT;
+
+void umfMemspaceLowestLatencyDestroy(void) {
+    if (UMF_MEMSPACE_LOWEST_LATENCY) {
+        umfMemspaceDestroy(UMF_MEMSPACE_LOWEST_LATENCY);
+        UMF_MEMSPACE_LOWEST_LATENCY = NULL;
+    }
+}
+
+static void umfMemspaceLowestLatencyInit(void) {
+    umf_result_t ret =
+        umfMemspaceLowestLatencyCreate(&UMF_MEMSPACE_LOWEST_LATENCY);
+    if (ret != UMF_RESULT_SUCCESS) {
+        LOG_ERR("Creating the lowest latency memspace failed with a %u error\n",
+                ret);
+        assert(ret == UMF_RESULT_ERROR_NOT_SUPPORTED);
+    }
+
+#if defined(_WIN32) && !defined(UMF_SHARED_LIBRARY)
+    atexit(umfMemspaceLowestLatencyDestroy);
+#endif
+}
+
+umf_memspace_handle_t umfMemspaceLowestLatencyGet(void) {
+    util_init_once(&UMF_MEMSPACE_LOWEST_LATENCY_INITIALIZED,
+                   umfMemspaceLowestLatencyInit);
+    return UMF_MEMSPACE_LOWEST_LATENCY;
+}