Add support for MAP_SHARED to OS memory provider for Linux only

Signed-off-by: Lukasz Dorau <lukasz.dorau@intel.com>

Author: ldorau

README.md

@@ -137,6 +137,9 @@ More detailed documentation is available here: https://oneapi-src.github.io/unif
 #### OS memory provider
 
 A memory provider that provides memory from an operating system.
+It supports two types of memory mappings
+1) private memory mapping (`UMF_MEM_MAP_PRIVATE`)
+2) shared memory mapping (`UMF_MEM_MAP_SHARED` - supported on Linux only yet)
 
 ##### Requirements
 

benchmark/ubench.c

@@ -107,6 +107,7 @@ UBENCH_EX(simple, glibc_malloc) {
 
 static umf_os_memory_provider_params_t UMF_OS_MEMORY_PROVIDER_PARAMS = {
     /* .protection = */ UMF_PROTECTION_READ | UMF_PROTECTION_WRITE,
+    /* .flag */ UMF_MEM_MAP_PRIVATE,
 
     // NUMA config
     /* .nodemask = */ NULL,

include/umf/providers/provider_os_memory.h

@@ -29,6 +29,14 @@ typedef enum umf_mem_protection_flags_t {
     /// @endcond
 } umf_mem_protection_flags_t;
 
+/// @brief Memory flags
+typedef enum umf_memory_flags_t {
+    UMF_MEM_NONE = 0,               ///< No flag
+    UMF_MEM_MAP_PRIVATE = (1 << 0), ///< private memory mapping
+    UMF_MEM_MAP_SHARED =
+        (1 << 1), ///< shared memory mapping (supported on Linux only)
+} umf_memory_flags_t;
+
 /// @brief Memory binding mode
 /// Specifies how memory is bound to NUMA nodes on systems that support NUMA.
 /// Not every mode is supported on every system.
@@ -61,6 +69,8 @@ typedef enum umf_numa_mode_t {
 typedef struct umf_os_memory_provider_params_t {
     /// Combination of 'umf_mem_protection_flags_t' flags
     unsigned protection;
+    /// 'umf_memory_flags_t' flag
+    unsigned flag;
 
     // NUMA config
     /// ordered list of numa nodes
@@ -91,6 +101,7 @@ static inline umf_os_memory_provider_params_t
 umfOsMemoryProviderParamsDefault(void) {
     umf_os_memory_provider_params_t params = {
         UMF_PROTECTION_READ | UMF_PROTECTION_WRITE, /* protection */
+        UMF_MEM_MAP_PRIVATE,                        /* flag */
         NULL,                                       /* numa_list */
         0,                                          /* numa_list_len */
         UMF_NUMA_MODE_DEFAULT,                      /* numa_mode */

src/provider/provider_os_memory.c

@@ -15,6 +15,7 @@
 #include <string.h>
 
 #include "base_alloc_global.h"
+#include "critnib.h"
 #include "provider_os_memory_internal.h"
 #include "utils_log.h"
 
@@ -24,8 +25,15 @@
 
 #define NODESET_STR_BUF_LEN 1024
 
+// maximum size of file used for memory mapping
+#define MAX_SIZE_FD (1ULL << 40)
+
 typedef struct os_memory_provider_t {
     unsigned protection; // combination of OS-specific protection flags
+    unsigned flag;       // memory mapping flag
+    int fd;              // file descriptor for memory mapping
+    size_t size_fd;      // size of file used for memory mapping
+    critnib *ptr_off;    // a critnib storing (ptr, fd_offset) pairs
 
     // NUMA config
     hwloc_bitmap_t nodeset;
@@ -191,6 +199,25 @@ static umf_result_t translate_params(umf_os_memory_provider_params_t *in_params,
         return result;
     }
 
+    result = os_translate_memory_flag(in_params->flag, &provider->flag);
+    if (result != UMF_RESULT_SUCCESS) {
+        LOG_ERR("incorrect memory flag: %u", in_params->flag);
+        return result;
+    }
+
+    provider->fd = os_create_memory_fd(provider->flag);
+    if (provider->fd == -1) {
+        LOG_PERR("creating a file descriptor for memory mapping failed");
+        return UMF_RESULT_ERROR_UNKNOWN;
+    }
+
+    provider->size_fd = 0; // will be increased during each allocation
+
+    if (provider->fd > 0 && os_set_file_size(provider->fd, MAX_SIZE_FD)) {
+        LOG_PDEBUG("growing file size failed");
+        return UMF_RESULT_ERROR_INVALID_ARGUMENT;
+    }
+
     // NUMA config
     int emptyNodeset = in_params->numa_list_len == 0;
     result = translate_numa_mode(in_params->numa_mode, emptyNodeset,
@@ -258,10 +285,19 @@ static umf_result_t os_initialize(void *params, void **provider) {
         }
     }
 
+    os_provider->ptr_off = critnib_new();
+    if (!os_provider->ptr_off) {
+        LOG_ERR("creating IPC cache failed");
+        ret = UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY;
+        goto err_free_nodeset_str_buf;
+    }
+
     *provider = os_provider;
 
     return UMF_RESULT_SUCCESS;
 
+err_free_nodeset_str_buf:
+    umf_ba_global_free(os_provider->nodeset_str_buf);
 err_destroy_hwloc_topology:
     hwloc_topology_destroy(os_provider->topo);
 err_free_os_provider:
@@ -276,6 +312,9 @@ static void os_finalize(void *provider) {
     }
 
     os_memory_provider_t *os_provider = provider;
+
+    critnib_delete(os_provider->ptr_off);
+
     if (os_provider->nodeset_str_buf) {
         umf_ba_global_free(os_provider->nodeset_str_buf);
     }
@@ -331,7 +370,8 @@ static inline void assert_is_page_aligned(uintptr_t ptr, size_t page_size) {
 }
 
 static int os_mmap_aligned(void *hint_addr, size_t length, size_t alignment,
-                           size_t page_size, int prot, void **out_addr) {
+                           size_t page_size, int prot, int flag, int fd,
+                           void **out_addr, size_t *fd_size) {
     assert(out_addr);
 
     size_t extended_length = length;
@@ -344,8 +384,20 @@ static int os_mmap_aligned(void *hint_addr, size_t length, size_t alignment,
         extended_length += alignment;
     }
 
-    void *ptr = os_mmap(hint_addr, extended_length, prot);
+    size_t fd_offset = 0;
+
+    if (fd > 0) {
+        fd_offset = *fd_size;
+        *fd_size += extended_length;
+        if (*fd_size > MAX_SIZE_FD) {
+            LOG_PDEBUG("growing file size failed");
+            return -1;
+        }
+    }
+
+    void *ptr = os_mmap(hint_addr, extended_length, prot, flag, fd, fd_offset);
     if (ptr == NULL) {
+        LOG_PDEBUG("memory mapping failed");
         return -1;
     }
 
@@ -411,15 +463,16 @@ static umf_result_t os_alloc(void *provider, size_t size, size_t alignment,
         return UMF_RESULT_ERROR_INVALID_ARGUMENT;
     }
 
-    int protection = os_provider->protection;
+    size_t fd_offset = os_provider->size_fd; // needed for critnib_insert()
 
     void *addr = NULL;
     errno = 0;
-    ret = os_mmap_aligned(NULL, size, alignment, page_size, protection, &addr);
+    ret = os_mmap_aligned(NULL, size, alignment, page_size,
+                          os_provider->protection, os_provider->flag,
+                          os_provider->fd, &addr, &os_provider->size_fd);
     if (ret) {
         os_store_last_native_error(UMF_OS_RESULT_ERROR_ALLOC_FAILED, errno);
         LOG_PERR("memory allocation failed");
-
         return UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC;
     }
 
@@ -429,7 +482,6 @@ static umf_result_t os_alloc(void *provider, size_t size, size_t alignment,
         LOG_ERR("allocated address 0x%llx is not aligned to %zu (0x%zx) "
                 "bytes",
                 (unsigned long long)addr, alignment, alignment);
-
         goto err_unmap;
     }
 
@@ -460,6 +512,16 @@ static umf_result_t os_alloc(void *provider, size_t size, size_t alignment,
         }
     }
 
+    if (os_provider->fd > 0) {
+        ret = critnib_insert(os_provider->ptr_off, (uintptr_t)addr,
+                             (void *)(uintptr_t)fd_offset, 0 /* update */);
+        if (ret) {
+            LOG_DEBUG("inserting a value to the IPC cache failed (addr=%p, "
+                      "offset=%zu)",
+                      addr, fd_offset);
+        }
+    }
+
     *resultPtr = addr;
 
     return UMF_RESULT_SUCCESS;
@@ -474,6 +536,12 @@ static umf_result_t os_free(void *provider, void *ptr, size_t size) {
         return UMF_RESULT_ERROR_INVALID_ARGUMENT;
     }
 
+    os_memory_provider_t *os_provider = (os_memory_provider_t *)provider;
+
+    if (os_provider->fd > 0) {
+        critnib_remove(os_provider->ptr_off, (uintptr_t)ptr);
+    }
+
     errno = 0;
     int ret = os_munmap(ptr, size);
     // ignore error when size == 0

src/provider/provider_os_memory_internal.h

@@ -28,7 +28,14 @@ umf_result_t os_translate_flags(unsigned in_flags, unsigned max,
 umf_result_t os_translate_mem_protection_flags(unsigned in_protection,
                                                unsigned *out_protection);
 
-void *os_mmap(void *hint_addr, size_t length, int prot);
+umf_result_t os_translate_memory_flag(unsigned in_flag, unsigned *out_flag);
+
+int os_create_memory_fd(unsigned translated_memory_flag);
+
+int os_set_file_size(int fd, size_t size);
+
+void *os_mmap(void *hint_addr, size_t length, int prot, int flag, int fd,
+              size_t fd_offset);
 
 int os_munmap(void *addr, size_t length);
 

src/provider/provider_os_memory_linux.c

@@ -6,13 +6,17 @@
 */
 
 #include <assert.h>
+#include <errno.h>
 #include <string.h>
 #include <sys/mman.h>
+#include <sys/syscall.h>
 #include <unistd.h>
 
-#include "provider_os_memory_internal.h"
 #include <umf/providers/provider_os_memory.h>
 
+#include "provider_os_memory_internal.h"
+#include "utils_log.h"
+
 umf_result_t os_translate_mem_protection_one_flag(unsigned in_protection,
                                                   unsigned *out_protection) {
     switch (in_protection) {
@@ -32,6 +36,70 @@ umf_result_t os_translate_mem_protection_one_flag(unsigned in_protection,
     return UMF_RESULT_ERROR_INVALID_ARGUMENT;
 }
 
+umf_result_t os_translate_memory_flag(unsigned in_flag, unsigned *out_flag) {
+    switch (in_flag) {
+    case UMF_MEM_MAP_PRIVATE:
+        *out_flag = MAP_PRIVATE;
+        return UMF_RESULT_SUCCESS;
+    case UMF_MEM_MAP_SHARED:
+#ifdef __APPLE__
+        return UMF_RESULT_ERROR_NOT_SUPPORTED; // not supported on MacOSX
+#else
+        *out_flag = MAP_SHARED;
+        return UMF_RESULT_SUCCESS;
+#endif
+    }
+    return UMF_RESULT_ERROR_INVALID_ARGUMENT;
+}
+
+int os_create_memory_fd(unsigned translated_memory_flag) {
+#ifdef __APPLE__
+    (void)translated_memory_flag; // unused
+    return 0;                     // ignored on MacOSX
+#else
+    // fd is created only for MAP_SHARED
+    if (translated_memory_flag != MAP_SHARED) {
+        return 0;
+    }
+
+    int fd = -1;
+
+#ifdef __NR_memfd_secret
+    // SYS_memfd_secret is supported since Linux 5.14
+    fd = syscall(SYS_memfd_secret, 0);
+    if (fd == -1) {
+        LOG_PDEBUG("memfd_secret()");
+    }
+    if (fd > 0) {
+        return fd;
+    }
+    // syscall(SYS_memfd_secret) can fail with errno == ENOTSYS (function not implemented).
+    // We should try to call syscall(SYS_memfd_create) in this case.
+#endif /* __NR_memfd_secret */
+
+#ifdef __NR_memfd_create
+    // SYS_memfd_create is supported since Linux 3.17, glibc 2.27
+    fd = syscall(SYS_memfd_create, "fd_name", 0);
+    if (fd == -1) {
+        LOG_PDEBUG("memfd_create()");
+    }
+#endif /* __NR_memfd_create */
+
+    return fd;
+
+#endif /* __APPLE__ */
+}
+
+int os_set_file_size(int fd, size_t size) {
+#ifdef __APPLE__
+    (void)fd;   // unused
+    (void)size; // unused
+    return 0;   // ignored on MacOSX
+#else
+    return ftruncate(fd, size);
+#endif /* __APPLE__ */
+}
+
 umf_result_t os_translate_mem_protection_flags(unsigned in_protection,
                                                unsigned *out_protection) {
     // translate protection - combination of 'umf_mem_protection_flags_t' flags
@@ -51,13 +119,19 @@ static int os_translate_purge_advise(umf_purge_advise_t advise) {
     return -1;
 }
 
-void *os_mmap(void *hint_addr, size_t length, int prot) {
-    // MAP_ANONYMOUS - the mapping is not backed by any file
-    void *ptr =
-        mmap(hint_addr, length, prot, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+void *os_mmap(void *hint_addr, size_t length, int prot, int flag, int fd,
+              size_t fd_offset) {
+    fd = (fd == 0) ? -1 : fd;
+    if (fd == -1) {
+        // MAP_ANONYMOUS - the mapping is not backed by any file
+        flag |= MAP_ANONYMOUS;
+    }
+
+    void *ptr = mmap(hint_addr, length, prot, flag, fd, fd_offset);
     if (ptr == MAP_FAILED) {
         return NULL;
     }
+
     return ptr;
 }
 

src/provider/provider_os_memory_windows.c

@@ -54,7 +54,33 @@ umf_result_t os_translate_mem_protection_flags(unsigned in_protection,
     return UMF_RESULT_ERROR_INVALID_ARGUMENT;
 }
 
-void *os_mmap(void *hint_addr, size_t length, int prot) {
+umf_result_t os_translate_memory_flag(unsigned in_flag, unsigned *out_flag) {
+    switch (in_flag) {
+    case UMF_MEM_MAP_PRIVATE:
+        *out_flag = 0; // ignored on Windows
+        return UMF_RESULT_SUCCESS;
+    case UMF_MEM_MAP_SHARED:
+        return UMF_RESULT_ERROR_NOT_SUPPORTED; // not supported on Windows yet
+    }
+    return UMF_RESULT_ERROR_INVALID_ARGUMENT;
+}
+
+int os_create_memory_fd(unsigned translated_memory_flag) {
+    (void)translated_memory_flag; // unused
+    return 0;                     // ignored on Windows
+}
+
+int os_set_file_size(int fd, size_t size) {
+    (void)fd;   // unused
+    (void)size; // unused
+    return 0;   // ignored on Windows
+}
+
+void *os_mmap(void *hint_addr, size_t length, int prot, int flag, int fd,
+              size_t fd_offset) {
+    (void)flag;      // ignored on Windows
+    (void)fd;        // ignored on Windows
+    (void)fd_offset; // ignored on Windows
     return VirtualAlloc(hint_addr, length, MEM_RESERVE | MEM_COMMIT, prot);
 }