Merge pull request #9693 from awlauria/more_hwloc_removal

awlauria · web-flow · commit 938f46dfa01d · 2021-11-22T22:27:54.000-05:00
opal/mca/hwloc: More cleanup/code removal.
diff --git a/opal/mca/hwloc/base/base.h b/opal/mca/hwloc/base/base.h
@@ -39,10 +39,7 @@ BEGIN_C_DECLS
  * hwloc_topology_load() can be expensive (and/or serialized by the
  * OS); it may not be desireable to call this function in every MPI
  * process on a machine.  Hence, it is the responsibility for an upper
- * layer to both fill opal_hwloc_topology in some scalable way, as
- * well as to invoke opal_hwloc_base_set_process_membind_policy()
- * (after opal_hwloc_topology has been loaded) to set the process-wide
- * memory affinity policy.
+ * layer to both fill opal_hwloc_topology in some scalable way.
  */
 
 /**
@@ -53,63 +50,6 @@ OPAL_DECLSPEC extern bool opal_hwloc_topology_inited;
 
 OPAL_DECLSPEC extern mca_base_framework_t opal_hwloc_base_framework;
 
-/* we always must have some minimal locality support */
-#define OPAL_HWLOC_PRINT_MAX_SIZE 50
-#define OPAL_HWLOC_PRINT_NUM_BUFS 16
-typedef struct {
-    char *buffers[OPAL_HWLOC_PRINT_NUM_BUFS];
-    int cntr;
-} opal_hwloc_print_buffers_t;
-opal_hwloc_print_buffers_t *opal_hwloc_get_print_buffer(void);
-
-/* convenience macro for debugging */
-#define OPAL_HWLOC_SHOW_BINDING(n, v, t)                                                        \
-    do {                                                                                        \
-        char tmp1[1024];                                                                        \
-        hwloc_cpuset_t bind;                                                                    \
-        bind = opal_hwloc_alloc();                                                              \
-        if (hwloc_get_cpubind(t, bind, HWLOC_CPUBIND_PROCESS) < 0) {                            \
-            opal_output_verbose(n, v, "CANNOT DETERMINE BINDING AT %s:%d", __FILE__, __LINE__); \
-        } else {                                                                                \
-            opal_hwloc_base_cset2mapstr(tmp1, sizeof(tmp1), t, bind);                           \
-            opal_output_verbose(n, v, "BINDINGS AT %s:%d: %s", __FILE__, __LINE__, tmp1);       \
-        }                                                                                       \
-        hwloc_bitmap_free(bind);                                                                \
-    } while (0);
-
-#if HWLOC_API_VERSION < 0x20000
-#    define OPAL_HWLOC_MAKE_OBJ_CACHE(level, obj, cache_level) \
-        do {                                                   \
-            obj = HWLOC_OBJ_CACHE;                             \
-            cache_level = level;                               \
-        } while (0)
-#else
-#    define OPAL_HWLOC_MAKE_OBJ_CACHE(level, obj, cache_level) \
-        do {                                                   \
-            obj = HWLOC_OBJ_L##level##CACHE;                   \
-            cache_level = 0;                                   \
-        } while (0)
-#endif
-
-struct opal_rmaps_numa_node_t {
-    opal_list_item_t super;
-    int index;
-    float dist_from_closed;
-};
-typedef struct opal_rmaps_numa_node_t opal_rmaps_numa_node_t;
-OBJ_CLASS_DECLARATION(opal_rmaps_numa_node_t);
-
-/**
- * Enum for what memory allocation policy we want for user allocations.
- * MAP = memory allocation policy.
- */
-typedef enum { OPAL_HWLOC_BASE_MAP_NONE, OPAL_HWLOC_BASE_MAP_LOCAL_ONLY } opal_hwloc_base_map_t;
-
-/**
- * Global reflecting the MAP (set by MCA param).
- */
-OPAL_DECLSPEC extern opal_hwloc_base_map_t opal_hwloc_base_map;
-
 /**
  * Enum for what to do if the hwloc framework tries to bind memory
  * and fails.  BFA = bind failure action.
@@ -120,21 +60,12 @@ typedef enum {
     OPAL_HWLOC_BASE_MBFA_ERROR
 } opal_hwloc_base_mbfa_t;
 
-/**
- * Global reflecting the BFA (set by MCA param).
- */
-OPAL_DECLSPEC extern opal_hwloc_base_mbfa_t opal_hwloc_base_mbfa;
-
 /**
  * Discover / load the hwloc topology (i.e., call hwloc_topology_init() and
  * hwloc_topology_load()).
  */
 OPAL_DECLSPEC int opal_hwloc_base_get_topology(void);
 
-/**
- * Free the hwloc topology.
- */
-OPAL_DECLSPEC void opal_hwloc_base_free_topology(hwloc_topology_t topo);
 OPAL_DECLSPEC unsigned int opal_hwloc_base_get_nbobjs_by_type(hwloc_topology_t topo,
                                                               hwloc_obj_type_t target,
                                                               unsigned cache_level,
@@ -146,29 +77,12 @@ OPAL_DECLSPEC hwloc_obj_t opal_hwloc_base_get_obj_by_type(hwloc_topology_t topo,
                                                           unsigned int instance,
                                                           opal_hwloc_resource_type_t rtype);
 
-/**
- * This function sets the process-wide memory affinity policy
- * according to opal_hwloc_base_map and opal_hwloc_base_mbfa.  It needs
- * to be a separate, standalone function (as opposed to being done
- * during opal_hwloc_base_open()) because opal_hwloc_topology is not
- * loaded by opal_hwloc_base_open().  Hence, an upper layer needs to
- * invoke this function after opal_hwloc_topology has been loaded.
- */
-OPAL_DECLSPEC int opal_hwloc_base_set_process_membind_policy(void);
-
 OPAL_DECLSPEC int opal_hwloc_base_membind(opal_hwloc_base_memory_segment_t *segs, size_t count,
                                           int node_id);
 
 OPAL_DECLSPEC int opal_hwloc_base_memory_set(opal_hwloc_base_memory_segment_t *segments,
                                              size_t num_segments);
 
-/**
- * Make a prettyprint string for a hwloc_cpuset_t (e.g., "socket
- * 2[core 3]").
- */
-OPAL_DECLSPEC int opal_hwloc_base_cset2str(char *str, int len, hwloc_topology_t topo,
-                                           hwloc_cpuset_t cpuset);
-
 /* extract a location from the locality string */
 OPAL_DECLSPEC char *opal_hwloc_base_get_location(char *locality, hwloc_obj_type_t type,
                                                  unsigned index);
diff --git a/opal/mca/hwloc/base/hwloc_base_frame.c b/opal/mca/hwloc/base/hwloc_base_frame.c
@@ -39,6 +39,8 @@ hwloc_topology_t opal_hwloc_topology = NULL;
 hwloc_cpuset_t opal_hwloc_my_cpuset = NULL;
 opal_hwloc_base_mbfa_t opal_hwloc_base_mbfa = OPAL_HWLOC_BASE_MBFA_WARN;
 
+extern bool opal_hwloc_topo_in_shmem;
+
 static mca_base_var_enum_value_t hwloc_failure_action[] = {{OPAL_HWLOC_BASE_MBFA_SILENT, "silent"},
                                                            {OPAL_HWLOC_BASE_MBFA_WARN, "warn"},
                                                            {OPAL_HWLOC_BASE_MBFA_ERROR, "error"},
@@ -95,6 +97,48 @@ static int opal_hwloc_base_open(mca_base_open_flag_t flags)
     return OPAL_SUCCESS;
 }
 
+static void free_object(hwloc_obj_t obj)
+{
+    opal_hwloc_obj_data_t *data;
+    unsigned k;
+
+    /* free any data hanging on this object */
+    if (NULL != obj->userdata) {
+        data = (opal_hwloc_obj_data_t *) obj->userdata;
+        OBJ_RELEASE(data);
+        obj->userdata = NULL;
+    }
+
+    /* loop thru our children */
+    for (k = 0; k < obj->arity; k++) {
+        free_object(obj->children[k]);
+    }
+}
+
+static void free_topology(hwloc_topology_t topo)
+{
+    hwloc_obj_t obj;
+    opal_hwloc_topo_data_t *rdata;
+    unsigned k;
+
+    if (!opal_hwloc_topo_in_shmem) {
+        obj = hwloc_get_root_obj(topo);
+        /* release the root-level userdata */
+        if (NULL != obj->userdata) {
+            rdata = (opal_hwloc_topo_data_t *) obj->userdata;
+            OBJ_RELEASE(rdata);
+            obj->userdata = NULL;
+        }
+        /* now recursively descend and release userdata
+         * in the rest of the objects
+         */
+        for (k = 0; k < obj->arity; k++) {
+            free_object(obj->children[k]);
+        }
+    }
+    hwloc_topology_destroy(topo);
+}
+
 static int opal_hwloc_base_close(void)
 {
     int ret;
@@ -118,7 +162,7 @@ static int opal_hwloc_base_close(void)
 
     /* destroy the topology */
     if (NULL != opal_hwloc_topology) {
-        opal_hwloc_base_free_topology(opal_hwloc_topology);
+        free_topology(opal_hwloc_topology);
         opal_hwloc_topology = NULL;
     }
 
@@ -171,5 +215,3 @@ static void topo_data_dest(opal_hwloc_topo_data_t *ptr)
     ptr->userdata = NULL;
 }
 OBJ_CLASS_INSTANCE(opal_hwloc_topo_data_t, opal_object_t, topo_data_const, topo_data_dest);
-
-OBJ_CLASS_INSTANCE(opal_rmaps_numa_node_t, opal_list_item_t, NULL, NULL);
diff --git a/opal/mca/hwloc/base/hwloc_base_maffinity.c b/opal/mca/hwloc/base/hwloc_base_maffinity.c
@@ -16,6 +16,11 @@
 #include "opal/mca/hwloc/base/base.h"
 #include "opal/mca/hwloc/hwloc-internal.h"
 
+/**
+ * Global reflecting the BFA (set by MCA param).
+ */
+extern opal_hwloc_base_mbfa_t opal_hwloc_base_mbfa;
+
 /**
  * Report a bind failure using the normal mechanisms if a component
  * fails to bind memory -- according to the value of the
diff --git a/opal/mca/hwloc/base/hwloc_base_util.c b/opal/mca/hwloc/base/hwloc_base_util.c
@@ -62,7 +62,7 @@
 #include "opal/mca/hwloc/base/base.h"
 #include "opal/mca/hwloc/hwloc-internal.h"
 
-static bool topo_in_shmem = false;
+bool opal_hwloc_topo_in_shmem = false;
 
 static void fill_cache_line_size(void)
 {
@@ -249,7 +249,7 @@ int opal_hwloc_base_get_topology(void)
         } else {
             opal_output_verbose(2, opal_hwloc_base_framework.framework_output,
                                 "hwloc:base: topology in shared memory");
-            topo_in_shmem = true;
+            opal_hwloc_topo_in_shmem = true;
             goto done;
         }
     }
@@ -332,48 +332,6 @@ int opal_hwloc_base_get_topology(void)
     return OPAL_SUCCESS;
 }
 
-static void free_object(hwloc_obj_t obj)
-{
-    opal_hwloc_obj_data_t *data;
-    unsigned k;
-
-    /* free any data hanging on this object */
-    if (NULL != obj->userdata) {
-        data = (opal_hwloc_obj_data_t *) obj->userdata;
-        OBJ_RELEASE(data);
-        obj->userdata = NULL;
-    }
-
-    /* loop thru our children */
-    for (k = 0; k < obj->arity; k++) {
-        free_object(obj->children[k]);
-    }
-}
-
-void opal_hwloc_base_free_topology(hwloc_topology_t topo)
-{
-    hwloc_obj_t obj;
-    opal_hwloc_topo_data_t *rdata;
-    unsigned k;
-
-    if (!topo_in_shmem) {
-        obj = hwloc_get_root_obj(topo);
-        /* release the root-level userdata */
-        if (NULL != obj->userdata) {
-            rdata = (opal_hwloc_topo_data_t *) obj->userdata;
-            OBJ_RELEASE(rdata);
-            obj->userdata = NULL;
-        }
-        /* now recursively descend and release userdata
-         * in the rest of the objects
-         */
-        for (k = 0; k < obj->arity; k++) {
-            free_object(obj->children[k]);
-        }
-    }
-    hwloc_topology_destroy(topo);
-}
-
 /* hwloc treats cache objects as special
  * cases. Instead of having a unique type for each cache level,
  * there is a single cache object type, and the level is encoded
@@ -442,7 +400,7 @@ static hwloc_obj_t df_search(hwloc_topology_t topo, hwloc_obj_t start, hwloc_obj
         hwloc_obj_t root;
         opal_hwloc_topo_data_t *rdata = NULL;
         root = hwloc_get_root_obj(topo);
-        if (false == topo_in_shmem) {
+        if (false == opal_hwloc_topo_in_shmem) {
             rdata = (opal_hwloc_topo_data_t *) root->userdata;
         }
         hwloc_cpuset_t constrained_cpuset;
@@ -514,12 +472,12 @@ unsigned int opal_hwloc_base_get_nbobjs_by_type(hwloc_topology_t topo, hwloc_obj
     obj = hwloc_get_root_obj(topo);
 
     /* first see if the topology already has this summary */
-    if (false == topo_in_shmem) {
+    if (false == opal_hwloc_topo_in_shmem) {
         data = (opal_hwloc_topo_data_t *) obj->userdata;
     }
     if (NULL == data) {
         data = OBJ_NEW(opal_hwloc_topo_data_t);
-        if (false == topo_in_shmem) {
+        if (false == opal_hwloc_topo_in_shmem) {
             // Can't touch userdata if in read-only shmem!
             // We have to protect here for the case where obj->userdata
             // is in shmem and it is NULL.
@@ -555,7 +513,7 @@ unsigned int opal_hwloc_base_get_nbobjs_by_type(hwloc_topology_t topo, hwloc_obj
     return num_objs;
 }
 
-/* as above, only return the Nth instance of the specified object
+/* Return the Nth instance of the specified object
  * type from inside the topology
  */
 hwloc_obj_t opal_hwloc_base_get_obj_by_type(hwloc_topology_t topo, hwloc_obj_type_t target,
@@ -708,4 +666,3 @@ opal_hwloc_locality_t opal_hwloc_compute_relative_locality(char *loc1, char *loc
     hwloc_bitmap_free(bit2);
     return locality;
 }
-
