add data structures

Author: xinzhao3

opal/mca/common/ucx/common_ucx.c

@@ -19,6 +19,74 @@
 
 /***********************************************************************/
 
+typedef struct  {
+    opal_mutex_t mutex;
+    ucp_worker_h worker;
+    ucp_ep_h *endpoints;
+    int comm_size;
+} _worker_engine_t;
+
+OBJ_CLASS_DECLARATION(_worker_engine_t);
+
+typedef struct {
+    int ctx_id;
+    opal_common_ucx_ctx_t *gctx;
+    _worker_engine_t *worker;
+} _tlocal_ctx_t;
+
+OBJ_CLASS_DECLARATION(_tlocal_ctx_t);
+
+typedef struct {
+    _worker_engine_t *worker;
+    ucp_rkey_h *rkeys;
+} _mem_info_t;
+
+OBJ_CLASS_DECLARATION(_mem_info_t);
+
+typedef struct {
+    int mem_id;
+    opal_common_ucx_mem_t *gmem;
+    _mem_info_t *mem;
+} _tlocal_mem_t;
+
+OBJ_CLASS_DECLARATION(_tlocal_mem_t);
+
+typedef struct {
+    opal_list_item_t super;
+    _worker_engine_t *ptr;
+} _idle_list_item_t;
+
+OBJ_CLASS_DECLARATION(_idle_list_item_t);
+OBJ_CLASS_INSTANCE(_idle_list_item_t, opal_list_item_t, NULL, NULL);
+
+typedef struct {
+    opal_list_item_t super;
+    _worker_engine_t *ptr;
+} _worker_list_item_t;
+
+OBJ_CLASS_DECLARATION(_worker_list_item_t);
+OBJ_CLASS_INSTANCE(_worker_list_item_t, opal_list_item_t, NULL, NULL);
+
+typedef struct {
+    opal_list_item_t super;
+    _mem_info_t *ptr;
+} _mem_region_list_item_t;
+
+OBJ_CLASS_DECLARATION(_mem_region_list_item_t);
+OBJ_CLASS_INSTANCE(_mem_region_list_item_t, opal_list_item_t, NULL, NULL);
+
+/* thread-local table */
+typedef struct {
+    _tlocal_ctx_t **ctx_tbl;
+    size_t ctx_tbl_size;
+    _tlocal_mem_t **mem_tbl;
+    size_t mem_tbl_size;
+} _tlocal_table_t;
+
+static pthread_key_t _tlocal_key = {0};
+
+/***********************************************************************/
+
 extern mca_base_framework_t opal_memory_base_framework;
 
 opal_common_ucx_module_t opal_common_ucx = {

opal/mca/common/ucx/common_ucx.h

@@ -96,6 +96,44 @@ typedef struct opal_common_ucx_del_proc {
 
 extern opal_common_ucx_module_t opal_common_ucx;
 
+typedef struct {
+    ucp_context_h ucp_ctx;
+    opal_mutex_t mutex;
+    opal_list_t idle_workers;
+    ucp_worker_h recv_worker;
+    ucp_address_t *recv_waddr;
+    size_t recv_waddr_len;
+    int cur_ctxid, cur_memid;
+} opal_common_ucx_wpool_t;
+
+typedef struct {
+    int ctx_id;
+    opal_mutex_t mutex;
+    opal_common_ucx_wpool_t *wpool; /* which wpool this ctx belongs to */
+    opal_list_t workers; /* active worker lists */
+    char *recv_worker_addrs;
+    int *recv_worker_displs;
+} opal_common_ucx_ctx_t;
+
+typedef struct {
+    int mem_id;
+    opal_mutex_t mutex;
+    opal_common_ucx_ctx_t *ctx; /* which ctx this mem_reg belongs to */
+    opal_list_t mem_regions; /* mem region lists */
+    char *mem_addrs;
+    int *mem_displs;
+} opal_common_ucx_mem_t;
+
+typedef enum {
+    OPAL_COMMON_UCX_PUT,
+    OPAL_COMMON_UCX_GET
+} opal_common_ucx_op_t;
+
+typedef enum {
+    OPAL_COMMON_UCX_SCOPE_EP,
+    OPAL_COMMON_UCX_SCOPE_WORKER
+} opal_common_ucx_flush_scope_t;
+
 OPAL_DECLSPEC void opal_common_ucx_mca_register(void);
 OPAL_DECLSPEC void opal_common_ucx_mca_deregister(void);
 OPAL_DECLSPEC void opal_common_ucx_empty_complete_cb(void *request, ucs_status_t status);