smcuda: fix edge case when using enable mca dso

turns out singleton and single process per node hit
an edge case - a segfault in MPI_Finalize - without this patch

related to #11627

Signed-off-by: Howard Pritchard <howardp@lanl.gov>

Author: hppritcha

opal/mca/btl/smcuda/btl_smcuda_accelerator.c

@@ -37,7 +37,7 @@ static int accelerator_event_max = 400;
 static int accelerator_event_ipc_most = 0;
 static bool smcuda_accelerator_initialized = false;
 
-static void mca_btl_smcuda_accelerator_fini(void);
+void mca_btl_smcuda_accelerator_fini(void);
 
 int mca_btl_smcuda_accelerator_init(void)
 {
@@ -83,14 +83,6 @@ int mca_btl_smcuda_accelerator_init(void)
         goto cleanup_and_error;
     }
 
-    /*
-     * add smcuda acclerator fini code to opal's list of cleanup functions.
-     * Cleanups are called before all the MCA frameworks are closed, so by
-     * adding this function to the callback list, we avoid issues with ordering
-     * of the closing of the BTL framework with the accelerator framework, etc. etc.
-     */
-    opal_finalize_register_cleanup(mca_btl_smcuda_accelerator_fini);
-
     smcuda_accelerator_initialized = true;
 
 cleanup_and_error:
@@ -115,7 +107,7 @@ int mca_btl_smcuda_accelerator_init(void)
     return rc;
 }
 
-static void mca_btl_smcuda_accelerator_fini(void)
+void mca_btl_smcuda_accelerator_fini(void)
 {
     int i;
 

opal/mca/btl/smcuda/btl_smcuda_accelerator.h

@@ -22,5 +22,6 @@ OPAL_DECLSPEC int mca_btl_smcuda_accelerator_init(void);
 OPAL_DECLSPEC int mca_btl_smcuda_progress_one_ipc_event(struct mca_btl_base_descriptor_t **frag);
 OPAL_DECLSPEC int mca_btl_smcuda_memcpy(void *dst, void *src, size_t amount, char *msg,
                            struct mca_btl_base_descriptor_t *frag);
+OPAL_DECLSPEC void mca_btl_smcuda_accelerator_fini(void);
 
 #endif /* MCA_BTL_SMCUDA_ACCELERATOR_H */

opal/mca/btl/smcuda/btl_smcuda_component.c

@@ -265,8 +265,6 @@ static int mca_btl_smcuda_component_open(void)
     OBJ_CONSTRUCT(&mca_btl_smcuda_component.sm_frags_user, opal_free_list_t);
     OBJ_CONSTRUCT(&mca_btl_smcuda_component.pending_send_fl, opal_free_list_t);
 
-    opal_finalize_register_cleanup(mca_btl_smcuda_component_fini);
-
     return OPAL_SUCCESS;
 }
 
@@ -283,6 +281,8 @@ static void mca_btl_smcuda_component_fini(void)
 {
     int rc;
 
+    mca_btl_smcuda_accelerator_fini();
+
     OBJ_DESTRUCT(&mca_btl_smcuda_component.sm_lock);
     /**
      * We don't have to destroy the fragment lists. They are allocated
@@ -892,6 +892,22 @@ mca_btl_smcuda_component_init(int *num_btls, bool enable_progress_threads, bool
     mca_btl_base_active_message_trigger[MCA_BTL_TAG_SMCUDA].cbfunc = btl_smcuda_control;
     mca_btl_base_active_message_trigger[MCA_BTL_TAG_SMCUDA].cbdata = NULL;
 
+    /*
+     * add smcuda component fini code to opal's list of cleanup functions.
+     * Cleanups are called before all the MCA frameworks are closed, so by
+     * of the closing of the BTL framework with the accelerator framework, etc. etc.
+     * We add it here in the btl_init routine as its possible under
+     * certain scenarios that one of the steps above in this routine will fail,
+     * resulting in a NULL return value, and the btl component selector to close
+     * the btl.  This can also happen in normal operation, for instance for singleton
+     * where the smcuda is closed during mpi initialization.  We don't want
+     * to add a cleanup callback if no btls were returned.
+     */
+
+    if (NULL !=  btls) {
+        opal_finalize_register_cleanup(mca_btl_smcuda_component_fini);
+    }
+
     return btls;
 }