Thread-shift errors reported by PMIx to the main MPI progress engine

Signed-off-by: Aurelien Bouteiller <bouteill@icl.utk.edu>

make things happen before the terminal call

Signed-off-by: Aurelien Bouteiller <bouteill@icl.utk.edu>

Author: abouteiller

ompi/errhandler/errhandler.c

@@ -290,6 +290,23 @@ ompi_errhandler_t *ompi_errhandler_create(ompi_errhandler_type_t object_type,
   return new_errhandler;
 }
 
+/* helper to move the error report back from the RTE thread to the MPI thread */
+typedef struct ompi_errhandler_event_s {
+    opal_event_t super;
+    opal_process_name_t procname;
+    int status;
+} ompi_errhandler_event_t;
+
+static void *ompi_errhandler_event_cb(int fd, int flags, void *context) {
+    ompi_errhandler_event_t *event = (ompi_errhandler_event_t*) context;
+    int status = event->status;
+    opal_event_del(&event->super);
+    free(event);
+    /* our default action is to abort */
+    OMPI_ERRHANDLER_NOHANDLE_INVOKE(status, "PMIx Event notification");
+    return NULL;
+}
+
 /* registration callback */
 void ompi_errhandler_registration_callback(int status,
                                            size_t errhandler_ref,
@@ -312,13 +329,37 @@ void ompi_errhandler_callback(size_t refid, pmix_status_t status,
                               pmix_event_notification_cbfunc_fn_t cbfunc,
                               void *cbdata)
 {
+    int rc;
+    /* an error has been found, report to the MPI layer and let it take
+     * further action. */
+    /* transition this from the RTE thread to the MPI progress engine */
+    ompi_errhandler_event_t *event = malloc(sizeof(*event));
+    if(NULL == event) {
+        OMPI_ERROR_LOG(OMPI_ERR_OUT_OF_RESOURCE);
+        goto error;
+    }
+    OPAL_PMIX_CONVERT_PROCT(rc, &event->procname, (pmix_proc_t*)source);
+    if(OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
+        OMPI_ERROR_LOG(rc);
+        free(event);
+        goto error;
+    }
+    event->status = status;
+    opal_event_set(opal_sync_event_base, &event->super, -1, OPAL_EV_READ,
+                   ompi_errhandler_event_cb, event);
+    opal_event_active(&event->super, OPAL_EV_READ, 1);
     /* tell the event chain engine to go no further - we
      * will handle this */
     if (NULL != cbfunc) {
         cbfunc(PMIX_EVENT_ACTION_COMPLETE, NULL, 0, NULL, NULL, cbdata);
     }
-    /* our default action is to abort */
-    OMPI_ERRHANDLER_NOHANDLE_INVOKE(status, "PMIx Event notification");
+    return;
+
+error:
+    if (NULL != cbfunc) {
+        /* We can't handle this, let the default action abort. */
+        cbfunc(PMIX_EVENT_NO_ACTION_TAKEN, NULL, 0, NULL, NULL, cbdata);
+    }
 }
 
 /**************************************************************************

ompi/errhandler/errhandler.h

@@ -339,19 +339,11 @@ struct ompi_request_t;
   }
 
 /* Same as OMPI_ERRHANDLER_RETURN for non-handle attached errors */
-#define OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, err_code, message) \
+#define OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, err_code, message) {\
+  OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, err_code, message) \
   OPAL_CR_EXIT_LIBRARY() \
-  if ( OPAL_UNLIKELY(OMPI_SUCCESS != rc) ) { \
-    int __mpi_err_code = ompi_errcode_get_mpi_code(err_code);         \
-    ompi_errhandler_invoke(NULL, \
-                           NULL, \
-                           -1, \
-                           (__mpi_err_code), \
-                           (message)); \
-    return (__mpi_err_code); \
-  } else { \
-    return MPI_SUCCESS; \
-  }
+  return MPI_SUCCESS; \
+}
 
   /**
    * Initialize the error handler interface.