Merge pull request #7798 from abouteiller/mpi-next/unbounderr-self

MPI-4 error handling: 'unbound' errors to MPI_COMM_SELF

Author: abouteiller

ompi/errhandler/errhandler.c

@@ -290,6 +290,26 @@ ompi_errhandler_t *ompi_errhandler_create(ompi_errhandler_type_t object_type,
   return new_errhandler;
 }
 
+/* helper to move the error report back from the RTE thread to the MPI thread */
+typedef struct ompi_errhandler_event_s {
+    opal_event_t super;
+    opal_process_name_t procname;
+    int status;
+} ompi_errhandler_event_t;
+
+static void *ompi_errhandler_event_cb(int fd, int flags, void *context) {
+    ompi_errhandler_event_t *event = (ompi_errhandler_event_t*) context;
+    int status = event->status;
+    opal_event_del(&event->super);
+    free(event);
+    /* our default action is to abort */
+    /* TODO: this error should return to the caller and invoke an error
+     * handler from the MPI API call.
+     * For now, it is fatal. */
+    ompi_mpi_errors_are_fatal_comm_handler(NULL, status, "PMIx Even Notification");
+    return NULL;
+}
+
 /* registration callback */
 void ompi_errhandler_registration_callback(int status,
                                            size_t errhandler_ref,
@@ -312,13 +332,37 @@ void ompi_errhandler_callback(size_t refid, pmix_status_t status,
                               pmix_event_notification_cbfunc_fn_t cbfunc,
                               void *cbdata)
 {
+    int rc;
+    /* an error has been found, report to the MPI layer and let it take
+     * further action. */
+    /* transition this from the RTE thread to the MPI progress engine */
+    ompi_errhandler_event_t *event = malloc(sizeof(*event));
+    if(NULL == event) {
+        OMPI_ERROR_LOG(OMPI_ERR_OUT_OF_RESOURCE);
+        goto error;
+    }
+    OPAL_PMIX_CONVERT_PROCT(rc, &event->procname, (pmix_proc_t*)source);
+    if(OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
+        OMPI_ERROR_LOG(rc);
+        free(event);
+        goto error;
+    }
+    event->status = status;
+    opal_event_set(opal_sync_event_base, &event->super, -1, OPAL_EV_READ,
+                   ompi_errhandler_event_cb, event);
+    opal_event_active(&event->super, OPAL_EV_READ, 1);
     /* tell the event chain engine to go no further - we
      * will handle this */
     if (NULL != cbfunc) {
         cbfunc(PMIX_EVENT_ACTION_COMPLETE, NULL, 0, NULL, NULL, cbdata);
     }
-    /* our default action is to abort */
-    ompi_mpi_abort(MPI_COMM_WORLD, status);
+    return;
+
+error:
+    if (NULL != cbfunc) {
+        /* We can't handle this, let the default action abort. */
+        cbfunc(PMIX_EVENT_NO_ACTION_TAKEN, NULL, 0, NULL, NULL, cbdata);
+    }
 }
 
 /**************************************************************************

ompi/errhandler/errhandler.h

@@ -38,6 +38,7 @@
 #include "opal/mca/pmix/pmix-internal.h"
 
 #include "ompi/runtime/mpiruntime.h"
+#include "ompi/runtime/params.h"
 #include "ompi/errhandler/errhandler_predefined.h"
 #include "ompi/errhandler/errcode-internal.h"
 
@@ -256,10 +257,19 @@ struct ompi_request_t;
  */
 #define OMPI_ERRHANDLER_INVOKE(mpi_object, err_code, message) \
   ompi_errhandler_invoke((mpi_object)->error_handler, \
-			 (mpi_object), \
+                         (mpi_object), \
                          (int)(mpi_object)->errhandler_type, \
                          ompi_errcode_get_mpi_code(err_code), \
-			 (message));
+                         (message));
+
+/**
+ * This is the macro to route errors to the 'default' communicator
+ * for non-handle attached errors (e.g., a datatype operation error).
+ */
+#define OMPI_ERRHANDLER_NOHANDLE_INVOKE(err_code, message) \
+    ompi_errhandler_invoke(NULL, NULL, -1, \
+                           ompi_errcode_get_mpi_code(err_code), \
+                           (message));
 
 /**
  * Conditionally invoke an MPI error handler.
@@ -279,13 +289,26 @@ struct ompi_request_t;
     int __mpi_err_code = ompi_errcode_get_mpi_code(err_code);         \
     OPAL_CR_EXIT_LIBRARY() \
     ompi_errhandler_invoke((mpi_object)->error_handler, \
-			   (mpi_object), \
+                           (mpi_object), \
                            (int) (mpi_object)->errhandler_type, \
                            (__mpi_err_code), \
                            (message)); \
     return (__mpi_err_code); \
   }
 
+/* Same as OMPI_ERRHANDLER_CHECK for non-handle attached errors */
+#define OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, err_code, message) \
+  if( OPAL_UNLIKELY(rc != OMPI_SUCCESS) ) { \
+    int __mpi_err_code = ompi_errcode_get_mpi_code(err_code);         \
+    OPAL_CR_EXIT_LIBRARY() \
+    ompi_errhandler_invoke(NULL, \
+                           NULL, \
+                           -1, \
+                           (__mpi_err_code), \
+                           (message)); \
+    return (__mpi_err_code); \
+  }
+
 /**
  * Conditionally invoke an MPI error handler; if there is no error,
  * return MPI_SUCCESS.
@@ -315,7 +338,12 @@ struct ompi_request_t;
     return MPI_SUCCESS; \
   }
 
-
+/* Same as OMPI_ERRHANDLER_RETURN for non-handle attached errors */
+#define OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, err_code, message) {\
+  OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, err_code, message) \
+  OPAL_CR_EXIT_LIBRARY() \
+  return MPI_SUCCESS; \
+}
 
   /**
    * Initialize the error handler interface.

ompi/errhandler/errhandler_invoke.c

@@ -47,7 +47,7 @@ int ompi_errhandler_invoke(ompi_errhandler_t *errhandler, void *mpi_object,
         int32_t state = ompi_mpi_state;
         if (state >= OMPI_MPI_STATE_INIT_COMPLETED &&
             state < OMPI_MPI_STATE_FINALIZE_PAST_COMM_SELF_DESTRUCT) {
-            comm = &ompi_mpi_comm_self.comm;
+            comm = (ompi_mpi_compat_mpi3)? &ompi_mpi_comm_world.comm: &ompi_mpi_comm_self.comm;
             comm->error_handler->eh_comm_fn(&comm, &err_code, message, NULL);
         }
         else {

ompi/mca/fcoll/two_phase/fcoll_two_phase_support_fns.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -171,12 +171,16 @@ int mca_fcoll_two_phase_calc_aggregator(ompio_file_t *fh,
     }
 
     if (rank_index >= num_aggregators || rank_index < 0) {
+       int err = MPI_ERR_INTERN;
        fprintf(stderr,
 	       "Error in mca_fcoll_two_phase_calc_aggregator:");
        fprintf(stderr,
 	       "rank_index(%d) >= num_aggregators(%d) fd_size=%lld off=%lld min_off=%lld striping_unit=%d\n",
 	       rank_index, num_aggregators, fd_size, off, min_off, striping_unit);
-       ompi_mpi_abort(&ompi_mpi_comm_world.comm, 1);
+       /* TODO: this error should return to the caller and invoke an error
+        * handler from the MPI API call.
+        * For now, it is fatal. */
+       ompi_mpi_errors_are_fatal_file_handler(NULL, &err, "Invalid rank in fcoll aggregator");
     }
 
 

ompi/mca/osc/base/osc_base_obj_convert.c

@@ -2,8 +2,9 @@
 /*
  * Copyright (c) 2004-2005 The Trustees of Indiana University.
  *                         All rights reserved.
- * Copyright (c) 2004-2006 The Trustees of the University of Tennessee.
- *                         All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
@@ -84,16 +85,20 @@ int ompi_osc_base_process_op (void *outbuf, void *inbuf, size_t inbuflen,
            MPI_DOUBLE_INT == datatype ||
            MPI_LONG_INT == datatype ||
            MPI_LONG_DOUBLE_INT == datatype) {
-           ompi_communicator_t *comm = &ompi_mpi_comm_world.comm;
-           opal_output(0, "Error: %s datatype is currently "
-                       "unsupported for MPI_MINLOC/MPI_MAXLOC "
-                       "operation\n", datatype->name);
-           opal_show_help("help-mpi-api.txt", "mpi-abort", true,
-                          comm->c_my_rank,
-                          ('\0' != comm->c_name[0]) ? comm->c_name : "<Unknown>",
-                          -1);
-
-           ompi_mpi_abort(comm, -1);
+            int err = MPI_ERR_UNSUPPORTED_DATAREP;
+            char *reason = NULL;
+            opal_asprintf(&reason,
+                    "%s datatype is currently "
+                    "unsupported for MPI_MINLOC/MPI_MAXLOC "
+                    "operation\n", datatype->name);
+            opal_show_help("help-mpi-api.txt", "MPI function not supported", true,
+                          (MPI_MINLOC==op)?"MPI_MINLOC":"MPI_MAXLOC",
+                          reason);
+            free(reason);
+            /* TODO: this error should return to the caller and invoke an error
+             * handler from the MPI API call.
+             * For now, it is fatal. */
+            ompi_mpi_errors_are_fatal_win_handler(NULL, &err, "OSC unsupported MINLOC/MAXLOC datatype");
         }
     }
 

ompi/mca/pml/ob1/pml_ob1.c

@@ -826,7 +826,10 @@ void mca_pml_ob1_error_handler(
         return;
     }
 
-    ompi_rte_abort(-1, btlinfo);
+    /* TODO: this error should return to the caller and invoke an error
+     * handler from the MPI API call.
+     * For now, it is fatal. */
+    ompi_mpi_errors_are_fatal_comm_handler(NULL, -1, btlinfo);
 }
 
 #if OPAL_ENABLE_FT_CR    == 0

ompi/mca/pml/ob1/pml_ob1_recvreq.h

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2016 The University of Tennessee and The University
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2007 High Performance Computing Center Stuttgart,
@@ -179,7 +179,9 @@ recv_request_pml_complete(mca_pml_ob1_recv_request_t *recvreq)
 
         if(true == recvreq->req_recv.req_base.req_free_called) {
             if( MPI_SUCCESS != recvreq->req_recv.req_base.req_ompi.req_status.MPI_ERROR ) {
-                ompi_mpi_abort(&ompi_mpi_comm_world.comm, MPI_ERR_REQUEST);
+                /* An error after freeing the request MUST be fatal
+                 * MPI3 ch3.7: MPI_REQUEST_FREE */
+                ompi_mpi_errors_are_fatal_comm_handler(NULL, MPI_ERR_REQUEST, "Recv error after request freed");
             }
             MCA_PML_OB1_RECV_REQUEST_RETURN(recvreq);
         } else {

ompi/mca/pml/ob1/pml_ob1_sendreq.h

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2016 The University of Tennessee and The University
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -276,7 +276,9 @@ send_request_pml_complete(mca_pml_ob1_send_request_t *sendreq)
                 MCA_PML_OB1_SEND_REQUEST_MPI_COMPLETE(sendreq, true);
             } else {
                 if( MPI_SUCCESS != sendreq->req_send.req_base.req_ompi.req_status.MPI_ERROR ) {
-                    ompi_mpi_abort(&ompi_mpi_comm_world.comm, MPI_ERR_REQUEST);
+                    /* An error after freeing the request MUST be fatal
+                     * MPI3 ch3.7: MPI_REQUEST_FREE */
+                    ompi_mpi_errors_are_fatal_comm_handler(NULL, MPI_ERR_REQUEST, "Send error after request freed");
                 }
             }
         } else {

ompi/mca/pml/ucx/pml_ucx_datatype.c

@@ -2,6 +2,9 @@
  * Copyright (C) Mellanox Technologies Ltd. 2001-2011.  ALL RIGHTS RESERVED.
  * Copyright (c) 2019      Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2020      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -176,8 +179,12 @@ pml_ucx_datatype_t *mca_pml_ucx_init_nbx_datatype(ompi_datatype_t *datatype,
 
     pml_datatype = malloc(sizeof(*pml_datatype));
     if (pml_datatype == NULL) {
+        int err = MPI_ERR_INTERN;
         PML_UCX_ERROR("Failed to allocate datatype structure");
-        ompi_mpi_abort(&ompi_mpi_comm_world.comm, 1);
+        /* TODO: this error should return to the caller and invoke an error
+         * handler from the MPI API call.
+         * For now, it is fatal. */
+        ompi_mpi_errors_are_fatal_comm_handler(NULL, &err, "Failed to allocate datatype structure");
     }
 
     pml_datatype->datatype                    = ucp_datatype;
@@ -219,8 +226,12 @@ ucp_datatype_t mca_pml_ucx_init_datatype(ompi_datatype_t *datatype)
     status = ucp_dt_create_generic(&pml_ucx_generic_datatype_ops,
                                    datatype, &ucp_datatype);
     if (status != UCS_OK) {
+        int err = MPI_ERR_INTERN;
         PML_UCX_ERROR("Failed to create UCX datatype for %s", datatype->name);
-        ompi_mpi_abort(&ompi_mpi_comm_world.comm, 1);
+        /* TODO: this error should return to the caller and invoke an error
+         * handler from the MPI API call.
+         * For now, it is fatal. */
+        ompi_mpi_errors_are_fatal_comm_handler(NULL, &err, "Failed to allocate datatype structure");
     }
 
     /* Add custom attribute, to clean up UCX resources when OMPI datatype is
@@ -234,9 +245,13 @@ ucp_datatype_t mca_pml_ucx_init_datatype(ompi_datatype_t *datatype)
                               ompi_pml_ucx.datatype_attr_keyval,
                               (void*)ucp_datatype, false);
         if (ret != OMPI_SUCCESS) {
+            int err = MPI_ERR_INTERN;
             PML_UCX_ERROR("Failed to add UCX datatype attribute for %s: %d",
                           datatype->name, ret);
-            ompi_mpi_abort(&ompi_mpi_comm_world.comm, 1);
+            /* TODO: this error should return to the caller and invoke an error
+             * handler from the MPI API call.
+             * For now, it is fatal. */
+            ompi_mpi_errors_are_fatal_comm_handler(NULL, &err, "Failed to allocate datatype structure");
         }
     }
 out:

ompi/mca/vprotocol/pessimist/vprotocol_pessimist_sender_based.c

@@ -1,6 +1,7 @@
 /*
- * Copyright (c) 2004-2014 The Trustees of the University of Tennessee.
- *                         All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
  * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
  * $COPYRIGHT$
  *
@@ -56,7 +57,7 @@ static void sb_mmap_alloc(void)
         V_OUTPUT_ERR("pml_v: vprotocol_pessimist: sender_based_alloc: ftruncate: %s",
                      strerror(errno));
         close(sb.sb_fd);
-        ompi_mpi_abort(MPI_COMM_NULL, MPI_ERR_NO_SPACE);
+        ompi_mpi_abort(MPI_COMM_SELF, MPI_ERR_NO_SPACE);
     }
     sb.sb_addr = (uintptr_t) mmap((void *) sb.sb_addr, sb.sb_length,
                                   PROT_WRITE | PROT_READ,
@@ -67,7 +68,7 @@ static void sb_mmap_alloc(void)
         V_OUTPUT_ERR("pml_v: vprotocol_pessimist: sender_based_alloc: mmap: %s",
                      strerror(errno));
         close(sb.sb_fd);
-        ompi_mpi_abort(MPI_COMM_NULL, MPI_ERR_NO_SPACE);
+        ompi_mpi_abort(MPI_COMM_SELF, MPI_ERR_NO_SPACE);
     }
 }