Add compliance mode with MPI-4 routing of errors to MPI_COMM_SELF by

default
And other streamlining of aborting behavior.

Signed-off-by: Aurélien Bouteiller <bouteill@icl.utk.edu>

Remove OMPI_COMM_ERRORS and use NOHANDLE macros instead.

Signed-off-by: Aurelien Bouteiller <bouteill@icl.utk.edu>

route unbound errors to self error handler

Signed-off-by: Aurelien Bouteiller <bouteill@icl.utk.edu>

Do not raise the error handler from within components

Signed-off-by: Aurelien Bouteiller <bouteill@icl.utk.edu>

Author: abouteiller

ompi/errhandler/errhandler.c

@@ -318,7 +318,7 @@ void ompi_errhandler_callback(size_t refid, pmix_status_t status,
         cbfunc(PMIX_EVENT_ACTION_COMPLETE, NULL, 0, NULL, NULL, cbdata);
     }
     /* our default action is to abort */
-    ompi_mpi_abort(MPI_COMM_WORLD, status);
+    OMPI_ERRHANDLER_NOHANDLE_INVOKE(status, "PMIx Event notification");
 }
 
 /**************************************************************************

ompi/errhandler/errhandler.h

@@ -38,6 +38,7 @@
 #include "opal/mca/pmix/pmix-internal.h"
 
 #include "ompi/runtime/mpiruntime.h"
+#include "ompi/runtime/params.h"
 #include "ompi/errhandler/errhandler_predefined.h"
 #include "ompi/errhandler/errcode-internal.h"
 
@@ -256,10 +257,19 @@ struct ompi_request_t;
  */
 #define OMPI_ERRHANDLER_INVOKE(mpi_object, err_code, message) \
   ompi_errhandler_invoke((mpi_object)->error_handler, \
-			 (mpi_object), \
+                         (mpi_object), \
                          (int)(mpi_object)->errhandler_type, \
                          ompi_errcode_get_mpi_code(err_code), \
-			 (message));
+                         (message));
+
+/**
+ * This is the macro to route errors to the 'default' communicator
+ * for non-handle attached errors (e.g., a datatype operation error).
+ */
+#define OMPI_ERRHANDLER_NOHANDLE_INVOKE(err_code, message) \
+    ompi_errhandler_invoke(NULL, NULL, -1, \
+                           ompi_errcode_get_mpi_code(err_code), \
+                           (message));
 
 /**
  * Conditionally invoke an MPI error handler.
@@ -279,13 +289,26 @@ struct ompi_request_t;
     int __mpi_err_code = ompi_errcode_get_mpi_code(err_code);         \
     OPAL_CR_EXIT_LIBRARY() \
     ompi_errhandler_invoke((mpi_object)->error_handler, \
-			   (mpi_object), \
+                           (mpi_object), \
                            (int) (mpi_object)->errhandler_type, \
                            (__mpi_err_code), \
                            (message)); \
     return (__mpi_err_code); \
   }
 
+/* Same as OMPI_ERRHANDLER_CHECK for non-handle attached errors */
+#define OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, err_code, message) \
+  if( OPAL_UNLIKELY(rc != OMPI_SUCCESS) ) { \
+    int __mpi_err_code = ompi_errcode_get_mpi_code(err_code);         \
+    OPAL_CR_EXIT_LIBRARY() \
+    ompi_errhandler_invoke(NULL, \
+                           NULL, \
+                           -1, \
+                           (__mpi_err_code), \
+                           (message)); \
+    return (__mpi_err_code); \
+  }
+
 /**
  * Conditionally invoke an MPI error handler; if there is no error,
  * return MPI_SUCCESS.
@@ -315,7 +338,20 @@ struct ompi_request_t;
     return MPI_SUCCESS; \
   }
 
-
+/* Same as OMPI_ERRHANDLER_RETURN for non-handle attached errors */
+#define OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, err_code, message) \
+  OPAL_CR_EXIT_LIBRARY() \
+  if ( OPAL_UNLIKELY(OMPI_SUCCESS != rc) ) { \
+    int __mpi_err_code = ompi_errcode_get_mpi_code(err_code);         \
+    ompi_errhandler_invoke(NULL, \
+                           NULL, \
+                           -1, \
+                           (__mpi_err_code), \
+                           (message)); \
+    return (__mpi_err_code); \
+  } else { \
+    return MPI_SUCCESS; \
+  }
 
   /**
    * Initialize the error handler interface.

ompi/errhandler/errhandler_invoke.c

@@ -47,7 +47,7 @@ int ompi_errhandler_invoke(ompi_errhandler_t *errhandler, void *mpi_object,
         int32_t state = ompi_mpi_state;
         if (state >= OMPI_MPI_STATE_INIT_COMPLETED &&
             state < OMPI_MPI_STATE_FINALIZE_PAST_COMM_SELF_DESTRUCT) {
-            comm = &ompi_mpi_comm_self.comm;
+            comm = (ompi_mpi_errors_mpi3)? &ompi_mpi_comm_world.comm: &ompi_mpi_comm_self.comm;
             comm->error_handler->eh_comm_fn(&comm, &err_code, message, NULL);
         }
         else {

ompi/mca/fcoll/two_phase/fcoll_two_phase_support_fns.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -171,12 +171,16 @@ int mca_fcoll_two_phase_calc_aggregator(ompio_file_t *fh,
     }
 
     if (rank_index >= num_aggregators || rank_index < 0) {
+       int err = MPI_ERR_INTERN;
        fprintf(stderr,
 	       "Error in mca_fcoll_two_phase_calc_aggregator:");
        fprintf(stderr,
 	       "rank_index(%d) >= num_aggregators(%d) fd_size=%lld off=%lld min_off=%lld striping_unit=%d\n",
 	       rank_index, num_aggregators, fd_size, off, min_off, striping_unit);
-       ompi_mpi_abort(&ompi_mpi_comm_world.comm, 1);
+       /* TODO: this error should return to the caller and invoke an error
+        * handler from the MPI API call.
+        * For now, it is fatal. */
+       ompi_mpi_errors_are_fatal_file_handler(NULL, &err, "Invalid rank in fcoll aggregator");
     }
 
 

ompi/mca/osc/base/osc_base_obj_convert.c

@@ -2,8 +2,9 @@
 /*
  * Copyright (c) 2004-2005 The Trustees of Indiana University.
  *                         All rights reserved.
- * Copyright (c) 2004-2006 The Trustees of the University of Tennessee.
- *                         All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
@@ -84,16 +85,20 @@ int ompi_osc_base_process_op (void *outbuf, void *inbuf, size_t inbuflen,
            MPI_DOUBLE_INT == datatype ||
            MPI_LONG_INT == datatype ||
            MPI_LONG_DOUBLE_INT == datatype) {
-           ompi_communicator_t *comm = &ompi_mpi_comm_world.comm;
-           opal_output(0, "Error: %s datatype is currently "
-                       "unsupported for MPI_MINLOC/MPI_MAXLOC "
-                       "operation\n", datatype->name);
-           opal_show_help("help-mpi-api.txt", "mpi-abort", true,
-                          comm->c_my_rank,
-                          ('\0' != comm->c_name[0]) ? comm->c_name : "<Unknown>",
-                          -1);
-
-           ompi_mpi_abort(comm, -1);
+            int err = MPI_ERR_UNSUPPORTED_DATAREP;
+            char *reason = NULL;
+            opal_asprintf(&reason,
+                    "%s datatype is currently "
+                    "unsupported for MPI_MINLOC/MPI_MAXLOC "
+                    "operation\n", datatype->name);
+            opal_show_help("help-mpi-api.txt", "MPI function not supported", true,
+                          (MPI_MINLOC==op)?"MPI_MINLOC":"MPI_MAXLOC",
+                          reason);
+            free(reason);
+            /* TODO: this error should return to the caller and invoke an error
+             * handler from the MPI API call.
+             * For now, it is fatal. */
+            ompi_mpi_errors_are_fatal_win_handler(NULL, &err, "OSC unsupported MINLOC/MAXLOC datatype");
         }
     }
 

ompi/mca/pml/ob1/pml_ob1_recvreq.h

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2016 The University of Tennessee and The University
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2007 High Performance Computing Center Stuttgart,
@@ -179,7 +179,7 @@ recv_request_pml_complete(mca_pml_ob1_recv_request_t *recvreq)
 
         if(true == recvreq->req_recv.req_base.req_free_called) {
             if( MPI_SUCCESS != recvreq->req_recv.req_base.req_ompi.req_status.MPI_ERROR ) {
-                ompi_mpi_abort(&ompi_mpi_comm_world.comm, MPI_ERR_REQUEST);
+                OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_REQUEST, "Recv error after request freed.");
             }
             MCA_PML_OB1_RECV_REQUEST_RETURN(recvreq);
         } else {

ompi/mca/pml/ob1/pml_ob1_sendreq.h

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2016 The University of Tennessee and The University
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -276,7 +276,7 @@ send_request_pml_complete(mca_pml_ob1_send_request_t *sendreq)
                 MCA_PML_OB1_SEND_REQUEST_MPI_COMPLETE(sendreq, true);
             } else {
                 if( MPI_SUCCESS != sendreq->req_send.req_base.req_ompi.req_status.MPI_ERROR ) {
-                    ompi_mpi_abort(&ompi_mpi_comm_world.comm, MPI_ERR_REQUEST);
+                    OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_REQUEST, "Send error after request freed");
                 }
             }
         } else {

ompi/mca/pml/ucx/pml_ucx_datatype.c

@@ -2,6 +2,9 @@
  * Copyright (C) Mellanox Technologies Ltd. 2001-2011.  ALL RIGHTS RESERVED.
  * Copyright (c) 2019      Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2020      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -176,8 +179,12 @@ pml_ucx_datatype_t *mca_pml_ucx_init_nbx_datatype(ompi_datatype_t *datatype,
 
     pml_datatype = malloc(sizeof(*pml_datatype));
     if (pml_datatype == NULL) {
+        int err = MPI_ERR_INTERN;
         PML_UCX_ERROR("Failed to allocate datatype structure");
-        ompi_mpi_abort(&ompi_mpi_comm_world.comm, 1);
+        /* TODO: this error should return to the caller and invoke an error
+         * handler from the MPI API call.
+         * For now, it is fatal. */
+        ompi_mpi_errors_are_fatal_comm_handler(NULL, &err, "Failed to allocate datatype structure");
     }
 
     pml_datatype->datatype                    = ucp_datatype;
@@ -219,8 +226,12 @@ ucp_datatype_t mca_pml_ucx_init_datatype(ompi_datatype_t *datatype)
     status = ucp_dt_create_generic(&pml_ucx_generic_datatype_ops,
                                    datatype, &ucp_datatype);
     if (status != UCS_OK) {
+        int err = MPI_ERR_INTERN;
         PML_UCX_ERROR("Failed to create UCX datatype for %s", datatype->name);
-        ompi_mpi_abort(&ompi_mpi_comm_world.comm, 1);
+        /* TODO: this error should return to the caller and invoke an error
+         * handler from the MPI API call.
+         * For now, it is fatal. */
+        ompi_mpi_errors_are_fatal_comm_handler(NULL, &err, "Failed to allocate datatype structure");
     }
 
     /* Add custom attribute, to clean up UCX resources when OMPI datatype is
@@ -234,9 +245,13 @@ ucp_datatype_t mca_pml_ucx_init_datatype(ompi_datatype_t *datatype)
                               ompi_pml_ucx.datatype_attr_keyval,
                               (void*)ucp_datatype, false);
         if (ret != OMPI_SUCCESS) {
+            int err = MPI_ERR_INTERN;
             PML_UCX_ERROR("Failed to add UCX datatype attribute for %s: %d",
                           datatype->name, ret);
-            ompi_mpi_abort(&ompi_mpi_comm_world.comm, 1);
+            /* TODO: this error should return to the caller and invoke an error
+             * handler from the MPI API call.
+             * For now, it is fatal. */
+            ompi_mpi_errors_are_fatal_comm_handler(NULL, &err, "Failed to allocate datatype structure");
         }
     }
 out:

ompi/mca/vprotocol/pessimist/vprotocol_pessimist_sender_based.c

@@ -1,6 +1,7 @@
 /*
- * Copyright (c) 2004-2014 The Trustees of the University of Tennessee.
- *                         All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
  * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
  * $COPYRIGHT$
  *
@@ -56,7 +57,7 @@ static void sb_mmap_alloc(void)
         V_OUTPUT_ERR("pml_v: vprotocol_pessimist: sender_based_alloc: ftruncate: %s",
                      strerror(errno));
         close(sb.sb_fd);
-        ompi_mpi_abort(MPI_COMM_NULL, MPI_ERR_NO_SPACE);
+        ompi_mpi_abort(MPI_COMM_SELF, MPI_ERR_NO_SPACE);
     }
     sb.sb_addr = (uintptr_t) mmap((void *) sb.sb_addr, sb.sb_length,
                                   PROT_WRITE | PROT_READ,
@@ -67,7 +68,7 @@ static void sb_mmap_alloc(void)
         V_OUTPUT_ERR("pml_v: vprotocol_pessimist: sender_based_alloc: mmap: %s",
                      strerror(errno));
         close(sb.sb_fd);
-        ompi_mpi_abort(MPI_COMM_NULL, MPI_ERR_NO_SPACE);
+        ompi_mpi_abort(MPI_COMM_SELF, MPI_ERR_NO_SPACE);
     }
 }
 

ompi/mpi/c/abort.c

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2014 The University of Tennessee and The University
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -56,6 +56,7 @@ int MPI_Abort(MPI_Comm comm, int errorcode)
     opal_show_help("help-mpi-api.txt", "mpi-abort", true,
                    ompi_comm_rank(comm),
                    ('\0' != comm->c_name[0]) ? comm->c_name : "<Unknown>",
+                   OMPI_NAME_PRINT(OMPI_PROC_MY_NAME),
                    errorcode);
     return ompi_mpi_abort(comm, errorcode);
 }