Ensure that libdir is always first in search path

Switch the ordering of both the -L and rpath/runpath arguments
added by the wrapper compiler so that the install's libdir
is always the first in the list of paths to search.  Previously,
the component-related paths were added first, which could cause
applications to find an errant (to us, anyway) libmpi.

Signed-off-by: Brian Barrett <bbarrett@amazon.com>

Author: bwbarrett

config/opal_setup_wrappers.m4

@@ -17,6 +17,8 @@ dnl                         and Technology (RIST). All rights reserved.
 dnl Copyright (c) 2016      IBM Corporation.  All rights reserved.
 dnl Copyright (c) 2020      Triad National Security, LLC. All rights
 dnl                         reserved.
+dnl Copyright (c) 2021      Amazon.com, Inc. or its affiliates.
+dnl                         All Rights reserved.
 dnl $COPYRIGHT$
 dnl
 dnl Additional copyrights may follow
@@ -291,9 +293,11 @@ AC_DEFUN([RPATHIFY_LDFLAGS_INTERNAL],[
                esac
            done
 
-           # Now add in the RPATH args for @{libdir}, and the RUNPATH args
+           # add in the RPATH args for @{libdir}, and the RUNPATH
+           # args.  The install libdir goes first, so that we prefer
+           # our libmpi over any imposter libmpi we might find.
            rpath_tmp=`echo ${$2} | sed -e s/LIBDIR/@{libdir}/`
-           $1="${$1} $rpath_out $rpath_tmp ${$3}"
+           $1="${$1} $rpath_tmp $rpath_out ${$3}"
           ])
     OPAL_VAR_SCOPE_POP
 ])

opal/tools/wrappers/opal_wrapper.c

@@ -15,7 +15,7 @@
  * Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2018-2021 Amazon.com, Inc. or its affiliates.  All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -359,7 +359,7 @@ static void data_callback(const char *key, const char *value)
                 char *line;
                 opal_asprintf(&line, OPAL_INCLUDE_FLAG "%s",
                               options_data[parse_options_idx].path_includedir);
-                opal_argv_append_nosize(&options_data[parse_options_idx].preproc_flags, line);
+                opal_argv_prepend_nosize(&options_data[parse_options_idx].preproc_flags, line);
                 free(line);
             }
         }
@@ -371,7 +371,7 @@ static void data_callback(const char *key, const char *value)
             char *line;
             opal_asprintf(&line, OPAL_LIBDIR_FLAG "%s",
                           options_data[parse_options_idx].path_libdir);
-            opal_argv_append_nosize(&options_data[parse_options_idx].link_flags, line);
+            opal_argv_prepend_nosize(&options_data[parse_options_idx].link_flags, line);
             free(line);
         }
     } else if (0 == strcmp(key, "opalincludedir")) {