Move from the use of regex to compression

We've been fighting the battle of trying to create a regex generator and
parser that can handle arbitrary hostname schemes - without long-term
success. The worst of it is that there is no way of checking to see if
the computed regex is correct short of parsing it and doing a
character-by-character comparison with the original string. Ugh...there
has to be a better solution.

One option is to investigate using 3rd-party regex libraries as
those are coming from communities whose sole focus is resolving that
problem. However, someone would need to spend the time to investigate
it, and we'd have to find a license-friendly implementation.

Another option is to quit beating our heads against the wall and just
compress the information. It won't be as much of a reduction, but we
also won't keep hitting scenarios where things break. In this case, it
seems that "perfection" is definitely the enemy of "good enough".

This PR implements the compression option while retaining the
possibility of people adding regex-generating components. The
compression code used in ORTE is consolidated into the opal/compress
framework. That framework currently held bzip and gzip components for
use in compressing checkpoint files - since we no longer support C/R, I
have .opal_ignore'd those components.

However, I have left the original framework APIs alone in case someone
ever decides to redo C/R. The APIs of interest here are added to the
framework - specifically, the "compress_block" and "decompress_block"
functions. I then moved the ORTE zlib compression code into a new
component in this framework.

Unfortunately, the framework currently is a single-select one - i.e.,
only one active component at a time. Since I .opal_ignore'd the other
two and made the priority of zlib high, this isn't a problem. However,
if someone wants to re-enable bzip/gzip or add another component, they
might need to transition opal/compress to a multi-select framework.

Included changes:

* Consolidate the compression code into the opal/compress framework

* Move the ORTE zlib compression code into a new opal/compress/zlib
  component

* Ignore the bzip and gzip components in opal/compress framework

* Add a "compress_base_limit" MCA param to set the threshold above which
  we compress data - defaults to 4096 bytes

* Delete stale brucks and rcd components from orte/grpcomm framework

* Delete the orte/regx framework

* Update the launch system to use opal/compress instead of string regex

* Provide a default module if no zlib is available

* Fix some misc multi-node issues

* Properly generate the nidmap in response to a "connection warmup"
  message so the remote daemon knows the children it needs to launch.

* Remove stale references to orte_node_regex

* opal_byte_object_t's are not OPAL objects - properly release allocated
  memory.

* Set the topology

* Currently only handling homogeneous case

* Update the compress framework files to conform

* Consolidate open/close into one "frame" file. Ensure we open/close the
  framework

Signed-off-by: Ralph Castain <rhc@pmix.org>

Author: rhc54

ompi/op/op.h

@@ -18,6 +18,8 @@
  * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
  * Copyright (c) 2019      Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2018      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow

opal/mca/compress/base/Makefile.am

@@ -3,6 +3,7 @@
 #                         University Research and Technology
 #                         Corporation.  All rights reserved.
 # Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2019      Intel, Inc.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -14,7 +15,6 @@ headers += \
         base/base.h
 
 libmca_compress_la_SOURCES += \
-        base/compress_base_open.c \
-        base/compress_base_close.c \
+        base/compress_base_frame.c \
         base/compress_base_select.c \
         base/compress_base_fns.c

opal/mca/compress/base/base.h

@@ -3,6 +3,7 @@
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
  *
+ * Copyright (c) 2019      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -27,6 +28,12 @@
 extern "C" {
 #endif
 
+typedef struct {
+    size_t compress_limit;
+} opal_compress_base_t;
+
+OPAL_DECLSPEC extern opal_compress_base_t opal_compress_base;
+
     /**
      * Initialize the COMPRESS MCA framework
      *

opal/mca/compress/base/compress_base_close.c

@@ -6,6 +6,7 @@
  *                         All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2019      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -23,14 +24,31 @@
 /*
  * Globals
  */
+static bool compress_block(uint8_t *inbytes,
+                           size_t inlen,
+                           uint8_t **outbytes,
+                           size_t *olen)
+{
+    return false;
+}
+
+static bool decompress_block(uint8_t **outbytes, size_t olen,
+                             uint8_t *inbytes, size_t len)
+{
+    return false;
+}
+
 opal_compress_base_module_t opal_compress = {
     NULL, /* init             */
     NULL, /* finalize         */
     NULL, /* compress         */
     NULL, /* compress_nb      */
     NULL, /* decompress       */
-    NULL  /* decompress_nb    */
+    NULL,  /* decompress_nb    */
+    compress_block,
+    decompress_block
 };
+opal_compress_base_t opal_compress_base = {0};
 
 opal_compress_base_component_t opal_compress_base_selected_component = {{0}};
 
@@ -42,6 +60,12 @@ MCA_BASE_FRAMEWORK_DECLARE(opal, compress, "COMPRESS MCA",
 
 static int opal_compress_base_register(mca_base_register_flag_t flags)
 {
+    opal_compress_base.compress_limit = 4096;
+    (void) mca_base_var_register("opal", "compress", "base", "limit",
+                                 "Threshold beyond which data will be compressed",
+                                 MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, OPAL_INFO_LVL_3,
+                                 MCA_BASE_VAR_SCOPE_READONLY, &opal_compress_base.compress_limit);
+
     return OPAL_SUCCESS;
 }
 
@@ -51,13 +75,17 @@ static int opal_compress_base_register(mca_base_register_flag_t flags)
  */
 int opal_compress_base_open(mca_base_open_flag_t flags)
 {
-    /* Compression currently only used with C/R */
-    if(!opal_cr_is_enabled) {
-        opal_output_verbose(10, opal_compress_base_framework.framework_output,
-                            "compress:open: FT is not enabled, skipping!");
-        return OPAL_SUCCESS;
-    }
-
     /* Open up all available components */
     return mca_base_framework_components_open(&opal_compress_base_framework, flags);
 }
+
+int opal_compress_base_close(void)
+{
+    /* Call the component's finalize routine */
+    if( NULL != opal_compress.finalize ) {
+        opal_compress.finalize();
+    }
+
+    /* Close all available modules that are open */
+    return mca_base_framework_components_close (&opal_compress_base_framework, NULL);
+}

opal/mca/compress/base/compress_base_open.c renamed to opal/mca/compress/base/compress_base_frame.c

@@ -7,6 +7,7 @@
  *
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2019      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -29,26 +30,19 @@
 
 int opal_compress_base_select(void)
 {
-    int ret, exit_status = OPAL_SUCCESS;
+    int ret = OPAL_SUCCESS;
     opal_compress_base_component_t *best_component = NULL;
     opal_compress_base_module_t *best_module = NULL;
 
-    /* Compression currently only used with C/R */
-    if( !opal_cr_is_enabled ) {
-        opal_output_verbose(10, opal_compress_base_framework.framework_output,
-                            "compress:open: FT is not enabled, skipping!");
-        return OPAL_SUCCESS;
-    }
-
     /*
      * Select the best component
      */
     if( OPAL_SUCCESS != mca_base_select("compress", opal_compress_base_framework.framework_output,
                                         &opal_compress_base_framework.framework_components,
                                         (mca_base_module_t **) &best_module,
                                         (mca_base_component_t **) &best_component, NULL) ) {
-        /* This will only happen if no component was selected */
-        exit_status = OPAL_ERROR;
+        /* This will only happen if no component was selected,
+         * in which case we use the default one */
         goto cleanup;
     }
 
@@ -58,12 +52,11 @@ int opal_compress_base_select(void)
     /* Initialize the winner */
     if (NULL != best_module) {
         if (OPAL_SUCCESS != (ret = best_module->init()) ) {
-            exit_status = ret;
             goto cleanup;
         }
         opal_compress = *best_module;
     }
 
  cleanup:
-    return exit_status;
+    return ret;
 }

opal/mca/compress/base/compress_base_select.c

@@ -4,6 +4,7 @@
  *                         All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2019      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -65,22 +66,39 @@ opal_compress_bzip_component_t mca_compress_bzip_component = {
     }
 };
 
+static bool nocompress(uint8_t *inbytes,
+                       size_t inlen,
+                       uint8_t **outbytes,
+                       size_t *olen)
+{
+    return false;
+}
+
+static bool nodecompress(uint8_t **outbytes, size_t olen,
+                         uint8_t *inbytes, size_t len)
+{
+    return false;
+}
+
 /*
  * Bzip module
  */
 static opal_compress_base_module_t loc_module = {
     /** Initialization Function */
-    opal_compress_bzip_module_init,
+    .init = opal_compress_bzip_module_init,
     /** Finalization Function */
-    opal_compress_bzip_module_finalize,
+    .finalize = opal_compress_bzip_module_finalize,
 
     /** Compress Function */
-    opal_compress_bzip_compress,
-    opal_compress_bzip_compress_nb,
+    .compress = opal_compress_bzip_compress,
+    .compress_nb = opal_compress_bzip_compress_nb,
 
     /** Decompress Function */
-    opal_compress_bzip_decompress,
-    opal_compress_bzip_decompress_nb
+    .decompress = opal_compress_bzip_decompress,
+    .decompress_nb = opal_compress_bzip_decompress_nb,
+
+    .compress_block = nocompress,
+    .decompress_block = nodecompress
 };
 
 static int compress_bzip_register (void)

orte/mca/grpcomm/brucks/.opal_ignore renamed to opal/mca/compress/bzip/.opal_ignore

@@ -6,6 +6,7 @@
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
  *
+ * Copyright (c) 2019      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -82,6 +83,20 @@ typedef int (*opal_compress_base_module_decompress_fn_t)
 typedef int (*opal_compress_base_module_decompress_nb_fn_t)
     (char * cname, char **fname, pid_t *child_pid);
 
+/**
+ * Compress a string
+ *
+ * Arguments:
+ *
+ */
+typedef bool (*opal_compress_base_module_compress_string_fn_t)(uint8_t *inbytes,
+                                                               size_t inlen,
+                                                               uint8_t **outbytes,
+                                                               size_t *olen);
+typedef bool (*opal_compress_base_module_decompress_string_fn_t)(uint8_t **outbytes, size_t olen,
+                                                                 uint8_t *inbytes, size_t len);
+
+
 /**
  * Structure for COMPRESS components.
  */
@@ -117,6 +132,10 @@ struct opal_compress_base_module_1_0_0_t {
     /** Decompress Interface */
     opal_compress_base_module_decompress_fn_t     decompress;
     opal_compress_base_module_decompress_nb_fn_t  decompress_nb;
+
+    /* COMPRESS STRING */
+    opal_compress_base_module_compress_string_fn_t      compress_block;
+    opal_compress_base_module_decompress_string_fn_t    decompress_block;
 };
 typedef struct opal_compress_base_module_1_0_0_t opal_compress_base_module_1_0_0_t;
 typedef struct opal_compress_base_module_1_0_0_t opal_compress_base_module_t;