Merge pull request #6350 from rhc54/topic/reg

Move from the use of regex to compression

Author: rhc54

ompi/op/op.h

@@ -18,6 +18,8 @@
  * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
  * Copyright (c) 2019      Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2018      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow

opal/mca/compress/base/Makefile.am

@@ -3,6 +3,7 @@
 #                         University Research and Technology
 #                         Corporation.  All rights reserved.
 # Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2019      Intel, Inc.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -14,7 +15,6 @@ headers += \
         base/base.h
 
 libmca_compress_la_SOURCES += \
-        base/compress_base_open.c \
-        base/compress_base_close.c \
+        base/compress_base_frame.c \
         base/compress_base_select.c \
         base/compress_base_fns.c

opal/mca/compress/base/base.h

@@ -3,6 +3,7 @@
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
  *
+ * Copyright (c) 2019      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -27,6 +28,12 @@
 extern "C" {
 #endif
 
+typedef struct {
+    size_t compress_limit;
+} opal_compress_base_t;
+
+OPAL_DECLSPEC extern opal_compress_base_t opal_compress_base;
+
     /**
      * Initialize the COMPRESS MCA framework
      *

opal/mca/compress/base/compress_base_close.c

@@ -6,6 +6,7 @@
  *                         All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2019      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -23,14 +24,31 @@
 /*
  * Globals
  */
+static bool compress_block(uint8_t *inbytes,
+                           size_t inlen,
+                           uint8_t **outbytes,
+                           size_t *olen)
+{
+    return false;
+}
+
+static bool decompress_block(uint8_t **outbytes, size_t olen,
+                             uint8_t *inbytes, size_t len)
+{
+    return false;
+}
+
 opal_compress_base_module_t opal_compress = {
     NULL, /* init             */
     NULL, /* finalize         */
     NULL, /* compress         */
     NULL, /* compress_nb      */
     NULL, /* decompress       */
-    NULL  /* decompress_nb    */
+    NULL,  /* decompress_nb    */
+    compress_block,
+    decompress_block
 };
+opal_compress_base_t opal_compress_base = {0};
 
 opal_compress_base_component_t opal_compress_base_selected_component = {{0}};
 
@@ -42,6 +60,12 @@ MCA_BASE_FRAMEWORK_DECLARE(opal, compress, "COMPRESS MCA",
 
 static int opal_compress_base_register(mca_base_register_flag_t flags)
 {
+    opal_compress_base.compress_limit = 4096;
+    (void) mca_base_var_register("opal", "compress", "base", "limit",
+                                 "Threshold beyond which data will be compressed",
+                                 MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, OPAL_INFO_LVL_3,
+                                 MCA_BASE_VAR_SCOPE_READONLY, &opal_compress_base.compress_limit);
+
     return OPAL_SUCCESS;
 }
 
@@ -51,13 +75,17 @@ static int opal_compress_base_register(mca_base_register_flag_t flags)
  */
 int opal_compress_base_open(mca_base_open_flag_t flags)
 {
-    /* Compression currently only used with C/R */
-    if(!opal_cr_is_enabled) {
-        opal_output_verbose(10, opal_compress_base_framework.framework_output,
-                            "compress:open: FT is not enabled, skipping!");
-        return OPAL_SUCCESS;
-    }
-
     /* Open up all available components */
     return mca_base_framework_components_open(&opal_compress_base_framework, flags);
 }
+
+int opal_compress_base_close(void)
+{
+    /* Call the component's finalize routine */
+    if( NULL != opal_compress.finalize ) {
+        opal_compress.finalize();
+    }
+
+    /* Close all available modules that are open */
+    return mca_base_framework_components_close (&opal_compress_base_framework, NULL);
+}

opal/mca/compress/base/compress_base_open.c renamed to opal/mca/compress/base/compress_base_frame.c

@@ -7,6 +7,7 @@
  *
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2019      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -29,26 +30,19 @@
 
 int opal_compress_base_select(void)
 {
-    int ret, exit_status = OPAL_SUCCESS;
+    int ret = OPAL_SUCCESS;
     opal_compress_base_component_t *best_component = NULL;
     opal_compress_base_module_t *best_module = NULL;
 
-    /* Compression currently only used with C/R */
-    if( !opal_cr_is_enabled ) {
-        opal_output_verbose(10, opal_compress_base_framework.framework_output,
-                            "compress:open: FT is not enabled, skipping!");
-        return OPAL_SUCCESS;
-    }
-
     /*
      * Select the best component
      */
     if( OPAL_SUCCESS != mca_base_select("compress", opal_compress_base_framework.framework_output,
                                         &opal_compress_base_framework.framework_components,
                                         (mca_base_module_t **) &best_module,
                                         (mca_base_component_t **) &best_component, NULL) ) {
-        /* This will only happen if no component was selected */
-        exit_status = OPAL_ERROR;
+        /* This will only happen if no component was selected,
+         * in which case we use the default one */
         goto cleanup;
     }
 
@@ -58,12 +52,11 @@ int opal_compress_base_select(void)
     /* Initialize the winner */
     if (NULL != best_module) {
         if (OPAL_SUCCESS != (ret = best_module->init()) ) {
-            exit_status = ret;
             goto cleanup;
         }
         opal_compress = *best_module;
     }
 
  cleanup:
-    return exit_status;
+    return ret;
 }

opal/mca/compress/base/compress_base_select.c

@@ -4,6 +4,7 @@
  *                         All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2019      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -65,22 +66,39 @@ opal_compress_bzip_component_t mca_compress_bzip_component = {
     }
 };
 
+static bool nocompress(uint8_t *inbytes,
+                       size_t inlen,
+                       uint8_t **outbytes,
+                       size_t *olen)
+{
+    return false;
+}
+
+static bool nodecompress(uint8_t **outbytes, size_t olen,
+                         uint8_t *inbytes, size_t len)
+{
+    return false;
+}
+
 /*
  * Bzip module
  */
 static opal_compress_base_module_t loc_module = {
     /** Initialization Function */
-    opal_compress_bzip_module_init,
+    .init = opal_compress_bzip_module_init,
     /** Finalization Function */
-    opal_compress_bzip_module_finalize,
+    .finalize = opal_compress_bzip_module_finalize,
 
     /** Compress Function */
-    opal_compress_bzip_compress,
-    opal_compress_bzip_compress_nb,
+    .compress = opal_compress_bzip_compress,
+    .compress_nb = opal_compress_bzip_compress_nb,
 
     /** Decompress Function */
-    opal_compress_bzip_decompress,
-    opal_compress_bzip_decompress_nb
+    .decompress = opal_compress_bzip_decompress,
+    .decompress_nb = opal_compress_bzip_decompress_nb,
+
+    .compress_block = nocompress,
+    .decompress_block = nodecompress
 };
 
 static int compress_bzip_register (void)

orte/mca/grpcomm/brucks/.opal_ignore renamed to opal/mca/compress/bzip/.opal_ignore

@@ -6,6 +6,7 @@
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
  *
+ * Copyright (c) 2019      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -82,6 +83,20 @@ typedef int (*opal_compress_base_module_decompress_fn_t)
 typedef int (*opal_compress_base_module_decompress_nb_fn_t)
     (char * cname, char **fname, pid_t *child_pid);
 
+/**
+ * Compress a string
+ *
+ * Arguments:
+ *
+ */
+typedef bool (*opal_compress_base_module_compress_string_fn_t)(uint8_t *inbytes,
+                                                               size_t inlen,
+                                                               uint8_t **outbytes,
+                                                               size_t *olen);
+typedef bool (*opal_compress_base_module_decompress_string_fn_t)(uint8_t **outbytes, size_t olen,
+                                                                 uint8_t *inbytes, size_t len);
+
+
 /**
  * Structure for COMPRESS components.
  */
@@ -117,6 +132,10 @@ struct opal_compress_base_module_1_0_0_t {
     /** Decompress Interface */
     opal_compress_base_module_decompress_fn_t     decompress;
     opal_compress_base_module_decompress_nb_fn_t  decompress_nb;
+
+    /* COMPRESS STRING */
+    opal_compress_base_module_compress_string_fn_t      compress_block;
+    opal_compress_base_module_decompress_string_fn_t    decompress_block;
 };
 typedef struct opal_compress_base_module_1_0_0_t opal_compress_base_module_1_0_0_t;
 typedef struct opal_compress_base_module_1_0_0_t opal_compress_base_module_t;