Add a test for very large data.

Signed-off-by: George Bosilca <bosilca@icl.utk.edu>

Author: bosilca

opal/datatype/opal_convertor.h

@@ -94,18 +94,22 @@ struct opal_convertor_t {
     const opal_datatype_t*        pDesc;          /**< the datatype description associated with the convertor */
     const dt_type_desc_t*         use_desc;       /**< the version used by the convertor (normal or optimized) */
     opal_datatype_count_t         count;          /**< the total number of full datatype elements */
-    uint32_t                      stack_size;     /**< size of the allocated stack */
 
     /* --- cacheline boundary (64 bytes - if 64bits arch and !OPAL_ENABLE_DEBUG) --- */
+    uint32_t                      stack_size;     /**< size of the allocated stack */
     unsigned char*                pBaseBuf;       /**< initial buffer as supplied by the user */
     dt_stack_t*                   pStack;         /**< the local stack for the actual conversion */
     convertor_advance_fct_t       fAdvance;       /**< pointer to the pack/unpack functions */
+
+    /* --- cacheline boundary (96 bytes - if 64bits arch and !OPAL_ENABLE_DEBUG) --- */
     struct opal_convertor_master_t* master;       /**< the master convertor */
 
     /* All others fields get modified for every call to pack/unpack functions */
     uint32_t                      stack_pos;      /**< the actual position on the stack */
     size_t                        partial_length; /**< amount of data left over from the last unpack */
     size_t                        bConverted;     /**< # of bytes already converted */
+
+    /* --- cacheline boundary (128 bytes - if 64bits arch and !OPAL_ENABLE_DEBUG) --- */
     uint32_t                      checksum;       /**< checksum computed by pack/unpack operation */
     uint32_t                      csum_ui1;       /**< partial checksum computed by pack/unpack operation */
     size_t                        csum_ui2;       /**< partial checksum computed by pack/unpack operation */

opal/datatype/opal_datatype.h

@@ -119,7 +119,6 @@ struct opal_datatype_t {
 
     /* Attribute fields */
     char               name[OPAL_MAX_OBJECT_NAME];  /**< name of the datatype */
-    /* --- cacheline 2 boundary (128 bytes) was 8-12 bytes ago --- */
     dt_type_desc_t     desc;     /**< the data description */
     dt_type_desc_t     opt_desc; /**< short description of the data used when conversion is useless
                                       or in the send case (without conversion) */

test/datatype/Makefile.am

@@ -15,7 +15,7 @@
 #
 
 if PROJECT_OMPI
-    MPI_TESTS = checksum position position_noncontig ddt_test ddt_raw unpack_ooo ddt_pack external32
+    MPI_TESTS = checksum position position_noncontig ddt_test ddt_raw unpack_ooo ddt_pack external32 large_data
     MPI_CHECKS = to_self
 endif
 TESTS = opal_datatype_test unpack_hetero $(MPI_TESTS)
@@ -68,6 +68,12 @@ to_self_SOURCES = to_self.c
 to_self_LDFLAGS = $(OMPI_PKG_CONFIG_LDFLAGS)
 to_self_LDADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la
 
+large_data_SOURCES = large_data.c
+large_data_LDFLAGS = $(OMPI_PKG_CONFIG_LDFLAGS)
+large_data_LDADD = \
+        $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
+        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+
 opal_datatype_test_SOURCES = opal_datatype_test.c opal_ddt_lib.c opal_ddt_lib.h
 opal_datatype_test_LDFLAGS = $(OMPI_PKG_CONFIG_LDFLAGS)
 opal_datatype_test_LDADD = \

test/datatype/large_data.c

@@ -0,0 +1,93 @@
+#include <mpi.h>
+#include <stdio.h>
+#include <stddef.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "ompi_config.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "opal/runtime/opal.h"
+#include "opal/datatype/opal_convertor.h"
+#include "opal/datatype/opal_datatype_internal.h"
+
+#define MAX_IOVEC 10
+#define MAX_CHUNK (1024*1024*1024)  /* 1GB */
+
+static size_t
+count_length_via_convertor_raw(MPI_Datatype dtype, int count)
+{
+    opal_convertor_t* pconv;
+    struct iovec iov[MAX_IOVEC];
+    uint32_t iov_count = MAX_IOVEC, i;
+    size_t length = MAX_CHUNK, packed_iovec = 0, packed = 0;
+
+    pconv = opal_convertor_create( opal_local_arch, 0 );
+    opal_convertor_prepare_for_send(pconv, (const struct opal_datatype_t *)dtype, 1, NULL);
+    while( 0 == opal_convertor_raw(pconv, iov, &iov_count, &length) ) {
+        printf("iov_count = %d packed_iovec = %"PRIsize_t"\n", iov_count, packed_iovec);
+        packed += length;
+        for( i = 0; i < iov_count; i++ ) {
+            packed_iovec += iov[i].iov_len;
+        }
+        if( packed != packed_iovec ) {
+            printf( "Packed send amount diverges %"PRIsize_t" != %"PRIsize_t"\n", packed, packed_iovec);
+            exit(-1);
+        }
+        iov_count = MAX_IOVEC;  /* number of available iov */
+        length = MAX_CHUNK;
+    }
+    packed += length;
+    for( i = 0; i < iov_count; i++ ) {
+        packed_iovec += iov[i].iov_len;
+    }
+    if( packed != packed_iovec ) {
+        printf( "Packed send amount diverges %"PRIsize_t" != %"PRIsize_t"\n", packed, packed_iovec);
+        exit(-1);
+    }
+    return packed_iovec;
+}
+
+int main(int argc, char * argv[])
+{
+
+    int const per_process = 192;
+    int const per_type = 20000000;
+
+    int scounts[2] = {per_process, per_process};
+    int sdispls[2] = {3*per_process, 0*per_process};
+    int rcounts[2] = {per_process, per_process};
+    int rdispls[2] = {1*per_process, 2*per_process};
+
+    MPI_Datatype ddt, stype, rtype;
+
+    opal_init_util(&argc, &argv);
+    ompi_datatype_init();
+
+    ompi_datatype_create_contiguous( per_type, MPI_FLOAT, &ddt);
+    ompi_datatype_create_indexed(2, scounts, sdispls, ddt, &stype);
+    ompi_datatype_commit(&stype);
+    ompi_datatype_create_indexed(2, rcounts, rdispls, ddt, &rtype);
+    ompi_datatype_commit(&rtype);
+
+    size_t packed = count_length_via_convertor_raw(stype, 1);
+    size_t length;
+    opal_datatype_type_size(&stype->super, &length);
+    if( length != packed ) {
+        printf("Mismatched length of packed data to datatype size (%"PRIsize_t" != %"PRIsize_t")\n",
+               packed, length);
+        exit(-2);
+    }
+
+    packed = count_length_via_convertor_raw(rtype, 1);
+    opal_datatype_type_size(&rtype->super, &length);
+    if( length != packed ) {
+        printf("Mismatched length of packed data to datatype size (%"PRIsize_t" != %"PRIsize_t")\n",
+               packed, length);
+        exit(-2);
+    }
+
+    ompi_datatype_destroy(&stype);
+    ompi_datatype_destroy(&rtype);
+
+    return 0;
+}