Merge pull request #275 from marklogic/feature/collated-string

Verifying that collatedString works

Author: rjrudin

build.gradle

@@ -21,6 +21,7 @@ java {
 
 repositories {
   mavenCentral()
+  mavenLocal()
   maven {
     url "https://bed-artifactory.bedford.progress.com:443/artifactory/ml-maven-snapshots/"
   }

src/main/java/com/marklogic/spark/reader/optic/SchemaInferrer.java

@@ -54,6 +54,7 @@ public abstract class SchemaInferrer {
         COLUMN_INFO_TYPES_TO_SPARK_TYPES.put("yearMonthDuration", DataTypes.StringType);
         COLUMN_INFO_TYPES_TO_SPARK_TYPES.put("dayTimeDuration", DataTypes.StringType);
         COLUMN_INFO_TYPES_TO_SPARK_TYPES.put("string", DataTypes.StringType);
+        COLUMN_INFO_TYPES_TO_SPARK_TYPES.put("collatedString", DataTypes.StringType);
         COLUMN_INFO_TYPES_TO_SPARK_TYPES.put("anyUri", DataTypes.StringType);
         COLUMN_INFO_TYPES_TO_SPARK_TYPES.put("point", DataTypes.StringType);
         COLUMN_INFO_TYPES_TO_SPARK_TYPES.put("boolean", DataTypes.BooleanType);

src/test/java/com/marklogic/spark/reader/optic/InferSchemaTest.java

@@ -36,7 +36,7 @@ void inferFromAllTdeTypes() throws Exception {
         String columnInfoResponse = readClasspathFile("allTypes-columnInfo-response.txt");
         StructType schema = SchemaInferrer.inferSchema(columnInfoResponse);
 
-        assertEquals(35, schema.size(), "The TDE has 35 columns, and the hidden 'rowid' column that's returned " +
+        assertEquals(36, schema.size(), "The TDE has 36 columns, and the hidden 'rowid' column that's returned " +
             "by /v1/rows should not be included in the Spark schema, as it will never be populated with a value.");
 
         String actualJson = schema.prettyJson();

src/test/java/com/marklogic/spark/reader/optic/ReadRowsWithInferredSchemaTest.java

@@ -73,25 +73,26 @@ void allTypes() {
         assertEquals("P2Y6M", row.getString(14)); // yearMonthDuration
         assertEquals("PT1M", row.getString(15)); // dayTimeDuration
         assertEquals("hello", row.getString(16));
-        assertEquals("http://example.org/", row.getString(17)); // anyURI
-        assertEquals("50,50", row.getString(18)); // point
-        assertEquals("50,50", row.getString(19)); // longLatPoint
-        assertTrue(row.getBoolean(20));
-        assertEquals("c2xpbmdzIGFuZCBhcnJvd3Mgb2Ygb3V0cmFnZW91cyBmb3J0dW5l", row.getString(21)); // base64Binary
-        assertEquals("499602D2", row.getString(22)); // hexBinary
-        assertEquals("1", row.getString(23), "Because MarkLogic defines the type of 'byte' as 'none', the Spark " +
+        assertEquals("hello collated", row.getString(17));
+        assertEquals("http://example.org/", row.getString(18)); // anyURI
+        assertEquals("50,50", row.getString(19)); // point
+        assertEquals("50,50", row.getString(20)); // longLatPoint
+        assertTrue(row.getBoolean(21));
+        assertEquals("c2xpbmdzIGFuZCBhcnJvd3Mgb2Ygb3V0cmFnZW91cyBmb3J0dW5l", row.getString(22)); // base64Binary
+        assertEquals("499602D2", row.getString(23)); // hexBinary
+        assertEquals("1", row.getString(24), "Because MarkLogic defines the type of 'byte' as 'none', the Spark " +
             "connector treats it as a string."); // byte
-        assertEquals("PT1M", row.getString(24)); // duration
-        assertEquals("--04-18", row.getString(25)); // gMonthDay
-        assertEquals(1, row.getInt(26));
-        assertEquals(-1, row.getInt(27)); // negativeInteger
-        assertEquals(11, row.getInt(28)); // nonNegativeInteger
-        assertEquals(-11, row.getInt(29)); // nonPositiveInteger
-        assertEquals(20, row.getInt(30)); // positiveInteger
-        assertEquals(7, row.getInt(31)); // short
-        assertEquals(4, row.getInt(32)); // unsignedByte
-        assertEquals(8, row.getInt(33)); // unsignedShort
-        assertEquals("http://example.org/", row.getString(34)); // IRI
+        assertEquals("PT1M", row.getString(25)); // duration
+        assertEquals("--04-18", row.getString(26)); // gMonthDay
+        assertEquals(1, row.getInt(27));
+        assertEquals(-1, row.getInt(28)); // negativeInteger
+        assertEquals(11, row.getInt(29)); // nonNegativeInteger
+        assertEquals(-11, row.getInt(30)); // nonPositiveInteger
+        assertEquals(20, row.getInt(31)); // positiveInteger
+        assertEquals(7, row.getInt(32)); // short
+        assertEquals(4, row.getInt(33)); // unsignedByte
+        assertEquals(8, row.getInt(34)); // unsignedShort
+        assertEquals("http://example.org/", row.getString(35)); // IRI
     }
 
     @Test
@@ -106,9 +107,9 @@ void allColumnsNullExceptRequiredOne() {
         assertEquals(1, rows.size());
 
         Row row = rows.get(0);
-        assertEquals(35, row.size(), "Expecting all 35 columns to still exist, even though all but one have a null value");
+        assertEquals(36, row.size(), "Expecting all 36 columns to still exist, even though all but one have a null value");
         assertEquals(2, row.getInt(0));
-        for (int i = 1; i < 35; i++) {
+        for (int i = 1; i < 36; i++) {
             assertNull(row.get(i));
         }
     }

src/test/ml-data/allTypes-data.json

@@ -18,6 +18,7 @@
       "yearMonthDurationValue": "P2Y6M",
       "dayTimeDurationValue": "PT1M",
       "stringValue": "hello",
+      "collatedStringValue": "hello collated",
       "anyURIValue": "http://example.org/",
       "pointValue": "50 50",
       "longLatPointValue": "50 50",

src/test/ml-schemas/tde/allTypes.json

@@ -108,6 +108,13 @@
             "val": "stringValue",
             "nullable": true
           },
+          {
+            "name": "collatedStringValue",
+            "scalarType": "string",
+            "collation": "http://marklogic.com/collation/",
+            "val": "collatedStringValue",
+            "nullable": true
+          },
           {
             "name": "anyURIValue",
             "scalarType": "anyURI",

src/test/resources/allTypes-columnInfo-response.txt

@@ -15,6 +15,7 @@
 {"schema":"sparkTest", "view":"AllTypeRows", "column":"yearMonthDurationValue", "type":"yearMonthDuration", "hidden":false, "nullable":true}
 {"schema":"sparkTest", "view":"AllTypeRows", "column":"dayTimeDurationValue", "type":"dayTimeDuration", "hidden":false, "nullable":true}
 {"schema":"sparkTest", "view":"AllTypeRows", "column":"stringValue", "type":"string", "hidden":false, "nullable":true}
+{"schema":"sparkTest", "view":"AllTypeRows", "column":"collatedStringValue", "type":"collatedString", "hidden":false, "nullable":true}
 {"schema":"sparkTest", "view":"AllTypeRows", "column":"anyURIValue", "type":"anyUri", "hidden":false, "nullable":true}
 {"schema":"sparkTest", "view":"AllTypeRows", "column":"pointValue", "type":"point", "hidden":false, "nullable":true, "coordinate-system":"wgs84"}
 {"schema":"sparkTest", "view":"AllTypeRows", "column":"longLatPointValue", "type":"point", "hidden":false, "nullable":true, "coordinate-system":"wgs84"}

src/test/resources/allTypes-expected-spark-schema.json

@@ -103,6 +103,12 @@
       "nullable": true,
       "metadata": {}
     },
+    {
+      "name": "sparkTest.AllTypeRows.collatedStringValue",
+      "type": "string",
+      "nullable": true,
+      "metadata": {}
+    },
     {
       "name": "sparkTest.AllTypeRows.anyURIValue",
       "type": "string",