Defaulting to including null fields.

This matches MLCP behavior for delimited files. And null fields are easy to query on and thus useful. And a user can easily drop them as well if desired.

Author: rjrudin

src/main/java/com/marklogic/spark/JsonRowSerializer.java

@@ -61,6 +61,8 @@ public String serializeRowToJson(InternalRow row) {
      */
     private Map<String, String> buildOptionsForJsonOptions(Map<String, String> connectorProperties) {
         Map<String, String> options = new HashMap<>();
+        // Default to include null fields, as they are easily queried in MarkLogic.
+        options.put("ignoreNullFields", "false");
         connectorProperties.forEach((key, value) -> {
             if (key.startsWith(Options.WRITE_JSON_SERIALIZATION_OPTION_PREFIX)) {
                 String optionName = key.substring(Options.WRITE_JSON_SERIALIZATION_OPTION_PREFIX.length());

src/test/java/com/marklogic/spark/writer/WriteNullValuesTest.java renamed to src/test/java/com/marklogic/spark/writer/IgnoreNullValuesTest.java

@@ -4,7 +4,6 @@
 package com.marklogic.spark.writer;
 
 import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.node.JsonNodeType;
 import com.marklogic.junit5.XmlNode;
 import com.marklogic.spark.AbstractIntegrationTest;
 import com.marklogic.spark.Options;
@@ -15,7 +14,7 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 
-class WriteNullValuesTest extends AbstractIntegrationTest {
+class IgnoreNullValuesTest extends AbstractIntegrationTest {
 
     @Test
     void jsonWithEmptyValues() {
@@ -29,21 +28,20 @@ void jsonWithEmptyValues() {
             .option(Options.CLIENT_URI, makeClientUri())
             .option(Options.WRITE_PERMISSIONS, DEFAULT_PERMISSIONS)
             .option(Options.WRITE_URI_TEMPLATE, "/a/{number}.json")
-            .option(Options.WRITE_JSON_SERIALIZATION_OPTION_PREFIX + "ignoreNullFields", "false")
+            .option(Options.WRITE_JSON_SERIALIZATION_OPTION_PREFIX + "ignoreNullFields", "true")
             .mode(SaveMode.Append)
             .save();
 
         JsonNode doc = readJsonDocument("/a/1.json");
         assertEquals(1, doc.get("number").asInt());
         assertEquals("blue", doc.get("color").asText());
-        assertEquals(JsonNodeType.NULL, doc.get("flag").getNodeType());
-        assertEquals(3, doc.size(), "The file path column should not be included in the serialization.");
+        assertEquals(2, doc.size(), "The flag column should not be included in the serialization.");
 
         doc = readJsonDocument("/a/2.json");
         assertEquals(2, doc.get("number").asInt());
         assertEquals(" ", doc.get("color").asText(), "Verifies that whitespace is retained by default.");
         assertFalse(doc.get("flag").asBoolean());
-        assertEquals(3, doc.size(), "The file path column should not be included in the serialization.");
+        assertEquals(3, doc.size());
     }
 
     @Test
@@ -58,14 +56,12 @@ void xmlWithEmptyValues() {
             .option(Options.WRITE_PERMISSIONS, DEFAULT_PERMISSIONS)
             .option(Options.WRITE_XML_ROOT_NAME, "test")
             .option(Options.WRITE_URI_TEMPLATE, "/a/{number}.xml")
-            .option(Options.WRITE_JSON_SERIALIZATION_OPTION_PREFIX + "ignoreNullFields", "false")
+            .option(Options.WRITE_JSON_SERIALIZATION_OPTION_PREFIX + "ignoreNullFields", "true")
             .mode(SaveMode.Append)
             .save();
 
         XmlNode doc = readXmlDocument("/a/1.xml");
-        doc.assertElementValue(
-            "The empty flag column should be retained due to ignoreNullFields=true",
-            "/test/flag", "");
+        doc.assertElementMissing("The empty flag column should be ignored", "/test/flag");
         doc.assertElementValue("/test/number", "1");
         doc.assertElementValue("/test/color", "blue");
 
@@ -87,7 +83,7 @@ void jsonLinesWithNestedFieldsConvertedToXml() {
             .option(Options.WRITE_PERMISSIONS, DEFAULT_PERMISSIONS)
             .option(Options.WRITE_XML_ROOT_NAME, "parent")
             .option(Options.WRITE_URI_TEMPLATE, "/a/{id}.xml")
-            .option(Options.WRITE_JSON_SERIALIZATION_OPTION_PREFIX + "ignoreNullFields", "false")
+            .option(Options.WRITE_JSON_SERIALIZATION_OPTION_PREFIX + "ignoreNullFields", "true")
             .mode(SaveMode.Append)
             .save();
 
@@ -99,9 +95,8 @@ void jsonLinesWithNestedFieldsConvertedToXml() {
         doc.assertElementValue("/parent/id", "1");
 
         doc = readXmlDocument("/a/2.xml");
-        doc.assertElementValue(
-            "'hello' is added even though it doesn't exist on the line. This is due to ignoreNullFields being false " +
-                "and Spark adding 'hello' to the schema since it appears on the first line.",
-            "/parent/hello", "");
+        doc.assertElementMissing("'hello' should not appear. Spark JSON will actually include it in the schema and " +
+                "give it a value of null. But with ignoreNullFields set to true, it should be discarded.",
+            "/parent/hello");
     }
 }

src/test/java/com/marklogic/spark/writer/WriteRowsWithFilePathTest.java

@@ -45,7 +45,9 @@ void test() {
 
             JsonNode doc = readJsonDocument(uri);
             assertEquals(2, doc.size(), "The marklogic_spark_file_path column should not have been used when " +
-                "constructing the JSON document.");
+                "constructing the JSON document. This includes when ignoreNullFields is set to false. We still want " +
+                "the column removed as the column is an implementation detail that should not be exposed to the user. " +
+                "If we ever want the file path to be included in the document, we'll add an explicit feature for that.");
             assertTrue(doc.has("docNum"));
             assertTrue(doc.has("docName"));
         });

src/test/java/com/marklogic/spark/writer/WriteSparkJsonTest.java

@@ -32,6 +32,7 @@ void eachLineInJsonLinesFileBecomesADocument() {
             .option(Options.CLIENT_URI, makeClientUri())
             .option(Options.WRITE_PERMISSIONS, DEFAULT_PERMISSIONS)
             .option(Options.WRITE_URI_TEMPLATE, "/spark-json/{number}.json")
+            .option(Options.WRITE_JSON_SERIALIZATION_OPTION_PREFIX + "ignoreNullFields", "true")
             .mode(SaveMode.Append)
             .save();
 
@@ -58,6 +59,7 @@ void eachObjectInArrayBecomesADocument() {
             .option(Options.CLIENT_URI, makeClientUri())
             .option(Options.WRITE_PERMISSIONS, DEFAULT_PERMISSIONS)
             .option(Options.WRITE_URI_TEMPLATE, "/spark-json/{number}.json")
+            .option(Options.WRITE_JSON_SERIALIZATION_OPTION_PREFIX + "ignoreNullFields", "true")
             .mode(SaveMode.Append)
             .save();
 
@@ -80,6 +82,7 @@ void singleObjectFileAndArrayOfObjectsFile() {
             .option(Options.CLIENT_URI, makeClientUri())
             .option(Options.WRITE_PERMISSIONS, DEFAULT_PERMISSIONS)
             .option(Options.WRITE_URI_TEMPLATE, "/spark-json/{number}.json")
+            .option(Options.WRITE_JSON_SERIALIZATION_OPTION_PREFIX + "ignoreNullFields", "true")
             .mode(SaveMode.Append)
             .save();