Merge pull request #350 from marklogic/feature/ignore-bad-file

Logging warning when text cannot be split from document

Author: rjrudin

src/main/java/com/marklogic/spark/writer/splitter/DOMTextSelector.java

@@ -5,6 +5,7 @@
 
 import com.marklogic.client.document.DocumentWriteOperation;
 import com.marklogic.spark.ConnectorException;
+import com.marklogic.spark.Util;
 import com.marklogic.spark.writer.dom.DOMHelper;
 import org.w3c.dom.Document;
 import org.w3c.dom.NodeList;
@@ -28,7 +29,14 @@ public DOMTextSelector(String textExpression, NamespaceContext namespaceContext)
 
     @Override
     public String selectTextToSplit(DocumentWriteOperation sourceDocument) {
-        Document doc = domHelper.extractDocument(sourceDocument);
+        Document doc;
+        try {
+            doc = domHelper.extractDocument(sourceDocument);
+        } catch (Exception ex) {
+            Util.MAIN_LOGGER.warn("Unable to select text to split in document: {}; cause: {}", sourceDocument.getUri(), ex.getMessage());
+            return null;
+        }
+
         NodeList items;
         try {
             items = (NodeList) this.textExpression.evaluate(doc, XPathConstants.NODESET);

src/main/java/com/marklogic/spark/writer/splitter/JsonPointerTextSelector.java

@@ -7,6 +7,7 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.marklogic.client.document.DocumentWriteOperation;
 import com.marklogic.spark.ConnectorException;
+import com.marklogic.spark.Util;
 import com.marklogic.spark.writer.JsonUtil;
 
 import java.util.ArrayList;
@@ -33,8 +34,14 @@ public JsonPointerTextSelector(String[] jsonPointerArray, String joinDelimiter)
     }
 
     @Override
-    public String selectTextToSplit(DocumentWriteOperation operation) {
-        JsonNode doc = JsonUtil.getJsonFromHandle(operation.getContent());
+    public String selectTextToSplit(DocumentWriteOperation sourceDocument) {
+        JsonNode doc;
+        try {
+            doc = JsonUtil.getJsonFromHandle(sourceDocument.getContent());
+        } catch (Exception ex) {
+            Util.MAIN_LOGGER.warn("Unable to select text to split in document: {}; cause: {}", sourceDocument.getUri(), ex.getMessage());
+            return null;
+        }
 
         return jsonPointers.stream()
             .map(jsonPointer -> {

src/test/java/com/marklogic/spark/writer/splitter/SplitJsonDocumentTest.java

@@ -315,6 +315,23 @@ void chunksFieldAlreadyExists() {
             "already, such that we do not need to provide a configuration option for defining the chunks array name.");
     }
 
+    @Test
+    void xpathOnJsonDocument() {
+        readDocument("/marklogic-docs/java-client-intro.json")
+            .write().format(CONNECTOR_IDENTIFIER)
+            .option(Options.CLIENT_URI, makeClientUri())
+            .option(Options.WRITE_SPLITTER_XPATH, "/text")
+            .option(Options.WRITE_PERMISSIONS, DEFAULT_PERMISSIONS)
+            .option(Options.WRITE_URI_TEMPLATE, "/split-test.json")
+            .mode(SaveMode.Append)
+            .save();
+
+        JsonNode doc = readJsonDocument("/split-test.json");
+        assertFalse(doc.has("chunks"), "If a user specifies an XPath split expression and the connector encounters a " +
+            "non-XML document, a warning should be logged and no chunks should be added. This scenario could happen " +
+            "when e.g. processing a zip file that contains mostly XML documents, but also a few non-XML documents.");
+    }
+
     private Dataset<Row> readDocument(String uri) {
         return newSparkSession().read().format(CONNECTOR_IDENTIFIER)
             .option(Options.CLIENT_URI, makeClientUri())

src/test/java/com/marklogic/spark/writer/splitter/SplitXmlDocumentTest.java

@@ -78,6 +78,24 @@ void undeclaredNamespace() {
         );
     }
 
+    @Test
+    void jsonPointerOnXmlDocument() {
+        readDocument("/marklogic-docs/java-client-intro.xml")
+            .write().format(CONNECTOR_IDENTIFIER)
+            .option(Options.CLIENT_URI, makeClientUri())
+            .option(Options.WRITE_SPLITTER_JSON_POINTERS, "/text")
+            .option(Options.WRITE_PERMISSIONS, DEFAULT_PERMISSIONS)
+            .option(Options.WRITE_URI_TEMPLATE, "/split-test.xml")
+            .mode(SaveMode.Append)
+            .save();
+
+        XmlNode doc = readXmlDocument("/split-test.xml");
+        doc.assertElementMissing("If a user specifies a JSON Pointer split expression and the connector encounters a " +
+            "non-JSON document, a warning should be logged and no chunks should be added. This scenario could happen " +
+            "when e.g. processing a zip file that contains mostly JSON documents, but also a few non-JSON documents.",
+            "//chunks");
+    }
+
     @Test
     void overlapSizeGreaterThanChunkSize() {
         DataFrameWriter writer = readDocument("/marklogic-docs/java-client-intro.xml")