Merge pull request #260 from marklogic/feature/file-fix

Fixing issue with URIs that cannot be parsed

Author: rjrudin

src/main/java/com/marklogic/spark/writer/file/FileUtil.java

@@ -1,22 +1,26 @@
 package com.marklogic.spark.writer.file;
 
-import com.marklogic.spark.ConnectorException;
+import com.marklogic.spark.Util;
 
 import java.net.URI;
 import java.net.URISyntaxException;
 
 abstract class FileUtil {
 
     static String makePathFromDocumentURI(String documentURI) {
-        // Copied from MLCP
-        URI uri;
+        // Mostly copied from MLCP.
         try {
-            uri = new URI(documentURI);
+            URI uri = new URI(documentURI);
+            // The isOpaque check is made because an opaque URI will not have a path.
+            return uri.isOpaque() ? uri.getSchemeSpecificPart() : uri.getPath();
         } catch (URISyntaxException e) {
-            throw new ConnectorException(String.format("Unable to construct URI from: %s", documentURI), e);
+            // MLCP logs errors from parsing the URI at the "WARN" level. That seems noisy, as large numbers of URIs
+            // could e.g. have spaces in them. So DEBUG is used instead.
+            if (Util.MAIN_LOGGER.isDebugEnabled()) {
+                Util.MAIN_LOGGER.debug("Unable to parse document URI: {}; will use unparsed URI as file path.", documentURI);
+            }
+            return documentURI;
         }
-        // The isOpaque check is made because an opaque URI will not have a path.
-        return uri.isOpaque() ? uri.getSchemeSpecificPart() : uri.getPath();
     }
 
     private FileUtil() {

src/test/java/com/marklogic/spark/writer/file/FileUtilTest.java

@@ -1,10 +1,8 @@
 package com.marklogic.spark.writer.file;
 
-import com.marklogic.spark.ConnectorException;
 import org.junit.jupiter.api.Test;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertThrows;
 
 class FileUtilTest {
 
@@ -22,7 +20,9 @@ void makePathFromOpaqueURI() {
 
     @Test
     void makePathWithInvalidURI() {
-        ConnectorException ex = assertThrows(ConnectorException.class, () -> FileUtil.makePathFromDocumentURI(":::"));
-        assertEquals("Unable to construct URI from: :::", ex.getMessage());
+        String uri = FileUtil.makePathFromDocumentURI("has space.json");
+        assertEquals("has space.json", uri, "If a java.net.URI cannot be constructed - in this case, it's due to " +
+            "the space in the string - then the error should be logged at the DEBUG level and the original value " +
+            "should be returned.");
     }
 }

src/test/java/com/marklogic/spark/writer/file/WriteDocumentFilesTest.java

@@ -1,7 +1,6 @@
 package com.marklogic.spark.writer.file;
 
 import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.marklogic.client.document.DocumentWriteSet;
 import com.marklogic.client.document.TextDocumentManager;
 import com.marklogic.client.io.DocumentMetadataHandle;
@@ -85,4 +84,36 @@ void variousURIs(@TempDir Path tempDir) throws Exception {
         content = new String(FileCopyUtils.copyToByteArray(files[filenames.indexOf("example2.txt")]));
         assertEquals("Opaque URI", content);
     }
+
+    @Test
+    void uriHasSpace(@TempDir Path tempDir) {
+        final String uri = "/has space.json";
+
+        newSparkSession().read().format(CONNECTOR_IDENTIFIER)
+            .load("src/test/resources/spark-json/single-object.json")
+            .write().format(CONNECTOR_IDENTIFIER)
+            .option(Options.CLIENT_URI, makeClientUri())
+            .option(Options.WRITE_COLLECTIONS, "char-test")
+            .option(Options.WRITE_PERMISSIONS, DEFAULT_PERMISSIONS)
+            .option(Options.WRITE_URI_TEMPLATE, uri)
+            .mode(SaveMode.Append)
+            .save();
+
+        sparkSession.read().format(CONNECTOR_IDENTIFIER)
+            .option(Options.CLIENT_URI, makeClientUri())
+            .option(Options.READ_DOCUMENTS_URIS, uri)
+            .load()
+            .write().format(CONNECTOR_IDENTIFIER)
+            .mode(SaveMode.Append)
+            .save(tempDir.toFile().getAbsolutePath());
+
+        File dir = tempDir.toFile();
+        assertEquals(1, dir.listFiles().length);
+        String filename = dir.listFiles()[0].getName();
+        System.out.println(filename);
+        assertEquals("has space.json", filename,
+            "Just like MLCP, if the connector cannot construct a java.net.URI from the document URI (it will fail " +
+                "due to a space), the error should be logged and the file should be written with its unaltered " +
+                "document URI used for the file path.");
+    }
 }