Merge pull request #274 from marklogic/feature/read-japanese-uris

Added test for URI with non-US-ASCII characters

Author: rjrudin

build.gradle

@@ -21,6 +21,9 @@ java {
 
 repositories {
   mavenCentral()
+  maven {
+    url "https://bed-artifactory.bedford.progress.com:443/artifactory/ml-maven-snapshots/"
+  }
 }
 
 configurations {
@@ -42,7 +45,7 @@ dependencies {
     exclude module: "rocksdbjni"
   }
 
-  shadowDependencies ("com.marklogic:marklogic-client-api:6.6.1") {
+  shadowDependencies ("com.marklogic:marklogic-client-api:7.0-SNAPSHOT") {
     // The Java Client uses Jackson 2.15.2; Scala 3.4.x does not yet support that and will throw the following error:
     // Scala module 2.14.2 requires Jackson Databind version >= 2.14.0 and < 2.15.0 - Found jackson-databind version 2.15.2
     // So the 4 Jackson modules are excluded to allow for Spark's to be used.
@@ -63,13 +66,20 @@ dependencies {
 
   shadowDependencies "org.jdom:jdom2:2.0.6.1"
 
-  testImplementation ('com.marklogic:ml-app-deployer:4.7.0') {
+  testImplementation ('com.marklogic:ml-app-deployer:4.8.0') {
     exclude group: "com.fasterxml.jackson.core"
     exclude group: "com.fasterxml.jackson.dataformat"
+
+    // Use the Java Client declared above.
+    exclude module: "marklogic-client-api"
   }
+
   testImplementation ('com.marklogic:marklogic-junit5:1.4.0') {
     exclude group: "com.fasterxml.jackson.core"
     exclude group: "com.fasterxml.jackson.dataformat"
+
+    // Use the Java Client declared above.
+    exclude module: "marklogic-client-api"
   }
 
   testImplementation "ch.qos.logback:logback-classic:1.3.14"

src/test/java/com/marklogic/spark/reader/document/ReadDocumentRowsByUrisTest.java

@@ -1,9 +1,12 @@
 package com.marklogic.spark.reader.document;
 
+import com.marklogic.junit5.XmlNode;
 import com.marklogic.spark.AbstractIntegrationTest;
 import com.marklogic.spark.Options;
 import org.apache.spark.sql.DataFrameReader;
+import org.apache.spark.sql.Dataset;
 import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SaveMode;
 import org.junit.jupiter.api.Test;
 
 import java.util.List;
@@ -79,6 +82,35 @@ void urisWithWrongCollection() {
         assertEquals(0, count, "This verifies that the collection impacts the list of URIs.");
     }
 
+    @Test
+    void nonUsAsciiUri() {
+        newSparkSession().read().format(CONNECTOR_IDENTIFIER)
+            .load("src/test/resources/encoding/太田佳伸のＸＭＬファイル.xml")
+            .write().format(CONNECTOR_IDENTIFIER)
+            .option(Options.CLIENT_URI, makeClientUri())
+            .option(Options.WRITE_PERMISSIONS, DEFAULT_PERMISSIONS)
+            .option(Options.WRITE_URI_REPLACE, ".*encoding,''")
+            .mode(SaveMode.Append)
+            .save();
+
+        final String expectedUri = "/太田佳伸のＸＭＬファイル.xml";
+        XmlNode doc = readXmlDocument(expectedUri);
+        doc.assertElementValue("/root/filename", "太田佳伸のＸＭＬファイル");
+
+        Dataset<Row> dataset = sparkSession.read().format(CONNECTOR_IDENTIFIER)
+            .option(Options.CLIENT_URI, makeClientUri())
+            .option(Options.READ_DOCUMENTS_URIS, expectedUri)
+            .load();
+
+        assertEquals(1, dataset.count());
+        Row row = dataset.collectAsList().get(0);
+        assertEquals(expectedUri, row.getString(0),
+            "As of 7.0.0, the Java Client should default to setting mail.mime.allowutf8=true so that the " +
+                "Jakarta Mail library allows UTF-8 characters in the header names of multipart response parts. " +
+                "Normally, it only allows US-ASCII characters. But since MarkLogic allows UTF-8 characters in " +
+                "URIs, we need the Jakarta Mail library (used by the Java Client) to be more permissive.");
+    }
+
     private DataFrameReader startRead() {
         return newSparkSession().read()
             .format(CONNECTOR_IDENTIFIER)

src/test/resources/encoding/太田佳伸のＸＭＬファイル.xml

@@ -0,0 +1,3 @@
+<root>
+<filename>太田佳伸のＸＭＬファイル</filename>
+</root>