Merge pull request #376 from marklogic/feature/splitter-xml-namespace

rjrudin · web-flow · commit 755e92b1e9ec · 2024-12-10T09:43:05.000-05:00
MLE-18275 Added default namespace for Flux XML
diff --git a/marklogic-langchain4j/src/main/java/com/marklogic/langchain4j/Util.java b/marklogic-langchain4j/src/main/java/com/marklogic/langchain4j/Util.java
@@ -20,6 +20,8 @@ public interface Util {
      */
     Logger LANGCHAIN4J_LOGGER = LoggerFactory.getLogger("com.marklogic.langchain4j");
 
+    String DEFAULT_XML_NAMESPACE = "http://marklogic.com/appservices/model";
+
     static JsonNode getJsonFromHandle(AbstractWriteHandle writeHandle) {
         if (writeHandle instanceof JacksonHandle) {
             return ((JacksonHandle) writeHandle).get();
diff --git a/marklogic-langchain4j/src/main/java/com/marklogic/langchain4j/embedding/XmlChunkConfig.java b/marklogic-langchain4j/src/main/java/com/marklogic/langchain4j/embedding/XmlChunkConfig.java
@@ -3,8 +3,14 @@
  */
 package com.marklogic.langchain4j.embedding;
 
+import com.marklogic.langchain4j.Util;
+
 import javax.xml.namespace.NamespaceContext;
 
+/**
+ * Captures configuration settings for the existing chunks in XML documents. Used to then add embeddings to each
+ * chunk.
+ */
 public class XmlChunkConfig {
 
     // The default expression ignores the namespace so that if a user is e.g. constructing a new XML document with a
@@ -24,7 +30,7 @@ public XmlChunkConfig() {
     public XmlChunkConfig(String textExpression, String embeddingName, String embeddingNamespace, NamespaceContext namespaceContext) {
         this.textExpression = textExpression != null ? textExpression : DEFAULT_TEXT_EXPRESSION;
         this.embeddingName = embeddingName != null ? embeddingName : "embedding";
-        this.embeddingNamespace = embeddingNamespace;
+        this.embeddingNamespace = embeddingNamespace != null ? embeddingNamespace : Util.DEFAULT_XML_NAMESPACE;
         this.namespaceContext = namespaceContext;
     }
 
diff --git a/marklogic-langchain4j/src/main/java/com/marklogic/langchain4j/splitter/ChunkConfig.java b/marklogic-langchain4j/src/main/java/com/marklogic/langchain4j/splitter/ChunkConfig.java
@@ -4,7 +4,12 @@
 package com.marklogic.langchain4j.splitter;
 
 import com.marklogic.client.io.DocumentMetadataHandle;
+import com.marklogic.langchain4j.Util;
 
+/**
+ * Captures configuration settings for producing chunks, either in a source document or in separate
+ * sidecar documents.
+ */
 public class ChunkConfig {
 
     private final DocumentMetadataHandle metadata;
@@ -30,7 +35,7 @@ public static class Builder {
         private int maxChunks;
         private String documentType;
         private String rootName;
-        private String xmlNamespace;
+        private String xmlNamespace = Util.DEFAULT_XML_NAMESPACE;
         private String uriPrefix;
         private String uriSuffix;
 
@@ -59,7 +64,9 @@ public Builder withRootName(String rootName) {
         }
 
         public Builder withXmlNamespace(String xmlNamespace) {
-            this.xmlNamespace = xmlNamespace;
+            if (xmlNamespace != null) {
+                this.xmlNamespace = xmlNamespace;
+            }
             return this;
         }
 
diff --git a/marklogic-langchain4j/src/main/java/com/marklogic/langchain4j/splitter/XmlChunkDocumentProducer.java b/marklogic-langchain4j/src/main/java/com/marklogic/langchain4j/splitter/XmlChunkDocumentProducer.java
@@ -7,6 +7,7 @@
 import com.marklogic.client.impl.DocumentWriteOperationImpl;
 import com.marklogic.client.io.DOMHandle;
 import com.marklogic.client.io.Format;
+import com.marklogic.langchain4j.Util;
 import com.marklogic.langchain4j.dom.DOMHelper;
 import com.marklogic.langchain4j.embedding.Chunk;
 import com.marklogic.langchain4j.embedding.DOMChunk;
@@ -92,7 +93,7 @@ private void addChunk(Document doc, TextSegment textSegment, Element chunksEleme
     }
 
     private String determineChunksElementName(Document doc) {
-        return doc.getDocumentElement().getElementsByTagName(DEFAULT_CHUNKS_ELEMENT_NAME).getLength() == 0 ?
+        return doc.getDocumentElement().getElementsByTagNameNS(Util.DEFAULT_XML_NAMESPACE, DEFAULT_CHUNKS_ELEMENT_NAME).getLength() == 0 ?
             DEFAULT_CHUNKS_ELEMENT_NAME : "splitter-chunks";
     }
 }
diff --git a/marklogic-spark-connector/src/test/java/com/marklogic/langchain4j/embedding/EmbedderTest.java b/marklogic-spark-connector/src/test/java/com/marklogic/langchain4j/embedding/EmbedderTest.java
@@ -19,6 +19,7 @@
 import com.marklogic.spark.writer.XmlUtil;
 import dev.langchain4j.data.document.splitter.DocumentSplitters;
 import dev.langchain4j.model.embedding.onnx.allminilml6v2.AllMiniLmL6V2EmbeddingModel;
+import org.jdom2.Namespace;
 import org.junit.jupiter.api.Test;
 
 import java.util.Iterator;
@@ -88,9 +89,10 @@ void xml() {
 
         docs.forEachRemaining(doc -> {
             XmlNode node = new XmlNode(XmlUtil.extractDocument(doc.getContent()));
-            node.assertElementCount("/root/chunks/chunk", 2);
-            node.assertElementExists("/root/chunks/chunk[1]/embedding");
-            node.assertElementExists("/root/chunks/chunk[2]/embedding");
+            node.setNamespaces(new Namespace[]{Namespace.getNamespace("model", "http://marklogic.com/appservices/model")});
+            node.assertElementCount("/model:root/model:chunks/model:chunk", 2);
+            node.assertElementExists("/model:root/model:chunks/model:chunk[1]/model:embedding");
+            node.assertElementExists("/model:root/model:chunks/model:chunk[2]/model:embedding");
         });
     }
 
diff --git a/marklogic-spark-connector/src/test/java/com/marklogic/langchain4j/splitter/SplitterTest.java b/marklogic-spark-connector/src/test/java/com/marklogic/langchain4j/splitter/SplitterTest.java
@@ -15,6 +15,7 @@
 import com.marklogic.junit5.XmlNode;
 import com.marklogic.spark.AbstractIntegrationTest;
 import dev.langchain4j.data.document.splitter.DocumentSplitters;
+import org.jdom2.Namespace;
 import org.junit.jupiter.api.Test;
 
 import java.util.Map;
@@ -34,16 +35,16 @@ void textPath() {
 
         doc.assertElementCount(
             "Expecting the default splitter to split the 'text' element into 4 chunks, each having its own 'text' element.",
-            "/root/chunks/chunk[text/text()]", 4);
+            "/root/model:chunks/model:chunk[model:text/text()]", 4);
     }
 
     @Test
     void elementPath() {
         XmlNode doc = splitTextDocument("/root/nested");
         doc.assertElementCount("Only expecting one chunk since the root/nested/text element has very little text",
-            "/root/chunks/chunk", 1);
+            "/root/model:chunks/model:chunk", 1);
 
-        String value = doc.getElementValue("/root/chunks/chunk/text");
+        String value = doc.getElementValue("/root/model:chunks/model:chunk/model:text");
         assertEquals("This is for testing.", value, "With our DOM-based implementation, we can easily return the " +
             "text content of any node selected by the user's path. We may eventually support an option to instead " +
             "serialize the selected node into a string.");
@@ -52,11 +53,11 @@ void elementPath() {
     @Test
     void attributePath() {
         XmlNode doc = splitTextDocument("/root/attribute-test/@text");
-        doc.assertElementCount("/root/chunks/chunk", 1);
+        doc.assertElementCount("/root/model:chunks/model:chunk", 1);
         doc.assertElementValue("It should be rare that a user wants to split the text in an attribute, but it should " +
                 "be feasible. We don't have a way though of preserving the attribute name in some sort of serialization " +
                 "with JDOM2; we can only get the attribute value.",
-            "/root/chunks/chunk/text",
+            "/root/model:chunks/model:chunk/model:text",
             "Some attribute text."
         );
     }
@@ -74,11 +75,11 @@ void multipleMatchingElements() {
 
         doc.assertElementCount(
             "Should have text from 2 elements, but that's small enough for 1 chunk",
-            "/root/chunks/chunk", 1);
+            "/root/model:chunks/model:chunk", 1);
 
         doc.assertElementValue(
             "The single chunk should have the concatenation of the two selected elements, joined with a space.",
-            "/root/chunks/chunk/text", "https://docs.marklogic.com/guide/java/intro This is for testing.");
+            "/root/model:chunks/model:chunk/model:text", "https://docs.marklogic.com/guide/java/intro This is for testing.");
     }
 
     @Test
@@ -138,7 +139,7 @@ private XmlNode splitTextDocument(String xpath) {
         DocumentWriteOperation sourceDocument = readXmlDocument();
         DocumentWriteOperation output = newXmlSplitter(xpath).apply(sourceDocument).next();
         String xml = HandleAccessor.contentAsString(output.getContent());
-        return new XmlNode(xml);
+        return new XmlNode(xml, Namespace.getNamespace("model", "http://marklogic.com/appservices/model"));
     }
 
     private DocumentTextSplitter newJsonSplitter(String... jsonPointers) {
diff --git a/marklogic-spark-connector/src/test/java/com/marklogic/spark/AbstractIntegrationTest.java b/marklogic-spark-connector/src/test/java/com/marklogic/spark/AbstractIntegrationTest.java
@@ -175,4 +175,15 @@ protected XmlNode readDocumentProperties(String uri) {
         props.setNamespaces(new Namespace[]{PROPERTIES_NAMESPACE});
         return props;
     }
+
+    @Override
+    protected XmlNode readXmlDocument(String uri) {
+        // Registers two frequently used namespaces in tests.
+        return readXmlDocument(uri,
+            Namespace.getNamespace("model", "http://marklogic.com/appservices/model"),
+            Namespace.getNamespace("ex", "org:example")
+        );
+    }
+
+
 }
diff --git a/marklogic-spark-connector/src/test/java/com/marklogic/spark/writer/embedding/AddEmbeddingsFromTextTest.java b/marklogic-spark-connector/src/test/java/com/marklogic/spark/writer/embedding/AddEmbeddingsFromTextTest.java
@@ -52,7 +52,7 @@ void xmlSidecarDocuments() {
         for (String uri : uris) {
             assertTrue(uri.endsWith(".xml"));
             XmlNode doc = readXmlDocument(uri);
-            doc.assertElementCount("/node()/chunks/chunk", 1);
+            doc.assertElementCount("/node()/model:chunks/model:chunk", 1);
         }
     }
 
diff --git a/marklogic-spark-connector/src/test/java/com/marklogic/spark/writer/embedding/AddEmbeddingsToXmlTest.java b/marklogic-spark-connector/src/test/java/com/marklogic/spark/writer/embedding/AddEmbeddingsToXmlTest.java
@@ -85,8 +85,14 @@ void sidecarWithNamespace() {
             .mode(SaveMode.Append)
             .save();
 
-        verifyChunksInNamespacedSidecar();
-        verifyEachChunkIsReturnedByAVectorQuery("namespaced_xml_chunks");
+        XmlNode doc = readXmlDocument("/split-test.xml-chunks-1.xml");
+        doc.assertElementCount("/ex:sidecar/ex:chunks/ex:chunk", 4);
+        for (XmlNode chunk : doc.getXmlNodes("/ex:sidecar/ex:chunks/ex:chunk")) {
+            chunk.assertElementExists("/ex:chunk/ex:text");
+            chunk.assertElementExists("For now, the embedding still defaults to the empty namespace. We may change " +
+                "this soon to be a MarkLogic-specific namespace to better distinguish it from the users " +
+                "content.", "/ex:chunk/model:embedding");
+        }
     }
 
     /**
@@ -133,10 +139,9 @@ void customChunks() {
 
         XmlNode doc = readXmlDocument("/split-test.xml");
         doc.assertElementCount("Each of the 2 custom chunks should have an 'embedding' element.",
-            "/envelope/my-chunks/my-chunk[my-text and embedding]", 2);
+            "/envelope/my-chunks/my-chunk[my-text and model:embedding]", 2);
     }
 
-
     @Test
     void namespacedCustomChunks() {
         readDocument("/marklogic-docs/namespaced-custom-chunks.xml")
@@ -151,9 +156,9 @@ void namespacedCustomChunks() {
             .mode(SaveMode.Append)
             .save();
 
-        XmlNode doc = readXmlDocument("/split-test.xml", Namespace.getNamespace("ex", "org:example"));
+        XmlNode doc = readXmlDocument("/split-test.xml");
         doc.assertElementCount("Each of the 2 custom chunks should have an 'embedding' element.",
-            "/ex:envelope/ex:my-chunks/ex:my-chunk[ex:my-text and embedding]", 2);
+            "/ex:envelope/ex:my-chunks/ex:my-chunk[ex:my-text and model:embedding]", 2);
     }
 
     @Test
@@ -243,7 +248,7 @@ private void verifyEachChunkOnDocumentHasAnEmbedding(String uri) {
         XmlNode doc = readXmlDocument(uri);
         doc.getXmlNodes("/node()/chunks/chunk").forEach(chunk -> {
             chunk.assertElementExists("/chunk/text");
-            chunk.assertElementExists("/chunk/embedding");
+            chunk.assertElementExists("/chunk/model:embedding");
         });
     }
 
@@ -270,13 +275,12 @@ private void verifyEachChunkIsReturnedByAVectorQuery(String viewName) {
     }
 
     private void verifyChunksInNamespacedSidecar() {
-        XmlNode doc = readXmlDocument("/split-test.xml-chunks-1.xml", Namespace.getNamespace("ex", "org:example"));
+        XmlNode doc = readXmlDocument("/split-test.xml-chunks-1.xml");
         doc.assertElementCount("/ex:sidecar/ex:chunks/ex:chunk", 4);
         for (XmlNode chunk : doc.getXmlNodes("/ex:sidecar/ex:chunks/ex:chunk")) {
             chunk.assertElementExists("/ex:chunk/ex:text");
-            chunk.assertElementExists("For now, the embedding still defaults to the empty namespace. We may change " +
-                "this soon to be a MarkLogic-specific namespace to better distinguish it from the users " +
-                "content.", "/ex:chunk/embedding");
+            chunk.assertElementExists("The embedding should default to the MarkLogic-specific namespace when not " +
+                "specified by the user.", "/ex:chunk/model:embedding");
         }
     }
 }
diff --git a/marklogic-spark-connector/src/test/java/com/marklogic/spark/writer/splitter/SplitJsonDocumentTest.java b/marklogic-spark-connector/src/test/java/com/marklogic/spark/writer/splitter/SplitJsonDocumentTest.java
@@ -244,10 +244,10 @@ void xmlChunks() {
         assertCollectionSize("chunks", 2);
 
         XmlNode doc = readXmlDocument("/split-test.json-chunks-1.xml");
-        doc.assertElementCount("/root/chunks/chunk", 2);
+        doc.assertElementCount("/model:root/model:chunks/model:chunk", 2);
 
         doc = readXmlDocument("/split-test.json-chunks-2.xml");
-        doc.assertElementCount("/root/chunks/chunk", 2);
+        doc.assertElementCount("/model:root/model:chunks/model:chunk", 2);
     }
 
     @Test
diff --git a/marklogic-spark-connector/src/test/java/com/marklogic/spark/writer/splitter/SplitTextDocumentTest.java b/marklogic-spark-connector/src/test/java/com/marklogic/spark/writer/splitter/SplitTextDocumentTest.java
@@ -48,9 +48,9 @@ void xmlChunks() {
         final String chunksUri = "/test/marklogic-docs/java-client-intro.txt-chunks-1.xml";
 
         XmlNode doc = readXmlDocument(chunksUri);
-        doc.assertElementValue("/root/source-uri", "/test/marklogic-docs/java-client-intro.txt");
+        doc.assertElementValue("/model:root/model:source-uri", "/test/marklogic-docs/java-client-intro.txt");
         doc.assertElementCount("Expecting 2 chunks based on the default max chunk size of 1000",
-            "/root/chunks/chunk", 2);
+            "/model:root/model:chunks/model:chunk", 2);
 
         PermissionsTester tester = readDocumentPermissions(chunksUri);
         tester.assertUpdatePermissionExists("This is just a temporary permission until we allow the URI and " +
@@ -95,12 +95,12 @@ void maxChunksOfThree() {
             "the second having 1 chunk.", "chunks", 2);
 
         XmlNode firstChunkDoc = readXmlDocument("/test/marklogic-docs/java-client-intro.txt-chunks-1.xml");
-        firstChunkDoc.assertElementValue("/root/source-uri", "/test/marklogic-docs/java-client-intro.txt");
-        firstChunkDoc.assertElementCount("/root/chunks/chunk", 3);
+        firstChunkDoc.assertElementValue("/model:root/model:source-uri", "/test/marklogic-docs/java-client-intro.txt");
+        firstChunkDoc.assertElementCount("/model:root/model:chunks/model:chunk", 3);
 
         XmlNode secondChunkDoc = readXmlDocument("/test/marklogic-docs/java-client-intro.txt-chunks-2.xml");
-        secondChunkDoc.assertElementValue("/root/source-uri", "/test/marklogic-docs/java-client-intro.txt");
-        secondChunkDoc.assertElementCount("/root/chunks/chunk", 1);
+        secondChunkDoc.assertElementValue("/model:root/model:source-uri", "/test/marklogic-docs/java-client-intro.txt");
+        secondChunkDoc.assertElementCount("/model:root/model:chunks/model:chunk", 1);
     }
 
     @Test
@@ -136,8 +136,8 @@ void maxChunksWithCustomUri() {
             assertTrue(uri.startsWith("/chunk/"), "Unexpected URI: " + uri);
             assertTrue(uri.endsWith(".xml"), "Unexpected URI: " + uri);
             XmlNode doc = readXmlDocument(uri);
-            doc.assertElementValue("/root/source-uri", "/test/marklogic-docs/java-client-intro.txt");
-            doc.assertElementCount("/root/chunks/chunk", 2);
+            doc.assertElementValue("/model:root/model:source-uri", "/test/marklogic-docs/java-client-intro.txt");
+            doc.assertElementCount("/model:root/model:chunks/model:chunk", 2);
         });
     }
 
diff --git a/marklogic-spark-connector/src/test/java/com/marklogic/spark/writer/splitter/SplitXmlDocumentTest.java b/marklogic-spark-connector/src/test/java/com/marklogic/spark/writer/splitter/SplitXmlDocumentTest.java
diff --git a/test-app/src/main/ml-data/marklogic-docs/has-chunks-already.xml b/test-app/src/main/ml-data/marklogic-docs/has-chunks-already.xml
diff --git a/test-app/src/main/ml-schemas-12/tde/namespaced-xml-vector-chunks.json b/test-app/src/main/ml-schemas-12/tde/namespaced-xml-vector-chunks.json
diff --git a/test-app/src/main/ml-schemas-12/tde/xml-vector-chunks.json b/test-app/src/main/ml-schemas-12/tde/xml-vector-chunks.json

Original file line number	Diff line number	Diff line change
`@@ -7,6 +7,7 @@`
`7`	`7`	`import com.marklogic.client.impl.DocumentWriteOperationImpl;`
`8`	`8`	`import com.marklogic.client.io.DOMHandle;`
`9`	`9`	`import com.marklogic.client.io.Format;`
	`10`	`+import com.marklogic.langchain4j.Util;`
`10`	`11`	`import com.marklogic.langchain4j.dom.DOMHelper;`
`11`	`12`	`import com.marklogic.langchain4j.embedding.Chunk;`
`12`	`13`	`import com.marklogic.langchain4j.embedding.DOMChunk;`
`@@ -92,7 +93,7 @@ private void addChunk(Document doc, TextSegment textSegment, Element chunksEleme`
`92`	`93`	`}`
`93`	`94`
`94`	`95`	`private String determineChunksElementName(Document doc) {`
`95`		`- return doc.getDocumentElement().getElementsByTagName(DEFAULT_CHUNKS_ELEMENT_NAME).getLength() == 0 ?`
	`96`	`+ return doc.getDocumentElement().getElementsByTagNameNS(Util.DEFAULT_XML_NAMESPACE, DEFAULT_CHUNKS_ELEMENT_NAME).getLength() == 0 ?`
`96`	`97`	`DEFAULT_CHUNKS_ELEMENT_NAME : "splitter-chunks";`
`97`	`98`	`}`
`98`	`99`	`}`
Original file line number	Diff line number	Diff line change
`@@ -52,7 +52,7 @@ void xmlSidecarDocuments() {`
`52`	`52`	`for (String uri : uris) {`
`53`	`53`	`assertTrue(uri.endsWith(".xml"));`
`54`	`54`	`XmlNode doc = readXmlDocument(uri);`
`55`		`- doc.assertElementCount("/node()/chunks/chunk", 1);`
	`55`	`+ doc.assertElementCount("/node()/model:chunks/model:chunk", 1);`
`56`	`56`	`}`
`57`	`57`	`}`
`58`	`58`