Refactor: Moving split/embed code into langchain4j package

No functional changes.

Next step will be to remove all "com.marklogic.spark" imports from "com.marklogic.langchain4j".

Author: rjrudin

src/main/java/com/marklogic/spark/writer/JsonUtil.java renamed to src/main/java/com/marklogic/langchain4j/JsonUtil.java

@@ -1,7 +1,7 @@
 /*
  * Copyright © 2024 MarkLogic Corporation. All Rights Reserved.
  */
-package com.marklogic.spark.writer;
+package com.marklogic.langchain4j;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;

src/main/java/com/marklogic/spark/writer/dom/DOMHelper.java renamed to src/main/java/com/marklogic/langchain4j/dom/DOMHelper.java

@@ -1,7 +1,7 @@
 /*
  * Copyright © 2024 MarkLogic Corporation. All Rights Reserved.
  */
-package com.marklogic.spark.writer.dom;
+package com.marklogic.langchain4j.dom;
 
 import com.marklogic.client.document.DocumentWriteOperation;
 import com.marklogic.client.impl.HandleAccessor;

src/main/java/com/marklogic/spark/writer/dom/XPathNamespaceContext.java renamed to src/main/java/com/marklogic/langchain4j/dom/XPathNamespaceContext.java

@@ -1,7 +1,7 @@
 /*
  * Copyright © 2024 MarkLogic Corporation. All Rights Reserved.
  */
-package com.marklogic.spark.writer.dom;
+package com.marklogic.langchain4j.dom;
 
 import com.marklogic.spark.Options;
 

src/main/java/com/marklogic/spark/writer/embedding/Chunk.java renamed to src/main/java/com/marklogic/langchain4j/embedding/Chunk.java

@@ -1,7 +1,7 @@
 /*
  * Copyright © 2024 MarkLogic Corporation. All Rights Reserved.
  */
-package com.marklogic.spark.writer.embedding;
+package com.marklogic.langchain4j.embedding;
 
 import dev.langchain4j.data.embedding.Embedding;
 

src/main/java/com/marklogic/spark/writer/embedding/ChunkSelector.java renamed to src/main/java/com/marklogic/langchain4j/embedding/ChunkSelector.java

@@ -1,7 +1,7 @@
 /*
  * Copyright © 2024 MarkLogic Corporation. All Rights Reserved.
  */
-package com.marklogic.spark.writer.embedding;
+package com.marklogic.langchain4j.embedding;
 
 import com.marklogic.client.document.DocumentWriteOperation;
 

src/main/java/com/marklogic/spark/writer/embedding/DOMChunk.java renamed to src/main/java/com/marklogic/langchain4j/embedding/DOMChunk.java

@@ -1,7 +1,7 @@
 /*
  * Copyright © 2024 MarkLogic Corporation. All Rights Reserved.
  */
-package com.marklogic.spark.writer.embedding;
+package com.marklogic.langchain4j.embedding;
 
 import com.marklogic.spark.ConnectorException;
 import dev.langchain4j.data.embedding.Embedding;

src/main/java/com/marklogic/spark/writer/embedding/DOMChunkSelector.java renamed to src/main/java/com/marklogic/langchain4j/embedding/DOMChunkSelector.java

@@ -1,13 +1,13 @@
 /*
  * Copyright © 2024 MarkLogic Corporation. All Rights Reserved.
  */
-package com.marklogic.spark.writer.embedding;
+package com.marklogic.langchain4j.embedding;
 
 import com.marklogic.client.document.DocumentWriteOperation;
 import com.marklogic.client.impl.DocumentWriteOperationImpl;
 import com.marklogic.client.io.DOMHandle;
+import com.marklogic.langchain4j.dom.DOMHelper;
 import com.marklogic.spark.ConnectorException;
-import com.marklogic.spark.writer.dom.DOMHelper;
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
 import org.w3c.dom.Node;
@@ -20,14 +20,14 @@
 import java.util.ArrayList;
 import java.util.List;
 
-class DOMChunkSelector implements ChunkSelector {
+public class DOMChunkSelector implements ChunkSelector {
 
     private final XPathFactory xpathFactory;
     private final XPathExpression chunksExpression;
     private final XmlChunkConfig xmlChunkConfig;
     private final DOMHelper domHelper;
 
-    DOMChunkSelector(String chunksExpression, XmlChunkConfig xmlChunkConfig) {
+    public DOMChunkSelector(String chunksExpression, XmlChunkConfig xmlChunkConfig) {
         this.xpathFactory = XPathFactory.newInstance();
         this.xmlChunkConfig = xmlChunkConfig;
         this.domHelper = new DOMHelper(xmlChunkConfig.getNamespaceContext());

src/main/java/com/marklogic/spark/writer/embedding/DocumentAndChunks.java renamed to src/main/java/com/marklogic/langchain4j/embedding/DocumentAndChunks.java

@@ -1,7 +1,7 @@
 /*
  * Copyright © 2024 MarkLogic Corporation. All Rights Reserved.
  */
-package com.marklogic.spark.writer.embedding;
+package com.marklogic.langchain4j.embedding;
 
 import com.marklogic.client.document.DocumentWriteOperation;
 import com.marklogic.client.io.marker.AbstractWriteHandle;

src/main/java/com/marklogic/spark/writer/embedding/EmbedderDocumentProcessor.java renamed to src/main/java/com/marklogic/langchain4j/embedding/EmbeddingAdder.java

@@ -1,35 +1,35 @@
 /*
  * Copyright © 2024 MarkLogic Corporation. All Rights Reserved.
  */
-package com.marklogic.spark.writer.embedding;
+package com.marklogic.langchain4j.embedding;
 
 import com.marklogic.client.document.DocumentWriteOperation;
+import com.marklogic.langchain4j.splitter.DocumentTextSplitter;
 import com.marklogic.spark.Util;
-import com.marklogic.spark.writer.DocumentProcessor;
-import com.marklogic.spark.writer.splitter.SplitterDocumentProcessor;
 
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
+import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.stream.Stream;
 
 /**
  * Supports a use case where a document already has chunks in it, which must be selected via a {@code ChunkSelector}.
  * The {@code EmbeddingModel} is then used to generate and add an embedding to each chunk in a given document.
  */
-class EmbedderDocumentProcessor implements DocumentProcessor, Supplier<Iterator<DocumentWriteOperation>> {
+public class EmbeddingAdder implements Function<DocumentWriteOperation, Iterator<DocumentWriteOperation>>, Supplier<Iterator<DocumentWriteOperation>> {
 
     private final ChunkSelector chunkSelector;
     private final EmbeddingGenerator embeddingGenerator;
-    private final SplitterDocumentProcessor splitterDocumentProcessor;
+    private final DocumentTextSplitter documentTextSplitter;
 
     private List<DocumentWriteOperation> pendingSourceDocuments = new ArrayList<>();
 
-    EmbedderDocumentProcessor(ChunkSelector chunkSelector, EmbeddingGenerator embeddingGenerator, SplitterDocumentProcessor splitterDocumentProcessor) {
+    public EmbeddingAdder(ChunkSelector chunkSelector, EmbeddingGenerator embeddingGenerator, DocumentTextSplitter documentTextSplitter) {
         this.chunkSelector = chunkSelector;
         this.embeddingGenerator = embeddingGenerator;
-        this.splitterDocumentProcessor = splitterDocumentProcessor;
+        this.documentTextSplitter = documentTextSplitter;
     }
 
     /**
@@ -40,7 +40,7 @@ class EmbedderDocumentProcessor implements DocumentProcessor, Supplier<Iterator<
      */
     @Override
     public Iterator<DocumentWriteOperation> apply(DocumentWriteOperation sourceDocument) {
-        if (splitterDocumentProcessor != null) {
+        if (documentTextSplitter != null) {
             return splitAndAddEmbeddings(sourceDocument);
         }
 
@@ -67,7 +67,7 @@ public Iterator<DocumentWriteOperation> get() {
     }
 
     private Iterator<DocumentWriteOperation> splitAndAddEmbeddings(DocumentWriteOperation sourceDocument) {
-        Iterator<DocumentWriteOperation> splitDocuments = splitterDocumentProcessor.apply(sourceDocument);
+        Iterator<DocumentWriteOperation> splitDocuments = documentTextSplitter.apply(sourceDocument);
 
         // Track the list of documents to return. A document won't be returned immediately if it has chunks but the
         // embedding generator doesn't receive enough chunks to meet its batch size threshold.

src/main/java/com/marklogic/spark/writer/embedding/EmbeddingGenerator.java renamed to src/main/java/com/marklogic/langchain4j/embedding/EmbeddingGenerator.java

@@ -1,7 +1,7 @@
 /*
  * Copyright © 2024 MarkLogic Corporation. All Rights Reserved.
  */
-package com.marklogic.spark.writer.embedding;
+package com.marklogic.langchain4j.embedding;
 
 import com.marklogic.spark.Util;
 import dev.langchain4j.data.document.Metadata;
@@ -16,7 +16,7 @@
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
 
-class EmbeddingGenerator {
+public class EmbeddingGenerator {
 
     // We don't have any use for metadata, so just need a single instance for constructing text segments.
     private static final Metadata TEXT_SEGMENT_METADATA = new Metadata();
@@ -30,11 +30,11 @@ class EmbeddingGenerator {
 
     private List<Chunk> pendingChunks = new ArrayList<>();
 
-    EmbeddingGenerator(EmbeddingModel embeddingModel) {
+    public EmbeddingGenerator(EmbeddingModel embeddingModel) {
         this(embeddingModel, 1);
     }
 
-    EmbeddingGenerator(EmbeddingModel embeddingModel, int batchSize) {
+    public EmbeddingGenerator(EmbeddingModel embeddingModel, int batchSize) {
         this.embeddingModel = embeddingModel;
         this.batchSize = batchSize;
     }