Merge pull request #276 from marklogic/feature/15791-refactor

MLE-15791 Refactored aggregate XML classes into a package

Author: rjrudin

src/main/java/com/marklogic/spark/reader/file/FileContext.java

@@ -16,12 +16,12 @@
 import java.util.Map;
 import java.util.zip.GZIPInputStream;
 
-class FileContext extends ContextSupport implements Serializable {
+public class FileContext extends ContextSupport implements Serializable {
 
     private SerializableConfiguration hadoopConfiguration;
     private final String encoding;
 
-    FileContext(Map<String, String> properties, SerializableConfiguration hadoopConfiguration) {
+    public FileContext(Map<String, String> properties, SerializableConfiguration hadoopConfiguration) {
         super(properties);
         this.hadoopConfiguration = hadoopConfiguration;
         this.encoding = getStringOption(Options.READ_FILES_ENCODING);
@@ -42,7 +42,7 @@ boolean isGzip() {
         return "gzip".equalsIgnoreCase(getStringOption(Options.READ_FILES_COMPRESSION));
     }
 
-    InputStream openFile(String filePath) {
+    public InputStream openFile(String filePath) {
         try {
             Path hadoopPath = new Path(filePath);
             FileSystem fileSystem = hadoopPath.getFileSystem(hadoopConfiguration.value());
@@ -54,7 +54,7 @@ InputStream openFile(String filePath) {
         }
     }
 
-    boolean isReadAbortOnFailure() {
+    public boolean isReadAbortOnFailure() {
         if (hasOption(Options.READ_FILES_ABORT_ON_FAILURE)) {
             return Boolean.parseBoolean(getStringOption(Options.READ_FILES_ABORT_ON_FAILURE));
         }

src/main/java/com/marklogic/spark/reader/file/FilePartition.java

@@ -4,7 +4,7 @@
 
 import java.util.List;
 
-class FilePartition implements InputPartition {
+public class FilePartition implements InputPartition {
 
     static final long serialVersionUID = 1;
 
@@ -14,7 +14,7 @@ public FilePartition(List<String> paths) {
         this.paths = paths;
     }
 
-    List<String> getPaths() {
+    public List<String> getPaths() {
         return paths;
     }
 

src/main/java/com/marklogic/spark/reader/file/FilePartitionReaderFactory.java

@@ -1,6 +1,8 @@
 package com.marklogic.spark.reader.file;
 
 import com.marklogic.spark.Options;
+import com.marklogic.spark.reader.file.xml.AggregateXmlFileReader;
+import com.marklogic.spark.reader.file.xml.ZipAggregateXmlFileReader;
 import org.apache.spark.sql.catalyst.InternalRow;
 import org.apache.spark.sql.connector.read.InputPartition;
 import org.apache.spark.sql.connector.read.PartitionReader;

src/main/java/com/marklogic/spark/reader/file/AggregateXmlFileReader.java renamed to src/main/java/com/marklogic/spark/reader/file/xml/AggregateXmlFileReader.java

@@ -1,14 +1,16 @@
-package com.marklogic.spark.reader.file;
+package com.marklogic.spark.reader.file.xml;
 
 import com.marklogic.spark.ConnectorException;
 import com.marklogic.spark.Util;
+import com.marklogic.spark.reader.file.FileContext;
+import com.marklogic.spark.reader.file.FilePartition;
 import org.apache.commons.io.IOUtils;
 import org.apache.spark.sql.catalyst.InternalRow;
 import org.apache.spark.sql.connector.read.PartitionReader;
 
 import java.io.InputStream;
 
-class AggregateXmlFileReader implements PartitionReader<InternalRow> {
+public class AggregateXmlFileReader implements PartitionReader<InternalRow> {
 
     private final FilePartition filePartition;
     private final FileContext fileContext;
@@ -18,7 +20,7 @@ class AggregateXmlFileReader implements PartitionReader<InternalRow> {
     private InternalRow nextRowToReturn;
     private int filePathIndex = 0;
 
-    AggregateXmlFileReader(FilePartition filePartition, FileContext fileContext) {
+    public AggregateXmlFileReader(FilePartition filePartition, FileContext fileContext) {
         this.filePartition = filePartition;
         this.fileContext = fileContext;
     }

src/main/java/com/marklogic/spark/reader/file/AggregateXmlSplitter.java renamed to src/main/java/com/marklogic/spark/reader/file/xml/AggregateXmlSplitter.java

@@ -1,9 +1,10 @@
-package com.marklogic.spark.reader.file;
+package com.marklogic.spark.reader.file.xml;
 
 import com.marklogic.client.datamovement.XMLSplitter;
 import com.marklogic.client.io.StringHandle;
 import com.marklogic.spark.ConnectorException;
 import com.marklogic.spark.Options;
+import com.marklogic.spark.reader.file.FileContext;
 import org.apache.spark.sql.catalyst.InternalRow;
 import org.apache.spark.sql.catalyst.expressions.GenericInternalRow;
 import org.apache.spark.unsafe.types.ByteArray;

src/main/java/com/marklogic/spark/reader/file/ZipAggregateXmlFileReader.java renamed to src/main/java/com/marklogic/spark/reader/file/xml/ZipAggregateXmlFileReader.java

@@ -1,7 +1,10 @@
-package com.marklogic.spark.reader.file;
+package com.marklogic.spark.reader.file.xml;
 
 import com.marklogic.spark.ConnectorException;
 import com.marklogic.spark.Util;
+import com.marklogic.spark.reader.file.FileContext;
+import com.marklogic.spark.reader.file.FilePartition;
+import com.marklogic.spark.reader.file.FileUtil;
 import org.apache.commons.io.IOUtils;
 import org.apache.spark.sql.catalyst.InternalRow;
 import org.apache.spark.sql.connector.read.PartitionReader;
@@ -12,7 +15,7 @@
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 
-class ZipAggregateXmlFileReader implements PartitionReader<InternalRow> {
+public class ZipAggregateXmlFileReader implements PartitionReader<InternalRow> {
 
     private static final Logger logger = LoggerFactory.getLogger(ZipAggregateXmlFileReader.class);
 
@@ -29,7 +32,7 @@ class ZipAggregateXmlFileReader implements PartitionReader<InternalRow> {
     private String currentFilePath;
     private ZipInputStream currentZipInputStream;
 
-    ZipAggregateXmlFileReader(FilePartition filePartition, FileContext fileContext) {
+    public ZipAggregateXmlFileReader(FilePartition filePartition, FileContext fileContext) {
         this.fileContext = fileContext;
         this.filePartition = filePartition;
         this.openNextFile();