Moving all connector code to subprojec

No functional changes. Will start separating things out into other subprojects next to modularize the codebase, including the test-app.

Author: rjrudin

build.gradle

@@ -1,251 +1,27 @@
-plugins {
-  id 'java-library'
-  id 'net.saliman.properties' version '1.5.2'
-  id 'com.github.johnrengelman.shadow' version '8.1.1'
-  id "com.marklogic.ml-gradle" version "5.0.0"
-  id 'maven-publish'
-  id "jacoco"
-  id "org.sonarqube" version "5.1.0.4882"
-}
-
-group 'com.marklogic'
-version '2.5-SNAPSHOT'
-
-java {
-  // To support reading RDF files, Apache Jena is used - but that requires Java 11.
-  sourceCompatibility = 11
-  targetCompatibility = 11
-}
-
-repositories {
-  mavenCentral()
-  mavenLocal()
-  maven {
-    url "https://bed-artifactory.bedford.progress.com:443/artifactory/ml-maven-snapshots/"
-  }
-}
-
-configurations {
-  // Defines all the implementation dependencies, but in such a way that they are not included as dependencies in the
-  // library's pom.xml file. This is due to the shadow jar being published instead of a jar only containing this
-  // project's classes. The shadow jar is published due to the need to relocate several packages to avoid conflicts
-  // with Spark.
-  shadowDependencies
-
-  // This approach allows for all of the dependencies to be available for compilation and for running tests.
-  compileOnly.extendsFrom(shadowDependencies)
-  testImplementation.extendsFrom(compileOnly)
-}
-
-configurations.all {
-  // Ensures that slf4j-api 1.x does not appear on the Flux classpath in particular, which can lead to this
-  // issue - https://www.slf4j.org/codes.html#StaticLoggerBinder .
-  resolutionStrategy {
-    force "org.slf4j:slf4j-api:2.0.13"
-  }
-}
-
-dependencies {
-  // This is compileOnly as any environment this is used in will provide the Spark dependencies itself.
-  compileOnly ('org.apache.spark:spark-sql_2.12:' + sparkVersion) {
-    // Excluded from our ETL tool for size reasons, so excluded here as well to ensure we don't need it.
-    exclude module: "rocksdbjni"
-  }
+subprojects {
+  apply plugin: "java-library"
 
-  shadowDependencies ("com.marklogic:marklogic-client-api:7.0.0") {
-    // The Java Client uses Jackson 2.15.2; Scala 3.4.x does not yet support that and will throw the following error:
-    // Scala module 2.14.2 requires Jackson Databind version >= 2.14.0 and < 2.15.0 - Found jackson-databind version 2.15.2
-    // So the 4 Jackson modules are excluded to allow for Spark's to be used.
-    exclude group: "com.fasterxml.jackson.core"
-    exclude group: "com.fasterxml.jackson.dataformat"
-  }
-
-  // Required for converting JSON to XML. Using 2.15.2 to align with Spark 3.5.3.
-  shadowDependencies ("com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.15.2") {
-    // Not needed, as the modules in this group that this dependency depends on are all provided by Spark.
-    exclude group: "com.fasterxml.jackson.core"
-  }
-
-  // Need this so that an OkHttpClientConfigurator can be created.
-  shadowDependencies 'com.squareup.okhttp3:okhttp:4.12.0'
+  group = "com.marklogic"
+  version '2.5-SNAPSHOT'
 
-  // Supports reading and writing RDF data.
-  shadowDependencies ("org.apache.jena:jena-arq:4.10.0") {
-    exclude group: "com.fasterxml.jackson.core"
-    exclude group: "com.fasterxml.jackson.dataformat"
+  java {
+    sourceCompatibility = 11
+    targetCompatibility = 11
   }
 
-  // Supports splitting documents.
-  shadowDependencies "dev.langchain4j:langchain4j:0.35.0"
-
-  // Supports testing the embedder feature.
-  testImplementation "dev.langchain4j:langchain4j-embeddings-all-minilm-l6-v2:0.35.0"
-
-  // Needed for some XML operations that are far easier with JDOM2 than with DOM.
-  shadowDependencies "org.jdom:jdom2:2.0.6.1"
-
-  // Needed for splitting XML documents via XPath.
-  shadowDependencies "jaxen:jaxen:2.0.0"
-
-  testImplementation ('com.marklogic:ml-app-deployer:5.0.0') {
-    exclude group: "com.fasterxml.jackson.core"
-    exclude group: "com.fasterxml.jackson.dataformat"
-
-    // Use the Java Client declared above.
-    exclude module: "marklogic-client-api"
-  }
-
-  testImplementation ('com.marklogic:marklogic-junit5:1.5.0') {
-    exclude group: "com.fasterxml.jackson.core"
-    exclude group: "com.fasterxml.jackson.dataformat"
-
-    // Use the Java Client declared above.
-    exclude module: "marklogic-client-api"
-  }
-
-  testImplementation "ch.qos.logback:logback-classic:1.3.14"
-  testImplementation "org.slf4j:jcl-over-slf4j:2.0.13"
-  testImplementation "org.skyscreamer:jsonassert:1.5.1"
-}
-
-test {
-  useJUnitPlatform()
-  finalizedBy jacocoTestReport
-  // Allows mlHost to override the value in gradle.properties, which the test plumbing will default to.
-  environment "mlHost", mlHost
-}
-
-// See https://docs.gradle.org/current/userguide/jacoco_plugin.html .
-jacocoTestReport {
-  dependsOn test
-  reports {
-    xml.required = true
-  }
-}
-
-sonar {
-  properties {
-    property "sonar.projectKey", "marklogic-spark"
-    property "sonar.host.url", "http://localhost:9000"
-  }
-}
-
-task reloadTestData(type: com.marklogic.gradle.task.MarkLogicTask) {
-  description = "Convenience task for clearing the test database and reloading the test data; only intended for a connector developer to use."
-  doLast {
-    new com.marklogic.mgmt.resource.databases.DatabaseManager(getManageClient()).clearDatabase("spark-test-test-content")
-  }
-}
-reloadTestData.finalizedBy mlLoadData
-
-if (JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_17)) {
-  test {
-    // See https://stackoverflow.com/questions/72724816/running-unit-tests-with-spark-3-3-0-on-java-17-fails-with-illegalaccesserror-cl
-    // for an explanation of why these are needed when running the tests on Java 17.
-    jvmArgs = [
-      '--add-exports=java.base/sun.nio.ch=ALL-UNNAMED',
-      '--add-opens=java.base/sun.util.calendar=ALL-UNNAMED',
-      '--add-opens=java.base/sun.security.action=ALL-UNNAMED'
-    ]
-  }
-}
-
-shadowJar {
-  configurations = [project.configurations.shadowDependencies]
-
-  // "all" is the default; no need for that in the connector filename. This also results in this becoming the library
-  // artifact that is published as a dependency. That is desirable as it includes the relocated packages listed below,
-  // which a dependent would otherwise have to manage themselves.
-  archiveClassifier.set("")
-
-  // Spark uses an older version of OkHttp; see
-  // https://stackoverflow.com/questions/61147800/how-to-override-spark-jars-while-running-spark-submit-command-in-cluster-mode
-  // for more information on why these are relocated.
-  relocate "okhttp3", "com.marklogic.okhttp3"
-  relocate "okio", "com.marklogic.okio"
-}
-
-// Publishing setup - see https://docs.gradle.org/current/userguide/publishing_setup.html .
-java {
-  withJavadocJar()
-  withSourcesJar()
-}
-
-javadoc.failOnError = false
-// Ignores warnings on params that don't have descriptions, which is a little too noisy
-javadoc.options.addStringOption('Xdoclint:none', '-quiet')
-
-publishing {
-  publications {
-    mainJava(MavenPublication) {
-      pom {
-        name = "${group}:${project.name}"
-        description = "Spark 3 connector for MarkLogic"
-        packaging = "jar"
-        from components.java
-        url = "https://github.com/marklogic/${project.name}"
-        licenses {
-          license {
-            name = "The Apache License, Version 2.0"
-            url = "http://www.apache.org/licenses/LICENSE-2.0.txt"
-          }
-        }
-        developers {
-          developer {
-            id = "marklogic"
-            name = "MarkLogic Github Contributors"
-            email = "general@developer.marklogic.com"
-            organization = "MarkLogic"
-            organizationUrl = "https://www.marklogic.com"
-          }
-        }
-        scm {
-          url = "git@github.com:marklogic/${project.name}.git"
-          connection = "scm:git@github.com:marklogic/${project.name}.git"
-          developerConnection = "scm:git@github.com:marklogic/${project.name}.git"
-        }
-      }
-    }
-  }
   repositories {
+    mavenCentral()
+    mavenLocal()
     maven {
-      if (project.hasProperty("mavenUser")) {
-        credentials {
-          username mavenUser
-          password mavenPassword
-        }
-        url publishUrl
-        allowInsecureProtocol = true
-      } else {
-        name = "central"
-        url = mavenCentralUrl
-        credentials {
-          username mavenCentralUsername
-          password mavenCentralPassword
-        }
-      }
+      url "https://bed-artifactory.bedford.progress.com:443/artifactory/ml-maven-snapshots/"
     }
   }
-}
 
-task gettingStartedZip(type: Zip) {
-  description = "Creates a zip of the getting-started project that is intended to be included as a downloadable file " +
-    "on the GitHub release page."
-  from "examples/getting-started"
-  exclude "build", ".gradle", "gradle-*.properties", ".venv", "venv", "docker"
-  into "marklogic-spark-getting-started-${version}"
-  archiveFileName = "marklogic-spark-getting-started-${version}.zip"
-  destinationDirectory = file("build")
-}
-
-tasks.register("addMarkLogic12SchemasIfNecessary", com.marklogic.gradle.task.MarkLogicTask) {
-  description = "If testing against MarkLogic 12, include schemas that will not work on MarkLogic 11 or earlier."
-  doLast {
-    def version = new com.marklogic.mgmt.resource.clusters.ClusterManager(getManageClient()).getVersion()
-    if (version.startsWith("12.")) {
-      mlAppConfig.getSchemaPaths().add(new File(getProjectDir(), "src/test/ml-schemas-12").getAbsolutePath())
+  test {
+    useJUnitPlatform()
+    testLogging {
+      events 'started', 'passed', 'skipped', 'failed'
+      exceptionFormat 'full'
     }
   }
 }
-mlDeploy.dependsOn addMarkLogic12SchemasIfNecessary
-mlLoadSchemas.dependsOn addMarkLogic12SchemasIfNecessary

gradle/wrapper/gradle-wrapper.jar

@@ -1,5 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-bin.zip
+networkTimeout=10000
+validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

gradle/wrapper/gradle-wrapper.properties

@@ -15,6 +15,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+# SPDX-License-Identifier: Apache-2.0
+#
 
 ##############################################################################
 #
@@ -55,7 +57,7 @@
 #       Darwin, MinGW, and NonStop.
 #
 #   (3) This script is generated from the Groovy template
-#       https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
+#       https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
 #       within the Gradle project.
 #
 #       You can find Gradle at https://github.com/gradle/gradle/.
@@ -80,13 +82,12 @@ do
     esac
 done
 
-APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
-
-APP_NAME="Gradle"
+# This is normally unused
+# shellcheck disable=SC2034
 APP_BASE_NAME=${0##*/}
-
-# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
+# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
+APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s
+' "$PWD" ) || exit
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD=maximum
@@ -133,22 +134,29 @@ location of your Java installation."
     fi
 else
     JAVACMD=java
-    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+    if ! command -v java >/dev/null 2>&1
+    then
+        die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
 
 Please set the JAVA_HOME variable in your environment to match the
 location of your Java installation."
+    fi
 fi
 
 # Increase the maximum file descriptors if we can.
 if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
     case $MAX_FD in #(
       max*)
+        # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
+        # shellcheck disable=SC2039,SC3045
         MAX_FD=$( ulimit -H -n ) ||
             warn "Could not query maximum file descriptor limit"
     esac
     case $MAX_FD in  #(
       '' | soft) :;; #(
       *)
+        # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
+        # shellcheck disable=SC2039,SC3045
         ulimit -n "$MAX_FD" ||
             warn "Could not set maximum file descriptor limit to $MAX_FD"
     esac
@@ -193,11 +201,15 @@ if "$cygwin" || "$msys" ; then
     done
 fi
 
-# Collect all arguments for the java command;
-#   * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
-#     shell script including quotes and variable substitutions, so put them in
-#     double quotes to make sure that they get re-expanded; and
-#   * put everything else in single quotes, so that it's not re-expanded.
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
+
+# Collect all arguments for the java command:
+#   * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
+#     and any embedded shellness will be escaped.
+#   * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
+#     treated as '${Hostname}' itself on the command line.
 
 set -- \
         "-Dorg.gradle.appname=$APP_BASE_NAME" \