PR for v1.1.2

.github/workflows/CI.yml

@@ -31,19 +31,15 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.13'
       - name: Get Python location
         run: python -c "import os, sys; print(sys.executable)"
       - name: Update pip
         run: python3 -m pip install --upgrade pip
       - name: Grant execute permission for gradlew
         run: chmod +x gradlew
       - name: Prepare
-        run: mkdir -p /tmp/mapping-service/{schemas,plugins}
-      - name: Copy Plugins
-        run: cp plugins/* /tmp/mapping-service/plugins
-      - name: List Plugins
-        run: ls -la /tmp/mapping-service/plugins
+        run: mkdir -p /tmp/mapping-service/{schemas}
       - name: Clean
         run: ./gradlew clean
 #      - if: matrix.os == 'windows-latest'

.github/workflows/codeql-analysis.yml

@@ -41,7 +41,7 @@ jobs:
     - name: Set up Python3
       uses: actions/setup-python@v5
       with:
-        python-version: '3.10'
+        python-version: '3.13'
     - name: Set up OpenJDK
       uses: actions/setup-java@v4
       with:
@@ -101,7 +101,7 @@ jobs:
       - name: Set up Python3
         uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.13'
 
       - name: Set up OpenJDK
         uses: actions/setup-java@v4

.gitignore

@@ -241,3 +241,4 @@ gradle-app.setting
 lib/gemma
 !/plugins/gemma-plugin-0.1.0-SNAPSHOT-plain.jar
 !/plugins/empty-plugin-0.0.0-SNAPSHOT-plain.jar
+custom

Dockerfile

@@ -11,7 +11,7 @@ ARG SERVICE_ROOT_DIRECTORY_DEFAULT=/spring/
 ####################################################
 # Building environment (java & git)
 ####################################################
-FROM eclipse-temurin:23 AS build-env-java
+FROM eclipse-temurin:24 AS build-env-java
 LABEL maintainer=webmaster@datamanager.kit.edu
 LABEL stage=build-env
 
@@ -48,7 +48,7 @@ RUN bash ./build.sh $SERVICE_DIRECTORY
 ####################################################
 # Runtime environment 4 metastore2
 ####################################################
-FROM eclipse-temurin:23 AS run-service-mapping-service
+FROM eclipse-temurin:24 AS run-service-mapping-service
 LABEL maintainer=webmaster@datamanager.kit.edu
 LABEL stage=run
 

HELP.md

@@ -1,8 +1,3 @@
-# Read Me First
-The following was discovered as part of building this project:
-
-* The original package name 'edu.kit.datamanager.mapping-service' is invalid and this project uses 'edu.kit.datamanager.mappingservice' instead.
-
 # Getting Started
 
 ### Reference Documentation

README.md

@@ -17,72 +17,58 @@ The REST-API is documented at the following link: [http://\<IP or hostname>:8095
 Dependencies that are needed to build and are not being downloaded via gradle:
 
 - OpenJDK 17
-- Python 3
-- pip (runtime only)
+- (Optional) Python 3
+- (Optional) pip (runtime only)
 
 `./gradlew build`
 
 ### Python Location
 
-Currently, mapping-service requires Python to be installed in order to build and to run. At runtime, the Python executable is configured in 
-`application.properties`(see below). For building the mapping-service Python executable is set to `/usr/bin/python3` by default. In case you want to build 
-the mapping-service on a machine on which the Python installation is located elsewhere, e.g., under Windows, you can provide the Python location 
-used at compile time externally, i.e.:
+The mapping-service supports plugins running Python code. To provide basic testing for this feature, some tests require configured Python in order to be executed.
+While at runtime, the Python executable is configured in application.properties, at build time the Python location may differ depending on the build environment.
+By default, '/usr/bin/python' is assumed as Python location. If you are using a different Python installation, e.g., under Windows or MacOS, you may either modify 
+'build.gradle' (look out for pythonExecutable) or you provide the Python executable as command line argument, e.g.,
 
 ```
 .\gradlew "-DpythonExecutable=file:///C:/Python310/python.exe" build
 ```
 
 ## How to start
 
-Before you can start the mapping-service, you  first have to create an `application.properties` file in the source folder. As an example you may use `config/application.default.properties`
+Before you can start the mapping-service, you  first have to create an `application.properties` file in the source folder. As an example you may use `settings/application.default.properties`
 and modify it according to your needs. Espacially the following properties (at the end of the file) are important:
-- `spring.datasource.url=jdbc:h2:file:/tmp/mapping-service/database`
-The path points to the location of the database in which your configured mappings are stored.
-- `mapping-service.pythonExecutable=${pythonExecutable:'file:///usr/bin/python3'}` \
-If no pythonExecutable is provided externally (see above) the default `/usr/bin/python3` is used.
-- `mapping-service.pluginLocation=file:///tmp/mapping-service/plugins` \
-The local folder where available plugins are located.
-- `mapping-service.mappingsLocation:file:///tmp/mapping-service/` \
-Enter the location where you want to store your mappings. This folder will be created if it does not exist yet.
 
-In order to provide the mapping-service with mapping functionality, there are already some pre-compiled plugins available under in the `plugins` folder of this repository.
-Copy them to your configured `mapping-service.pluginLocation` to make them available to the mapping-service. 
-The source code of the gemma-plugin can be found [here](https://github.com/maximilianiKIT/gemma-plugin). The plugin shows how to integrate Python mappings easily.
+| Property | Description | Default |
+|----------|-------------|---------|
+| spring.datasource.url | The path points to the location of the database in which your configured mappings are stored. For production use it is not recommended to use the pre-configured H2 database!    | jdbc:h2:file:/tmp/mapping-service/database   |
+| mapping-service.pythonExecutable | The path to your local Python executable. The default uses the pythonExecutable system property provided via -DpythonExecutable= or  file:///usr/bin/python3 if no such system property is provided. | ${pythonExecutable:'file:///usr/bin/python3'}  |
+| mapping-service.pluginLocation | The local folder from where plugins are loaded. The folder will be created on startup if it does not exist. | None  |
+| mapping-service.mappingSchemasLocation | The local folder where the mapping files are stored. The folder will be created on startup if it does not exist. | None  |
+| mapping-service.jobOutput | The local folder where asynchronous mapping execution job outputs are stored. The folder will be created on startup if it does not exist. | None  |
+| mapping-service.packagesToScan | Packages scanned for mapping plugins in addition to plugins located in mapping-service.pluginLocation. Typically, this property has not the be changed. | edu.kit.datamanager.mappingservice.plugins.impl  |
+| mapping-service.executionTimeout | The timeout in seconds a plugin process, i.e., Python of Shell, may take before it is assumed to be stale. | 30 |
 
-There is also the possibility to add new plugins directly at the source tree and create a pluggable Jar out of them. Therefor, check 
-`src/main/java/edu/kit/datamanager/mappingservice/plugins/impl`. Just add your new plugin, e.g., based on the `TestPlugin` example. 
-In order to make the plugin usable by the mapping service, you then have to build a plugin Jar out of it. In order to do that, just call:
+## Starting the Mapping-Service
 
-```
-./gradlew buildPluginJar
-```
-
-This task creates a file `default-plugins-<VERSION>` at `build/libs` which has to be copied to `mapping-service.pluginLocation` to make it available. 
-
-After doing this, the mapping-service is ready for the first start. This can be achieved by executing:
-
-`java -jar build/lib/mapping-service-<VERSION>.jar`
+The executable jar of the mapping-service is located at 'build/libs/mapping-service-<VERSION>.jar' You should copy it to some dedicated folder, 
+place 'application.properties' next to it, adapt it according to your needs, and startt he mapping-service by calling:
 
-This assumes, that the command is called from the source folder and that your `application.properties` is located in the same folder. 
-Otherwise, you may use:
-
-`java -jar build/lib/mapping-service-<VERSION>.jar --spring.config.location=/tmp/application.properties`
+`java -jar mapping-service-<VERSION>.jar`
 
-Ideally, for production use, you place everything (`mapping-service-<VERSION>.jar`, `application.properties`, `mapping-service.pluginLocation`, `mapping-service.mappingsLocation`,
-and `spring.datasource.url`) in a separate folder from where you then call the mapping-service via: 
+If your 'application.properties' is located in another folder, you may use the following call:
 
-`java -jar mapping-service-<VERSION>.jar`
+`java -jar mapping-service-<VERSION>.jar --spring.config.location=/myConfigFolder/application.properties`
 
 ## Installation
-There are three ways to install metaStore2 as a microservice:
+There are three ways to install the mapping-service as a system service:
+
 - [Using](#Installation-via-GitHub-Packages) the image available via [GitHub Packages](https://github.com/orgs/kit-data-manager/packages?repo_name=mapping-service) (***recommended***)
 - [Building](#Build-docker-container-locally) docker image locally
 - [Building](#Build-and-run-locally) and running locally
 
 ## Installation via GitHub Packages
 ### Prerequisites
-In order to run this microservice via docker you'll need:
+In order to run the mapping-service via docker you'll need:
 
 * [Docker](https://www.docker.com/) 
 
@@ -110,7 +96,7 @@ user@localhost:/home/user/mapping-service$
 ```
 
 #### Create image
-Now you'll have to create an image containing the microservice. This can be done via a script.
+Now you'll have to create an image containing the mapping-service. This can be done via a script.
 On default the created images will be tagged as follows:
 
 *'latest tag'-'actual date(yyyy-mm-dd)'* (e.g.: 1.1.0-2023-06-27)
@@ -128,7 +114,7 @@ user@localhost:/home/user/mapping-service$
 ```
 
 #### Build docker container
-After building image you have to create (and start) a container for executing microservice:
+After building image you have to create (and start) a container for executing the mapping-service:
 ```
 # If you want to use a specific image you may list all possible tags first.
 user@localhost:/home/user/mapping-service$ docker images ghcr.io/kit-data-manager/mapping-service --format {{.Tag}}

build.gradle

@@ -1,14 +1,19 @@
 plugins {
-    id 'org.springframework.boot' version '3.4.1'
+    id 'org.springframework.boot' version '3.4.5'
     id 'io.spring.dependency-management' version '1.1.7'
     id 'org.asciidoctor.jvm.convert' version '4.0.4'
-    id 'io.freefair.maven-publish-java' version '8.11'
-    id "org.owasp.dependencycheck" version "12.0.1"
+    id 'io.freefair.maven-publish-java' version '8.13.1'
+    id "org.owasp.dependencycheck" version "12.1.1"
     id 'net.researchgate.release' version '3.1.0'
-    id "com.gorylenko.gradle-git-properties" version "2.4.2"
-    id 'io.freefair.lombok' version '8.11'
+    id "com.gorylenko.gradle-git-properties" version "2.5.0"
+    id 'io.freefair.lombok' version '8.13.1'
     id 'java'
+    id 'application'
     id 'jacoco'
+    // plugins for release and publishing to maven repo
+    id "signing"
+    id "io.github.gradle-nexus.publish-plugin" version "2.0.0"
+    id 'maven-publish'
 }
 
 description = 'Generic mapping service supporting different mapping implementations.'
@@ -45,23 +50,27 @@ repositories {
 ext {
     set('snippetsDir', file('build/generated-snippets'))
     applicationProperties = System.getProperty('applicationProperties', './src/test/resources/test-config/application-test.properties')
-    pythonExecutable = System.getProperty('pythonExecutable', 'file:///usr/bin/python')
+    pythonExecutable = System.getProperty('pythonExecutable', "file:///usr/bin/python3")
     userDir = System.getProperty('user.dir')
     set('springBootVersion', "3.2.1")
-    set('springDocVersion', "2.8.3")
-    set('javersVersion', "7.7.0")
+    set('springDocVersion', "2.8.8")
+    set('javersVersion', "7.8.0")
     set('keycloakVersion', "19.0.0")
 }
 
 dependencies {
-    implementation 'org.eclipse.jgit:org.eclipse.jgit:7.1.0.202411261347-r'
+    implementation 'org.eclipse.jgit:org.eclipse.jgit:7.2.1.202505142326-r'
     implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
     implementation 'org.springframework.boot:spring-boot-starter-security'
     implementation "org.springframework.boot:spring-boot-starter-data-rest"
     implementation 'org.springframework.boot:spring-boot-starter-validation'
     implementation 'org.springframework.boot:spring-boot-starter-web'
     implementation 'org.springframework.boot:spring-boot-starter-actuator'
 
+    // monitoring
+    implementation 'io.micronaut.micrometer:micronaut-micrometer-registry-prometheus:5.10.2'
+    implementation 'org.springframework.boot:spring-boot-starter-actuator:3.3.4'
+
      // springdoc
     implementation "org.springdoc:springdoc-openapi-starter-webmvc-ui:${springDocVersion}"
     implementation "org.springdoc:springdoc-openapi-starter-common:${springDocVersion}"
@@ -74,15 +83,20 @@ dependencies {
 
     implementation "org.javers:javers-spring-boot-starter-sql:${javersVersion}"
     implementation 'org.apache.httpcomponents:httpclient:4.5.14'
-    implementation 'org.apache.commons:commons-collections4:4.4'
-    implementation 'org.json:json:20250107'
+    implementation 'org.apache.commons:commons-collections4:4.5.0'
+    implementation 'org.apache.maven:maven-artifact:3.9.9'
+    implementation 'org.json:json:20250517'
     implementation 'com.github.jknack:handlebars:4.4.0'
-    implementation 'com.google.guava:guava:33.4.0-jre'
-    implementation 'commons-io:commons-io:2.18.0'
+    implementation 'com.google.guava:guava:33.4.8-jre'
+    implementation 'commons-io:commons-io:2.19.0'
     implementation 'javax.validation:validation-api:2.0.1.Final'
-    implementation 'edu.kit.datamanager:service-base:1.3.3'
+    implementation ('edu.kit.datamanager:service-base:1.3.4'){
+        //exclude dependency as spring boot includes 
+        //org.glassfish.jaxb:jaxb-core:4.0.5 which leads to a duplication conflict
+       	exclude group: "com.sun.xml.bind"
+    }
     // apache
-    implementation "org.apache.tika:tika-core:2.9.2"
+    implementation "org.apache.tika:tika-core:3.1.0"
 
     testImplementation platform('org.junit:junit-bom')
     testImplementation 'org.junit.jupiter:junit-jupiter'
@@ -94,10 +108,10 @@ dependencies {
     //testImplementation 'org.springframework:spring-test'
 
     testImplementation 'org.springframework.restdocs:spring-restdocs-mockmvc:3.0.3'
-    testImplementation 'org.mockito:mockito-core:5.15.2'
+    testImplementation 'org.mockito:mockito-core:5.17.0'
     testImplementation 'org.powermock:powermock-module-junit4:2.0.9'
     testImplementation 'org.powermock:powermock-api-mockito2:2.0.9'
-    testImplementation 'net.bytebuddy:byte-buddy:1.16.1'
+    testImplementation 'net.bytebuddy:byte-buddy:1.17.5'
 
     annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
 
@@ -125,7 +139,7 @@ test {
 }
 
 jacoco {
-    toolVersion = "0.8.12"
+    toolVersion = "0.8.13"
 }
 
 jacocoTestReport{
@@ -142,27 +156,31 @@ asciidoctor {
     dependsOn test
 }
 
-/*jar {
-    manifest {
-        attributes 'Main-Class': 'edu.kit.datamanager.mapping-service.MappingServiceApplication'
-    }
-    archiveBaseName = 'mapping-service'
-    // version is defined in file 'gradle.properties'
-    archiveVersion = System.getenv('version')
-    // disable plain jar file
-    enabled = false
-}*/
-
 bootJar {
-    println 'Create bootable jar...'
+    dependsOn asciidoctor
+    //dependsOn 'buildPluginJar'
+
+     from ("${asciidoctor.outputDir}/html5") {
+        into 'static/docs'
+    }
     archiveFileName = "${archiveBaseName.get()}.${archiveExtension.get()}"
     duplicatesStrategy = DuplicatesStrategy.EXCLUDE
     manifest {
         attributes 'Main-Class': 'org.springframework.boot.loader.launch.PropertiesLauncher'
     }
+   //exclude '**/plugins/impl/**'
+
+   // from('build/libs/default-plugins-1.1.2-SNAPSHOT.jar') {
+   //     into('BOOT-INF/lib/plugins' )
+   //  }
     launchScript()
 }
 
+if (project.hasProperty('release')) {
+    println 'Using \'release\' profile for building ' + project.getName()
+    apply from: 'gradle/profile-deploy.gradle'
+}
+
 springBoot {    
     buildInfo()
 }
@@ -172,25 +190,29 @@ bootRun {
     systemProperty "pythonLocation", pythonExecutable
 }
 
-bootJar {
-    dependsOn asciidoctor
-    from ("${asciidoctor.outputDir}/html5") {
-        into 'static/docs'
-    }
-    launchScript()
-}
-
 release {
     tagTemplate = 'v${version}'
 }
 
-task buildPluginJar(type: Jar) {
- description = 'Bundeling only plugin classes'
- archiveFileName.set("default-plugins-${version}.jar")
- from sourceSets.main.output
- include '**/plugins/impl/*.class'
+tasks.named("jar") {
+    enabled = false
 }
 
+//task buildPluginJar(type: Jar) {
+//  description = 'Bundeling plugin core classes'
+//    archiveFileName.set("plugin-core-${version}.jar")
+//    from sourceSets.main.output
+//   include '**/configuration/ApplicationProperties.class', 
+//            '**/exception.BadExitCodeException.class', 
+//            '**/plugins/AbstractPythonMappingPlugin.class', 
+//            '**/plugins/IMappingPlugin.class', 
+//            '**/plugins/MappingPluginException.class', 
+//            '**/plugins/MappingPluginState.class', 
+//            '**/util/PythonRunnerUtil.class', 
+//            '**/util/ShellRunnerUtil.class'
+//    includeEmptyDirs false
+//}
+
 // task for printing project name.
 // should be the last task inside file
 task printProjectName {