Merge branch 'development' into renovate/gradle-8.x

Author: ThomasJejkal

build.gradle

@@ -8,6 +8,7 @@ plugins {
     id "com.gorylenko.gradle-git-properties" version "2.5.0"
     id 'io.freefair.lombok' version '8.13.1'
     id 'java'
+    id 'application'
     id 'jacoco'
 }
 
@@ -75,14 +76,19 @@ dependencies {
     implementation "org.javers:javers-spring-boot-starter-sql:${javersVersion}"
     implementation 'org.apache.httpcomponents:httpclient:4.5.14'
     implementation 'org.apache.commons:commons-collections4:4.4'
+    implementation 'org.apache.maven:maven-artifact:3.6.3'
     implementation 'org.json:json:20250107'
     implementation 'com.github.jknack:handlebars:4.4.0'
     implementation 'com.google.guava:guava:33.4.8-jre'
     implementation 'commons-io:commons-io:2.19.0'
     implementation 'javax.validation:validation-api:2.0.1.Final'
-    implementation 'edu.kit.datamanager:service-base:1.3.3'
+    implementation ('edu.kit.datamanager:service-base:1.3.3'){
+        //exclude dependency as spring boot includes 
+        //org.glassfish.jaxb:jaxb-core:4.0.5 which leads to a duplication conflict
+       	exclude group: "com.sun.xml.bind"
+    }
     // apache
-    implementation "org.apache.tika:tika-core:2.9.3"
+    implementation "org.apache.tika:tika-core:3.1.0"
 
     testImplementation platform('org.junit:junit-bom')
     testImplementation 'org.junit.jupiter:junit-jupiter'

src/main/java/edu/kit/datamanager/mappingservice/configuration/ApplicationProperties.java

@@ -15,7 +15,6 @@
  */
 package edu.kit.datamanager.mappingservice.configuration;
 
-import edu.kit.datamanager.annotations.ExecutableFileURL;
 import edu.kit.datamanager.annotations.LocalFolderURL;
 import edu.kit.datamanager.validator.ExecutableFileValidator;
 import lombok.Data;
@@ -60,6 +59,14 @@ public class ApplicationProperties {
     @Value("${mapping-service.mappingSchemasLocation}")
     private URL mappingsLocation;
 
+    /**
+     * The absolute path where mapping plugin code is checked out into, i.e.,
+     * for Python-based plugins.
+     */
+    @LocalFolderURL
+    @Value("${mapping-service.codeLocation}")
+    private URL codeLocation;
+
     /**
      * The absolute path where job data is stored.
      */

src/main/java/edu/kit/datamanager/mappingservice/impl/MappingService.java

@@ -498,6 +498,11 @@ private void init(ApplicationProperties applicationProperties) {
             } catch (IOException e) {
                 throw new MappingServiceException(String.format("Could not initialize job output directory '%s'.", applicationProperties.getJobOutputLocation()), e);
             }
+            try {
+                Files.createDirectories(new File(applicationProperties.getCodeLocation().getPath()).getAbsoluteFile().toPath());
+            } catch (IOException e) {
+                throw new MappingServiceException(String.format("Could not initialize code target directory '%s'.", applicationProperties.getCodeLocation()), e);
+            }
         } else {
             throw new MappingServiceException("Cannot configure MappingService due to missing application.properties.");
         }

src/main/java/edu/kit/datamanager/mappingservice/plugins/AbstractPythonMappingPlugin.java

@@ -0,0 +1,288 @@
+/*
+ * Copyright 2025 Karlsruhe Institute of Technology.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.kit.datamanager.mappingservice.plugins;
+
+import edu.kit.datamanager.mappingservice.configuration.ApplicationProperties;
+import edu.kit.datamanager.mappingservice.exception.PluginInitializationFailedException;
+import edu.kit.datamanager.mappingservice.util.FileUtil;
+import edu.kit.datamanager.mappingservice.util.PythonRunnerUtil;
+import edu.kit.datamanager.mappingservice.util.ShellRunnerUtil;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Properties;
+import org.apache.maven.artifact.versioning.ComparableVersion;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+/**
+ *
+ * @author jejkal
+ */
+public abstract class AbstractPythonMappingPlugin implements IMappingPlugin {
+
+    private final Logger LOGGER = LoggerFactory.getLogger(AbstractPythonMappingPlugin.class);
+
+    /**
+     * The plugin name.
+     */
+    private String name;
+    /**
+     * The URL of the Git repository where the plugin code is located.
+     */
+    private String repositoryUrl;
+
+    /**
+     * The tag which should be used to checkout a specific version from
+     * repositoryUrl.
+     */
+    private String tag;
+
+    /**
+     * The minimal python version required by the plugin
+     */
+    private String minPython;
+
+    /**
+     * The folder where the code is checked out from repositoryUrl.
+     */
+    private Path dir;
+
+    private String pluginVenv = "venv/PluginVenv";
+    private String venvInterpreter;
+
+    /**
+     * Default constructor for instantiating a Python-based mapping plugin. It
+     * is assumed, that the code for the plugin is stored in a Git repository
+     * located at 'repositoryUrl'. Furthermore, it is assumed, that the plugin
+     * itself is delivered as a single Jar file which contains at least the
+     * plugin class and a file PLUGIN_NAME.properties, where PLUGIN_NAME must be
+     * identical to the 'pluginName' argument and the file must be located in
+     * the root of the Jar file. The properties file must contain one property
+     * 'version' which represents an existing Git tag matching a released
+     * version of the Python plugin code, e.g., version=v1.0.0
+     *
+     * Furthermore, the properties file may contain a minimal Python version
+     * that is required by the plugin to work. The minimal Python version is set
+     * via the 'min.python' property, e.g.., min.python=3.10.0 If the minimal
+     * Python version is not met, the plugin will be ignored.
+     *
+     * @param pluginName The name of the plugin.
+     * @param repositoryUrl The Git repository where the plugin Python code is
+     * located.
+     */
+    public AbstractPythonMappingPlugin(String pluginName, String repositoryUrl) {
+        try {
+            name = pluginName;
+            this.repositoryUrl = repositoryUrl;
+            // Get the context class loader
+            ClassLoader classLoader = this.getClass().getClassLoader();
+            // TODO: do we need to make sure that the resource path is somehow related to the current plugin to avoid loading the wrong property file in case of identical property names?
+            URL resource = classLoader.getResource(pluginName.toLowerCase() + ".properties");
+            LOGGER.info("Resource file: {}", resource);
+            if (resource != null) {
+                // Load the properties file
+                try (InputStream input = resource.openStream()) {
+                    Properties properties = new Properties();
+                    properties.load(input);
+                    tag = properties.getProperty("version");
+                    minPython = properties.getProperty("min.python");
+                }
+            } else {
+                System.err.println("Properties file not found!");
+                tag = "unavailable";
+            }
+
+            if (System.getProperty("os.name").startsWith("Windows")) {
+                venvInterpreter = pluginVenv + "/Scripts/python.exe";
+            } else {
+                venvInterpreter = pluginVenv + "/bin/python3";
+            }
+        } catch (IOException e) {
+            throw new PluginInitializationFailedException("Failed to instantiate plugin class.", e);
+        }
+    }
+
+    /**
+     * Abstract method that is supposed to be implemented by each Python mapping
+     * plugin to gather all information required for starting a Python process
+     * executing the mapping script. The returned array must contain at least
+     * the following information:
+     *
+     * <ul> <li>The absolute path of the main script. It must start
+     * with the working dir received as argument, where all checked-out code is
+     * located.</li> <li>Script-specific parameters to provide
+     * mappingFile, inputFile, and outputFile to the script execution. Depending
+     * on the script implementation, the number and kind of required arguments
+     * may differ.</li> </ul>
+     *
+     * Example: In standalone mode, a script is called via `plugin_wrapper.py
+     * sem -m mappingFile -i inputFile -o outputFile -debug`. In that case, the
+     * resulting array should look as follows: [workingDir +
+     * "plugin_wrapper.py", "sem", "-m", mappingFile.toString(), "-i",
+     * inputFile.toString(), "-o", outputFile.toString(), "-debug"].
+     *
+     * The Python call itself will be added according to the Venv used for
+     * plugin execution and must not be included.
+     *
+     * @param workingDir The working directory, i.e., where the plugin code was
+     * checked-out into.
+     * @param mappingFile The file which contains the mapping rules registered
+     * at the mapping-service and used by the script.
+     * @param inputFile The file which was uploaded by the user, i.e., the
+     * source of the mapping process.
+     * @param outputFile The destination where mapping results must be written
+     * to in order to allow the mapping-service to return the result to the
+     * user.
+     *
+     * @return A string array containing the single elements of the command line
+     * call of the script.
+     */
+    public abstract String[] getCommandArray(Path workingDir, Path mappingFile, Path inputFile, Path outputFile);
+
+    @Override
+    public String name() {
+        return this.name;
+    }
+
+    @Override
+    public String version() {
+        return this.tag;
+    }
+
+    @Override
+    public String description() {
+        return "Plugin " + name() + ", Version " + version() + ", Implementation: " + uri();
+    }
+
+    @Override
+    public String uri() {
+        return this.repositoryUrl;
+    }
+
+    @Override
+    public void setup(ApplicationProperties applicationProperties) {
+        LOGGER.trace("Setting up mapping plugin {} {}", name(), version());
+
+        //testing minimal Python version
+        if (minPython != null) {
+            if (!hasMinimalPythonVersion(minPython)) {
+                throw new PluginInitializationFailedException("Minimal Python version '" + minPython + "' required by plugin not met.");
+            }
+        }
+
+        //checkout and install plugin
+        try {
+            LOGGER.info("Cloning git repository {}, tag {}", repositoryUrl, tag);
+            Path path = Paths.get(applicationProperties.getCodeLocation().toURI());
+            path = path.resolve(repositoryUrl.trim().replace("https://", "").replace("http://", "").replace(".git", "") + "_" + version());
+            LOGGER.info("Target path: {}", path);
+            dir = FileUtil.cloneGitRepository(repositoryUrl, tag, path.toAbsolutePath().toString());
+            // Install Python dependencies
+            MappingPluginState venvState = PythonRunnerUtil.runPythonScript("-m", "venv", "--system-site-packages", dir + "/" + pluginVenv);
+            if (MappingPluginState.SUCCESS().getState().equals(venvState.getState())) {
+                LOGGER.info("Venv for plugin installed successfully. Installing requirements.");
+
+                Path requirementsFile = Paths.get(dir + "/" + "requirements.dist.txt");
+                if (requirementsFile.toFile().exists()) {
+                    MappingPluginState requirementsInstallState = ShellRunnerUtil.run(dir + "/" + venvInterpreter, "-m", "pip", "install", "-r", dir + "/" + "requirements.dist.txt");
+                    if (MappingPluginState.SUCCESS().getState().equals(requirementsInstallState.getState())) {
+                        LOGGER.info("Requirements for plugin installed successfully. Setup complete.");
+                    } else {
+                        throw new PluginInitializationFailedException("Failed to install plugin requirements. Status: " + venvState.getState());
+                    }
+                } else {
+                    LOGGER.info("No requirements file found. Skipping dependency installation.");
+                }
+            } else {
+                throw new PluginInitializationFailedException("Venv installation has failed. Status: " + venvState.getState());
+            }
+        } catch (URISyntaxException e) {
+            throw new PluginInitializationFailedException("Invalid codeLocation configured in application.properties.", e);
+        } catch (MappingPluginException e) {
+            throw new PluginInitializationFailedException("Unexpected error during plugin setup.", e);
+        }
+    }
+
+    @Override
+    public MappingPluginState mapFile(Path mappingFile, Path inputFile, Path outputFile) throws MappingPluginException {
+        long startTime = System.currentTimeMillis();
+        LOGGER.trace("Run mapping plugin {} {} on '{}' with mapping '{}' -> '{}'", name(), version(), mappingFile, inputFile, outputFile);
+        String[] commandArray = getCommandArray(dir, mappingFile, inputFile, outputFile);
+        List<String> command = new LinkedList<>();
+        command.add(dir + "/" + venvInterpreter);
+        command.addAll(Arrays.asList(commandArray));
+        MappingPluginState result = ShellRunnerUtil.run(command.toArray(String[]::new));
+        long endTime = System.currentTimeMillis();
+        long totalTime = endTime - startTime;
+        LOGGER.info("Execution time of mapFile: {} milliseconds", totalTime);
+        return result;
+    }
+
+    /**
+     * This method checks if the local Python installation version is larger or
+     * equal the provided version number. The version should be provided as
+     * semantic version number, i.e., 3.13.2
+     *
+     * The method will return TRUE if the minimal requirements are met and false
+     * otherwise. False is also returned if obtaining/parsing the local python
+     * version fails. for any reason.
+     *
+     * @param versionString The semantic version string to compare the local
+     * Python version against.
+     *
+     * @return True if versionString is smaller or equal the local Python
+     * version, false otherwise.
+     */
+    private boolean hasMinimalPythonVersion(String versionString) {
+        boolean result = false;
+        try {
+            LOGGER.trace("Checking for minimal Python version {}.", versionString);
+            ByteArrayOutputStream bout = new ByteArrayOutputStream();
+            MappingPluginState state = PythonRunnerUtil.runPythonScript("--version", bout, System.err);
+
+            if (!MappingPluginState.StateEnum.SUCCESS.equals(state.getState())) {
+                LOGGER.error("Failed to obtain Python version. python --version returned with status {}.", state.getState());
+            } else {
+
+                LOGGER.trace("Version command output: {}", bout.toString());
+
+                String[] split = bout.toString().split(" ");
+
+                if (split.length == 2) {
+                    String localPythonVersion = bout.toString().split(" ")[1].trim();
+                    LOGGER.trace("Obtained local Python version: {}", localPythonVersion);
+                    ComparableVersion localVersion = new ComparableVersion(localPythonVersion);
+                    ComparableVersion minimalVersion = new ComparableVersion(versionString);
+                    result = minimalVersion.compareTo(localVersion) <= 0;
+                } else {
+                    LOGGER.info("Unexpected Python version output. Unable to check for minimal version.");
+                }
+            }
+        } catch (MappingPluginException e) {
+            LOGGER.error("Failed to obtain Python version.", e);
+        }
+        return result;
+    }
+}