Merge branch 'development' into renovate/springdocversion

Author: ThomasJejkal

build.gradle

@@ -82,9 +82,13 @@ dependencies {
     implementation 'com.google.guava:guava:33.4.8-jre'
     implementation 'commons-io:commons-io:2.19.0'
     implementation 'javax.validation:validation-api:2.0.1.Final'
-    implementation 'edu.kit.datamanager:service-base:1.3.3'
+    implementation ('edu.kit.datamanager:service-base:1.3.3'){
+        //exclude dependency as spring boot includes 
+        //org.glassfish.jaxb:jaxb-core:4.0.5 which leads to a duplication conflict
+       	exclude group: "com.sun.xml.bind"
+    }
     // apache
-    implementation "org.apache.tika:tika-core:2.9.3"
+    implementation "org.apache.tika:tika-core:3.1.0"
 
     testImplementation platform('org.junit:junit-bom')
     testImplementation 'org.junit.jupiter:junit-jupiter'

src/main/java/edu/kit/datamanager/mappingservice/configuration/ApplicationProperties.java

@@ -59,6 +59,14 @@ public class ApplicationProperties {
     @Value("${mapping-service.mappingSchemasLocation}")
     private URL mappingsLocation;
 
+    /**
+     * The absolute path where mapping plugin code is checked out into, i.e.,
+     * for Python-based plugins.
+     */
+    @LocalFolderURL
+    @Value("${mapping-service.codeLocation}")
+    private URL codeLocation;
+
     /**
      * The absolute path where job data is stored.
      */

src/main/java/edu/kit/datamanager/mappingservice/impl/MappingService.java

@@ -498,6 +498,11 @@ private void init(ApplicationProperties applicationProperties) {
             } catch (IOException e) {
                 throw new MappingServiceException(String.format("Could not initialize job output directory '%s'.", applicationProperties.getJobOutputLocation()), e);
             }
+            try {
+                Files.createDirectories(new File(applicationProperties.getCodeLocation().getPath()).getAbsoluteFile().toPath());
+            } catch (IOException e) {
+                throw new MappingServiceException(String.format("Could not initialize code target directory '%s'.", applicationProperties.getCodeLocation()), e);
+            }
         } else {
             throw new MappingServiceException("Cannot configure MappingService due to missing application.properties.");
         }

src/main/java/edu/kit/datamanager/mappingservice/plugins/AbstractPythonMappingPlugin.java

@@ -1,23 +1,40 @@
 /*
- * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license
- * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template
+ * Copyright 2025 Karlsruhe Institute of Technology.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 package edu.kit.datamanager.mappingservice.plugins;
 
+import edu.kit.datamanager.mappingservice.configuration.ApplicationProperties;
 import edu.kit.datamanager.mappingservice.exception.PluginInitializationFailedException;
 import edu.kit.datamanager.mappingservice.util.FileUtil;
 import edu.kit.datamanager.mappingservice.util.PythonRunnerUtil;
 import edu.kit.datamanager.mappingservice.util.ShellRunnerUtil;
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.Arrays;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Properties;
+import org.apache.maven.artifact.versioning.ComparableVersion;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
 
 /**
  *
@@ -26,6 +43,7 @@
 public abstract class AbstractPythonMappingPlugin implements IMappingPlugin {
 
     private final Logger LOGGER = LoggerFactory.getLogger(AbstractPythonMappingPlugin.class);
+
     /**
      * The plugin name.
      */
@@ -108,27 +126,28 @@ public AbstractPythonMappingPlugin(String pluginName, String repositoryUrl) {
 
     /**
      * Abstract method that is supposed to be implemented by each Python mapping
-     * plugin the gather all information required for starting a Python process
-     * executing the mapping script. The returned array should contain at least
+     * plugin to gather all information required for starting a Python process
+     * executing the mapping script. The returned array must contain at least
      * the following information:
      *
      * <ul> <li>The absolute path of the main script. It must start
-     * with the working dir given as argument, where all checked out code is
+     * with the working dir received as argument, where all checked-out code is
      * located.</li> <li>Script-specific parameters to provide
-     * mappingFile, inputFile, and outputFile. Depending on the script
-     * implementation the number of required arguments may differ.</li>
-     * </ul>
+     * mappingFile, inputFile, and outputFile to the script execution. Depending
+     * on the script implementation, the number and kind of required arguments
+     * may differ.</li> </ul>
      *
-     * Example: In standalone mode, your script is called via `plugin_wrapper.py
-     * sem -m mappingFile -i inputFile -o outputFile`. In that case, the
+     * Example: In standalone mode, a script is called via `plugin_wrapper.py
+     * sem -m mappingFile -i inputFile -o outputFile -debug`. In that case, the
      * resulting array should look as follows: [workingDir +
      * "plugin_wrapper.py", "sem", "-m", mappingFile.toString(), "-i",
-     * inputFile.toString(), "-o", outputFile.toString()]. The Python call
-     * itself will be added depending on the local installation and must not be
-     * included.
+     * inputFile.toString(), "-o", outputFile.toString(), "-debug"].
+     *
+     * The Python call itself will be added according to the Venv used for
+     * plugin execution and must not be included.
      *
      * @param workingDir The working directory, i.e., where the plugin code was
-     * checked out into.
+     * checked-out into.
      * @param mappingFile The file which contains the mapping rules registered
      * at the mapping-service and used by the script.
      * @param inputFile The file which was uploaded by the user, i.e., the
@@ -163,25 +182,44 @@ public String uri() {
     }
 
     @Override
-    public void setup() {
+    public void setup(ApplicationProperties applicationProperties) {
         LOGGER.trace("Setting up mapping plugin {} {}", name(), version());
-        
+
         //testing minimal Python version
-        
-        
-        
-        
+        if (minPython != null) {
+            if (!hasMinimalPythonVersion(minPython)) {
+                throw new PluginInitializationFailedException("Minimal Python version '" + minPython + "' required by plugin not met.");
+            }
+        }
+
+        //checkout and install plugin
         try {
-            LOGGER.info("Cloning git repository {}, Tag {}", repositoryUrl, tag);
-            dir = FileUtil.cloneGitRepository(repositoryUrl, tag);
+            LOGGER.info("Cloning git repository {}, tag {}", repositoryUrl, tag);
+            Path path = Paths.get(applicationProperties.getCodeLocation().toURI());
+            path = path.resolve(repositoryUrl.trim().replace("https://", "").replace("http://", "").replace(".git", "") + "_" + version());
+            LOGGER.info("Target path: {}", path);
+            dir = FileUtil.cloneGitRepository(repositoryUrl, tag, path.toAbsolutePath().toString());
             // Install Python dependencies
             MappingPluginState venvState = PythonRunnerUtil.runPythonScript("-m", "venv", "--system-site-packages", dir + "/" + pluginVenv);
             if (MappingPluginState.SUCCESS().getState().equals(venvState.getState())) {
-                LOGGER.info("Venv for plugin installed successfully. Installing packages.");
-                ShellRunnerUtil.run(dir + "/" + venvInterpreter, "-m", "pip", "install", "-r", dir + "/" + "requirements.dist.txt");
+                LOGGER.info("Venv for plugin installed successfully. Installing requirements.");
+
+                Path requirementsFile = Paths.get(dir + "/" + "requirements.dist.txt");
+                if (requirementsFile.toFile().exists()) {
+                    MappingPluginState requirementsInstallState = ShellRunnerUtil.run(dir + "/" + venvInterpreter, "-m", "pip", "install", "-r", dir + "/" + "requirements.dist.txt");
+                    if (MappingPluginState.SUCCESS().getState().equals(requirementsInstallState.getState())) {
+                        LOGGER.info("Requirements for plugin installed successfully. Setup complete.");
+                    } else {
+                        throw new PluginInitializationFailedException("Failed to install plugin requirements. Status: " + venvState.getState());
+                    }
+                } else {
+                    LOGGER.info("No requirements file found. Skipping dependency installation.");
+                }
             } else {
-                throw new PluginInitializationFailedException("Venv installation was not successful. Status: " + venvState.getState());
+                throw new PluginInitializationFailedException("Venv installation has failed. Status: " + venvState.getState());
             }
+        } catch (URISyntaxException e) {
+            throw new PluginInitializationFailedException("Invalid codeLocation configured in application.properties.", e);
         } catch (MappingPluginException e) {
             throw new PluginInitializationFailedException("Unexpected error during plugin setup.", e);
         }
@@ -190,16 +228,61 @@ public void setup() {
     @Override
     public MappingPluginState mapFile(Path mappingFile, Path inputFile, Path outputFile) throws MappingPluginException {
         long startTime = System.currentTimeMillis();
-        LOGGER.trace("Run SEM-Mapping-Tool on '{}' with mapping '{}' -> '{}'", mappingFile, inputFile, outputFile);
+        LOGGER.trace("Run mapping plugin {} {} on '{}' with mapping '{}' -> '{}'", name(), version(), mappingFile, inputFile, outputFile);
         String[] commandArray = getCommandArray(dir, mappingFile, inputFile, outputFile);
         List<String> command = new LinkedList<>();
         command.add(dir + "/" + venvInterpreter);
         command.addAll(Arrays.asList(commandArray));
-        //MappingPluginState result = ShellRunnerUtil.run(dir + "/" + venvInterpreter, dir + "/plugin_wrapper.py", "sem", "-m", mappingFile.toString(), "-i", inputFile.toString(), "-o", outputFile.toString());
         MappingPluginState result = ShellRunnerUtil.run(command.toArray(String[]::new));
         long endTime = System.currentTimeMillis();
         long totalTime = endTime - startTime;
         LOGGER.info("Execution time of mapFile: {} milliseconds", totalTime);
         return result;
     }
+
+    /**
+     * This method checks if the local Python installation version is larger or
+     * equal the provided version number. The version should be provided as
+     * semantic version number, i.e., 3.13.2
+     *
+     * The method will return TRUE if the minimal requirements are met and false
+     * otherwise. False is also returned if obtaining/parsing the local python
+     * version fails. for any reason.
+     *
+     * @param versionString The semantic version string to compare the local
+     * Python version against.
+     *
+     * @return True if versionString is smaller or equal the local Python
+     * version, false otherwise.
+     */
+    private boolean hasMinimalPythonVersion(String versionString) {
+        boolean result = false;
+        try {
+            LOGGER.trace("Checking for minimal Python version {}.", versionString);
+            ByteArrayOutputStream bout = new ByteArrayOutputStream();
+            MappingPluginState state = PythonRunnerUtil.runPythonScript("--version", bout, System.err);
+
+            if (!MappingPluginState.StateEnum.SUCCESS.equals(state.getState())) {
+                LOGGER.error("Failed to obtain Python version. python --version returned with status {}.", state.getState());
+            } else {
+
+                LOGGER.trace("Version command output: {}", bout.toString());
+
+                String[] split = bout.toString().split(" ");
+
+                if (split.length == 2) {
+                    String localPythonVersion = bout.toString().split(" ")[1].trim();
+                    LOGGER.trace("Obtained local Python version: {}", localPythonVersion);
+                    ComparableVersion localVersion = new ComparableVersion(localPythonVersion);
+                    ComparableVersion minimalVersion = new ComparableVersion(versionString);
+                    result = minimalVersion.compareTo(localVersion) <= 0;
+                } else {
+                    LOGGER.info("Unexpected Python version output. Unable to check for minimal version.");
+                }
+            }
+        } catch (MappingPluginException e) {
+            LOGGER.error("Failed to obtain Python version.", e);
+        }
+        return result;
+    }
 }

src/main/java/edu/kit/datamanager/mappingservice/plugins/IMappingPlugin.java

@@ -12,16 +12,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package edu.kit.datamanager.mappingservice.plugins;
 
+import edu.kit.datamanager.mappingservice.configuration.ApplicationProperties;
 import org.springframework.util.MimeType;
 
 import java.nio.file.Path;
 
 /**
- * Interface for mapping plugins.
- * Every plugin which implements this interface and is placed in the plugins folder will be loaded and usable via the REST-API.
+ * Interface for mapping plugins. Every plugin which implements this interface
+ * and is placed in the plugins folder will be loaded and usable via the
+ * REST-API.
  *
  * @author maximilianiKIT
  */
@@ -42,15 +43,17 @@ public interface IMappingPlugin {
     String description();
 
     /**
-     * The version of the plugin which gets displayed in the UI and is part of the id.
+     * The version of the plugin which gets displayed in the UI and is part of
+     * the id.
      *
      * @return The version of the plugin.
      */
     String version();
 
     /**
-     * A URI which refers to the plugin or the technology used by the plugin (e.g. a link to a GitHub repository).
-     * This URI will be displayed in the UI.
+     * A URI which refers to the plugin or the technology used by the plugin
+     * (e.g. a link to a GitHub repository). This URI will be displayed in the
+     * UI.
      *
      * @return The URI of the plugin.
      */
@@ -71,8 +74,9 @@ public interface IMappingPlugin {
     MimeType[] outputTypes();
 
     /**
-     * The id of the plugin which is used to identify the plugin.
-     * By default, the id is composed of the name and the version of the plugin (e.g. testPlugin_2.1.0).
+     * The id of the plugin which is used to identify the plugin. By default,
+     * the id is composed of the name and the version of the plugin (e.g.
+     * testPlugin_2.1.0).
      *
      * @return The id of the plugin.
      */
@@ -81,19 +85,22 @@ default String id() {
     }
 
     /**
-     * This method is called when the plugin is loaded.
-     * It can be used to initialize the plugin and install dependencies.
+     * This method is called when the plugin is loaded. It can be used to
+     * initialize the plugin and install dependencies.
+     *
+     * @param applicationProperties Properties object containing all
+     * mapping-service settings.
      */
-    void setup();
+    void setup(ApplicationProperties applicationProperties);
 
     /**
      * The method which is called to execute the plugin.
      *
-     * @param inputFile   The path to the output document.
-     * @param outputFile  The path to the output document.
+     * @param inputFile The path to the output document.
+     * @param outputFile The path to the output document.
      * @param mappingFile The path to the mapping schema.
      * @return The exit code of the plugin.
-     * 
+     *
      * @throws MappingPluginException If the mapping execution fails.
      */
     MappingPluginState mapFile(Path mappingFile, Path inputFile, Path outputFile) throws MappingPluginException;

src/main/java/edu/kit/datamanager/mappingservice/plugins/PluginLoader.java

@@ -100,12 +100,12 @@ public Map<String, IMappingPlugin> loadPlugins(File pluginDir, String[] packages
             cl = Thread.currentThread().getContextClassLoader();
         }
 
-        List<Class<IMappingPlugin>> plugClasses = extractClassesFromJARs(pluginJars, packagesToScan, cl);
-        List<IMappingPlugin> IMappingPluginList = createPluggableObjects(plugClasses);
+        List<Class<IMappingPlugin>> pluginClasses = extractClassesFromJARs(pluginJars, packagesToScan, cl);
+        List<IMappingPlugin> IMappingPluginList = createPluggableObjects(pluginClasses);
 
         for (IMappingPlugin i : IMappingPluginList) {
             try {
-                i.setup();
+                i.setup(applicationProperties);
                 LOG.trace(" - Adding new plugin {}, v{} to available list", i.name(), i.version());
                 result.put(i.id(), i);
             } catch (PluginInitializationFailedException re) {

src/main/java/edu/kit/datamanager/mappingservice/plugins/impl/GemmaPlugin.java

@@ -15,22 +15,14 @@
  */
 package edu.kit.datamanager.mappingservice.plugins.impl;
 
-import edu.kit.datamanager.mappingservice.exception.PluginInitializationFailedException;
 import edu.kit.datamanager.mappingservice.plugins.*;
-import edu.kit.datamanager.mappingservice.util.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.springframework.util.MimeType;
 import org.springframework.util.MimeTypeUtils;
 import java.nio.file.Path;
 
 public class GemmaPlugin extends AbstractPythonMappingPlugin {
 
-    private final Logger LOGGER = LoggerFactory.getLogger(GemmaPlugin.class);
     private static final String GEMMA_REPOSITORY = "https://github.com/kit-data-manager/gemma.git";
-    private static final String GEMMA_BRANCH = "master";
-    private static Path gemmaDir;
-    private boolean initialized = false;
 
     public GemmaPlugin() {
         super("GEMMA", GEMMA_REPOSITORY);
@@ -60,17 +52,4 @@ public String[] getCommandArray(Path workingDir, Path mappingFile, Path inputFil
             outputFile.toString()
         };
     }
-
-  /*  @Override
-    public MappingPluginState mapFile(Path mappingFile, Path inputFile, Path outputFile) throws MappingPluginException {
-        if (initialized) {
-            LOGGER.trace("Running plugin '{}' v{} on '{}' with mapping '{}' -> '{}'", name(), version(), inputFile, mappingFile, outputFile);
-            return PythonRunnerUtil.runPythonScript(gemmaDir + "/mapping_single.py", mappingFile.toString(), inputFile.toString(), outputFile.toString());
-        } else {
-            LOGGER.error("Plugin '" + name() + "' " + version() + " not initialized. Returning EXECUTION_ERROR.");
-            MappingPluginState result = MappingPluginState.EXECUTION_ERROR();
-            result.setDetails("Plugin not initialized, probably due to missing dependencies or external plugin repository.");
-            return result;
-        }
-    }*/
 }