Continued with AbstractPythonMappingPlugin, integrated Python version check, added new property for codeLocation, still required proper integration

Author: ThomasJejkal

src/main/java/edu/kit/datamanager/mappingservice/configuration/ApplicationProperties.java

@@ -59,6 +59,14 @@ public class ApplicationProperties {
     @Value("${mapping-service.mappingSchemasLocation}")
     private URL mappingsLocation;
 
+    /**
+     * The absolute path where mapping plugin code is checked out into, i.e.,
+     * for Python-based plugins.
+     */
+    @LocalFolderURL
+    @Value("${mapping-service.codeLocation}")
+    private URL codeLocation;
+
     /**
      * The absolute path where job data is stored.
      */

src/main/java/edu/kit/datamanager/mappingservice/plugins/AbstractPythonMappingPlugin.java

@@ -1,31 +1,55 @@
 /*
- * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license
- * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template
+ * Copyright 2025 Karlsruhe Institute of Technology.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 package edu.kit.datamanager.mappingservice.plugins;
 
+import edu.kit.datamanager.mappingservice.configuration.ApplicationProperties;
 import edu.kit.datamanager.mappingservice.exception.PluginInitializationFailedException;
 import edu.kit.datamanager.mappingservice.util.FileUtil;
 import edu.kit.datamanager.mappingservice.util.PythonRunnerUtil;
 import edu.kit.datamanager.mappingservice.util.ShellRunnerUtil;
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.Arrays;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Properties;
+import org.apache.maven.artifact.versioning.ComparableVersion;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
 
 /**
  *
  * @author jejkal
  */
+@Component
 public abstract class AbstractPythonMappingPlugin implements IMappingPlugin {
 
     private final Logger LOGGER = LoggerFactory.getLogger(AbstractPythonMappingPlugin.class);
+
+    /**
+     * Application properties autowired at instantiation time.
+     */
+    @Autowired
+    private final ApplicationProperties applicationProperties;
     /**
      * The plugin name.
      */
@@ -108,27 +132,28 @@ public AbstractPythonMappingPlugin(String pluginName, String repositoryUrl) {
 
     /**
      * Abstract method that is supposed to be implemented by each Python mapping
-     * plugin the gather all information required for starting a Python process
-     * executing the mapping script. The returned array should contain at least
+     * plugin to gather all information required for starting a Python process
+     * executing the mapping script. The returned array must contain at least
      * the following information:
      *
      * <ul> <li>The absolute path of the main script. It must start
-     * with the working dir given as argument, where all checked out code is
+     * with the working dir received as argument, where all checked-out code is
      * located.</li> <li>Script-specific parameters to provide
-     * mappingFile, inputFile, and outputFile. Depending on the script
-     * implementation the number of required arguments may differ.</li>
-     * </ul>
+     * mappingFile, inputFile, and outputFile to the script execution. Depending
+     * on the script implementation, the number and kind of required arguments
+     * may differ.</li> </ul>
      *
-     * Example: In standalone mode, your script is called via `plugin_wrapper.py
-     * sem -m mappingFile -i inputFile -o outputFile`. In that case, the
+     * Example: In standalone mode, a script is called via `plugin_wrapper.py
+     * sem -m mappingFile -i inputFile -o outputFile -debug`. In that case, the
      * resulting array should look as follows: [workingDir +
      * "plugin_wrapper.py", "sem", "-m", mappingFile.toString(), "-i",
-     * inputFile.toString(), "-o", outputFile.toString()]. The Python call
-     * itself will be added depending on the local installation and must not be
-     * included.
+     * inputFile.toString(), "-o", outputFile.toString(), "-debug"].
+     *
+     * The Python call itself will be added according to the Venv used for
+     * plugin execution and must not be included.
      *
      * @param workingDir The working directory, i.e., where the plugin code was
-     * checked out into.
+     * checked-out into.
      * @param mappingFile The file which contains the mapping rules registered
      * at the mapping-service and used by the script.
      * @param inputFile The file which was uploaded by the user, i.e., the
@@ -165,23 +190,33 @@ public String uri() {
     @Override
     public void setup() {
         LOGGER.trace("Setting up mapping plugin {} {}", name(), version());
-        
+
         //testing minimal Python version
-        
-        
-        
-        
+        if (minPython != null) {
+            if (!hasMinimalPythonVersion(minPython)) {
+                throw new PluginInitializationFailedException("Minimal Python version '" + minPython + "' required by plugin not met.");
+            }
+        }
+
+        //checkout and install plugin
         try {
-            LOGGER.info("Cloning git repository {}, Tag {}", repositoryUrl, tag);
-            dir = FileUtil.cloneGitRepository(repositoryUrl, tag);
+            LOGGER.info("Cloning git repository {}, tag {}", repositoryUrl, tag);
+            dir = FileUtil.cloneGitRepository(repositoryUrl, tag, Paths.get(applicationProperties.getCodeLocation().toURI()).toAbsolutePath().toString());
             // Install Python dependencies
             MappingPluginState venvState = PythonRunnerUtil.runPythonScript("-m", "venv", "--system-site-packages", dir + "/" + pluginVenv);
             if (MappingPluginState.SUCCESS().getState().equals(venvState.getState())) {
                 LOGGER.info("Venv for plugin installed successfully. Installing packages.");
-                ShellRunnerUtil.run(dir + "/" + venvInterpreter, "-m", "pip", "install", "-r", dir + "/" + "requirements.dist.txt");
+                MappingPluginState requirementsInstallState = ShellRunnerUtil.run(dir + "/" + venvInterpreter, "-m", "pip", "install", "-r", dir + "/" + "requirements.dist.txt");
+                if (MappingPluginState.SUCCESS().getState().equals(requirementsInstallState.getState())) {
+                    LOGGER.info("Requirements for plugin installed successfully. Setup complete.");
+                } else {
+                    throw new PluginInitializationFailedException("Failed to install plugin requirements. Status: " + venvState.getState());
+                }
             } else {
-                throw new PluginInitializationFailedException("Venv installation was not successful. Status: " + venvState.getState());
+                throw new PluginInitializationFailedException("Venv installation has failed. Status: " + venvState.getState());
             }
+        } catch (URISyntaxException e) {
+            throw new PluginInitializationFailedException("Invalid codeLocation configured in application.properties.", e);
         } catch (MappingPluginException e) {
             throw new PluginInitializationFailedException("Unexpected error during plugin setup.", e);
         }
@@ -190,16 +225,64 @@ public void setup() {
     @Override
     public MappingPluginState mapFile(Path mappingFile, Path inputFile, Path outputFile) throws MappingPluginException {
         long startTime = System.currentTimeMillis();
-        LOGGER.trace("Run SEM-Mapping-Tool on '{}' with mapping '{}' -> '{}'", mappingFile, inputFile, outputFile);
+        LOGGER.trace("Run mapping plugin {} {} on '{}' with mapping '{}' -> '{}'", name(), version(), mappingFile, inputFile, outputFile);
         String[] commandArray = getCommandArray(dir, mappingFile, inputFile, outputFile);
         List<String> command = new LinkedList<>();
         command.add(dir + "/" + venvInterpreter);
         command.addAll(Arrays.asList(commandArray));
-        //MappingPluginState result = ShellRunnerUtil.run(dir + "/" + venvInterpreter, dir + "/plugin_wrapper.py", "sem", "-m", mappingFile.toString(), "-i", inputFile.toString(), "-o", outputFile.toString());
         MappingPluginState result = ShellRunnerUtil.run(command.toArray(String[]::new));
         long endTime = System.currentTimeMillis();
         long totalTime = endTime - startTime;
         LOGGER.info("Execution time of mapFile: {} milliseconds", totalTime);
         return result;
     }
+
+    /**
+     * This method checks if the local Python installation version is larger or
+     * equal the provided version number. The version should be provided as
+     * semantic version number, i.e., 3.13.2
+     *
+     * The method will return TRUE if the minimal requirements are met and false
+     * otherwise. False is also returned if obtaining/parsing the local python
+     * version fails. for any reason.
+     *
+     * @param versionString The semantic version string to compare the local
+     * Python version against.
+     *
+     * @return True if versionString is smaller or equal the local Python
+     * version, false otherwise.
+     */
+    private boolean hasMinimalPythonVersion(String versionString) {
+        boolean result = false;
+        try {
+            LOGGER.trace("Checking for minimal Python version {}.", versionString);
+            ByteArrayOutputStream bout = new ByteArrayOutputStream();
+            List<String> command = new LinkedList<>();
+            command.add(dir + "/" + venvInterpreter);
+            command.addAll(Arrays.asList("--version"));
+            MappingPluginState state = ShellRunnerUtil.run(bout, System.err, command.toArray(String[]::new));
+
+            if (!MappingPluginState.StateEnum.SUCCESS.equals(state.getState())) {
+                LOGGER.error("Failed to obtain Python version. python --version returned with status {}.", state.getState());
+            } else {
+
+                LOGGER.trace("Version command output: {}", bout.toString());
+
+                String[] split = bout.toString().split(" ");
+
+                if (split.length == 2) {
+                    String localPythonVersion = bout.toString().split(" ")[1].trim();
+                    LOGGER.trace("Obtained local Python version: {}", localPythonVersion);
+                    ComparableVersion localVersion = new ComparableVersion(localPythonVersion);
+                    ComparableVersion minimalVersion = new ComparableVersion(versionString);
+                    result = minimalVersion.compareTo(localVersion) <= 0;
+                } else {
+                    LOGGER.info("Unexpected Python version output. Unable to check for minimal version.");
+                }
+            }
+        } catch (MappingPluginException e) {
+            LOGGER.error("Failed to obtain Python version.", e);
+        }
+        return result;
+    }
 }

src/main/java/edu/kit/datamanager/mappingservice/plugins/impl/GemmaPlugin.java

@@ -15,9 +15,7 @@
  */
 package edu.kit.datamanager.mappingservice.plugins.impl;
 
-import edu.kit.datamanager.mappingservice.exception.PluginInitializationFailedException;
 import edu.kit.datamanager.mappingservice.plugins.*;
-import edu.kit.datamanager.mappingservice.util.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.util.MimeType;

src/main/java/edu/kit/datamanager/mappingservice/util/FileUtil.java

@@ -42,7 +42,6 @@
 import org.apache.tika.mime.MimeType;
 import org.apache.tika.mime.MimeTypeException;
 import org.apache.tika.mime.MimeTypes;
-import org.eclipse.jgit.errors.RepositoryNotFoundException;
 
 /**
  * Various utility methods for file handling.
@@ -282,20 +281,10 @@ private static String guessFileExtension(String filename, byte[] fewKilobytesOfF
     }
 
     /**
-     * This method clones a git repository into the 'lib' folder.
-     *
-     * @param repositoryUrl the url of the repository to clone
-     * @param branch the branch to clone
-     * @return the path to the cloned repository
-     */
-    public static Path cloneGitRepository(String repositoryUrl, String branch) {
-        String target = "lib/" + repositoryUrl.trim().replace("https://", "").replace("http://", "").replace(".git", "") + "_" + branch;
-        return cloneGitRepository(repositoryUrl, branch, target);
-    }
-
-    /**
-     * This method clones a git repository into the 'lib' folder. If the folder
-     * already exists, a pull is performed.
+     * This method clones a git repository into the provided target folder. If
+     * the folder already exists, a pull is performed, otherwise it is created
+     * before. Typically, the target folder should be takes from property
+     * 'mapping-service.codeLocation' obtained from ApplicationProperties.
      *
      * @param repositoryUrl the url of the repository to clone
      * @param branch the branch to clone