Merge branch 'development' into renovate/edu.kit.datamanager-service-base-1.x

Author: ThomasJejkal

build.gradle

@@ -88,7 +88,7 @@ dependencies {
        	exclude group: "com.sun.xml.bind"
     }
     // apache
-    implementation "org.apache.tika:tika-core:2.9.3"
+    implementation "org.apache.tika:tika-core:3.1.0"
 
     testImplementation platform('org.junit:junit-bom')
     testImplementation 'org.junit.jupiter:junit-jupiter'

src/main/java/edu/kit/datamanager/mappingservice/impl/MappingService.java

@@ -498,6 +498,11 @@ private void init(ApplicationProperties applicationProperties) {
             } catch (IOException e) {
                 throw new MappingServiceException(String.format("Could not initialize job output directory '%s'.", applicationProperties.getJobOutputLocation()), e);
             }
+            try {
+                Files.createDirectories(new File(applicationProperties.getCodeLocation().getPath()).getAbsoluteFile().toPath());
+            } catch (IOException e) {
+                throw new MappingServiceException(String.format("Could not initialize code target directory '%s'.", applicationProperties.getCodeLocation()), e);
+            }
         } else {
             throw new MappingServiceException("Cannot configure MappingService due to missing application.properties.");
         }

src/main/java/edu/kit/datamanager/mappingservice/plugins/AbstractPythonMappingPlugin.java

@@ -44,11 +44,6 @@ public abstract class AbstractPythonMappingPlugin implements IMappingPlugin {
 
     private final Logger LOGGER = LoggerFactory.getLogger(AbstractPythonMappingPlugin.class);
 
-    /**
-     * Application properties autowired at instantiation time via
-     * setApplicationProperties.
-     */
-    private ApplicationProperties applicationProperties;
     /**
      * The plugin name.
      */
@@ -129,17 +124,6 @@ public AbstractPythonMappingPlugin(String pluginName, String repositoryUrl) {
         }
     }
 
-    /**
-     * Setter to autowire ApplicationProperties into all implementations of this
-     * abstract class.
-     *
-     * @param applicationProperties The applicationProperties bean.
-     */
-    @Autowired
-    public final void setApplicationProperties(ApplicationProperties applicationProperties) {
-        this.applicationProperties = applicationProperties;
-    }
-
     /**
      * Abstract method that is supposed to be implemented by each Python mapping
      * plugin to gather all information required for starting a Python process
@@ -198,7 +182,7 @@ public String uri() {
     }
 
     @Override
-    public void setup() {
+    public void setup(ApplicationProperties applicationProperties) {
         LOGGER.trace("Setting up mapping plugin {} {}", name(), version());
 
         //testing minimal Python version
@@ -211,16 +195,25 @@ public void setup() {
         //checkout and install plugin
         try {
             LOGGER.info("Cloning git repository {}, tag {}", repositoryUrl, tag);
-            dir = FileUtil.cloneGitRepository(repositoryUrl, tag, Paths.get(applicationProperties.getCodeLocation().toURI()).toAbsolutePath().toString());
+            Path path = Paths.get(applicationProperties.getCodeLocation().toURI());
+            path = path.resolve(repositoryUrl.trim().replace("https://", "").replace("http://", "").replace(".git", "") + "_" + version());
+            LOGGER.info("Target path: {}", path);
+            dir = FileUtil.cloneGitRepository(repositoryUrl, tag, path.toAbsolutePath().toString());
             // Install Python dependencies
             MappingPluginState venvState = PythonRunnerUtil.runPythonScript("-m", "venv", "--system-site-packages", dir + "/" + pluginVenv);
             if (MappingPluginState.SUCCESS().getState().equals(venvState.getState())) {
-                LOGGER.info("Venv for plugin installed successfully. Installing packages.");
-                MappingPluginState requirementsInstallState = ShellRunnerUtil.run(dir + "/" + venvInterpreter, "-m", "pip", "install", "-r", dir + "/" + "requirements.dist.txt");
-                if (MappingPluginState.SUCCESS().getState().equals(requirementsInstallState.getState())) {
-                    LOGGER.info("Requirements for plugin installed successfully. Setup complete.");
+                LOGGER.info("Venv for plugin installed successfully. Installing requirements.");
+
+                Path requirementsFile = Paths.get(dir + "/" + "requirements.dist.txt");
+                if (requirementsFile.toFile().exists()) {
+                    MappingPluginState requirementsInstallState = ShellRunnerUtil.run(dir + "/" + venvInterpreter, "-m", "pip", "install", "-r", dir + "/" + "requirements.dist.txt");
+                    if (MappingPluginState.SUCCESS().getState().equals(requirementsInstallState.getState())) {
+                        LOGGER.info("Requirements for plugin installed successfully. Setup complete.");
+                    } else {
+                        throw new PluginInitializationFailedException("Failed to install plugin requirements. Status: " + venvState.getState());
+                    }
                 } else {
-                    throw new PluginInitializationFailedException("Failed to install plugin requirements. Status: " + venvState.getState());
+                    LOGGER.info("No requirements file found. Skipping dependency installation.");
                 }
             } else {
                 throw new PluginInitializationFailedException("Venv installation has failed. Status: " + venvState.getState());
@@ -267,10 +260,7 @@ private boolean hasMinimalPythonVersion(String versionString) {
         try {
             LOGGER.trace("Checking for minimal Python version {}.", versionString);
             ByteArrayOutputStream bout = new ByteArrayOutputStream();
-            List<String> command = new LinkedList<>();
-            command.add(dir + "/" + venvInterpreter);
-            command.addAll(Arrays.asList("--version"));
-            MappingPluginState state = ShellRunnerUtil.run(bout, System.err, command.toArray(String[]::new));
+            MappingPluginState state = PythonRunnerUtil.runPythonScript("--version", bout, System.err);
 
             if (!MappingPluginState.StateEnum.SUCCESS.equals(state.getState())) {
                 LOGGER.error("Failed to obtain Python version. python --version returned with status {}.", state.getState());

src/main/java/edu/kit/datamanager/mappingservice/plugins/IMappingPlugin.java

@@ -12,16 +12,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package edu.kit.datamanager.mappingservice.plugins;
 
+import edu.kit.datamanager.mappingservice.configuration.ApplicationProperties;
 import org.springframework.util.MimeType;
 
 import java.nio.file.Path;
 
 /**
- * Interface for mapping plugins.
- * Every plugin which implements this interface and is placed in the plugins folder will be loaded and usable via the REST-API.
+ * Interface for mapping plugins. Every plugin which implements this interface
+ * and is placed in the plugins folder will be loaded and usable via the
+ * REST-API.
  *
  * @author maximilianiKIT
  */
@@ -42,15 +43,17 @@ public interface IMappingPlugin {
     String description();
 
     /**
-     * The version of the plugin which gets displayed in the UI and is part of the id.
+     * The version of the plugin which gets displayed in the UI and is part of
+     * the id.
      *
      * @return The version of the plugin.
      */
     String version();
 
     /**
-     * A URI which refers to the plugin or the technology used by the plugin (e.g. a link to a GitHub repository).
-     * This URI will be displayed in the UI.
+     * A URI which refers to the plugin or the technology used by the plugin
+     * (e.g. a link to a GitHub repository). This URI will be displayed in the
+     * UI.
      *
      * @return The URI of the plugin.
      */
@@ -71,8 +74,9 @@ public interface IMappingPlugin {
     MimeType[] outputTypes();
 
     /**
-     * The id of the plugin which is used to identify the plugin.
-     * By default, the id is composed of the name and the version of the plugin (e.g. testPlugin_2.1.0).
+     * The id of the plugin which is used to identify the plugin. By default,
+     * the id is composed of the name and the version of the plugin (e.g.
+     * testPlugin_2.1.0).
      *
      * @return The id of the plugin.
      */
@@ -81,19 +85,22 @@ default String id() {
     }
 
     /**
-     * This method is called when the plugin is loaded.
-     * It can be used to initialize the plugin and install dependencies.
+     * This method is called when the plugin is loaded. It can be used to
+     * initialize the plugin and install dependencies.
+     *
+     * @param applicationProperties Properties object containing all
+     * mapping-service settings.
      */
-    void setup();
+    void setup(ApplicationProperties applicationProperties);
 
     /**
      * The method which is called to execute the plugin.
      *
-     * @param inputFile   The path to the output document.
-     * @param outputFile  The path to the output document.
+     * @param inputFile The path to the output document.
+     * @param outputFile The path to the output document.
      * @param mappingFile The path to the mapping schema.
      * @return The exit code of the plugin.
-     * 
+     *
      * @throws MappingPluginException If the mapping execution fails.
      */
     MappingPluginState mapFile(Path mappingFile, Path inputFile, Path outputFile) throws MappingPluginException;

src/main/java/edu/kit/datamanager/mappingservice/plugins/PluginLoader.java

@@ -100,12 +100,12 @@ public Map<String, IMappingPlugin> loadPlugins(File pluginDir, String[] packages
             cl = Thread.currentThread().getContextClassLoader();
         }
 
-        List<Class<IMappingPlugin>> plugClasses = extractClassesFromJARs(pluginJars, packagesToScan, cl);
-        List<IMappingPlugin> IMappingPluginList = createPluggableObjects(plugClasses);
+        List<Class<IMappingPlugin>> pluginClasses = extractClassesFromJARs(pluginJars, packagesToScan, cl);
+        List<IMappingPlugin> IMappingPluginList = createPluggableObjects(pluginClasses);
 
         for (IMappingPlugin i : IMappingPluginList) {
             try {
-                i.setup();
+                i.setup(applicationProperties);
                 LOG.trace(" - Adding new plugin {}, v{} to available list", i.name(), i.version());
                 result.put(i.id(), i);
             } catch (PluginInitializationFailedException re) {

src/main/java/edu/kit/datamanager/mappingservice/plugins/impl/IdentifyPlugin.java

@@ -15,6 +15,7 @@
  */
 package edu.kit.datamanager.mappingservice.plugins.impl;
 
+import edu.kit.datamanager.mappingservice.configuration.ApplicationProperties;
 import edu.kit.datamanager.mappingservice.exception.PluginInitializationFailedException;
 import edu.kit.datamanager.mappingservice.plugins.IMappingPlugin;
 import edu.kit.datamanager.mappingservice.plugins.MappingPluginException;
@@ -71,7 +72,7 @@ public MimeType[] outputTypes() {
     }
 
     @Override
-    public void setup() {
+    public void setup(ApplicationProperties applicationProperties) {
         if (Paths.get("/usr/bin/identify").toFile().exists()) {
             initialized = true;
         } else {

src/main/java/edu/kit/datamanager/mappingservice/plugins/impl/InOutPlugin.java

@@ -15,6 +15,7 @@
  */
 package edu.kit.datamanager.mappingservice.plugins.impl;
 
+import edu.kit.datamanager.mappingservice.configuration.ApplicationProperties;
 import edu.kit.datamanager.mappingservice.exception.MappingException;
 import edu.kit.datamanager.mappingservice.plugins.IMappingPlugin;
 import edu.kit.datamanager.mappingservice.plugins.MappingPluginException;
@@ -66,7 +67,7 @@ public MimeType[] outputTypes() {
     }
 
     @Override
-    public void setup() {
+    public void setup(ApplicationProperties applicationProperties) {
         //nothing to do here
         LOG.trace("Plugin {} {} successfully set up.", name(), version());
     }

src/main/java/edu/kit/datamanager/mappingservice/util/FileUtil.java

@@ -294,27 +294,34 @@ private static String guessFileExtension(String filename, byte[] fewKilobytesOfF
     public static Path cloneGitRepository(String repositoryUrl, String branch, String targetFolder) {
         File target = new File(targetFolder);
         if (target.exists()) {
+            Git g = null;
             try {
-                try (Git g = Git.open(target)) {
-                    LOGGER.trace("Repository already exists at {}. Active branch is: {}", target, g.getRepository().getBranch());
-                    g.getRepository().close();
-                }
+                g = Git.open(target);
+                LOGGER.trace("Repository already exists at {}. Active branch is: {}", target, g.getRepository().getBranch());
             } catch (IOException e) {
                 String message = String.format("Folder '%s' already exists but contains not Git repository.", target);
                 LOGGER.error(message, e);
                 throw new MappingServiceException("Failed to prepare plugin. Plugin code destination already exists but is empty.");
+            } finally {
+                if (g != null) {
+                    g.getRepository().close();
+                }
             }
         } else {
             target.mkdirs();
 
             LOGGER.info("Cloning branch '{}' of repository '{}' to '{}'", branch, repositoryUrl, target.getPath());
+            Git g = null;
             try {
-                try (Git res = Git.cloneRepository().setURI(repositoryUrl).setBranch(branch).setDirectory(target).call()) {
-                    res.getRepository().close();
-                }
+                g = Git.cloneRepository().setURI(repositoryUrl).setBranch(branch).setDirectory(target).call();
+                LOGGER.trace("Repository successfully cloned to {}.", target);
             } catch (JGitInternalException | GitAPIException e) {
                 LOGGER.error("Error cloning git repository '" + repositoryUrl + "' to '" + target + "'!", e);
                 throw new MappingServiceException("Failed to prepare plugin. Plugin code destination not accessible.");
+            } finally {
+                if (g != null) {
+                    g.getRepository().close();
+                }
             }
         }
         return target.toPath();

src/main/resources/gemma.properties

@@ -1,2 +1,2 @@
-version=master
+version=v1.0.0
 min.python=3.0.0

src/test/java/edu/kit/datamanager/mappingservice/plugins/PluginLoaderTest.java

@@ -73,7 +73,7 @@ void valid() {
             assertEquals("https://github.com/kit-data-manager/mapping-service", plugins.get("InOutPlugin_1.1.2").uri());
             assertEquals("application/*", plugins.get("InOutPlugin_1.1.2").inputTypes()[0].toString());
             assertEquals("application/*", plugins.get("InOutPlugin_1.1.2").outputTypes()[0].toString());
-            plugins.get("InOutPlugin_1.1.2").setup();
+            plugins.get("InOutPlugin_1.1.2").setup(applicationProperties);
             File inputFile = new File("/tmp/imputFile");
             if (!inputFile.exists()) {
                 Assertions.assertTrue(inputFile.createNewFile());

src/test/java/edu/kit/datamanager/mappingservice/rest/impl/MappingAdministrationControllerTest.java

@@ -92,7 +92,7 @@ public class MappingAdministrationControllerTest {
 
     private final static String TEMP_DIR_4_MAPPING = "/tmp/mapping-service/";
     private static final String MAPPING_ID = "my_dc";
-    private static final String MAPPING_TYPE = "GEMMA_1.0.0";
+    private static final String MAPPING_TYPE = "GEMMA_v1.0.0";
     private static final String MAPPING_TITLE = "TITEL";
     private static final String MAPPING_DESCRIPTION = "DESCRIPTION";
 
@@ -890,7 +890,7 @@ public void testDeleteMappingUnknownMappingId() throws JsonProcessingException,
         result = this.mockMvc.perform(delete(deleteMappingIdUrl).header("If-Match", etag)).andDo(print()).andExpect(status().isNoContent()).andReturn();
         // assertEquals(1, mappingsDir.list().length);
         String expectedFilename = mappingId + "_" + mappingType + ".mapping";
-        assertEquals("my_dc_GEMMA_1.0.0.mapping", expectedFilename);
+        assertEquals("my_dc_" + MAPPING_TYPE + ".mapping", expectedFilename);
         assertEquals(1, mappingRecordDao.count());
     }
 
@@ -935,7 +935,7 @@ public void testDeleteMappingMissingEtag() throws JsonProcessingException, Excep
         result = this.mockMvc.perform(delete(deleteMappingIdUrl)).andDo(print()).andExpect(status().isPreconditionRequired()).andReturn();
         // assertEquals(1, mappingsDir.list().length);
         String expectedFilename = mappingId + "_" + mappingType + ".mapping";
-        assertEquals("my_dc_GEMMA_1.0.0.mapping", expectedFilename);
+        assertEquals("my_dc_" + MAPPING_TYPE + ".mapping", expectedFilename);
         result = this.mockMvc.perform(get(getMappingIdUrl).header("Accept", MappingRecord.MAPPING_RECORD_MEDIA_TYPE)).andDo(print()).andExpect(status().isOk()).andReturn();
         assertEquals(1, mappingRecordDao.count());
     }
@@ -959,7 +959,7 @@ public void testDeleteMappingWrongEtag() throws JsonProcessingException, Excepti
         result = this.mockMvc.perform(delete(deleteMappingIdUrl).header("If-Match", etag)).andDo(print()).andExpect(status().isPreconditionFailed()).andReturn();
         //   assertEquals(1, mappingsDir.list().length);
         String expectedFilename = mappingId + "_" + mappingType + ".mapping";
-        assertEquals("my_dc_GEMMA_1.0.0.mapping", expectedFilename);
+        assertEquals("my_dc_" + MAPPING_TYPE + ".mapping", expectedFilename);
         result = this.mockMvc.perform(get(getMappingIdUrl).header("Accept", MappingRecord.MAPPING_RECORD_MEDIA_TYPE)).andDo(print()).andExpect(status().isOk()).andReturn();
         assertEquals(1, mappingRecordDao.count());
     }